Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): fail over on model-level 403 instead of 502 (#256)

Tashfeenfb69fbcef193
diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.ts
index c75f7584..cbda67ea 100644
--- a/server/src/__tests__/routes/proxy-retry.test.ts
+++ b/server/src/__tests__/routes/proxy-retry.test.ts
@@ -1,5 +1,23 @@
import { describe, it, expect } from 'vitest';
-import { isRetryableError, isPaymentRequiredError, isModelNotFoundError } from '../../routes/proxy.js';
+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError } from '../../routes/proxy.js';
+
+describe('isModelAccessForbiddenError (403 model-not-on-tier, drives whole-model skip — issue #256)', () => {
+ it('flags a 403 reaching the proxy by message or attached status', () => {
+ // GitHub Models / Cloudflare 403 a model the key's free tier can't reach.
+ expect(isModelAccessForbiddenError(new Error('GitHub Models API error 403: Model not available on your plan'))).toBe(true);
+ expect(isModelAccessForbiddenError(new Error('Cloudflare API error 403: this model requires a subscription'))).toBe(true);
+ expect(isModelAccessForbiddenError(new Error('Forbidden'))).toBe(true);
+ // #261 attaches the upstream status to the thrown error; honor it even if
+ // the message phrasing omits the code.
+ expect(isModelAccessForbiddenError(Object.assign(new Error('access denied'), { status: 403 }))).toBe(true);
+ });
+
+ it('does not flag rate limits, 404s, or payment errors', () => {
+ expect(isModelAccessForbiddenError(new Error('429 Too Many Requests'))).toBe(false);
+ expect(isModelAccessForbiddenError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(false);
+ expect(isModelAccessForbiddenError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);
+ });
+});
describe('isModelNotFoundError (drives whole-model skip within a request)', () => {
it('flags 404 / not-found / no-endpoints phrasings', () => {
@@ -67,6 +85,22 @@ describe('isRetryableError', () => {
});
});
+ describe('403 model not on this key\'s tier fails over instead of 502 (issue #256)', () => {
+ it('treats a 403 from GitHub Models / Cloudflare as retryable', () => {
+ expect(isRetryableError(new Error('GitHub Models API error 403: Model not available on your plan'))).toBe(true);
+ expect(isRetryableError(new Error('Cloudflare API error 403: this model requires a subscription'))).toBe(true);
+ });
+
+ it('treats a bare "Forbidden" / attached 403 status as retryable', () => {
+ expect(isRetryableError(new Error('Forbidden'))).toBe(true);
+ expect(isRetryableError(Object.assign(new Error('access denied'), { status: 403 }))).toBe(true);
+ });
+
+ it('still treats a bare 400 validation error as non-retryable', () => {
+ expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);
+ });
+ });
+
describe('402 Payment Required out-of-credits fails over (graceful degradation)', () => {
it('treats a HuggingFace Router 402 as retryable (same model lives on other providers)', () => {
expect(isRetryableError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(true);
@@ -100,11 +134,14 @@ describe('isRetryableError', () => {
expect(isRetryableError(new Error('ECONNREFUSED'))).toBe(true);
});
- it('4xx auth/validation errors are NOT retryable', () => {
+ it('401 / bare-400 auth & validation errors are NOT retryable', () => {
expect(isRetryableError(new Error('401 Unauthorized'))).toBe(false);
- expect(isRetryableError(new Error('403 Forbidden'))).toBe(false);
expect(isRetryableError(new Error('400 Bad Request'))).toBe(false);
expect(isRetryableError(new Error('Invalid API key'))).toBe(false);
+ // 403 is deliberately NOT here anymore: a request-time 403 on a key that
+ // passed validateKey is a model-not-on-tier gate, so it fails over to the
+ // next model rather than 502-ing the request (issue #256). The 403 cases
+ // are covered in the dedicated describe block above.
});
});
});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 8b6a0b84..1fb5e283 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -4,7 +4,7 @@ import type { Request, Response } from 'express';
import { z } from 'zod';
import type { ChatMessage, ModelListRow } from '@freellmapi/shared/types.js';
import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, type RouteResult } from '../services/router.js';
-import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS } from '../services/ratelimit.js';
+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS } from '../services/ratelimit.js';
import { pruneRequestAnalytics } from '../services/request-retention.js';
import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
import { getDb, getUnifiedApiKey } from '../db/index.js';
@@ -322,6 +322,13 @@ export function isRetryableError(err: any): boolean {
// for a model that's been pulled). Rotate to the next model in the chain —
// setCooldown + the health checker will avoid this model on subsequent requests.
|| msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found')
+ // 403: the key is valid (it passed validateKey, and the health checker
+ // disables truly-forbidden keys) but this specific model is off-limits to
+ // the key's tier — e.g. gpt-4o on GitHub Models' free tier, subscription-only
+ // models on Cloudflare. Another model in the chain is reachable, so fail over
+ // instead of 502-ing the whole request. Paired with isModelAccessForbiddenError
+ // to rule the model out for this request and a day-long bench. See issue #256.
+ || isModelAccessForbiddenError(err)
// 400: one provider may reject parameters another accepts (e.g. max_tokens
// limits, unsupported params). The matching pattern is "api error 400"
// which comes from the OpenAI-compat provider's error formatting, not
@@ -363,6 +370,18 @@ export function isModelNotFoundError(err: any): boolean {
return msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found');
}
+// A 403 Forbidden returned for a specific model behind an otherwise-valid key.
+// Drives the same whole-model skip as a 404: every key on this platform's tier
+// would be forbidden the same model, so rule it out for the rest of the request
+// rather than trying it again with a sibling key. Distinct from a dead key —
+// validateKey returns false on 401/403, so the health checker disables genuinely
+// forbidden keys; a 403 reaching here is model-not-on-this-tier. See issue #256.
+export function isModelAccessForbiddenError(err: any): boolean {
+ if (err?.status === 403) return true;
+ const msg = (err?.message ?? '').toLowerCase();
+ return msg.includes('403') || msg.includes('forbidden');
+}
+
// Pull the incremental text out of a streaming chunk for token counting.
// Must tolerate chunks that carry no `choices` array at all: some providers
// (e.g. Groq) emit usage/keepalive frames shaped like `{usage:{...}}` with no
@@ -959,7 +978,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// out for the rest of this request — its other keys would 404 the same
// way. The per-key cooldown below still applies, so cross-request
// behavior (#66/#76) is unchanged. (PR #111, credits @barbotkonv.)
- if (isModelNotFoundError(err)) skipModels.add(route.modelDbId);
+ // 404 (removed upstream) and 403 (model off-limits to this key's tier)
+ // both rule the model out: a sibling key on the same platform would
+ // fail it identically, so skip it for the rest of this request.
+ if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) skipModels.add(route.modelDbId);
// Put this model+key on cooldown and try the next one
const skipId = `${route.platform}:${route.modelId}:${route.keyId}`;
@@ -970,6 +992,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
route.keyId,
isPaymentRequiredError(err)
? PAYMENT_REQUIRED_COOLDOWN_MS
+ // A 403 won't clear on the next window (it's a tier/subscription gate,
+ // not a transient limit), so bench this model+key for a day like a 402
+ // instead of re-trying it every request. See issue #256.
+ : isModelAccessForbiddenError(err)
+ ? MODEL_FORBIDDEN_COOLDOWN_MS
: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, {
rpd: route.rpdLimit,
tpd: route.tpdLimit,
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index 252ed09c..2f9540d6 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -308,6 +308,14 @@ const TRANSIENT_COOLDOWN_MS = 90 * 1000;
// on the next 402 after expiry if still unpaid; a restart re-benches on first hit.
export const PAYMENT_REQUIRED_COOLDOWN_MS = DAY;
+// Long cooldown for a 403 Forbidden on a key that already passed validateKey
+// (so it is not a dead key — the health checker disables those). A request-time
+// 403 means this key's tier can't reach this specific model (e.g. gpt-4o on
+// GitHub Models' free tier, subscription-only models on Cloudflare). That won't
+// change within a minute window, so bench this model+key for a full day and let
+// the router fail over to a model the key can actually serve. See issue #256.
+export const MODEL_FORBIDDEN_COOLDOWN_MS = DAY;
+
// Decide how long to bench a model+key after an upstream 429. Escalate to the
// long quarantine (getNextCooldownDuration, up to 24h) ONLY when the model is
// genuinely at its DAILY limit (RPD or TPD) — that won't recover until the