Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(router): skip the whole model for the rest of a request on 404 (from #111, credits @barbotkonv)

Tashfeen6c2a0ba43d03
diff --git a/README.md b/README.md
index a31c8af8..a2299ada 100644
--- a/README.md
+++ b/README.md
@@ -462,6 +462,7 @@ PRs should include a test, keep the existing test suite green, and match the `.e
<a href="https://github.com/m1nuzz"><img src="https://images.weserv.nl/?url=github.com/m1nuzz.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@m1nuzz" /></a>
<a href="https://github.com/LoneRifle"><img src="https://images.weserv.nl/?url=github.com/LoneRifle.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@LoneRifle" /></a>
<a href="https://github.com/ita333"><img src="https://images.weserv.nl/?url=github.com/ita333.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@ita333" /></a>
+<a href="https://github.com/barbotkonv"><img src="https://images.weserv.nl/?url=github.com/barbotkonv.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@barbotkonv" /></a>
## Terms of Service review
diff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.ts
index 2eea16ac..c75f7584 100644
--- a/server/src/__tests__/routes/proxy-retry.test.ts
+++ b/server/src/__tests__/routes/proxy-retry.test.ts
@@ -1,5 +1,19 @@
import { describe, it, expect } from 'vitest';
-import { isRetryableError, isPaymentRequiredError } from '../../routes/proxy.js';
+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError } from '../../routes/proxy.js';
+
+describe('isModelNotFoundError (drives whole-model skip within a request)', () => {
+ it('flags 404 / not-found / no-endpoints phrasings', () => {
+ expect(isModelNotFoundError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(true);
+ expect(isModelNotFoundError(new Error('Model not found'))).toBe(true);
+ expect(isModelNotFoundError(new Error('No endpoints found for openrouter/minimax/minimax-m2.5:free'))).toBe(true);
+ });
+
+ it('does not flag rate limits, 5xx, or payment errors', () => {
+ expect(isModelNotFoundError(new Error('429 Too Many Requests'))).toBe(false);
+ expect(isModelNotFoundError(new Error('503 Service Unavailable'))).toBe(false);
+ expect(isModelNotFoundError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);
+ });
+});
describe('isRetryableError', () => {
describe('413 Payload Too Large', () => {
diff --git a/server/src/__tests__/services/routing-exhaustion.test.ts b/server/src/__tests__/services/routing-exhaustion.test.ts
index c2df83f9..72097b2d 100644
--- a/server/src/__tests__/services/routing-exhaustion.test.ts
+++ b/server/src/__tests__/services/routing-exhaustion.test.ts
@@ -115,4 +115,43 @@ describe('Routing Key Exhaustion', () => {
const result = routeRequest(100);
expect(result.modelId).toBe('gemini-1.5-flash');
});
+
+ // 404 model-removed handling: a dead model is skipped ENTIRELY for the rest
+ // of the request instead of burning one fallback attempt per key on the same
+ // dead route. (PR #111, credits @barbotkonv.)
+ describe('skipModels (model-level 404 skip)', () => {
+ it('skips every key of a skipped model and routes to the next model', () => {
+ const db = getDb();
+ const proId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-pro'").get().id;
+
+ // Both keys have quota — without skipModels, Pro would be chosen.
+ (ratelimit.canMakeRequest as any).mockReturnValue(true);
+ (ratelimit.canUseTokens as any).mockReturnValue(true);
+
+ const result = routeRequest(100, undefined, undefined, false, false, new Set([proId]));
+ expect(result.modelId).toBe('gemini-1.5-flash');
+ });
+
+ it('throws when every model is in skipModels', () => {
+ const db = getDb();
+ const ids = db.prepare('SELECT id FROM models WHERE enabled = 1').all().map((r: any) => r.id);
+
+ (ratelimit.canMakeRequest as any).mockReturnValue(true);
+ (ratelimit.canUseTokens as any).mockReturnValue(true);
+
+ expect(() => routeRequest(100, undefined, undefined, false, false, new Set(ids))).toThrow();
+ });
+
+ it('overrides a sticky/preferred model that has been skipped', () => {
+ const db = getDb();
+ const proId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-pro'").get().id;
+
+ (ratelimit.canMakeRequest as any).mockReturnValue(true);
+ (ratelimit.canUseTokens as any).mockReturnValue(true);
+
+ // Sticky session prefers Pro, but Pro 404ed earlier in this request.
+ const result = routeRequest(100, undefined, proId, false, false, new Set([proId]));
+ expect(result.modelId).toBe('gemini-1.5-flash');
+ });
+ });
});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index cbb416e1..23881a7b 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -344,6 +344,16 @@ export function isPaymentRequiredError(err: any): boolean {
|| msg.includes('insufficient balance');
}
+// A 404 "model removed/deprecated upstream" error. It's a MODEL-level failure,
+// not a key-level one: every key for the platform will 404 the same way, so the
+// retry loop skips the entire model for the rest of the request instead of
+// burning one fallback attempt per key on the same dead route.
+// (PR #111, credits @barbotkonv.)
+export function isModelNotFoundError(err: any): boolean {
+ const msg = (err.message ?? '').toLowerCase();
+ return msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found');
+}
+
// Pull the incremental text out of a streaming chunk for token counting.
// Must tolerate chunks that carry no `choices` array at all: some providers
// (e.g. Groq) emit usage/keepalive frames shaped like `{usage:{...}}` with no
@@ -609,12 +619,13 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Retry loop: on 429/rate limit, skip that model+key and try the next one
const skipKeys = new Set<string>();
+ const skipModels = new Set<number>();
let lastError: any = null;
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
let route: RouteResult;
try {
- route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage, wantsTools);
+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage, wantsTools, skipModels.size > 0 ? skipModels : undefined);
} catch (err: any) {
// No more models available
if (lastError) {
@@ -936,6 +947,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError, null, pinnedModelId);
if (isRetryableError(err)) {
+ // Model-level 404 (removed/deprecated upstream): rule the whole model
+ // out for the rest of this request — its other keys would 404 the same
+ // way. The per-key cooldown below still applies, so cross-request
+ // behavior (#66/#76) is unchanged. (PR #111, credits @barbotkonv.)
+ if (isModelNotFoundError(err)) skipModels.add(route.modelDbId);
+
// Put this model+key on cooldown and try the next one
const skipId = `${route.platform}:${route.modelId}:${route.keyId}`;
skipKeys.add(skipId);
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index ce8f7826..f337d82d 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -17,6 +17,7 @@ import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarke
import {
isRetryableError,
isPaymentRequiredError,
+ isModelNotFoundError,
timingSafeStringEqual,
extractApiToken,
getStickyModel,
@@ -340,6 +341,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
const responseId = newId('resp');
const skipKeys = new Set<string>();
+ const skipModels = new Set<number>();
let lastError: any = null;
// Stream bookkeeping (used only when stream === true).
@@ -353,7 +355,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
let route: RouteResult;
try {
- route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, false, wantsTools);
+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, false, wantsTools, skipModels.size > 0 ? skipModels : undefined);
} catch (err: any) {
const status = lastError ? 429 : (err.status ?? 503);
const message = lastError
@@ -655,6 +657,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
if (isRetryableError(err)) {
+ // Model-level 404: rule out the whole model for this request — its
+ // other keys would 404 the same way. (PR #111, credits @barbotkonv.)
+ if (isModelNotFoundError(err)) skipModels.add(route.modelDbId);
skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
setCooldown(route.platform, route.modelId, route.keyId, isPaymentRequiredError(err)
? PAYMENT_REQUIRED_COOLDOWN_MS
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 8dad647a..3e0e08bb 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -387,7 +387,7 @@ function orderChain(chain: ChainRow[], strategy: RoutingStrategy): ChainRow[] {
* @param requireVision - only consider models that accept image input (#118)
* @param requireTools - only consider models that emit structured tool_calls
*/
-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false): RouteResult {
+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>): RouteResult {
const db = getDb();
const strategy = getRoutingStrategy();
@@ -417,6 +417,12 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
}
for (const entry of sortedChain) {
+ // Models the caller has ruled out for this request — e.g. a 404
+ // "model removed upstream" already seen this request: trying the same
+ // model again on a different key would just burn another attempt on the
+ // same dead route (PR #111, credits @barbotkonv).
+ if (skipModels?.has(entry.model_db_id)) continue;
+
// Vision requests skip text-only models — including a sticky/preferred one,
// which is correct: don't pin an image turn to a model that can't see it.
if (requireVision && !entry.supports_vision) continue;