7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(router): skip the whole model for the rest of a request on 404 (from #111, credits @barbotkonv)
Tashfeen6c2a0ba43d03
diff --git a/README.md b/README.mdindex a31c8af8..a2299ada 100644--- a/README.md+++ b/README.md@@ -462,6 +462,7 @@ PRs should include a test, keep the existing test suite green, and match the `.e<a href="https://github.com/m1nuzz"><img src="https://images.weserv.nl/?url=github.com/m1nuzz.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@m1nuzz" /></a><a href="https://github.com/LoneRifle"><img src="https://images.weserv.nl/?url=github.com/LoneRifle.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@LoneRifle" /></a><a href="https://github.com/ita333"><img src="https://images.weserv.nl/?url=github.com/ita333.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@ita333" /></a>+<a href="https://github.com/barbotkonv"><img src="https://images.weserv.nl/?url=github.com/barbotkonv.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@barbotkonv" /></a>## Terms of Service reviewdiff --git a/server/src/__tests__/routes/proxy-retry.test.ts b/server/src/__tests__/routes/proxy-retry.test.tsindex 2eea16ac..c75f7584 100644--- a/server/src/__tests__/routes/proxy-retry.test.ts+++ b/server/src/__tests__/routes/proxy-retry.test.ts@@ -1,5 +1,19 @@import { describe, it, expect } from 'vitest';-import { isRetryableError, isPaymentRequiredError } from '../../routes/proxy.js';+import { isRetryableError, isPaymentRequiredError, isModelNotFoundError } from '../../routes/proxy.js';++describe('isModelNotFoundError (drives whole-model skip within a request)', () => {+ it('flags 404 / not-found / no-endpoints phrasings', () => {+ expect(isModelNotFoundError(new Error('OpenRouter API error 404: Provider returned error'))).toBe(true);+ expect(isModelNotFoundError(new Error('Model not found'))).toBe(true);+ expect(isModelNotFoundError(new Error('No endpoints found for openrouter/minimax/minimax-m2.5:free'))).toBe(true);+ });++ it('does not flag rate limits, 5xx, or payment errors', () => {+ expect(isModelNotFoundError(new Error('429 Too Many Requests'))).toBe(false);+ expect(isModelNotFoundError(new Error('503 Service Unavailable'))).toBe(false);+ expect(isModelNotFoundError(new Error('HuggingFace Router API error 402: Payment required'))).toBe(false);+ });+});describe('isRetryableError', () => {describe('413 Payload Too Large', () => {diff --git a/server/src/__tests__/services/routing-exhaustion.test.ts b/server/src/__tests__/services/routing-exhaustion.test.tsindex c2df83f9..72097b2d 100644--- a/server/src/__tests__/services/routing-exhaustion.test.ts+++ b/server/src/__tests__/services/routing-exhaustion.test.ts@@ -115,4 +115,43 @@ describe('Routing Key Exhaustion', () => {const result = routeRequest(100);expect(result.modelId).toBe('gemini-1.5-flash');});++ // 404 model-removed handling: a dead model is skipped ENTIRELY for the rest+ // of the request instead of burning one fallback attempt per key on the same+ // dead route. (PR #111, credits @barbotkonv.)+ describe('skipModels (model-level 404 skip)', () => {+ it('skips every key of a skipped model and routes to the next model', () => {+ const db = getDb();+ const proId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-pro'").get().id;++ // Both keys have quota — without skipModels, Pro would be chosen.+ (ratelimit.canMakeRequest as any).mockReturnValue(true);+ (ratelimit.canUseTokens as any).mockReturnValue(true);++ const result = routeRequest(100, undefined, undefined, false, false, new Set([proId]));+ expect(result.modelId).toBe('gemini-1.5-flash');+ });++ it('throws when every model is in skipModels', () => {+ const db = getDb();+ const ids = db.prepare('SELECT id FROM models WHERE enabled = 1').all().map((r: any) => r.id);++ (ratelimit.canMakeRequest as any).mockReturnValue(true);+ (ratelimit.canUseTokens as any).mockReturnValue(true);++ expect(() => routeRequest(100, undefined, undefined, false, false, new Set(ids))).toThrow();+ });++ it('overrides a sticky/preferred model that has been skipped', () => {+ const db = getDb();+ const proId = db.prepare("SELECT id FROM models WHERE model_id = 'gemini-1.5-pro'").get().id;++ (ratelimit.canMakeRequest as any).mockReturnValue(true);+ (ratelimit.canUseTokens as any).mockReturnValue(true);++ // Sticky session prefers Pro, but Pro 404ed earlier in this request.+ const result = routeRequest(100, undefined, proId, false, false, new Set([proId]));+ expect(result.modelId).toBe('gemini-1.5-flash');+ });+ });});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex cbb416e1..23881a7b 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -344,6 +344,16 @@ export function isPaymentRequiredError(err: any): boolean {|| msg.includes('insufficient balance');}+// A 404 "model removed/deprecated upstream" error. It's a MODEL-level failure,+// not a key-level one: every key for the platform will 404 the same way, so the+// retry loop skips the entire model for the rest of the request instead of+// burning one fallback attempt per key on the same dead route.+// (PR #111, credits @barbotkonv.)+export function isModelNotFoundError(err: any): boolean {+ const msg = (err.message ?? '').toLowerCase();+ return msg.includes('404') || msg.includes('not found') || msg.includes('no endpoints found');+}+// Pull the incremental text out of a streaming chunk for token counting.// Must tolerate chunks that carry no `choices` array at all: some providers// (e.g. Groq) emit usage/keepalive frames shaped like `{usage:{...}}` with no@@ -609,12 +619,13 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Retry loop: on 429/rate limit, skip that model+key and try the next oneconst skipKeys = new Set<string>();+ const skipModels = new Set<number>();let lastError: any = null;for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {let route: RouteResult;try {- route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage, wantsTools);+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, hasImage, wantsTools, skipModels.size > 0 ? skipModels : undefined);} catch (err: any) {// No more models availableif (lastError) {@@ -936,6 +947,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, latency, safeError, null, pinnedModelId);if (isRetryableError(err)) {+ // Model-level 404 (removed/deprecated upstream): rule the whole model+ // out for the rest of this request — its other keys would 404 the same+ // way. The per-key cooldown below still applies, so cross-request+ // behavior (#66/#76) is unchanged. (PR #111, credits @barbotkonv.)+ if (isModelNotFoundError(err)) skipModels.add(route.modelDbId);+// Put this model+key on cooldown and try the next oneconst skipId = `${route.platform}:${route.modelId}:${route.keyId}`;skipKeys.add(skipId);diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex ce8f7826..f337d82d 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -17,6 +17,7 @@ import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarkeimport {isRetryableError,isPaymentRequiredError,+ isModelNotFoundError,timingSafeStringEqual,extractApiToken,getStickyModel,@@ -340,6 +341,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const responseId = newId('resp');const skipKeys = new Set<string>();+ const skipModels = new Set<number>();let lastError: any = null;// Stream bookkeeping (used only when stream === true).@@ -353,7 +355,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {let route: RouteResult;try {- route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, false, wantsTools);+ route = routeRequest(estimatedTotal, skipKeys.size > 0 ? skipKeys : undefined, preferredModel, false, wantsTools, skipModels.size > 0 ? skipModels : undefined);} catch (err: any) {const status = lastError ? 429 : (err.status ?? 503);const message = lastError@@ -655,6 +657,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}if (isRetryableError(err)) {+ // Model-level 404: rule out the whole model for this request — its+ // other keys would 404 the same way. (PR #111, credits @barbotkonv.)+ if (isModelNotFoundError(err)) skipModels.add(route.modelDbId);skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);setCooldown(route.platform, route.modelId, route.keyId, isPaymentRequiredError(err)? PAYMENT_REQUIRED_COOLDOWN_MSdiff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 8dad647a..3e0e08bb 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -387,7 +387,7 @@ function orderChain(chain: ChainRow[], strategy: RoutingStrategy): ChainRow[] {* @param requireVision - only consider models that accept image input (#118)* @param requireTools - only consider models that emit structured tool_calls*/-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false): RouteResult {+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>): RouteResult {const db = getDb();const strategy = getRoutingStrategy();@@ -417,6 +417,12 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre}for (const entry of sortedChain) {+ // Models the caller has ruled out for this request — e.g. a 404+ // "model removed upstream" already seen this request: trying the same+ // model again on a different key would just burn another attempt on the+ // same dead route (PR #111, credits @barbotkonv).+ if (skipModels?.has(entry.model_db_id)) continue;+// Vision requests skip text-only models — including a sticky/preferred one,// which is correct: don't pin an image turn to a model that can't see it.if (requireVision && !entry.supports_vision) continue;