7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(fallback): short-circuit a platform after distinct model_not_found misses (#1222)
suantea (asuan_suan)2a54c980d3ff
diff --git a/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts b/server/src/__tests__/lib/fallback-loop-account-suspended.test.tsindex c851d98e..477d5be3 100644--- a/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts+++ b/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts@@ -166,3 +166,56 @@ describe('a suspended account benches the whole key, not one model', () => {expect(classifyAttemptError(SUSPENDED())).toBe('forbidden');});});++// A stale CATALOG misses on every sibling model in a row (#1218): the per-model+// skip can't see the pattern (each miss is a different model), so from+// MODEL_NOT_FOUND_PLATFORM_LIMIT distinct model_not_found hops on one platform+// — within one request — the whole platform is ruled out for the rest of it.+describe('distinct model_not_found hops short-circuit the platform within one request', () => {+ // Plain phrasing that the EXISTING isModelNotFoundError substring checks+ // already match — this PR is independent of the quoted-id regex widening+ // (#1219); together the two cover NavyAI's exact wording.+ const notFoundErr = () => Object.assign(+ new Error('NavyAI API error 400: model does not exist'),+ { status: 400 },+ );++ it('rules the platform out from the Nth DISTINCT model miss', () => {+ const state = newFallbackState();+ recordRetryableFailure(routeFor(models[0], keyA), notFoundErr(), state);+ recordRetryableFailure(routeFor(models[1], keyA), notFoundErr(), state);+ // Under the limit: the platform stays reachable for the next hop.+ expect(state.skipPlatforms.has(PLATFORM)).toBe(false);+ recordRetryableFailure(routeFor(models[2], keyA), notFoundErr(), state);+ expect(state.skipPlatforms.has(PLATFORM)).toBe(true);+ });++ it('repeated misses on the SAME model do not count toward the limit', () => {+ const state = newFallbackState();+ for (let i = 0; i < 5; i++) {+ recordRetryableFailure(routeFor(models[0], keyA), notFoundErr(), state);+ }+ expect(state.skipPlatforms.has(PLATFORM)).toBe(false);+ });++ it('never rules out the shared custom platform: each relay has its own catalog', () => {+ const state = newFallbackState();+ // Three different relays, one stale model each. They all carry the platform+ // id 'custom', so a platform-wide skip would take healthy relays with it.+ models.slice(0, 3).forEach((model, i) => {+ const route = { ...routeFor(model, keyA), platform: 'custom', endpointScope: `https://relay-${i}.example/v1` };+ recordRetryableFailure(route, notFoundErr(), state);+ });+ expect(state.skipPlatforms.has('custom')).toBe(false);+ // The per-model skip still applies.+ expect(state.skipModels.size).toBe(3);+ });++ it('misses on sibling keys of the same platform tally together (the catalog is shared)', () => {+ const state = newFallbackState();+ recordRetryableFailure(routeFor(models[0], keyA), notFoundErr(), state);+ recordRetryableFailure(routeFor(models[1], keyB), notFoundErr(), state);+ recordRetryableFailure(routeFor(models[2], keyA), notFoundErr(), state);+ expect(state.skipPlatforms.has(PLATFORM)).toBe(true);+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 6ec28d7f..41e083ab 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -212,6 +212,12 @@ export interface FallbackState {// reading is the same request, not a smaller one.observedTotalTokens?: number;observedInputTokens?: number;+ // Per-request tally of DISTINCT models that answered model_not_found, keyed+ // by platform (#1218): a stale catalog misses on every sibling model in a+ // row, and the per-model skip can't see the pattern. From+ // MODEL_NOT_FOUND_PLATFORM_LIMIT distinct misses the platform joins+ // skipPlatforms for the rest of the request.+ modelNotFoundPlatforms: Map<string, Set<number>>;}/** Total for the next dispatch. Input-only observations still need output space. */@@ -224,9 +230,16 @@ export function fallbackRoutingTokens(state: FallbackState, estimatedTotal: numb}export function newFallbackState(): FallbackState {- return { skipKeys: new Set<string>(), skipModels: new Set<number>(), skipPlatforms: new Set<string>() };+ return { skipKeys: new Set<string>(), skipModels: new Set<number>(), skipPlatforms: new Set<string>(), modelNotFoundPlatforms: new Map() };}+// DISTINCT model_not_found hops on one platform, within one request, before the+// whole platform is ruled out: a healthy catalog never misses three different+// models in a single failover chain — a stale/broken one does it routinely+// (NavyAI: 24 models tried in one session, 0 ok, #1218). 3 keeps one fluke+// removal from condemning a provider.+export const MODEL_NOT_FOUND_PLATFORM_LIMIT = 3;+// Milliseconds until the next UTC midnight — when most providers' daily free// allocations reset. Floored at one minute so a hit seconds before midnight// still records a real bench instead of a no-op.@@ -398,6 +411,25 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fallif (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || isContextTooLargeError(err) || err?.skipModelForRequest === true) {state.skipModels.add(route.modelDbId);}+ // A stale CATALOG (not a single dead model) answers model-not-found on every+ // sibling model in a row: NavyAI served 24 models over one session, 0 ok,+ // each re-paying a 2.5–10s round trip (#1218). The per-model skip above+ // can't see the pattern — each miss is a different model. Count DISTINCT+ // model_not_found hops per platform within one request; from the threshold,+ // rule the whole platform out for the rest of the request: the catalog (or+ // the key's access to it) is broken, and the next PROVIDER is the better+ // hop. Same request-scoped lifetime as skipModels (#111/#256 semantics).+ // Custom relays are exempt: every relay shares the one platform id 'custom'+ // (#651), so three misses spread over three different relays would rule out+ // every healthy relay too. Their misses stay per-model.+ if (isModelNotFoundError(err) && !route.endpointScope) {+ const seen = state.modelNotFoundPlatforms.get(route.platform) ?? new Set<number>();+ seen.add(route.modelDbId);+ state.modelNotFoundPlatforms.set(route.platform, seen);+ if (seen.size >= MODEL_NOT_FOUND_PLATFORM_LIMIT) {+ state.skipPlatforms.add(route.platform);+ }+ }// A model-level 404/410 that says the model is GONE (not merely missing right// now) outlives this request: persist it once the evidence is strong enough,// or the retired model burns a fallback slot on every request forever (#634).