Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(fallback): short-circuit a platform after distinct model_not_found misses (#1222)

suantea (asuan_suan)2a54c980d3ff
diff --git a/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts b/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts
index c851d98e..477d5be3 100644
--- a/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts
+++ b/server/src/__tests__/lib/fallback-loop-account-suspended.test.ts
@@ -166,3 +166,56 @@ describe('a suspended account benches the whole key, not one model', () => {
expect(classifyAttemptError(SUSPENDED())).toBe('forbidden');
});
});
+
+// A stale CATALOG misses on every sibling model in a row (#1218): the per-model
+// skip can't see the pattern (each miss is a different model), so from
+// MODEL_NOT_FOUND_PLATFORM_LIMIT distinct model_not_found hops on one platform
+// — within one request — the whole platform is ruled out for the rest of it.
+describe('distinct model_not_found hops short-circuit the platform within one request', () => {
+ // Plain phrasing that the EXISTING isModelNotFoundError substring checks
+ // already match — this PR is independent of the quoted-id regex widening
+ // (#1219); together the two cover NavyAI's exact wording.
+ const notFoundErr = () => Object.assign(
+ new Error('NavyAI API error 400: model does not exist'),
+ { status: 400 },
+ );
+
+ it('rules the platform out from the Nth DISTINCT model miss', () => {
+ const state = newFallbackState();
+ recordRetryableFailure(routeFor(models[0], keyA), notFoundErr(), state);
+ recordRetryableFailure(routeFor(models[1], keyA), notFoundErr(), state);
+ // Under the limit: the platform stays reachable for the next hop.
+ expect(state.skipPlatforms.has(PLATFORM)).toBe(false);
+ recordRetryableFailure(routeFor(models[2], keyA), notFoundErr(), state);
+ expect(state.skipPlatforms.has(PLATFORM)).toBe(true);
+ });
+
+ it('repeated misses on the SAME model do not count toward the limit', () => {
+ const state = newFallbackState();
+ for (let i = 0; i < 5; i++) {
+ recordRetryableFailure(routeFor(models[0], keyA), notFoundErr(), state);
+ }
+ expect(state.skipPlatforms.has(PLATFORM)).toBe(false);
+ });
+
+ it('never rules out the shared custom platform: each relay has its own catalog', () => {
+ const state = newFallbackState();
+ // Three different relays, one stale model each. They all carry the platform
+ // id 'custom', so a platform-wide skip would take healthy relays with it.
+ models.slice(0, 3).forEach((model, i) => {
+ const route = { ...routeFor(model, keyA), platform: 'custom', endpointScope: `https://relay-${i}.example/v1` };
+ recordRetryableFailure(route, notFoundErr(), state);
+ });
+ expect(state.skipPlatforms.has('custom')).toBe(false);
+ // The per-model skip still applies.
+ expect(state.skipModels.size).toBe(3);
+ });
+
+ it('misses on sibling keys of the same platform tally together (the catalog is shared)', () => {
+ const state = newFallbackState();
+ recordRetryableFailure(routeFor(models[0], keyA), notFoundErr(), state);
+ recordRetryableFailure(routeFor(models[1], keyB), notFoundErr(), state);
+ recordRetryableFailure(routeFor(models[2], keyA), notFoundErr(), state);
+ expect(state.skipPlatforms.has(PLATFORM)).toBe(true);
+ });
+});
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 6ec28d7f..41e083ab 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -212,6 +212,12 @@ export interface FallbackState {
// reading is the same request, not a smaller one.
observedTotalTokens?: number;
observedInputTokens?: number;
+ // Per-request tally of DISTINCT models that answered model_not_found, keyed
+ // by platform (#1218): a stale catalog misses on every sibling model in a
+ // row, and the per-model skip can't see the pattern. From
+ // MODEL_NOT_FOUND_PLATFORM_LIMIT distinct misses the platform joins
+ // skipPlatforms for the rest of the request.
+ modelNotFoundPlatforms: Map<string, Set<number>>;
}
/** Total for the next dispatch. Input-only observations still need output space. */
@@ -224,9 +230,16 @@ export function fallbackRoutingTokens(state: FallbackState, estimatedTotal: numb
}
export function newFallbackState(): FallbackState {
- return { skipKeys: new Set<string>(), skipModels: new Set<number>(), skipPlatforms: new Set<string>() };
+ return { skipKeys: new Set<string>(), skipModels: new Set<number>(), skipPlatforms: new Set<string>(), modelNotFoundPlatforms: new Map() };
}
+// DISTINCT model_not_found hops on one platform, within one request, before the
+// whole platform is ruled out: a healthy catalog never misses three different
+// models in a single failover chain — a stale/broken one does it routinely
+// (NavyAI: 24 models tried in one session, 0 ok, #1218). 3 keeps one fluke
+// removal from condemning a provider.
+export const MODEL_NOT_FOUND_PLATFORM_LIMIT = 3;
+
// Milliseconds until the next UTC midnight — when most providers' daily free
// allocations reset. Floored at one minute so a hit seconds before midnight
// still records a real bench instead of a no-op.
@@ -398,6 +411,25 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || isContextTooLargeError(err) || err?.skipModelForRequest === true) {
state.skipModels.add(route.modelDbId);
}
+ // A stale CATALOG (not a single dead model) answers model-not-found on every
+ // sibling model in a row: NavyAI served 24 models over one session, 0 ok,
+ // each re-paying a 2.5–10s round trip (#1218). The per-model skip above
+ // can't see the pattern — each miss is a different model. Count DISTINCT
+ // model_not_found hops per platform within one request; from the threshold,
+ // rule the whole platform out for the rest of the request: the catalog (or
+ // the key's access to it) is broken, and the next PROVIDER is the better
+ // hop. Same request-scoped lifetime as skipModels (#111/#256 semantics).
+ // Custom relays are exempt: every relay shares the one platform id 'custom'
+ // (#651), so three misses spread over three different relays would rule out
+ // every healthy relay too. Their misses stay per-model.
+ if (isModelNotFoundError(err) && !route.endpointScope) {
+ const seen = state.modelNotFoundPlatforms.get(route.platform) ?? new Set<number>();
+ seen.add(route.modelDbId);
+ state.modelNotFoundPlatforms.set(route.platform, seen);
+ if (seen.size >= MODEL_NOT_FOUND_PLATFORM_LIMIT) {
+ state.skipPlatforms.add(route.platform);
+ }
+ }
// A model-level 404/410 that says the model is GONE (not merely missing right
// now) outlives this request: persist it once the evidence is strong enough,
// or the retired model burns a fallback slot on every request forever (#634).