7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(output-cap): key learned output ceilings per endpoint, not just model (#1532)
tashdroid53f01e871a81
diff --git a/server/src/__tests__/lib/module-purity.test.ts b/server/src/__tests__/lib/module-purity.test.tsindex bad6b841..7592e9fd 100644--- a/server/src/__tests__/lib/module-purity.test.ts+++ b/server/src/__tests__/lib/module-purity.test.ts@@ -38,8 +38,6 @@ const PURE_MODULES = [// Signal handling for the container's PID 1; the server, DB and process are injected.'graceful-shutdown.ts','header-value.ts',- // Learned output ceilings; the fallback loop and every route import it.- 'output-cap.ts','provider-identity.ts','provider-size-parser.ts','retry-hint.ts',diff --git a/server/src/__tests__/lib/output-cap.test.ts b/server/src/__tests__/lib/output-cap.test.tsindex e2688237..cc84127e 100644--- a/server/src/__tests__/lib/output-cap.test.ts+++ b/server/src/__tests__/lib/output-cap.test.ts@@ -72,3 +72,38 @@ describe('learned output caps', () => {expect(routeOutputBudget({ ...route, contextWindow: null }, 20000)).toBe(65536);});});++describe('learned caps are per endpoint (#651 identity, #1230 precedent)', () => {+ // Two custom relays serving the same model id share platform 'custom'; only+ // endpoint_scope tells them apart. A ceiling learned from relay A must not+ // clamp relay B's replies (and a catalog platform must never be clamped by+ // anything a relay learned).+ const relayA = { platform: 'custom', modelId: 'deepseek-v3.1', contextWindow: 131072, endpointScope: 'https://relay-a.example.com/v1' };+ const relayB = { platform: 'custom', modelId: 'deepseek-v3.1', contextWindow: 131072, endpointScope: 'https://relay-b.example.com/v1' };+ const rejection = (ceiling: number) => Object.assign(+ new Error(`max_tokens (128000) exceeds model's maximum output tokens (${ceiling})`),+ { status: 400 },+ );++ it('keeps one relay ceiling out of a sibling relay on the same model id', () => {+ expect(learnOutputCapFromError(relayA, rejection(32768))).toBe(32768);+ expect(routeOutputBudget(relayA, 1000)).toBe(32768);+ // Relay B serves the same model id but never rejected anything.+ expect(learnedOutputCap(relayB.platform, relayB.modelId, relayB.endpointScope)).toBeUndefined();+ expect(routeOutputBudget(relayB, 1000)).toBe(130072);+ });++ it('keeps a relay ceiling out of the catalog row for the same model id', () => {+ learnOutputCapFromError(relayA, rejection(32768));+ expect(learnedOutputCap('custom', 'deepseek-v3.1')).toBeUndefined();+ expect(routeOutputBudget({ platform: 'custom', modelId: 'deepseek-v3.1', contextWindow: 131072 }, 1000)).toBe(130072);+ });++ it('treats a missing scope and an empty scope as the same identity', () => {+ learnOutputCapFromError({ platform: 'custom', modelId: 'm', endpointScope: '' }, rejection(8192));+ expect(learnedOutputCap('custom', 'm', undefined)).toBe(8192);+ expect(learnedOutputCap('custom', 'm', null)).toBe(8192);+ // And a scoped reading never leaks into the un-scoped identity either.+ expect(learnedOutputCap('custom', 'm', 'https://relay-x.example.com')).toBeUndefined();+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex ac725f5c..eb2bf24e 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -519,7 +519,7 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall// learned just now makes the same route usable again at once (the next// attempt is clamped to it); one that was already applied and still got// rejected rules the model out for this request.- const capBefore = learnedOutputCap(route.platform, route.modelId);+ const capBefore = learnedOutputCap(route.platform, route.modelId, route.endpointScope);const ceiling = learnOutputCapFromError(route, err);if (ceiling != null) {if (capBefore == null || ceiling < capBefore) {diff --git a/server/src/lib/output-cap.ts b/server/src/lib/output-cap.tsindex 18b13295..c02dc4b2 100644--- a/server/src/lib/output-cap.ts+++ b/server/src/lib/output-cap.ts@@ -12,6 +12,14 @@// per-route output budget (resolveMaxTokens clamps max_tokens to it). A// restart forgets the ceilings; the first request after it pays one failover// hop to relearn each one.+//+// Keyed per model AND per endpoint (`modelStatsKey`), not per model: every+// custom relay shares platform 'custom', so a ceiling learned from relay A+// would silently clamp relay B's replies to relay A's output limit until+// restart. Same reasoning as lib/tool-capability.ts (#1230) and the+// endpoint_scope identity migration (#651): output ceilings are a property of+// the model behind ONE relay, and sibling relays do not share it.+import { modelStatsKey } from './endpoint-scope.js';const learned = new Map<string, number>();@@ -32,8 +40,8 @@ const CEILING = new RegExp(`\\b(?:max_tokens|max_completion_tokens|max_output_to// wrong low cap would truncate every reply on the model until restart.export const MIN_LEARNED_OUTPUT_CAP = 1024;-function capKey(platform: string, modelId: string): string {- return `${platform}:${modelId}`;+function capKey(platform: string, modelId: string, endpointScope?: string | null): string {+ return modelStatsKey(platform, modelId, endpointScope);}/** The output ceiling a provider names when it rejects max_tokens as too@@ -52,30 +60,30 @@ export function parseMaxTokensCeiling(message: unknown): number | null {/** Learn the ceiling from a max_tokens rejection. Returns it, or null when the* error is not one, or when the reading is below MIN_LEARNED_OUTPUT_CAP. The* lowest reading wins. */-export function learnOutputCapFromError(route: { platform: string; modelId: string }, err: any): number | null {+export function learnOutputCapFromError(route: { platform: string; modelId: string; endpointScope?: string | null }, err: any): number | null {const status = typeof err?.status === 'number' ? err.status : 0;if (status !== 0 && status !== 400 && status !== 422) return null;const ceiling = parseMaxTokensCeiling(err?.message);if (ceiling == null || ceiling < MIN_LEARNED_OUTPUT_CAP) return null;- const key = capKey(route.platform, route.modelId);+ const key = capKey(route.platform, route.modelId, route.endpointScope);const current = learned.get(key);learned.set(key, current == null ? ceiling : Math.min(current, ceiling));return ceiling;}-export function learnedOutputCap(platform: string, modelId: string): number | undefined {- return learned.get(capKey(platform, modelId));+export function learnedOutputCap(platform: string, modelId: string, endpointScope?: string | null): number | undefined {+ return learned.get(capKey(platform, modelId, endpointScope));}/** The output budget for one route: what is left of its context window after* the input, lowered to any learned output ceiling. Adapters pass it to* resolveMaxTokens as `contextBudget`. */export function routeOutputBudget(- route: { platform: string; modelId: string; contextWindow?: number | null },+ route: { platform: string; modelId: string; contextWindow?: number | null; endpointScope?: string | null },estimatedInputTokens: number,): number | undefined {const windowLeft = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;- const cap = learnedOutputCap(route.platform, route.modelId);+ const cap = learnedOutputCap(route.platform, route.modelId, route.endpointScope);if (cap == null) return windowLeft;return windowLeft == null ? cap : Math.min(windowLeft, cap);}