Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(output-cap): key learned output ceilings per endpoint, not just model (#1532)

tashdroid53f01e871a81
diff --git a/server/src/__tests__/lib/module-purity.test.ts b/server/src/__tests__/lib/module-purity.test.ts
index bad6b841..7592e9fd 100644
--- a/server/src/__tests__/lib/module-purity.test.ts
+++ b/server/src/__tests__/lib/module-purity.test.ts
@@ -38,8 +38,6 @@ const PURE_MODULES = [
// Signal handling for the container's PID 1; the server, DB and process are injected.
'graceful-shutdown.ts',
'header-value.ts',
- // Learned output ceilings; the fallback loop and every route import it.
- 'output-cap.ts',
'provider-identity.ts',
'provider-size-parser.ts',
'retry-hint.ts',
diff --git a/server/src/__tests__/lib/output-cap.test.ts b/server/src/__tests__/lib/output-cap.test.ts
index e2688237..cc84127e 100644
--- a/server/src/__tests__/lib/output-cap.test.ts
+++ b/server/src/__tests__/lib/output-cap.test.ts
@@ -72,3 +72,38 @@ describe('learned output caps', () => {
expect(routeOutputBudget({ ...route, contextWindow: null }, 20000)).toBe(65536);
});
});
+
+describe('learned caps are per endpoint (#651 identity, #1230 precedent)', () => {
+ // Two custom relays serving the same model id share platform 'custom'; only
+ // endpoint_scope tells them apart. A ceiling learned from relay A must not
+ // clamp relay B's replies (and a catalog platform must never be clamped by
+ // anything a relay learned).
+ const relayA = { platform: 'custom', modelId: 'deepseek-v3.1', contextWindow: 131072, endpointScope: 'https://relay-a.example.com/v1' };
+ const relayB = { platform: 'custom', modelId: 'deepseek-v3.1', contextWindow: 131072, endpointScope: 'https://relay-b.example.com/v1' };
+ const rejection = (ceiling: number) => Object.assign(
+ new Error(`max_tokens (128000) exceeds model's maximum output tokens (${ceiling})`),
+ { status: 400 },
+ );
+
+ it('keeps one relay ceiling out of a sibling relay on the same model id', () => {
+ expect(learnOutputCapFromError(relayA, rejection(32768))).toBe(32768);
+ expect(routeOutputBudget(relayA, 1000)).toBe(32768);
+ // Relay B serves the same model id but never rejected anything.
+ expect(learnedOutputCap(relayB.platform, relayB.modelId, relayB.endpointScope)).toBeUndefined();
+ expect(routeOutputBudget(relayB, 1000)).toBe(130072);
+ });
+
+ it('keeps a relay ceiling out of the catalog row for the same model id', () => {
+ learnOutputCapFromError(relayA, rejection(32768));
+ expect(learnedOutputCap('custom', 'deepseek-v3.1')).toBeUndefined();
+ expect(routeOutputBudget({ platform: 'custom', modelId: 'deepseek-v3.1', contextWindow: 131072 }, 1000)).toBe(130072);
+ });
+
+ it('treats a missing scope and an empty scope as the same identity', () => {
+ learnOutputCapFromError({ platform: 'custom', modelId: 'm', endpointScope: '' }, rejection(8192));
+ expect(learnedOutputCap('custom', 'm', undefined)).toBe(8192);
+ expect(learnedOutputCap('custom', 'm', null)).toBe(8192);
+ // And a scoped reading never leaks into the un-scoped identity either.
+ expect(learnedOutputCap('custom', 'm', 'https://relay-x.example.com')).toBeUndefined();
+ });
+});
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index ac725f5c..eb2bf24e 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -519,7 +519,7 @@ export function recordRetryableFailure(route: RouteResult, err: any, state: Fall
// learned just now makes the same route usable again at once (the next
// attempt is clamped to it); one that was already applied and still got
// rejected rules the model out for this request.
- const capBefore = learnedOutputCap(route.platform, route.modelId);
+ const capBefore = learnedOutputCap(route.platform, route.modelId, route.endpointScope);
const ceiling = learnOutputCapFromError(route, err);
if (ceiling != null) {
if (capBefore == null || ceiling < capBefore) {
diff --git a/server/src/lib/output-cap.ts b/server/src/lib/output-cap.ts
index 18b13295..c02dc4b2 100644
--- a/server/src/lib/output-cap.ts
+++ b/server/src/lib/output-cap.ts
@@ -12,6 +12,14 @@
// per-route output budget (resolveMaxTokens clamps max_tokens to it). A
// restart forgets the ceilings; the first request after it pays one failover
// hop to relearn each one.
+//
+// Keyed per model AND per endpoint (`modelStatsKey`), not per model: every
+// custom relay shares platform 'custom', so a ceiling learned from relay A
+// would silently clamp relay B's replies to relay A's output limit until
+// restart. Same reasoning as lib/tool-capability.ts (#1230) and the
+// endpoint_scope identity migration (#651): output ceilings are a property of
+// the model behind ONE relay, and sibling relays do not share it.
+import { modelStatsKey } from './endpoint-scope.js';
const learned = new Map<string, number>();
@@ -32,8 +40,8 @@ const CEILING = new RegExp(`\\b(?:max_tokens|max_completion_tokens|max_output_to
// wrong low cap would truncate every reply on the model until restart.
export const MIN_LEARNED_OUTPUT_CAP = 1024;
-function capKey(platform: string, modelId: string): string {
- return `${platform}:${modelId}`;
+function capKey(platform: string, modelId: string, endpointScope?: string | null): string {
+ return modelStatsKey(platform, modelId, endpointScope);
}
/** The output ceiling a provider names when it rejects max_tokens as too
@@ -52,30 +60,30 @@ export function parseMaxTokensCeiling(message: unknown): number | null {
/** Learn the ceiling from a max_tokens rejection. Returns it, or null when the
* error is not one, or when the reading is below MIN_LEARNED_OUTPUT_CAP. The
* lowest reading wins. */
-export function learnOutputCapFromError(route: { platform: string; modelId: string }, err: any): number | null {
+export function learnOutputCapFromError(route: { platform: string; modelId: string; endpointScope?: string | null }, err: any): number | null {
const status = typeof err?.status === 'number' ? err.status : 0;
if (status !== 0 && status !== 400 && status !== 422) return null;
const ceiling = parseMaxTokensCeiling(err?.message);
if (ceiling == null || ceiling < MIN_LEARNED_OUTPUT_CAP) return null;
- const key = capKey(route.platform, route.modelId);
+ const key = capKey(route.platform, route.modelId, route.endpointScope);
const current = learned.get(key);
learned.set(key, current == null ? ceiling : Math.min(current, ceiling));
return ceiling;
}
-export function learnedOutputCap(platform: string, modelId: string): number | undefined {
- return learned.get(capKey(platform, modelId));
+export function learnedOutputCap(platform: string, modelId: string, endpointScope?: string | null): number | undefined {
+ return learned.get(capKey(platform, modelId, endpointScope));
}
/** The output budget for one route: what is left of its context window after
* the input, lowered to any learned output ceiling. Adapters pass it to
* resolveMaxTokens as `contextBudget`. */
export function routeOutputBudget(
- route: { platform: string; modelId: string; contextWindow?: number | null },
+ route: { platform: string; modelId: string; contextWindow?: number | null; endpointScope?: string | null },
estimatedInputTokens: number,
): number | undefined {
const windowLeft = route.contextWindow != null ? route.contextWindow - estimatedInputTokens : undefined;
- const cap = learnedOutputCap(route.platform, route.modelId);
+ const cap = learnedOutputCap(route.platform, route.modelId, route.endpointScope);
if (cap == null) return windowLeft;
return windowLeft == null ? cap : Math.min(windowLeft, cap);
}