7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(gateway): request guardrails - per-request token budget + failover circuit breaker (#511)
Tashfeencdce07d193fc
diff --git a/.env.example b/.env.exampleindex 78cfc33b..c576c562 100644--- a/.env.example+++ b/.env.example@@ -46,6 +46,21 @@ PORT=3001# fallback_time_budget_ms settings key, which takes precedence.# FALLBACK_TIME_BUDGET_MS=45000+# Request guardrails, both OFF (0) by default. Also runtime-tunable via the+# request_max_tokens_budget / max_consecutive_upstream_fails settings keys+# (dashboard API PUT /api/settings/guardrails), which take precedence.+#+# Per-request token budget: estimated input + requested max_tokens must fit+# this ceiling or the request is rejected with a 413 before any provider is+# tried; a request that sent no max_tokens gets its output capped to the+# remainder instead.+# REQUEST_MAX_TOKENS_BUDGET=0+#+# Failover circuit breaker: after this many consecutive upstream failures in+# one request, stop the failover chain with a 503 instead of trying every+# remaining candidate of an unhealthy pool.+# MAX_CONSECUTIVE_UPSTREAM_FAILS=0+# Opt-in response cache. When on, a successful non-streaming /v1/chat/completions# answer is stored in a bounded in-memory LRU keyed by a canonical hash of the# request, so an IDENTICAL later request is served from memory without spendingdiff --git a/README.md b/README.mdindex 9f535328..15f906d8 100644--- a/README.md+++ b/README.md@@ -795,6 +795,7 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full migration CLI and workflow<a href="https://github.com/danscMax"><img src="https://images.weserv.nl/?url=github.com/danscMax.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@danscMax" /></a><a href="https://github.com/jhash"><img src="https://images.weserv.nl/?url=github.com/jhash.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@jhash" /></a><a href="https://github.com/JammyJames1234"><img src="https://images.weserv.nl/?url=github.com/JammyJames1234.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@JammyJames1234" /></a>+<a href="https://github.com/coffcoe"><img src="https://images.weserv.nl/?url=github.com/coffcoe.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@coffcoe" /></a><a href="https://github.com/Sumit4codes"><img src="https://images.weserv.nl/?url=github.com/Sumit4codes.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@Sumit4codes" /></a><a href="https://github.com/meliani"><img src="https://images.weserv.nl/?url=github.com/meliani.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@meliani" /></a><a href="https://github.com/thedavidweng"><img src="https://images.weserv.nl/?url=github.com/thedavidweng.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@thedavidweng" /></a>diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.tsindex 091d9842..eb8bb20e 100644--- a/server/src/__tests__/lib/fallback-loop.test.ts+++ b/server/src/__tests__/lib/fallback-loop.test.ts@@ -349,3 +349,68 @@ describe('runFallbackLoop: dispatch outcome contract', () => {consoleError.mockRestore();});});++describe('runFallbackLoop: circuit-breaker guardrail (max_consecutive_upstream_fails)', () => {+ const retryable429 = () => Object.assign(new Error('429 Too Many Requests'), { status: 429 });++ it('is off by default: a doomed chain still runs to the attempt cap', async () => {+ const onExhausted = vi.fn();+ const dispatch = vi.fn(async () => { throw retryable429(); });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 5, dispatch, onExhausted }));++ expect(dispatch).toHaveBeenCalledTimes(5);+ expect(onExhausted).toHaveBeenCalledTimes(1);+ expect(onExhausted.mock.calls[0][0].status).toBe(429); // ordinary exhaustion, no breaker+ });++ it('trips after N consecutive retryable failures and renders a 503 with the trail', async () => {+ const onExhausted = vi.fn();+ const dispatch = vi.fn(async () => { throw retryable429(); });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 3, dispatch, onExhausted }));++ expect(dispatch).toHaveBeenCalledTimes(3); // stopped at the limit, not the cap+ expect(onExhausted).toHaveBeenCalledTimes(1);+ const [body, info] = onExhausted.mock.calls[0];+ expect(body.kind).toBe('unavailable');+ expect(body.status).toBe(503);+ expect(body.type).toBe('service_unavailable');+ expect(body.message).toContain('circuit-breaker');+ expect(body.message).toContain('3 consecutive upstream failures');+ expect(body.message).toContain('Attempt trail:');+ expect(info.attempts).toHaveLength(3);+ expect(info.timedOut).toBe(false);+ });++ it('auth failures count toward the breaker, and an all-auth trip keeps the 502 diagnosis', async () => {+ mockCheckKeyHealth.mockResolvedValue(undefined);+ const onExhausted = vi.fn();+ const dispatch = vi.fn(async () => {+ throw Object.assign(new Error('Unauthorized'), { status: 401 });+ });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 2, dispatch, onExhausted }));++ expect(dispatch).toHaveBeenCalledTimes(2); // breaker stopped the rotation+ const [body, info] = onExhausted.mock.calls[0];+ expect(body.kind).toBe('auth'); // the more specific all-auth body wins over the breaker body+ expect(body.status).toBe(502);+ expect(info.attempts).toHaveLength(2);+ });++ it('a success below the limit ends the request normally (breaker untouched)', async () => {+ const onExhausted = vi.fn();+ let calls = 0;+ const dispatch = vi.fn(async () => {+ calls += 1;+ if (calls < 3) throw retryable429();+ return 'done' as const;+ });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 3, dispatch, onExhausted }));++ expect(dispatch).toHaveBeenCalledTimes(3);+ expect(onExhausted).not.toHaveBeenCalled();+ });+});diff --git a/server/src/__tests__/lib/guardrails.test.ts b/server/src/__tests__/lib/guardrails.test.tsnew file mode 100644index 00000000..c2c684cc--- /dev/null+++ b/server/src/__tests__/lib/guardrails.test.ts@@ -0,0 +1,135 @@+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';++// Unit tests for the request guardrails (ported from @coffcoe's fork): the+// per-request token budget (pre-flight reject / cap-absent-max_tokens) and+// the consecutive-upstream-failure circuit breaker, plus the+// setting → env → 0 precedence of both knobs.++// Mock the settings store so the getters can be driven deterministically.+const settingStore = new Map<string, string>();+vi.mock('../../db/index.js', () => ({+ getSetting: (key: string) => settingStore.get(key),+}));++import {+ getRequestMaxTokensBudget,+ getMaxConsecutiveUpstreamFails,+ applyTokenBudget,+ tokenBudgetMessage,+ newBreaker,+ recordBreakerFailure,+ REQUEST_MAX_TOKENS_BUDGET_SETTING,+ MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING,+} from '../../lib/guardrails.js';++beforeEach(() => {+ settingStore.clear();+});++afterEach(() => {+ delete process.env.REQUEST_MAX_TOKENS_BUDGET;+ delete process.env.MAX_CONSECUTIVE_UPSTREAM_FAILS;+});++describe('guardrail setting getters (setting → env → 0)', () => {+ it('default to 0 (disabled) when nothing is configured', () => {+ expect(getRequestMaxTokensBudget()).toBe(0);+ expect(getMaxConsecutiveUpstreamFails()).toBe(0);+ });++ it('read the settings-table value', () => {+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');+ settingStore.set(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, '5');+ expect(getRequestMaxTokensBudget()).toBe(8000);+ expect(getMaxConsecutiveUpstreamFails()).toBe(5);+ });++ it('fall back to the env var when the setting is unset', () => {+ process.env.REQUEST_MAX_TOKENS_BUDGET = '4096';+ process.env.MAX_CONSECUTIVE_UPSTREAM_FAILS = '3';+ expect(getRequestMaxTokensBudget()).toBe(4096);+ expect(getMaxConsecutiveUpstreamFails()).toBe(3);+ });++ it('the settings-table value wins over the env var', () => {+ process.env.REQUEST_MAX_TOKENS_BUDGET = '4096';+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '100');+ expect(getRequestMaxTokensBudget()).toBe(100);+ });++ it('treats garbage, negative, and non-integer values as unset (fail-safe off)', () => {+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, 'abc');+ expect(getRequestMaxTokensBudget()).toBe(0);+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '-5');+ expect(getRequestMaxTokensBudget()).toBe(0);+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8.5');+ expect(getRequestMaxTokensBudget()).toBe(0);+ });++ it('a garbage setting falls through to a valid env var', () => {+ settingStore.set(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, 'lots');+ process.env.MAX_CONSECUTIVE_UPSTREAM_FAILS = '7';+ expect(getMaxConsecutiveUpstreamFails()).toBe(7);+ });+});++describe('applyTokenBudget', () => {+ it('passes everything through untouched when the budget is disabled', () => {+ expect(applyTokenBudget(1_000_000, undefined)).toEqual({ rejection: null, maxTokens: undefined });+ expect(applyTokenBudget(1_000_000, 32_000)).toEqual({ rejection: null, maxTokens: 32_000 });+ });++ it('rejects when estimated input + requested output exceeds the budget', () => {+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');+ const r = applyTokenBudget(5000, 4000);+ expect(r.rejection).toEqual({ budget: 8000, estimatedTotal: 9000 });+ });++ it('an exact fit passes', () => {+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');+ expect(applyTokenBudget(5000, 3000)).toEqual({ rejection: null, maxTokens: 3000 });+ });++ it('caps an absent max_tokens to the budget remainder instead of rejecting', () => {+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');+ expect(applyTokenBudget(5000, undefined)).toEqual({ rejection: null, maxTokens: 3000 });+ });++ it('rejects an absent max_tokens when the input alone fills the budget', () => {+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');+ expect(applyTokenBudget(8000, undefined).rejection).toEqual({ budget: 8000, estimatedTotal: 8000 });+ expect(applyTokenBudget(9000, undefined).rejection).toEqual({ budget: 8000, estimatedTotal: 9000 });+ });++ it('tokenBudgetMessage names the numbers and the setting', () => {+ const msg = tokenBudgetMessage({ budget: 8000, estimatedTotal: 9000 });+ expect(msg).toContain('9000');+ expect(msg).toContain('8000');+ expect(msg).toContain(REQUEST_MAX_TOKENS_BUDGET_SETTING);+ });+});++describe('circuit breaker state machine', () => {+ it('never trips when disabled (limit 0)', () => {+ const b = newBreaker(0);+ expect(recordBreakerFailure(b)).toBe(false);+ expect(recordBreakerFailure(b)).toBe(false);+ expect(b.consecutive).toBe(0);+ });++ it('trips exactly on the Nth failure', () => {+ const b = newBreaker(3);+ expect(recordBreakerFailure(b)).toBe(false);+ expect(recordBreakerFailure(b)).toBe(false);+ expect(recordBreakerFailure(b)).toBe(true);+ expect(b.consecutive).toBe(3);+ });++ it('newBreaker defaults its limit from the setting', () => {+ settingStore.set(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, '2');+ const b = newBreaker();+ expect(b.limit).toBe(2);+ expect(recordBreakerFailure(b)).toBe(false);+ expect(recordBreakerFailure(b)).toBe(true);+ });+});diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 4fa1d0e4..402649ed 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -39,6 +39,7 @@ import {import { sanitizeProviderErrorMessage } from './error-redaction.js';import { checkKeyHealth } from '../services/health.js';import { getSetting } from '../db/index.js';+import { newBreaker, recordBreakerFailure } from './guardrails.js';// Every surface caps failover hops at the same number.export const FALLBACK_MAX_RETRIES = 20;@@ -266,8 +267,9 @@ export interface ExhaustionBody {type: string;message: string;// Coarse class of the exhaustion, for surfaces that need to remap `type` to- // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error').- kind: 'auth' | 'bad_request' | 'rate_limit';+ // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error'+ // and 'unavailable' → 'overloaded_error').+ kind: 'auth' | 'bad_request' | 'rate_limit' | 'unavailable';}export interface ExhaustionContext {@@ -275,6 +277,9 @@ export interface ExhaustionContext {// True when the wall-clock retry budget stopped the loop before maxRetries.timedOut?: boolean;budgetMs?: number;+ // Set (to the failure count) when the circuit-breaker guardrail stopped the+ // loop; renders as a 503 instead of a rate-limit exhaustion.+ breakerFails?: number;}/**@@ -318,6 +323,22 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E};}+ // Circuit-breaker guardrail stop. Checked after the all-auth and bad-request+ // diagnoses, which are more specific about WHY the pool is failing.+ if (ctx?.breakerFails) {+ const breakerEta = formatResetEta(getSoonestCooldownExpiry());+ const breakerEtaNote = breakerEta ? ` Soonest cooldown reset ${breakerEta}.` : '';+ return {+ kind: 'unavailable',+ status: 503,+ type: 'service_unavailable',+ message: `Failover stopped early by the circuit-breaker guardrail: ${ctx.breakerFails} consecutive upstream ` ++ `failure${ctx.breakerFails === 1 ? '' : 's'} (max_consecutive_upstream_fails). The enabled pool looks ` ++ `unhealthy right now, so the remaining candidates were skipped instead of burning quota on them.` ++ `${breakerEtaNote}${trail} Last error: ${safeLastError}`,+ };+ }+const attemptCount = attempts.length > 0 ? attempts.length : maxRetries;const scope = attemptCount == null? 'All models rate-limited'@@ -354,6 +375,9 @@ export interface FallbackHooks {// Wall-clock retry budget override, mostly for tests. Defaults to// getFallbackTimeBudgetMs() (setting → env → 45s; 0 disables).timeBudgetMs?: number;+ // Circuit-breaker threshold override, mostly for tests. Defaults to+ // getMaxConsecutiveUpstreamFails() (setting → env → 0 = disabled).+ breakerLimit?: number;// Skip state; recordRetryableFailure / recordAuthFailure (called by the loop)// mutate it, and the surface's route() reads it to exclude failed keys/models.state: FallbackState;@@ -421,6 +445,22 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {};let lastError: any = null;+ // Circuit-breaker guardrail (default off): the Nth consecutive upstream+ // failure aborts the chain with a 503 instead of grinding the remaining+ // candidates of a pool that is failing across the board. Auth failures count+ // too — a wall of dead keys is exactly the "stop early" case. Within one+ // request every recorded failure is consecutive by construction (a success+ // ends the loop), so this is "max upstream failures per request".+ const breaker = newBreaker(hooks.breakerLimit);+ const stopIfBreakerTripped = (): boolean => {+ if (!recordBreakerFailure(breaker)) return false;+ hooks.onExhausted(+ exhaustedRetryError(lastError, maxRetries, { attempts, breakerFails: breaker.consecutive }),+ { attempts, timedOut: false },+ );+ return true;+ };+for (let attempt = 0; attempt < maxRetries; attempt++) {// Wall-clock budget: refuse to START another retry once spent. The first// attempt always runs; a slow attempt is never aborted mid-flight (that is@@ -455,12 +495,14 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {recordAuthFailure(route, hooks.state);attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass: 'auth' });lastError = err;+ if (stopIfBreakerTripped()) return;continue;}if (isRetryableError(err)) {recordRetryableFailure(route, err, hooks.state);attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass: classifyAttemptError(err) });lastError = err;+ if (stopIfBreakerTripped()) return;continue;}hooks.onFatal(route, err, attempt);diff --git a/server/src/lib/guardrails.ts b/server/src/lib/guardrails.tsnew file mode 100644index 00000000..7995be6a--- /dev/null+++ b/server/src/lib/guardrails.ts@@ -0,0 +1,118 @@+// Request-level guardrails: two hard limits, both OFF by default so an+// unconfigured install behaves exactly as before.+//+// 1. request_max_tokens_budget — a per-request token-cost ceiling. Pre-flight:+// estimated input (plus the flat per-image estimate) plus the requested+// max_tokens must fit the budget, else the request is rejected with a 413+// before any provider is tried. A request that sent NO max_tokens has its+// output capped to the budget remainder instead of being rejected — with+// the cap forwarded upstream, the provider's own max_tokens enforcement+// bounds the spend, so no mid-stream truncation machinery is needed.+//+// 2. max_consecutive_upstream_fails — a per-request circuit breaker. When the+// Nth consecutive upstream attempt fails, the whole failover chain stops+// with a 503 instead of grinding through the remaining candidates of a+// pool that is failing across the board (the observed worst case for a+// doomed chain was 38.8s of serial retries; the wall-clock budget bounds+// the time, this bounds the wasted attempts/quota).+//+// Ported from @coffcoe's fork (coffcoe/freellmapi@e5024d53) and adapted to the+// unified fallback loop: the original patched each surface's copy of the retry+// loop; here the breaker is wired once in runFallbackLoop and the budget is a+// shared pre-flight helper each surface renders in its own wire format.+//+// Precedence mirrors the response cache and the fallback time budget: the+// settings-table value wins when present (runtime-tunable, no restart), then+// the env var, then the default (0 = disabled).++import { getSetting } from '../db/index.js';++export const REQUEST_MAX_TOKENS_BUDGET_SETTING = 'request_max_tokens_budget';+export const MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING = 'max_consecutive_upstream_fails';++function readGuardrailValue(settingKey: string, envKey: string): number {+ let stored: string | undefined;+ try {+ stored = getSetting(settingKey);+ } catch {+ stored = undefined; // DB not ready — never throw on the proxy hot path+ }+ for (const raw of [stored, process.env[envKey]]) {+ if (raw === undefined || raw.trim() === '') continue;+ const n = Number(raw);+ if (Number.isInteger(n) && n >= 0) return n;+ }+ return 0;+}++/** Per-request token budget ceiling; 0 = disabled. */+export function getRequestMaxTokensBudget(): number {+ return readGuardrailValue(REQUEST_MAX_TOKENS_BUDGET_SETTING, 'REQUEST_MAX_TOKENS_BUDGET');+}++/** Consecutive-upstream-failure breaker threshold; 0 = disabled. */+export function getMaxConsecutiveUpstreamFails(): number {+ return readGuardrailValue(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, 'MAX_CONSECUTIVE_UPSTREAM_FAILS');+}++// ── Token budget (pre-flight) ────────────────────────────────────────────────++export interface TokenBudgetRejection {+ budget: number;+ estimatedTotal: number;+}++export interface TokenBudgetResult {+ rejection: TokenBudgetRejection | null;+ /** The max_tokens to forward upstream: unchanged when the client set one,+ * capped to the budget remainder when it didn't. */+ maxTokens: number | undefined;+}++/**+ * Apply the per-request token budget to one request. `estimatedInputTokens`+ * is the surface's existing input estimate (~4 chars/token, images at the+ * flat per-image estimate); `maxTokens` is the client-requested output cap,+ * possibly undefined on the OpenAI-shaped surfaces.+ */+export function applyTokenBudget(estimatedInputTokens: number, maxTokens: number | undefined): TokenBudgetResult {+ const budget = getRequestMaxTokensBudget();+ if (budget <= 0) return { rejection: null, maxTokens };+ const estimatedTotal = estimatedInputTokens + (maxTokens ?? 0);+ if (estimatedTotal > budget) return { rejection: { budget, estimatedTotal }, maxTokens };+ if (maxTokens == null) {+ const remaining = budget - estimatedInputTokens;+ // Input alone fills the whole budget: nothing left for output.+ if (remaining < 1) return { rejection: { budget, estimatedTotal }, maxTokens };+ return { rejection: null, maxTokens: remaining };+ }+ return { rejection: null, maxTokens };+}++export function tokenBudgetMessage(r: TokenBudgetRejection): string {+ return `Request exceeds the per-request token budget guardrail: ~${r.estimatedTotal} tokens ` ++ `(estimated input + requested output) over a budget of ${r.budget}. Trim the prompt or lower ` ++ `max_tokens, or raise the ${REQUEST_MAX_TOKENS_BUDGET_SETTING} setting (0 disables it).`;+}++// ── Circuit breaker (one per request, threaded through the fallback loop) ────++export interface BreakerState {+ limit: number;+ consecutive: number;+}++export function newBreaker(limit: number = getMaxConsecutiveUpstreamFails()): BreakerState {+ return { limit, consecutive: 0 };+}++/**+ * Record one failed upstream attempt. Returns true when the breaker trips+ * (the recorded failure reached the limit). No-op returning false when the+ * guardrail is disabled (limit <= 0).+ */+export function recordBreakerFailure(state: BreakerState): boolean {+ if (state.limit <= 0) return false;+ state.consecutive += 1;+ return state.consecutive >= state.limit;+}diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex bafcc123..4c57a1a3 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -18,6 +18,7 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { logRequest } from '../lib/request-log.js';import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody } from '../lib/fallback-loop.js';+import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { resolveAnthropicModel } from '../services/anthropic-map.js';import { buildModelListing } from '../services/model-listing.js';@@ -110,10 +111,14 @@ function sendError(res: Response, status: number, errorType: string, message: st// The shared exhaustion body's `type` strings are chosen to be valid on the// OpenAI-shaped surfaces; the all-keys-failed-auth exhaustion carries-// 'provider_error', which is not an Anthropic error type. Remap it onto-// Anthropic's generic 'api_error' so this surface stays wire-correct.+// 'provider_error' and the circuit-breaker stop carries 'service_unavailable',+// neither of which is an Anthropic error type. Remap them onto Anthropic's+// vocabulary ('api_error' / 'overloaded_error') so this surface stays+// wire-correct.function anthropicErrorType(body: ExhaustionBody): string {- return body.kind === 'auth' ? 'api_error' : body.type;+ if (body.kind === 'auth') return 'api_error';+ if (body.kind === 'unavailable') return 'overloaded_error';+ return body.type;}function newMessageId(): string {@@ -374,6 +379,15 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {// pool (#470); input + images count in full.const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);+ // Guardrail: per-request token budget (request_max_tokens_budget, default+ // off). max_tokens is always set on this surface (Anthropic requires it),+ // so a violation can only reject — no capping branch.+ const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, max_tokens);+ if (budgetCheck.rejection) {+ sendError(res, 413, 'invalid_request_error', tokenBudgetMessage(budgetCheck.rejection));+ return;+ }+// Resolve the model through the operator's Claude-family map (opus/sonnet/// haiku/default → auto | a pinned catalog model). A concrete catalog id pins// directly. `pinned` drives the analytics requested-model label.diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex ac0499ff..52d23494 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -18,6 +18,7 @@ import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelimport { logRequest } from '../lib/request-log.js';import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError } from '../lib/fallback-loop.js';+import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import type { Platform } from '@freellmapi/shared/types.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from '../services/model-groups.js';@@ -654,6 +655,17 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {// exclude the whole model pool (#470); input is still counted in full.const estimatedTotal = estimatedInputTokens + routingReserveTokens(max_tokens);+ // Guardrail: per-request token budget (request_max_tokens_budget, default+ // off). max_tokens always has a value on this surface (default 128), so a+ // violation can only reject — no capping branch.+ const budgetCheck = applyTokenBudget(estimatedInputTokens, max_tokens);+ if (budgetCheck.rejection) {+ res.status(413).json({+ error: { message: tokenBudgetMessage(budgetCheck.rejection), type: 'invalid_request_error', code: 'request_token_budget' },+ });+ return;+ }+let resolvedChain: ResolvedChain | undefined;if (isAutoModel(requestedModel)) {resolvedChain = resolveRoutingChain(requestedModel);@@ -964,7 +976,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Agent-tolerant knob normalization (#200): max_tokens <= 0 means "no// limit" in several clients → unset; tool_choice 'any' is OpenAI's// 'required'; tool definitions get their 'function' type re-defaulted.- const max_tokens = parsed.data.max_tokens != null && parsed.data.max_tokens > 0+ // `let`: the token-budget guardrail below may cap an absent max_tokens to+ // the budget remainder before the options objects are built from it.+ let max_tokens = parsed.data.max_tokens != null && parsed.data.max_tokens > 0? parsed.data.max_tokens : undefined;const stop = providerSafeStop(parsed.data.stop);const tool_choice = parsed.data.tool_choice === 'any' ? 'required' as const : parsed.data.tool_choice ?? undefined;@@ -1109,6 +1123,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {return;}+ // Guardrail: per-request token budget (request_max_tokens_budget, default+ // off). Estimated input (incl. images) + requested output must fit the+ // ceiling; a request with no max_tokens gets its output capped to the+ // remainder instead. Sits before the Fusion branch so fan-out inherits the+ // capped max_tokens too.+ const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, max_tokens);+ if (budgetCheck.rejection) {+ res.status(413).json({+ error: { message: tokenBudgetMessage(budgetCheck.rejection), type: 'invalid_request_error', code: 'request_token_budget' },+ });+ return;+ }+ max_tokens = budgetCheck.maxTokens;+// ── Fusion: multi-model synthesis ──────────────────────────────────────────// The virtual "fusion" model fans the prompt out to a panel of diverse models// in parallel, then a judge synthesizes one answer. It routes each panel/judgediff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex ca189aa7..9e4ff7ea 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -24,6 +24,7 @@ import {logRequest,} from './proxy.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess } from '../lib/fallback-loop.js';+import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';@@ -342,6 +343,18 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// Capped output reserve so a large max_output_tokens can't falsely exclude the// model pool (#470); input counts in full.const estimatedTotal = estimatedInputTokens + routingReserveTokens(reqData.max_output_tokens);++ // Guardrail: per-request token budget (request_max_tokens_budget, default+ // off). A request with no max_output_tokens gets its output capped to the+ // budget remainder instead of a rejection.+ const budgetCheck = applyTokenBudget(estimatedInputTokens, completionOpts.max_tokens);+ if (budgetCheck.rejection) {+ res.status(413).json({+ error: { message: tokenBudgetMessage(budgetCheck.rejection), type: 'invalid_request_error', code: 'request_token_budget' },+ });+ return;+ }+ completionOpts.max_tokens = budgetCheck.maxTokens;// Optional client-managed session affinity (mirrors /chat/completions).const rawSessionId = req.headers['x-session-id'];const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;diff --git a/server/src/routes/settings.ts b/server/src/routes/settings.tsindex b2c3fda8..b5b0011a 100644--- a/server/src/routes/settings.ts+++ b/server/src/routes/settings.ts@@ -5,6 +5,12 @@ import { applyProxyUrl, applyProxyEnabled, applyProxyBypass, isProxyActive, getPimport { getSavedFusionConfig, setSavedFusionConfig, savedFusionConfigSchema, getFusionMaxK } from '../services/fusion.js';import { isUnifyEnabled, setUnifyEnabled, getUnifyOverrides, setUnifyOverrides, unifyOverridesSchema } from '../services/model-groups.js';import { getClaudeModelMap, setClaudeModelMap } from '../services/anthropic-map.js';+import {+ getRequestMaxTokensBudget,+ getMaxConsecutiveUpstreamFails,+ REQUEST_MAX_TOKENS_BUDGET_SETTING,+ MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING,+} from '../lib/guardrails.js';import { z } from 'zod';export const settingsRouter = Router();@@ -73,6 +79,41 @@ settingsRouter.put('/anthropic-map', (req: Request, res: Response) => {}});+// Get the request guardrails (per-request token budget + failover circuit+// breaker). Both default to 0 = disabled; see lib/guardrails.ts.+settingsRouter.get('/guardrails', (_req: Request, res: Response) => {+ res.json({+ requestMaxTokensBudget: getRequestMaxTokensBudget(),+ maxConsecutiveUpstreamFails: getMaxConsecutiveUpstreamFails(),+ });+});++const guardrailsPutSchema = z.object({+ requestMaxTokensBudget: z.number().int().min(0).optional(),+ maxConsecutiveUpstreamFails: z.number().int().min(0).optional(),+});++// Update the guardrails. Partial: send just the knob you want to change.+// Takes effect on the next request — no restart needed. 0 disables a knob.+settingsRouter.put('/guardrails', (req: Request, res: Response) => {+ const parsed = guardrailsPutSchema.safeParse(req.body);+ if (!parsed.success) {+ const detail = parsed.error.errors.map(e => (e.path.length ? `${e.path.join('.')}: ${e.message}` : e.message)).slice(0, 5).join(', ');+ res.status(400).json({ error: { message: `Invalid guardrail settings: ${detail}`, type: 'invalid_request_error' } });+ return;+ }+ if (parsed.data.requestMaxTokensBudget !== undefined) {+ setSetting(REQUEST_MAX_TOKENS_BUDGET_SETTING, String(parsed.data.requestMaxTokensBudget));+ }+ if (parsed.data.maxConsecutiveUpstreamFails !== undefined) {+ setSetting(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, String(parsed.data.maxConsecutiveUpstreamFails));+ }+ res.json({+ requestMaxTokensBudget: getRequestMaxTokensBudget(),+ maxConsecutiveUpstreamFails: getMaxConsecutiveUpstreamFails(),+ });+});+// Get the unified API keysettingsRouter.get('/api-key', (_req: Request, res: Response) => {res.json({ apiKey: getUnifiedApiKey() });