Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(gateway): request guardrails - per-request token budget + failover circuit breaker (#511)

Tashfeencdce07d193fc
diff --git a/.env.example b/.env.example
index 78cfc33b..c576c562 100644
--- a/.env.example
+++ b/.env.example
@@ -46,6 +46,21 @@ PORT=3001
# fallback_time_budget_ms settings key, which takes precedence.
# FALLBACK_TIME_BUDGET_MS=45000
+# Request guardrails, both OFF (0) by default. Also runtime-tunable via the
+# request_max_tokens_budget / max_consecutive_upstream_fails settings keys
+# (dashboard API PUT /api/settings/guardrails), which take precedence.
+#
+# Per-request token budget: estimated input + requested max_tokens must fit
+# this ceiling or the request is rejected with a 413 before any provider is
+# tried; a request that sent no max_tokens gets its output capped to the
+# remainder instead.
+# REQUEST_MAX_TOKENS_BUDGET=0
+#
+# Failover circuit breaker: after this many consecutive upstream failures in
+# one request, stop the failover chain with a 503 instead of trying every
+# remaining candidate of an unhealthy pool.
+# MAX_CONSECUTIVE_UPSTREAM_FAILS=0
+
# Opt-in response cache. When on, a successful non-streaming /v1/chat/completions
# answer is stored in a bounded in-memory LRU keyed by a canonical hash of the
# request, so an IDENTICAL later request is served from memory without spending
diff --git a/README.md b/README.md
index 9f535328..15f906d8 100644
--- a/README.md
+++ b/README.md
@@ -795,6 +795,7 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full migration CLI and workflow
<a href="https://github.com/danscMax"><img src="https://images.weserv.nl/?url=github.com/danscMax.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@danscMax" /></a>
<a href="https://github.com/jhash"><img src="https://images.weserv.nl/?url=github.com/jhash.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@jhash" /></a>
<a href="https://github.com/JammyJames1234"><img src="https://images.weserv.nl/?url=github.com/JammyJames1234.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@JammyJames1234" /></a>
+<a href="https://github.com/coffcoe"><img src="https://images.weserv.nl/?url=github.com/coffcoe.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@coffcoe" /></a>
<a href="https://github.com/Sumit4codes"><img src="https://images.weserv.nl/?url=github.com/Sumit4codes.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@Sumit4codes" /></a>
<a href="https://github.com/meliani"><img src="https://images.weserv.nl/?url=github.com/meliani.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@meliani" /></a>
<a href="https://github.com/thedavidweng"><img src="https://images.weserv.nl/?url=github.com/thedavidweng.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@thedavidweng" /></a>
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.ts
index 091d9842..eb8bb20e 100644
--- a/server/src/__tests__/lib/fallback-loop.test.ts
+++ b/server/src/__tests__/lib/fallback-loop.test.ts
@@ -349,3 +349,68 @@ describe('runFallbackLoop: dispatch outcome contract', () => {
consoleError.mockRestore();
});
});
+
+describe('runFallbackLoop: circuit-breaker guardrail (max_consecutive_upstream_fails)', () => {
+ const retryable429 = () => Object.assign(new Error('429 Too Many Requests'), { status: 429 });
+
+ it('is off by default: a doomed chain still runs to the attempt cap', async () => {
+ const onExhausted = vi.fn();
+ const dispatch = vi.fn(async () => { throw retryable429(); });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 5, dispatch, onExhausted }));
+
+ expect(dispatch).toHaveBeenCalledTimes(5);
+ expect(onExhausted).toHaveBeenCalledTimes(1);
+ expect(onExhausted.mock.calls[0][0].status).toBe(429); // ordinary exhaustion, no breaker
+ });
+
+ it('trips after N consecutive retryable failures and renders a 503 with the trail', async () => {
+ const onExhausted = vi.fn();
+ const dispatch = vi.fn(async () => { throw retryable429(); });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 3, dispatch, onExhausted }));
+
+ expect(dispatch).toHaveBeenCalledTimes(3); // stopped at the limit, not the cap
+ expect(onExhausted).toHaveBeenCalledTimes(1);
+ const [body, info] = onExhausted.mock.calls[0];
+ expect(body.kind).toBe('unavailable');
+ expect(body.status).toBe(503);
+ expect(body.type).toBe('service_unavailable');
+ expect(body.message).toContain('circuit-breaker');
+ expect(body.message).toContain('3 consecutive upstream failures');
+ expect(body.message).toContain('Attempt trail:');
+ expect(info.attempts).toHaveLength(3);
+ expect(info.timedOut).toBe(false);
+ });
+
+ it('auth failures count toward the breaker, and an all-auth trip keeps the 502 diagnosis', async () => {
+ mockCheckKeyHealth.mockResolvedValue(undefined);
+ const onExhausted = vi.fn();
+ const dispatch = vi.fn(async () => {
+ throw Object.assign(new Error('Unauthorized'), { status: 401 });
+ });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 2, dispatch, onExhausted }));
+
+ expect(dispatch).toHaveBeenCalledTimes(2); // breaker stopped the rotation
+ const [body, info] = onExhausted.mock.calls[0];
+ expect(body.kind).toBe('auth'); // the more specific all-auth body wins over the breaker body
+ expect(body.status).toBe(502);
+ expect(info.attempts).toHaveLength(2);
+ });
+
+ it('a success below the limit ends the request normally (breaker untouched)', async () => {
+ const onExhausted = vi.fn();
+ let calls = 0;
+ const dispatch = vi.fn(async () => {
+ calls += 1;
+ if (calls < 3) throw retryable429();
+ return 'done' as const;
+ });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 3, dispatch, onExhausted }));
+
+ expect(dispatch).toHaveBeenCalledTimes(3);
+ expect(onExhausted).not.toHaveBeenCalled();
+ });
+});
diff --git a/server/src/__tests__/lib/guardrails.test.ts b/server/src/__tests__/lib/guardrails.test.ts
new file mode 100644
index 00000000..c2c684cc
--- /dev/null
+++ b/server/src/__tests__/lib/guardrails.test.ts
@@ -0,0 +1,135 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+// Unit tests for the request guardrails (ported from @coffcoe's fork): the
+// per-request token budget (pre-flight reject / cap-absent-max_tokens) and
+// the consecutive-upstream-failure circuit breaker, plus the
+// setting → env → 0 precedence of both knobs.
+
+// Mock the settings store so the getters can be driven deterministically.
+const settingStore = new Map<string, string>();
+vi.mock('../../db/index.js', () => ({
+ getSetting: (key: string) => settingStore.get(key),
+}));
+
+import {
+ getRequestMaxTokensBudget,
+ getMaxConsecutiveUpstreamFails,
+ applyTokenBudget,
+ tokenBudgetMessage,
+ newBreaker,
+ recordBreakerFailure,
+ REQUEST_MAX_TOKENS_BUDGET_SETTING,
+ MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING,
+} from '../../lib/guardrails.js';
+
+beforeEach(() => {
+ settingStore.clear();
+});
+
+afterEach(() => {
+ delete process.env.REQUEST_MAX_TOKENS_BUDGET;
+ delete process.env.MAX_CONSECUTIVE_UPSTREAM_FAILS;
+});
+
+describe('guardrail setting getters (setting → env → 0)', () => {
+ it('default to 0 (disabled) when nothing is configured', () => {
+ expect(getRequestMaxTokensBudget()).toBe(0);
+ expect(getMaxConsecutiveUpstreamFails()).toBe(0);
+ });
+
+ it('read the settings-table value', () => {
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');
+ settingStore.set(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, '5');
+ expect(getRequestMaxTokensBudget()).toBe(8000);
+ expect(getMaxConsecutiveUpstreamFails()).toBe(5);
+ });
+
+ it('fall back to the env var when the setting is unset', () => {
+ process.env.REQUEST_MAX_TOKENS_BUDGET = '4096';
+ process.env.MAX_CONSECUTIVE_UPSTREAM_FAILS = '3';
+ expect(getRequestMaxTokensBudget()).toBe(4096);
+ expect(getMaxConsecutiveUpstreamFails()).toBe(3);
+ });
+
+ it('the settings-table value wins over the env var', () => {
+ process.env.REQUEST_MAX_TOKENS_BUDGET = '4096';
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '100');
+ expect(getRequestMaxTokensBudget()).toBe(100);
+ });
+
+ it('treats garbage, negative, and non-integer values as unset (fail-safe off)', () => {
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, 'abc');
+ expect(getRequestMaxTokensBudget()).toBe(0);
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '-5');
+ expect(getRequestMaxTokensBudget()).toBe(0);
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8.5');
+ expect(getRequestMaxTokensBudget()).toBe(0);
+ });
+
+ it('a garbage setting falls through to a valid env var', () => {
+ settingStore.set(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, 'lots');
+ process.env.MAX_CONSECUTIVE_UPSTREAM_FAILS = '7';
+ expect(getMaxConsecutiveUpstreamFails()).toBe(7);
+ });
+});
+
+describe('applyTokenBudget', () => {
+ it('passes everything through untouched when the budget is disabled', () => {
+ expect(applyTokenBudget(1_000_000, undefined)).toEqual({ rejection: null, maxTokens: undefined });
+ expect(applyTokenBudget(1_000_000, 32_000)).toEqual({ rejection: null, maxTokens: 32_000 });
+ });
+
+ it('rejects when estimated input + requested output exceeds the budget', () => {
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');
+ const r = applyTokenBudget(5000, 4000);
+ expect(r.rejection).toEqual({ budget: 8000, estimatedTotal: 9000 });
+ });
+
+ it('an exact fit passes', () => {
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');
+ expect(applyTokenBudget(5000, 3000)).toEqual({ rejection: null, maxTokens: 3000 });
+ });
+
+ it('caps an absent max_tokens to the budget remainder instead of rejecting', () => {
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');
+ expect(applyTokenBudget(5000, undefined)).toEqual({ rejection: null, maxTokens: 3000 });
+ });
+
+ it('rejects an absent max_tokens when the input alone fills the budget', () => {
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');
+ expect(applyTokenBudget(8000, undefined).rejection).toEqual({ budget: 8000, estimatedTotal: 8000 });
+ expect(applyTokenBudget(9000, undefined).rejection).toEqual({ budget: 8000, estimatedTotal: 9000 });
+ });
+
+ it('tokenBudgetMessage names the numbers and the setting', () => {
+ const msg = tokenBudgetMessage({ budget: 8000, estimatedTotal: 9000 });
+ expect(msg).toContain('9000');
+ expect(msg).toContain('8000');
+ expect(msg).toContain(REQUEST_MAX_TOKENS_BUDGET_SETTING);
+ });
+});
+
+describe('circuit breaker state machine', () => {
+ it('never trips when disabled (limit 0)', () => {
+ const b = newBreaker(0);
+ expect(recordBreakerFailure(b)).toBe(false);
+ expect(recordBreakerFailure(b)).toBe(false);
+ expect(b.consecutive).toBe(0);
+ });
+
+ it('trips exactly on the Nth failure', () => {
+ const b = newBreaker(3);
+ expect(recordBreakerFailure(b)).toBe(false);
+ expect(recordBreakerFailure(b)).toBe(false);
+ expect(recordBreakerFailure(b)).toBe(true);
+ expect(b.consecutive).toBe(3);
+ });
+
+ it('newBreaker defaults its limit from the setting', () => {
+ settingStore.set(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, '2');
+ const b = newBreaker();
+ expect(b.limit).toBe(2);
+ expect(recordBreakerFailure(b)).toBe(false);
+ expect(recordBreakerFailure(b)).toBe(true);
+ });
+});
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 4fa1d0e4..402649ed 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -39,6 +39,7 @@ import {
import { sanitizeProviderErrorMessage } from './error-redaction.js';
import { checkKeyHealth } from '../services/health.js';
import { getSetting } from '../db/index.js';
+import { newBreaker, recordBreakerFailure } from './guardrails.js';
// Every surface caps failover hops at the same number.
export const FALLBACK_MAX_RETRIES = 20;
@@ -266,8 +267,9 @@ export interface ExhaustionBody {
type: string;
message: string;
// Coarse class of the exhaustion, for surfaces that need to remap `type` to
- // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error').
- kind: 'auth' | 'bad_request' | 'rate_limit';
+ // their own wire vocabulary (the Anthropic route maps 'auth' → 'api_error'
+ // and 'unavailable' → 'overloaded_error').
+ kind: 'auth' | 'bad_request' | 'rate_limit' | 'unavailable';
}
export interface ExhaustionContext {
@@ -275,6 +277,9 @@ export interface ExhaustionContext {
// True when the wall-clock retry budget stopped the loop before maxRetries.
timedOut?: boolean;
budgetMs?: number;
+ // Set (to the failure count) when the circuit-breaker guardrail stopped the
+ // loop; renders as a 503 instead of a rate-limit exhaustion.
+ breakerFails?: number;
}
/**
@@ -318,6 +323,22 @@ export function exhaustedRetryError(lastError: any, maxRetries?: number, ctx?: E
};
}
+ // Circuit-breaker guardrail stop. Checked after the all-auth and bad-request
+ // diagnoses, which are more specific about WHY the pool is failing.
+ if (ctx?.breakerFails) {
+ const breakerEta = formatResetEta(getSoonestCooldownExpiry());
+ const breakerEtaNote = breakerEta ? ` Soonest cooldown reset ${breakerEta}.` : '';
+ return {
+ kind: 'unavailable',
+ status: 503,
+ type: 'service_unavailable',
+ message: `Failover stopped early by the circuit-breaker guardrail: ${ctx.breakerFails} consecutive upstream ` +
+ `failure${ctx.breakerFails === 1 ? '' : 's'} (max_consecutive_upstream_fails). The enabled pool looks ` +
+ `unhealthy right now, so the remaining candidates were skipped instead of burning quota on them.` +
+ `${breakerEtaNote}${trail} Last error: ${safeLastError}`,
+ };
+ }
+
const attemptCount = attempts.length > 0 ? attempts.length : maxRetries;
const scope = attemptCount == null
? 'All models rate-limited'
@@ -354,6 +375,9 @@ export interface FallbackHooks {
// Wall-clock retry budget override, mostly for tests. Defaults to
// getFallbackTimeBudgetMs() (setting → env → 45s; 0 disables).
timeBudgetMs?: number;
+ // Circuit-breaker threshold override, mostly for tests. Defaults to
+ // getMaxConsecutiveUpstreamFails() (setting → env → 0 = disabled).
+ breakerLimit?: number;
// Skip state; recordRetryableFailure / recordAuthFailure (called by the loop)
// mutate it, and the surface's route() reads it to exclude failed keys/models.
state: FallbackState;
@@ -421,6 +445,22 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
};
let lastError: any = null;
+ // Circuit-breaker guardrail (default off): the Nth consecutive upstream
+ // failure aborts the chain with a 503 instead of grinding the remaining
+ // candidates of a pool that is failing across the board. Auth failures count
+ // too — a wall of dead keys is exactly the "stop early" case. Within one
+ // request every recorded failure is consecutive by construction (a success
+ // ends the loop), so this is "max upstream failures per request".
+ const breaker = newBreaker(hooks.breakerLimit);
+ const stopIfBreakerTripped = (): boolean => {
+ if (!recordBreakerFailure(breaker)) return false;
+ hooks.onExhausted(
+ exhaustedRetryError(lastError, maxRetries, { attempts, breakerFails: breaker.consecutive }),
+ { attempts, timedOut: false },
+ );
+ return true;
+ };
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
// Wall-clock budget: refuse to START another retry once spent. The first
// attempt always runs; a slow attempt is never aborted mid-flight (that is
@@ -455,12 +495,14 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
recordAuthFailure(route, hooks.state);
attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass: 'auth' });
lastError = err;
+ if (stopIfBreakerTripped()) return;
continue;
}
if (isRetryableError(err)) {
recordRetryableFailure(route, err, hooks.state);
attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass: classifyAttemptError(err) });
lastError = err;
+ if (stopIfBreakerTripped()) return;
continue;
}
hooks.onFatal(route, err, attempt);
diff --git a/server/src/lib/guardrails.ts b/server/src/lib/guardrails.ts
new file mode 100644
index 00000000..7995be6a
--- /dev/null
+++ b/server/src/lib/guardrails.ts
@@ -0,0 +1,118 @@
+// Request-level guardrails: two hard limits, both OFF by default so an
+// unconfigured install behaves exactly as before.
+//
+// 1. request_max_tokens_budget — a per-request token-cost ceiling. Pre-flight:
+// estimated input (plus the flat per-image estimate) plus the requested
+// max_tokens must fit the budget, else the request is rejected with a 413
+// before any provider is tried. A request that sent NO max_tokens has its
+// output capped to the budget remainder instead of being rejected — with
+// the cap forwarded upstream, the provider's own max_tokens enforcement
+// bounds the spend, so no mid-stream truncation machinery is needed.
+//
+// 2. max_consecutive_upstream_fails — a per-request circuit breaker. When the
+// Nth consecutive upstream attempt fails, the whole failover chain stops
+// with a 503 instead of grinding through the remaining candidates of a
+// pool that is failing across the board (the observed worst case for a
+// doomed chain was 38.8s of serial retries; the wall-clock budget bounds
+// the time, this bounds the wasted attempts/quota).
+//
+// Ported from @coffcoe's fork (coffcoe/freellmapi@e5024d53) and adapted to the
+// unified fallback loop: the original patched each surface's copy of the retry
+// loop; here the breaker is wired once in runFallbackLoop and the budget is a
+// shared pre-flight helper each surface renders in its own wire format.
+//
+// Precedence mirrors the response cache and the fallback time budget: the
+// settings-table value wins when present (runtime-tunable, no restart), then
+// the env var, then the default (0 = disabled).
+
+import { getSetting } from '../db/index.js';
+
+export const REQUEST_MAX_TOKENS_BUDGET_SETTING = 'request_max_tokens_budget';
+export const MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING = 'max_consecutive_upstream_fails';
+
+function readGuardrailValue(settingKey: string, envKey: string): number {
+ let stored: string | undefined;
+ try {
+ stored = getSetting(settingKey);
+ } catch {
+ stored = undefined; // DB not ready — never throw on the proxy hot path
+ }
+ for (const raw of [stored, process.env[envKey]]) {
+ if (raw === undefined || raw.trim() === '') continue;
+ const n = Number(raw);
+ if (Number.isInteger(n) && n >= 0) return n;
+ }
+ return 0;
+}
+
+/** Per-request token budget ceiling; 0 = disabled. */
+export function getRequestMaxTokensBudget(): number {
+ return readGuardrailValue(REQUEST_MAX_TOKENS_BUDGET_SETTING, 'REQUEST_MAX_TOKENS_BUDGET');
+}
+
+/** Consecutive-upstream-failure breaker threshold; 0 = disabled. */
+export function getMaxConsecutiveUpstreamFails(): number {
+ return readGuardrailValue(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, 'MAX_CONSECUTIVE_UPSTREAM_FAILS');
+}
+
+// ── Token budget (pre-flight) ────────────────────────────────────────────────
+
+export interface TokenBudgetRejection {
+ budget: number;
+ estimatedTotal: number;
+}
+
+export interface TokenBudgetResult {
+ rejection: TokenBudgetRejection | null;
+ /** The max_tokens to forward upstream: unchanged when the client set one,
+ * capped to the budget remainder when it didn't. */
+ maxTokens: number | undefined;
+}
+
+/**
+ * Apply the per-request token budget to one request. `estimatedInputTokens`
+ * is the surface's existing input estimate (~4 chars/token, images at the
+ * flat per-image estimate); `maxTokens` is the client-requested output cap,
+ * possibly undefined on the OpenAI-shaped surfaces.
+ */
+export function applyTokenBudget(estimatedInputTokens: number, maxTokens: number | undefined): TokenBudgetResult {
+ const budget = getRequestMaxTokensBudget();
+ if (budget <= 0) return { rejection: null, maxTokens };
+ const estimatedTotal = estimatedInputTokens + (maxTokens ?? 0);
+ if (estimatedTotal > budget) return { rejection: { budget, estimatedTotal }, maxTokens };
+ if (maxTokens == null) {
+ const remaining = budget - estimatedInputTokens;
+ // Input alone fills the whole budget: nothing left for output.
+ if (remaining < 1) return { rejection: { budget, estimatedTotal }, maxTokens };
+ return { rejection: null, maxTokens: remaining };
+ }
+ return { rejection: null, maxTokens };
+}
+
+export function tokenBudgetMessage(r: TokenBudgetRejection): string {
+ return `Request exceeds the per-request token budget guardrail: ~${r.estimatedTotal} tokens ` +
+ `(estimated input + requested output) over a budget of ${r.budget}. Trim the prompt or lower ` +
+ `max_tokens, or raise the ${REQUEST_MAX_TOKENS_BUDGET_SETTING} setting (0 disables it).`;
+}
+
+// ── Circuit breaker (one per request, threaded through the fallback loop) ────
+
+export interface BreakerState {
+ limit: number;
+ consecutive: number;
+}
+
+export function newBreaker(limit: number = getMaxConsecutiveUpstreamFails()): BreakerState {
+ return { limit, consecutive: 0 };
+}
+
+/**
+ * Record one failed upstream attempt. Returns true when the breaker trips
+ * (the recorded failure reached the limit). No-op returning false when the
+ * guardrail is disabled (limit <= 0).
+ */
+export function recordBreakerFailure(state: BreakerState): boolean {
+ if (state.limit <= 0) return false;
+ state.consecutive += 1;
+ return state.consecutive >= state.limit;
+}
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index bafcc123..4c57a1a3 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -18,6 +18,7 @@ import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { logRequest } from '../lib/request-log.js';
import { extractApiToken, timingSafeStringEqual, getStickyModel, setStickyModel } from './proxy.js';
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, type ExhaustionBody } from '../lib/fallback-loop.js';
+import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { resolveAnthropicModel } from '../services/anthropic-map.js';
import { buildModelListing } from '../services/model-listing.js';
@@ -110,10 +111,14 @@ function sendError(res: Response, status: number, errorType: string, message: st
// The shared exhaustion body's `type` strings are chosen to be valid on the
// OpenAI-shaped surfaces; the all-keys-failed-auth exhaustion carries
-// 'provider_error', which is not an Anthropic error type. Remap it onto
-// Anthropic's generic 'api_error' so this surface stays wire-correct.
+// 'provider_error' and the circuit-breaker stop carries 'service_unavailable',
+// neither of which is an Anthropic error type. Remap them onto Anthropic's
+// vocabulary ('api_error' / 'overloaded_error') so this surface stays
+// wire-correct.
function anthropicErrorType(body: ExhaustionBody): string {
- return body.kind === 'auth' ? 'api_error' : body.type;
+ if (body.kind === 'auth') return 'api_error';
+ if (body.kind === 'unavailable') return 'overloaded_error';
+ return body.type;
}
function newMessageId(): string {
@@ -374,6 +379,15 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
// pool (#470); input + images count in full.
const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);
+ // Guardrail: per-request token budget (request_max_tokens_budget, default
+ // off). max_tokens is always set on this surface (Anthropic requires it),
+ // so a violation can only reject — no capping branch.
+ const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, max_tokens);
+ if (budgetCheck.rejection) {
+ sendError(res, 413, 'invalid_request_error', tokenBudgetMessage(budgetCheck.rejection));
+ return;
+ }
+
// Resolve the model through the operator's Claude-family map (opus/sonnet/
// haiku/default → auto | a pinned catalog model). A concrete catalog id pins
// directly. `pinned` drives the analytics requested-model label.
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index ac0499ff..52d23494 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -18,6 +18,7 @@ import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModel
import { logRequest } from '../lib/request-log.js';
import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError } from '../lib/fallback-loop.js';
+import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import type { Platform } from '@freellmapi/shared/types.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from '../services/model-groups.js';
@@ -654,6 +655,17 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
// exclude the whole model pool (#470); input is still counted in full.
const estimatedTotal = estimatedInputTokens + routingReserveTokens(max_tokens);
+ // Guardrail: per-request token budget (request_max_tokens_budget, default
+ // off). max_tokens always has a value on this surface (default 128), so a
+ // violation can only reject — no capping branch.
+ const budgetCheck = applyTokenBudget(estimatedInputTokens, max_tokens);
+ if (budgetCheck.rejection) {
+ res.status(413).json({
+ error: { message: tokenBudgetMessage(budgetCheck.rejection), type: 'invalid_request_error', code: 'request_token_budget' },
+ });
+ return;
+ }
+
let resolvedChain: ResolvedChain | undefined;
if (isAutoModel(requestedModel)) {
resolvedChain = resolveRoutingChain(requestedModel);
@@ -964,7 +976,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Agent-tolerant knob normalization (#200): max_tokens <= 0 means "no
// limit" in several clients → unset; tool_choice 'any' is OpenAI's
// 'required'; tool definitions get their 'function' type re-defaulted.
- const max_tokens = parsed.data.max_tokens != null && parsed.data.max_tokens > 0
+ // `let`: the token-budget guardrail below may cap an absent max_tokens to
+ // the budget remainder before the options objects are built from it.
+ let max_tokens = parsed.data.max_tokens != null && parsed.data.max_tokens > 0
? parsed.data.max_tokens : undefined;
const stop = providerSafeStop(parsed.data.stop);
const tool_choice = parsed.data.tool_choice === 'any' ? 'required' as const : parsed.data.tool_choice ?? undefined;
@@ -1109,6 +1123,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
return;
}
+ // Guardrail: per-request token budget (request_max_tokens_budget, default
+ // off). Estimated input (incl. images) + requested output must fit the
+ // ceiling; a request with no max_tokens gets its output capped to the
+ // remainder instead. Sits before the Fusion branch so fan-out inherits the
+ // capped max_tokens too.
+ const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, max_tokens);
+ if (budgetCheck.rejection) {
+ res.status(413).json({
+ error: { message: tokenBudgetMessage(budgetCheck.rejection), type: 'invalid_request_error', code: 'request_token_budget' },
+ });
+ return;
+ }
+ max_tokens = budgetCheck.maxTokens;
+
// ── Fusion: multi-model synthesis ──────────────────────────────────────────
// The virtual "fusion" model fans the prompt out to a panel of diverse models
// in parallel, then a judge synthesizes one answer. It routes each panel/judge
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index ca189aa7..9e4ff7ea 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -24,6 +24,7 @@ import {
logRequest,
} from './proxy.js';
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess } from '../lib/fallback-loop.js';
+import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
@@ -342,6 +343,18 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// Capped output reserve so a large max_output_tokens can't falsely exclude the
// model pool (#470); input counts in full.
const estimatedTotal = estimatedInputTokens + routingReserveTokens(reqData.max_output_tokens);
+
+ // Guardrail: per-request token budget (request_max_tokens_budget, default
+ // off). A request with no max_output_tokens gets its output capped to the
+ // budget remainder instead of a rejection.
+ const budgetCheck = applyTokenBudget(estimatedInputTokens, completionOpts.max_tokens);
+ if (budgetCheck.rejection) {
+ res.status(413).json({
+ error: { message: tokenBudgetMessage(budgetCheck.rejection), type: 'invalid_request_error', code: 'request_token_budget' },
+ });
+ return;
+ }
+ completionOpts.max_tokens = budgetCheck.maxTokens;
// Optional client-managed session affinity (mirrors /chat/completions).
const rawSessionId = req.headers['x-session-id'];
const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;
diff --git a/server/src/routes/settings.ts b/server/src/routes/settings.ts
index b2c3fda8..b5b0011a 100644
--- a/server/src/routes/settings.ts
+++ b/server/src/routes/settings.ts
@@ -5,6 +5,12 @@ import { applyProxyUrl, applyProxyEnabled, applyProxyBypass, isProxyActive, getP
import { getSavedFusionConfig, setSavedFusionConfig, savedFusionConfigSchema, getFusionMaxK } from '../services/fusion.js';
import { isUnifyEnabled, setUnifyEnabled, getUnifyOverrides, setUnifyOverrides, unifyOverridesSchema } from '../services/model-groups.js';
import { getClaudeModelMap, setClaudeModelMap } from '../services/anthropic-map.js';
+import {
+ getRequestMaxTokensBudget,
+ getMaxConsecutiveUpstreamFails,
+ REQUEST_MAX_TOKENS_BUDGET_SETTING,
+ MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING,
+} from '../lib/guardrails.js';
import { z } from 'zod';
export const settingsRouter = Router();
@@ -73,6 +79,41 @@ settingsRouter.put('/anthropic-map', (req: Request, res: Response) => {
}
});
+// Get the request guardrails (per-request token budget + failover circuit
+// breaker). Both default to 0 = disabled; see lib/guardrails.ts.
+settingsRouter.get('/guardrails', (_req: Request, res: Response) => {
+ res.json({
+ requestMaxTokensBudget: getRequestMaxTokensBudget(),
+ maxConsecutiveUpstreamFails: getMaxConsecutiveUpstreamFails(),
+ });
+});
+
+const guardrailsPutSchema = z.object({
+ requestMaxTokensBudget: z.number().int().min(0).optional(),
+ maxConsecutiveUpstreamFails: z.number().int().min(0).optional(),
+});
+
+// Update the guardrails. Partial: send just the knob you want to change.
+// Takes effect on the next request — no restart needed. 0 disables a knob.
+settingsRouter.put('/guardrails', (req: Request, res: Response) => {
+ const parsed = guardrailsPutSchema.safeParse(req.body);
+ if (!parsed.success) {
+ const detail = parsed.error.errors.map(e => (e.path.length ? `${e.path.join('.')}: ${e.message}` : e.message)).slice(0, 5).join(', ');
+ res.status(400).json({ error: { message: `Invalid guardrail settings: ${detail}`, type: 'invalid_request_error' } });
+ return;
+ }
+ if (parsed.data.requestMaxTokensBudget !== undefined) {
+ setSetting(REQUEST_MAX_TOKENS_BUDGET_SETTING, String(parsed.data.requestMaxTokensBudget));
+ }
+ if (parsed.data.maxConsecutiveUpstreamFails !== undefined) {
+ setSetting(MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING, String(parsed.data.maxConsecutiveUpstreamFails));
+ }
+ res.json({
+ requestMaxTokensBudget: getRequestMaxTokensBudget(),
+ maxConsecutiveUpstreamFails: getMaxConsecutiveUpstreamFails(),
+ });
+});
+
// Get the unified API key
settingsRouter.get('/api-key', (_req: Request, res: Response) => {
res.json({ apiKey: getUnifiedApiKey() });