Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix: harden 7/11 features — disconnect misclassification, JSON heal, breaker, MCP, wake flush (#528)

Tashfeen90c84daab798
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.ts
index da69ed2c..bea71005 100644
--- a/server/src/__tests__/lib/fallback-loop.test.ts
+++ b/server/src/__tests__/lib/fallback-loop.test.ts
@@ -163,6 +163,30 @@ describe('recordRetryableFailure skipBench exemption (reasoning truncation)', ()
// modelDbId 424242 has no sibling key rows, so the penalty fires.
expect(getAllPenalties().some(p => p.modelDbId === route.modelDbId)).toBe(true);
});
+
+ it('skipModelForRequest rules out the whole model, not just the key', () => {
+ // Format-ignore is MODEL behavior: a sibling key reproduces it exactly, so
+ // burning one failover hop per key on the same model was pure waste.
+ const route = fakeRoute();
+ const state = newFallbackState();
+ const err = Object.assign(
+ new Error(`${route.displayName} ignored response_format (returned non-JSON despite json_schema)`),
+ { skipBench: true, skipModelForRequest: true },
+ );
+ recordRetryableFailure(route, err, state);
+
+ expect(state.skipModels.has(route.modelDbId)).toBe(true);
+ expect(state.skipKeys.has(`fake:fake-model:${route.keyId}`)).toBe(true);
+ // skipBench still applies: no cooldown, no penalty.
+ const cooldown = getDb().prepare('SELECT 1 FROM rate_limit_cooldowns WHERE platform = ? AND key_id = ?').get('fake', route.keyId);
+ expect(cooldown).toBeUndefined();
+ expect(getAllPenalties().some(p => p.modelDbId === route.modelDbId)).toBe(false);
+ });
+
+ it('classifyAttemptError buckets format-ignore and truncated-JSON as format_ignored', () => {
+ expect(classifyAttemptError(new Error('X ignored response_format (returned non-JSON despite json_object)'))).toBe('format_ignored');
+ expect(classifyAttemptError(new Error('truncated JSON from X (finish_reason=length — raise max_tokens for this json_schema request)'))).toBe('format_ignored');
+ });
});
describe('exhaustedRetryError attempt trail + auth exhaustion', () => {
@@ -399,6 +423,42 @@ describe('runFallbackLoop: circuit-breaker guardrail (max_consecutive_upstream_f
expect(info.attempts).toHaveLength(2);
});
+ it('skipBench failures (format ignored, hidden reasoning) never trip the breaker', async () => {
+ // Three prose answers to a json_schema request say nothing about pool
+ // health — candidate four must still get its shot. (#511 x #516)
+ const onExhausted = vi.fn();
+ let calls = 0;
+ const dispatch = vi.fn(async () => {
+ calls += 1;
+ if (calls <= 3) {
+ throw Object.assign(
+ new Error(`Fake ${calls} ignored response_format (returned non-JSON despite json_schema)`),
+ { skipBench: true },
+ );
+ }
+ return 'done' as const;
+ });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 3, dispatch, onExhausted }));
+
+ expect(dispatch).toHaveBeenCalledTimes(4); // 3 skipBench failures did not trip the 3-limit breaker
+ expect(onExhausted).not.toHaveBeenCalled();
+ });
+
+ it('a breaker trip after client disconnect stops the chain without rendering', async () => {
+ const onExhausted = vi.fn();
+ let gone = false;
+ const dispatch = vi.fn(async () => {
+ gone = true; // client hangs up during the (only) attempt
+ throw retryable429();
+ });
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 1, clientGone: () => gone, dispatch, onExhausted }));
+
+ expect(dispatch).toHaveBeenCalledTimes(1);
+ expect(onExhausted).not.toHaveBeenCalled(); // no body to a dead socket
+ });
+
it('a success below the limit ends the request normally (breaker untouched)', async () => {
const onExhausted = vi.fn();
let calls = 0;
diff --git a/server/src/__tests__/lib/guardrails.test.ts b/server/src/__tests__/lib/guardrails.test.ts
index c2c684cc..39dd8883 100644
--- a/server/src/__tests__/lib/guardrails.test.ts
+++ b/server/src/__tests__/lib/guardrails.test.ts
@@ -20,6 +20,7 @@ import {
recordBreakerFailure,
REQUEST_MAX_TOKENS_BUDGET_SETTING,
MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING,
+ TOKEN_BUDGET_OUTPUT_CAP,
} from '../../lib/guardrails.js';
beforeEach(() => {
@@ -95,6 +96,16 @@ describe('applyTokenBudget', () => {
expect(applyTokenBudget(5000, undefined)).toEqual({ rejection: null, maxTokens: 3000 });
});
+ it('clamps the injected max_tokens to the output cap under a generous budget', () => {
+ // Regression: budget 100000 minus a 1000-token prompt forwarded
+ // max_tokens=99000 verbatim, and providers that validate max_tokens
+ // against the model's limits 400'd every candidate in the chain.
+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '100000');
+ expect(applyTokenBudget(1000, undefined)).toEqual({ rejection: null, maxTokens: TOKEN_BUDGET_OUTPUT_CAP });
+ // A client-set max_tokens is never rewritten, only validated.
+ expect(applyTokenBudget(1000, 50_000)).toEqual({ rejection: null, maxTokens: 50_000 });
+ });
+
it('rejects an absent max_tokens when the input alone fills the budget', () => {
settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');
expect(applyTokenBudget(8000, undefined).rejection).toEqual({ budget: 8000, estimatedTotal: 8000 });
diff --git a/server/src/__tests__/lib/structured-output.test.ts b/server/src/__tests__/lib/structured-output.test.ts
index 589d66f9..fbebd8b1 100644
--- a/server/src/__tests__/lib/structured-output.test.ts
+++ b/server/src/__tests__/lib/structured-output.test.ts
@@ -45,4 +45,26 @@ describe('enforceJsonContent', () => {
const clean = '{"code":"if (a) { return b; }"}';
expect(enforceJsonContent(`prefix ${clean} suffix`)).toEqual({ ok: true, content: clean, healed: true });
});
+
+ it('a citation marker in leading prose does not shadow the real object', () => {
+ // Regression: first-bracket-wins sliced this to `[1]` — a valid parse of
+ // the WRONG value, delivered silently as the structured output.
+ const r = enforceJsonContent('According to source [1], here is the JSON: {"city":"Paris"}');
+ expect(r).toEqual({ ok: true, content: '{"city":"Paris"}', healed: true });
+ });
+
+ it('a bracket token in trailing prose does not shadow the real object', () => {
+ const r = enforceJsonContent('{"city":"Paris"} as shown in [2]');
+ expect(r).toEqual({ ok: true, content: '{"city":"Paris"}', healed: true });
+ });
+
+ it('markdown links around a JSON array still heal to the array', () => {
+ const r = enforceJsonContent('See [docs](https://x) first. ["a","b","c"] covers it.');
+ expect(r).toEqual({ ok: false }); // first-[ to last-] spans the link — unparseable, honest reject
+ });
+
+ it('prose-wrapped array with no other brackets heals', () => {
+ const r = enforceJsonContent('Here are your items: [1, 2, 3]');
+ expect(r).toEqual({ ok: true, content: '[1, 2, 3]', healed: true });
+ });
});
diff --git a/server/src/__tests__/lib/wake-detect.test.ts b/server/src/__tests__/lib/wake-detect.test.ts
index 050508d5..1582100c 100644
--- a/server/src/__tests__/lib/wake-detect.test.ts
+++ b/server/src/__tests__/lib/wake-detect.test.ts
@@ -40,6 +40,29 @@ describe('wake-detect', () => {
expect(onWake.mock.calls[0][0]).toMatchObject({ reason: 'signal', signal: 'SIGUSR2' });
});
+ it('debounces wake events: one recovery pass per resume, not one per signal', () => {
+ // A drift tick and an operator SIGCONT for the SAME wake (or signal spam)
+ // used to stack full key re-probes on top of each other.
+ const onWake = vi.fn();
+ startWakeDetect({ onWake });
+ process.emit('SIGUSR2' as any);
+ process.emit('SIGCONT' as any);
+ process.emit('SIGUSR2' as any);
+ expect(onWake).toHaveBeenCalledTimes(1);
+ });
+
+ it('stop removes only its OWN signal listeners, not other modules’', () => {
+ // removeAllListeners('SIGUSR2') silently unhooked any other registrant
+ // (log rotation, heap-dump triggers).
+ const foreign = vi.fn();
+ process.on('SIGUSR2', foreign);
+ startWakeDetect({ onWake: vi.fn() });
+ stopWakeDetect();
+ process.emit('SIGUSR2' as any);
+ expect(foreign).toHaveBeenCalledTimes(1);
+ process.removeListener('SIGUSR2', foreign);
+ });
+
it('fires a drift wake when the wall clock jumps past the threshold (host suspend)', () => {
vi.useFakeTimers();
const onWake = vi.fn();
diff --git a/server/src/__tests__/providers/google-params.test.ts b/server/src/__tests__/providers/google-params.test.ts
index 195d54e1..18fd454c 100644
--- a/server/src/__tests__/providers/google-params.test.ts
+++ b/server/src/__tests__/providers/google-params.test.ts
@@ -46,6 +46,27 @@ describe('toGeminiExtendedConfig', () => {
expect(cfg).not.toHaveProperty('responseSchema');
});
+ it('grounding-only pseudo-tools do NOT suppress structured output', () => {
+ // google_search converts to a grounding block, not functionDeclarations —
+ // raw tools.length was silently dropping responseMimeType for it.
+ const cfg = toGeminiExtendedConfig({
+ response_format: { type: 'json_object' },
+ tools: [{ type: 'function', function: { name: 'google_search', parameters: {} } }],
+ });
+ expect(cfg.responseMimeType).toBe('application/json');
+ });
+
+ it('a grounding tool alongside a real function still suppresses structured output', () => {
+ const cfg = toGeminiExtendedConfig({
+ response_format: { type: 'json_object' },
+ tools: [
+ { type: 'function', function: { name: 'google_search', parameters: {} } },
+ { type: 'function', function: { name: 'f', parameters: {} } },
+ ],
+ });
+ expect(cfg).not.toHaveProperty('responseMimeType');
+ });
+
it('returns all-undefined fields for empty options (JSON.stringify drops them)', () => {
const cfg = toGeminiExtendedConfig(undefined);
expect(cfg.topK).toBeUndefined();
diff --git a/server/src/__tests__/routes/mcp.test.ts b/server/src/__tests__/routes/mcp.test.ts
index 7d427b0c..a110d5ef 100644
--- a/server/src/__tests__/routes/mcp.test.ts
+++ b/server/src/__tests__/routes/mcp.test.ts
@@ -1,7 +1,7 @@
import { describe, it, expect, beforeAll } from 'vitest';
import type { Express } from 'express';
import { createApp } from '../../app.js';
-import { initDb, getUnifiedApiKey } from '../../db/index.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
let app: Express;
@@ -41,20 +41,20 @@ describe('MCP server (/mcp, stateless Streamable HTTP)', () => {
expect(body.error.code).toBe(-32001);
});
- it('initialize negotiates a supported protocol version and advertises tools', async () => {
+ it('initialize always negotiates 2025-06-18 (batch-free transport) and advertises tools', async () => {
+ // Echoing an older requested revision (2025-03-26 allowed batching) while
+ // the transport rejects batches promised clients something they never got.
const { status, body } = await rpc({
jsonrpc: '2.0', id: 1, method: 'initialize',
params: { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 't', version: '0' } },
});
expect(status).toBe(200);
- expect(body.result.protocolVersion).toBe('2025-03-26');
+ expect(body.result.protocolVersion).toBe('2025-06-18');
expect(body.result.capabilities.tools).toBeDefined();
expect(body.result.serverInfo.name).toBe('freellmapi');
- });
- it('falls back to the default protocol version for an unknown one', async () => {
- const { body } = await rpc({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '1999-01-01' } });
- expect(body.result.protocolVersion).toBe('2025-06-18');
+ const unknown = await rpc({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '1999-01-01' } });
+ expect(unknown.body.result.protocolVersion).toBe('2025-06-18');
});
it('accepts notifications with a 202 and no body', async () => {
@@ -63,6 +63,67 @@ describe('MCP server (/mcp, stateless Streamable HTTP)', () => {
expect(body).toBeNull();
});
+ it('a notification is any message without an id — never answered, even for regular methods', async () => {
+ // JSON-RPC 2.0: no `id` member = notification. Detecting notifications by
+ // the method prefix answered no-id pings, which violates the spec.
+ const { status, body } = await rpc({ jsonrpc: '2.0', method: 'ping' });
+ expect(status).toBe(202);
+ expect(body).toBeNull();
+ });
+
+ it('echoes id:0 (falsy ids must not collapse to null)', async () => {
+ const { body } = await rpc({ jsonrpc: '2.0', id: 0, method: 'ping' });
+ expect(body.id).toBe(0);
+ expect(body.result).toEqual({});
+ });
+
+ it('echoes the request id on auth failures', async () => {
+ const { status, body } = await rpc({ jsonrpc: '2.0', id: 42, method: 'tools/list' }, { auth: false });
+ expect(status).toBe(401);
+ expect(body.id).toBe(42);
+ });
+
+ it('prototype-key tool names are unknown tools, not confusing tool errors', async () => {
+ for (const name of ['constructor', 'toString', '__proto__', 'hasOwnProperty']) {
+ const { body } = await rpc({ jsonrpc: '2.0', id: 10, method: 'tools/call', params: { name } });
+ expect(body.error?.code).toBe(-32602);
+ }
+ });
+
+ it('a prototype-key usage range falls back to 24h instead of throwing', async () => {
+ const { body } = await rpc({
+ jsonrpc: '2.0', id: 11, method: 'tools/call',
+ params: { name: 'usage_summary', arguments: { range: 'constructor' } },
+ });
+ expect(body.result.isError).toBeUndefined();
+ expect(toolResultJson(body).range).toBe('24h');
+ });
+
+ it('usage_summary counts rows stored in SQLite datetime format (boundary day included)', async () => {
+ // Regression: the ISO 'T' in the since-param sorted GREATER than every
+ // same-day SQLite-format row (space < 'T'), so the window's boundary day
+ // was silently dropped and "24h" behaved like "since UTC midnight".
+ const db = getDb();
+ // 23h old: still inside the 24h window but (almost always) on the same
+ // UTC date as the since-boundary — exactly the rows the bug dropped.
+ const boundaryDayRow = new Date(Date.now() - 23 * 3600_000);
+ const sqlite = (d: Date) => d.toISOString().slice(0, 19).replace('T', ' ');
+ db.prepare(`INSERT INTO requests (platform, model_id, status, input_tokens, output_tokens, created_at)
+ VALUES ('groq', 'test-model', 'success', 100, 50, ?)`).run(sqlite(boundaryDayRow));
+ const hour = sqlite(boundaryDayRow).slice(0, 13) + ':00:00';
+ db.prepare(`INSERT INTO request_hourly (hour, total_requests, success_count, input_tokens, output_tokens)
+ VALUES (?, 1, 1, 100, 50)
+ ON CONFLICT(hour) DO UPDATE SET total_requests = total_requests + 1`).run(hour);
+
+ const { body } = await rpc({
+ jsonrpc: '2.0', id: 12, method: 'tools/call',
+ params: { name: 'usage_summary', arguments: { range: '24h' } },
+ });
+ const data = toolResultJson(body);
+ expect(data.requests).toBeGreaterThanOrEqual(1);
+ expect(data.top_models.some((m: any) => m.model_id === 'test-model')).toBe(true);
+ });
+
it('lists the six gateway tools with schemas', async () => {
const { body } = await rpc({ jsonrpc: '2.0', id: 2, method: 'tools/list' });
const names = body.result.tools.map((t: any) => t.name).sort();
diff --git a/server/src/app.ts b/server/src/app.ts
index 3956007b..2494da81 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -104,7 +104,10 @@ export function createApp(config?: Config) {
// MCP server (Model Context Protocol over stateless Streamable HTTP):
// gateway introspection tools for MCP-speaking agents. Unified-key auth,
- // like /v1 — NOT behind the dashboard session gate.
+ // like /v1 — NOT behind the dashboard session gate. Same per-IP limiter as
+ // /v1 (its own bucket): both surfaces guard the same unified key, so an
+ // unauthenticated brute-force must not get a free throttle-less oracle here.
+ app.use('/mcp', createProxyRateLimiter(cfg.proxyRateLimitRpm));
app.use('/mcp', mcpRouter);
// Health check
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 5e3d41d3..41f1b1a3 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -66,6 +66,11 @@ export function isRetryableError(err: any): boolean {
// JSON couldn't be healed out of the text) — thrown before any byte
// reached the client, so the next candidate can serve it invisibly.
|| msg.includes('ignored response_format')
+ // The model DID emit JSON but ran out of max_tokens mid-value
+ // (finish_reason 'length'). A terser model may fit the same schema in the
+ // same budget, so fail over — but the thrower marks the model skipped for
+ // this request: retrying the same model would truncate identically.
+ || msg.includes('truncated json')
|| msg.includes('in-band provider error')
|| msg.includes('stream ended unexpectedly')
|| msg.includes('stream stalled')
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 369391b5..dd23f61c 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -147,7 +147,11 @@ export function cooldownForError(route: RouteResult, err: any): number {
* Callers add the just-failed key to skipKeys via this function (do not pre-add).
*/
export function recordRetryableFailure(route: RouteResult, err: any, state: FallbackState): void {
- if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) {
+ // `skipModelForRequest: true` = the failure is MODEL behavior, not key
+ // state (ignored response_format, JSON truncated at max_tokens): a sibling
+ // key would reproduce it exactly, so rule out the whole model for this
+ // request instead of burning one failover hop per key.
+ if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || err?.skipModelForRequest === true) {
state.skipModels.add(route.modelDbId);
}
state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
@@ -224,6 +228,7 @@ export type AttemptErrorClass =
| 'forbidden'
| 'provider_bad_request'
| 'empty_completion'
+ | 'format_ignored'
| 'timeout'
| 'rate_limited'
| 'upstream_error'
@@ -245,6 +250,7 @@ export function classifyAttemptError(err: any): AttemptErrorClass {
if (isProviderBadRequestError(err)) return 'provider_bad_request';
const msg = (err?.message ?? '').toLowerCase();
if (msg.includes('empty completion')) return 'empty_completion';
+ if (msg.includes('ignored response_format') || msg.includes('truncated json')) return 'format_ignored';
if (msg.includes('timeout') || msg.includes('stalled') || msg.includes('etimedout') || msg.includes('aborted')) return 'timeout';
if (msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') || msg.includes('quota')) return 'rate_limited';
const status = typeof err?.status === 'number' ? err.status : 0;
@@ -488,6 +494,13 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
const breaker = newBreaker(hooks.breakerLimit);
const stopIfBreakerTripped = (): boolean => {
if (!recordBreakerFailure(breaker)) return false;
+ // Tripped after the client already hung up: stop the chain, but there is
+ // no socket to render an exhaustion body to (mirrors the loop-top
+ // clientGone check).
+ if (hooks.clientGone?.()) {
+ console.log(`[FallbackLoop] breaker tripped after client disconnect — stopping without rendering (${attempts.length} failed attempt(s))`);
+ return true;
+ }
hooks.onExhausted(
exhaustedRetryError(lastError, maxRetries, { attempts, breakerFails: breaker.consecutive }),
{ attempts, timedOut: false },
@@ -544,7 +557,12 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {
recordRetryableFailure(route, err, hooks.state);
attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass: classifyAttemptError(err) });
lastError = err;
- if (stopIfBreakerTripped()) return;
+ // skipBench failures (format ignored, hidden-reasoning truncation) are
+ // model behavior, not provider health — recordRetryableFailure already
+ // skips the cooldown/penalty for them, and they must not count toward
+ // the "pool looks unhealthy" breaker either: three prose answers to a
+ // json_schema request say nothing about whether candidate four is up.
+ if (err?.skipBench !== true && stopIfBreakerTripped()) return;
continue;
}
hooks.onFatal(route, err, attempt);
diff --git a/server/src/lib/guardrails.ts b/server/src/lib/guardrails.ts
index 7995be6a..64b88bfc 100644
--- a/server/src/lib/guardrails.ts
+++ b/server/src/lib/guardrails.ts
@@ -65,10 +65,21 @@ export interface TokenBudgetRejection {
export interface TokenBudgetResult {
rejection: TokenBudgetRejection | null;
/** The max_tokens to forward upstream: unchanged when the client set one,
- * capped to the budget remainder when it didn't. */
+ * capped to min(budget remainder, TOKEN_BUDGET_OUTPUT_CAP) when it didn't. */
maxTokens: number | undefined;
}
+// Ceiling on the max_tokens the budget gate INJECTS when the client sent none.
+// Without it, a generous budget (say 100000) minus a small prompt forwarded
+// max_tokens≈99000 verbatim, and providers that validate max_tokens against
+// the model's context/completion limit 400'd a request that previously worked —
+// on every candidate in the chain, since the same value rode every retry.
+// Routing only reserves a capped ~2000 tokens for output (routingReserveTokens,
+// #470), so it never protects against this. A client that genuinely wants a
+// bigger completion can always set max_tokens explicitly; the budget then only
+// validates, never rewrites.
+export const TOKEN_BUDGET_OUTPUT_CAP = 4096;
+
/**
* Apply the per-request token budget to one request. `estimatedInputTokens`
* is the surface's existing input estimate (~4 chars/token, images at the
@@ -84,7 +95,7 @@ export function applyTokenBudget(estimatedInputTokens: number, maxTokens: number
const remaining = budget - estimatedInputTokens;
// Input alone fills the whole budget: nothing left for output.
if (remaining < 1) return { rejection: { budget, estimatedTotal }, maxTokens };
- return { rejection: null, maxTokens: remaining };
+ return { rejection: null, maxTokens: Math.min(remaining, TOKEN_BUDGET_OUTPUT_CAP) };
}
return { rejection: null, maxTokens };
}
diff --git a/server/src/lib/proxy.ts b/server/src/lib/proxy.ts
index 1011c835..d6f360e5 100644
--- a/server/src/lib/proxy.ts
+++ b/server/src/lib/proxy.ts
@@ -400,10 +400,31 @@ export function isProxyActive(): boolean {
return _proxyEnabled && !!_proxyUrl;
}
-/** Force-rebuild the proxy dispatcher on the next request. Called on
+/** Force-rebuild the outbound connection pools on the next request. Called on
* sleep/wake recovery to drop pooled TCP connections that died while the
* host was suspended (undici keeps them warm and would hand a dead socket
* to the first post-wake request). */
export function flushProxyCache(): void {
+ // Outbound-proxy dispatcher (only in play when a proxy URL is configured).
cached = null;
+ // The default no-proxy path is bare fetch() on Node's GLOBAL undici
+ // dispatcher — exactly the pool the headline laptop-lid scenario rides — so
+ // nulling the proxy cache alone left the flush a no-op for most
+ // deployments. Node keeps that dispatcher in the global symbol registry
+ // (getGlobalDispatcher/setGlobalDispatcher read and write the same key), so
+ // swap in a fresh instance of its own constructor: new requests get new
+ // sockets, in-flight requests keep a reference to the old dispatcher and
+ // complete undisturbed. Deliberately NOT `import('undici')`: the built-in
+ // fetch uses Node's bundled copy, and the npm package isn't installed in
+ // the production image (verified live — the import throws there).
+ try {
+ const sym = Symbol.for('undici.globalDispatcher.1');
+ const current = (globalThis as Record<symbol, unknown>)[sym] as { constructor: new () => unknown } | undefined;
+ // Symbol unset = no fetch has run yet, so there are no pooled sockets to drop.
+ if (current?.constructor) {
+ (globalThis as Record<symbol, unknown>)[sym] = new current.constructor();
+ }
+ } catch (err: any) {
+ console.warn(`[proxy] could not replace the global fetch dispatcher on wake: ${err?.message ?? err}`);
+ }
}
diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.ts
index 717b95dc..0e832ae8 100644
--- a/server/src/lib/sampling-params.ts
+++ b/server/src/lib/sampling-params.ts
@@ -22,6 +22,7 @@
// droplist entry is one line to add.
import { z } from 'zod';
+import type { Platform } from '@freellmapi/shared/types.js';
// Every field is `.nullable()` because real clients serialize their whole
// request struct and send explicit nulls for unset knobs (#200); null is
@@ -118,7 +119,10 @@ export interface PlatformParamPolicy {
jsonObjectToSchema?: boolean;
}
-export const PLATFORM_PARAM_POLICIES: Record<string, PlatformParamPolicy> = {
+// Keyed by Platform (not string) so a typo'd platform id fails tsc instead of
+// silently no-op'ing the policy; the string-typed accessors below cast at the
+// boundary since routes carry platform ids as plain strings.
+export const PLATFORM_PARAM_POLICIES: Partial<Record<Platform, PlatformParamPolicy>> = {
// Mistral's API is strict (422 on unknown body keys) and names its seed
// `random_seed`. It has no top_k/min_p/logit_bias/logprobs equivalents.
mistral: {
@@ -168,7 +172,7 @@ const ANY_OBJECT_SCHEMA = {
*/
export function extendedBodyParams(platform: string, options: ExtendedSamplingOptions | undefined): Record<string, unknown> {
if (!options) return {};
- const policy = PLATFORM_PARAM_POLICIES[platform];
+ const policy = PLATFORM_PARAM_POLICIES[platform as Platform];
const dropped = new Set<string>(policy?.drop ?? []);
const out: Record<string, unknown> = {};
for (const key of EXTENDED_SAMPLING_KEYS) {
@@ -187,14 +191,14 @@ export function extendedBodyParams(platform: string, options: ExtendedSamplingOp
/** True when this platform's policy strips response_format before send — the
* router uses it to skip such platforms for structured-output requests. */
export function platformDropsResponseFormat(platform: string): boolean {
- return PLATFORM_PARAM_POLICIES[platform]?.drop?.includes('response_format') ?? false;
+ return PLATFORM_PARAM_POLICIES[platform as Platform]?.drop?.includes('response_format') ?? false;
}
/** The advertised parameter list for a model on `platform` — the base set
* every surface supports, plus tools when the model does, minus the
* platform's droplist. */
export function supportedParametersFor(platform: string, caps: { tools?: boolean } = {}): string[] {
- const policy = PLATFORM_PARAM_POLICIES[platform];
+ const policy = PLATFORM_PARAM_POLICIES[platform as Platform];
const dropped = new Set<string>(policy?.drop ?? []);
const params = [
'temperature', 'top_p', 'max_tokens', 'max_completion_tokens', 'stop', 'stream',
diff --git a/server/src/lib/structured-output.ts b/server/src/lib/structured-output.ts
index 28b4fcb6..c6707fe4 100644
--- a/server/src/lib/structured-output.ts
+++ b/server/src/lib/structured-output.ts
@@ -33,17 +33,23 @@ function parses(text: string): boolean {
}
}
-/** Slice from the first '{' or '[' to the matching last '}' or ']'. */
-function outermostJsonSlice(text: string): string | null {
- const firstObj = text.indexOf('{');
- const firstArr = text.indexOf('[');
- const first = firstObj === -1 ? firstArr : (firstArr === -1 ? firstObj : Math.min(firstObj, firstArr));
- if (first === -1) return null;
- const open = text[first];
- const close = open === '{' ? '}' : ']';
- const last = text.lastIndexOf(close);
- if (last <= first) return null;
- return text.slice(first, last + 1);
+/**
+ * Candidate slices from the first '{' to the last '}' and from the first '['
+ * to the last ']', longest first. Both bracket types must be tried
+ * independently: picking whichever opener appears FIRST turned
+ * `According to [1], here is the JSON: {"city":"Paris"}` into the citation
+ * marker `[1]` — a valid parse of the wrong value, silently delivered as the
+ * structured output. Longest-first means an incidental bracket token in prose
+ * can never shadow the real payload next to it.
+ */
+function candidateJsonSlices(text: string): string[] {
+ const out: string[] = [];
+ for (const [open, close] of [['{', '}'], ['[', ']']] as const) {
+ const first = text.indexOf(open);
+ const last = text.lastIndexOf(close);
+ if (first !== -1 && last > first) out.push(text.slice(first, last + 1));
+ }
+ return out.sort((a, b) => b.length - a.length);
}
export function enforceJsonContent(content: string): JsonEnforcement {
@@ -60,8 +66,9 @@ export function enforceJsonContent(content: string): JsonEnforcement {
}
// Leading/trailing prose around one JSON value ("Here is your JSON: {...}").
- const slice = outermostJsonSlice(trimmed);
- if (slice && parses(slice)) return { ok: true, content: slice, healed: true };
+ for (const slice of candidateJsonSlices(trimmed)) {
+ if (parses(slice)) return { ok: true, content: slice, healed: true };
+ }
return { ok: false };
}
diff --git a/server/src/lib/wake-detect.ts b/server/src/lib/wake-detect.ts
index 56b79be0..a70850b8 100644
--- a/server/src/lib/wake-detect.ts
+++ b/server/src/lib/wake-detect.ts
@@ -27,11 +27,21 @@ export interface WakeEvent {
const TICK_MS = 5_000;
const DRIFT_THRESHOLD_MS = 30_000;
+// One recovery pass per resume: a drift tick and an operator SIGCONT/SIGUSR2
+// for the SAME wake (or signal spam) must not stack flushes + full key
+// re-probes on top of each other.
+const WAKE_DEBOUNCE_MS = 15_000;
let hooks: WakeHooks | null = null;
let timer: NodeJS.Timeout | null = null;
let lastTickAt = 0;
+let lastWakeAt = 0;
let installed = false;
+// Kept so stopWakeDetect can remove exactly the listeners this module added —
+// removeAllListeners would silently unhook any OTHER module's SIGUSR2 handler
+// (log rotation, heap-dump triggers).
+const sigcontHandler = () => handleSignal('SIGCONT');
+const sigusr2Handler = () => handleSignal('SIGUSR2');
function tick(): void {
const now = Date.now();
@@ -57,6 +67,9 @@ function handleSignal(name: string): void {
// logged and swallowed, sync or async.
function invokeHooks(event: WakeEvent): void {
if (!hooks) return;
+ const now = Date.now();
+ if (now - lastWakeAt < WAKE_DEBOUNCE_MS) return;
+ lastWakeAt = now;
try {
Promise.resolve(hooks.onWake(event)).catch((err) => {
console.error(`[wake-detect] onWake handler error: ${err?.message ?? err}`);
@@ -73,8 +86,8 @@ export function startWakeDetect(h: WakeHooks): void {
lastTickAt = Date.now();
timer = setTimeout(tick, TICK_MS);
if (timer.unref) timer.unref();
- process.on('SIGCONT', () => handleSignal('SIGCONT'));
- process.on('SIGUSR2', () => handleSignal('SIGUSR2'));
+ process.on('SIGCONT', sigcontHandler);
+ process.on('SIGUSR2', sigusr2Handler);
}
export function stopWakeDetect(): void {
@@ -84,10 +97,11 @@ export function stopWakeDetect(): void {
clearTimeout(timer);
timer = null;
}
- process.removeAllListeners('SIGCONT');
- process.removeAllListeners('SIGUSR2');
+ process.removeListener('SIGCONT', sigcontHandler);
+ process.removeListener('SIGUSR2', sigusr2Handler);
hooks = null;
lastTickAt = 0;
+ lastWakeAt = 0;
}
export function _resetForTests(): void {
diff --git a/server/src/providers/google.ts b/server/src/providers/google.ts
index c6cf7f39..eb94e340 100644
--- a/server/src/providers/google.ts
+++ b/server/src/providers/google.ts
@@ -202,7 +202,12 @@ export function toGeminiExtendedConfig(options?: CompletionOptions): Record<stri
frequencyPenalty: options?.frequency_penalty,
};
const rf = options?.response_format;
- const hasTools = (options?.tools?.length ?? 0) > 0;
+ // Count only real function declarations, mirroring hasFunctionDeclarations:
+ // grounding pseudo-tools (google_search etc.) are converted to a grounding
+ // block by toGeminiTools and never conflict with responseMimeType — raw
+ // tools.length was silently dropping structured output for grounding-only
+ // requests.
+ const hasTools = (options?.tools ?? []).some(t => !GROUNDING_TOOL_NAMES.has(t.function.name.toLowerCase()));
if (rf && !hasTools) {
out.responseMimeType = 'application/json';
const schema = rf.type === 'json_schema' ? rf.json_schema?.schema : undefined;
@@ -603,20 +608,76 @@ export class GoogleProvider extends BaseProvider {
const seenToolCallKeys = new Set<string>();
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split('\n');
+ buffer = lines.pop() ?? '';
+
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed || !trimmed.startsWith('data: ')) continue;
+ const raw = trimmed.slice(6);
+ if (raw === '[DONE]') {
+ if (!emittedFinish) {
+ emittedFinish = true;
+ yield {
+ id,
+ object: 'chat.completion.chunk',
+ created: Math.floor(Date.now() / 1000),
+ model: modelId,
+ choices: [{
+ index: 0,
+ delta: {},
+ finish_reason: sawToolCalls ? 'tool_calls' : 'stop',
+ }],
+ };
+ }
+ return;
+ }
+
+ // Skip malformed SSE frames instead of aborting the whole stream.
+ // Matches the defensive parse in openai-compat / cohere / cloudflare:
+ // a single corrupt chunk shouldn't take down the rest of the response.
+ let chunk: GeminiResponse;
+ try {
+ chunk = JSON.parse(raw) as GeminiResponse;
+ } catch {
+ continue;
+ }
+ const candidate = chunk.candidates?.[0];
+ const parts = candidate?.content?.parts ?? [];
+
+ const text = extractText(parts);
+ const toolCalls = extractToolCalls(parts).filter(call => {
+ const key = `${call.id}:${call.function.name}:${call.function.arguments}`;
+ if (seenToolCallKeys.has(key)) return false;
+ seenToolCallKeys.add(key);
+ return true;
+ });
- buffer += decoder.decode(value, { stream: true });
- const lines = buffer.split('\n');
- buffer = lines.pop() ?? '';
+ if ((text && text.length > 0) || toolCalls.length > 0) {
+ sawToolCalls = sawToolCalls || toolCalls.length > 0;
+ yield {
+ id,
+ object: 'chat.completion.chunk',
+ created: Math.floor(Date.now() / 1000),
+ model: modelId,
+ choices: [{
+ index: 0,
+ delta: {
+ ...(text ? { content: text } : {}),
+ ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
+ },
+ finish_reason: null,
+ }],
+ };
+ }
- for (const line of lines) {
- const trimmed = line.trim();
- if (!trimmed || !trimmed.startsWith('data: ')) continue;
- const raw = trimmed.slice(6);
- if (raw === '[DONE]') {
- if (!emittedFinish) {
+ if (candidate?.finishReason && !emittedFinish) {
emittedFinish = true;
yield {
id,
@@ -626,67 +687,21 @@ export class GoogleProvider extends BaseProvider {
choices: [{
index: 0,
delta: {},
- finish_reason: sawToolCalls ? 'tool_calls' : 'stop',
+ finish_reason: sawToolCalls ? 'tool_calls' : toGeminiFinishReason(candidate.finishReason),
}],
};
+ return;
}
- return;
- }
-
- // Skip malformed SSE frames instead of aborting the whole stream.
- // Matches the defensive parse in openai-compat / cohere / cloudflare:
- // a single corrupt chunk shouldn't take down the rest of the response.
- let chunk: GeminiResponse;
- try {
- chunk = JSON.parse(raw) as GeminiResponse;
- } catch {
- continue;
- }
- const candidate = chunk.candidates?.[0];
- const parts = candidate?.content?.parts ?? [];
-
- const text = extractText(parts);
- const toolCalls = extractToolCalls(parts).filter(call => {
- const key = `${call.id}:${call.function.name}:${call.function.arguments}`;
- if (seenToolCallKeys.has(key)) return false;
- seenToolCallKeys.add(key);
- return true;
- });
-
- if ((text && text.length > 0) || toolCalls.length > 0) {
- sawToolCalls = sawToolCalls || toolCalls.length > 0;
- yield {
- id,
- object: 'chat.completion.chunk',
- created: Math.floor(Date.now() / 1000),
- model: modelId,
- choices: [{
- index: 0,
- delta: {
- ...(text ? { content: text } : {}),
- ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),
- },
- finish_reason: null,
- }],
- };
- }
-
- if (candidate?.finishReason && !emittedFinish) {
- emittedFinish = true;
- yield {
- id,
- object: 'chat.completion.chunk',
- created: Math.floor(Date.now() / 1000),
- model: modelId,
- choices: [{
- index: 0,
- delta: {},
- finish_reason: sawToolCalls ? 'tool_calls' : toGeminiFinishReason(candidate.finishReason),
- }],
- };
- return;
}
}
+ } finally {
+ // Runs on normal completion, on the early returns above, AND when the
+ // consumer abandons the generator mid-stream (a client disconnect breaks
+ // the route pump's for-await, which calls gen.return() at the yield
+ // point). Without it the Gemini generation kept running upstream —
+ // quota burning with nobody reading — because this adapter reads the
+ // body itself instead of going through readSseStream's shared cleanup.
+ reader.cancel().catch(() => { /* upstream already gone */ });
}
if (!emittedFinish) {
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 92da2945..3abd4ec4 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -369,27 +369,35 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
const body = parsed.data;
const requestedModel = body.model ?? 'auto';
- const max_tokens = body.max_tokens != null && body.max_tokens > 0 ? body.max_tokens : DEFAULT_MAX_TOKENS;
+ // The Anthropic wire format requires max_tokens, but OUR default injection
+ // must not change the budget gate's verdict: gating AFTER defaulting made a
+ // no-max_tokens request 413 here (input + 1024 over budget) where the chat
+ // surface would cap it and serve. Gate on what the client actually sent,
+ // then resolve the default against the budget remainder.
+ const clientMaxTokens = body.max_tokens != null && body.max_tokens > 0 ? body.max_tokens : undefined;
const { temperature, top_p, stream } = body;
const { messages, tools, tool_choice, hasImage, wantsTools } = convertRequest(body);
- const completionOptions = { temperature, max_tokens, top_p, top_k: body.top_k ?? undefined, tools, tool_choice };
const estimatedInputTokens = estimateTokens(messages);
const imageCount = messages.reduce((n, m) =>
n + (Array.isArray(m.content) ? m.content.filter(b => (b as any)?.type === 'image_url').length : 0), 0);
- // Capped output reserve so a large max_tokens can't falsely exclude the model
- // pool (#470); input + images count in full.
- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);
// Guardrail: per-request token budget (request_max_tokens_budget, default
- // off). max_tokens is always set on this surface (Anthropic requires it),
- // so a violation can only reject — no capping branch.
- const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, max_tokens);
+ // off). A client-set max_tokens can only pass or reject; an omitted one is
+ // capped to the budget remainder, mirroring /v1/chat/completions.
+ const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, clientMaxTokens);
if (budgetCheck.rejection) {
sendError(res, 413, 'invalid_request_error', tokenBudgetMessage(budgetCheck.rejection));
return;
}
+ const max_tokens = clientMaxTokens
+ ?? (budgetCheck.maxTokens != null ? Math.min(DEFAULT_MAX_TOKENS, budgetCheck.maxTokens) : DEFAULT_MAX_TOKENS);
+
+ const completionOptions = { temperature, max_tokens, top_p, top_k: body.top_k ?? undefined, tools, tool_choice };
+ // Capped output reserve so a large max_tokens can't falsely exclude the model
+ // pool (#470); input + images count in full.
+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);
// Resolve the model through the operator's Claude-family map (opus/sonnet/
// haiku/default → auto | a pinned catalog model). A concrete catalog id pins
@@ -713,6 +721,15 @@ async function streamCompletion(
// Nothing usable came out — fail over (message_start was never sent, so the
// client never saw this attempt).
if (!messageStarted && completedCalls.length === 0) {
+ // Disconnect before the commit point: the pump loop broke with nothing
+ // accumulated because the CLIENT left, not because the provider failed —
+ // benching a healthy model+key for it was wrong. StreamAlreadyStarted
+ // maps to 'committed' in the dispatcher: stop, no failover, no
+ // exhaustion render (the socket is gone anyway).
+ if (ctx.clientGone()) {
+ console.log(`[Anthropic] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);
+ throw new StreamAlreadyStarted();
+ }
// finish_reason 'length' = the model spent the whole output budget on
// hidden reasoning before any visible text: fail over, but skip the
// cooldown/penalty (not a provider-health signal).
diff --git a/server/src/routes/mcp.ts b/server/src/routes/mcp.ts
index 41592763..fc1dacff 100644
--- a/server/src/routes/mcp.ts
+++ b/server/src/routes/mcp.ts
@@ -30,8 +30,12 @@ import { getCacheStats } from '../services/cache.js';
export const mcpRouter = Router();
-const SUPPORTED_PROTOCOL_VERSIONS = new Set(['2025-06-18', '2025-03-26', '2024-11-05']);
-const DEFAULT_PROTOCOL_VERSION = '2025-06-18';
+// Always negotiated to 2025-06-18: this transport rejects JSON-RPC batches,
+// which the 2025-03-26 and 2024-11-05 revisions still allowed — echoing an
+// older requested version while enforcing the newer transport rule promised
+// clients batching they'd never get. Per the MCP spec the server answers with
+// the latest version it supports; clients that can't speak it disconnect.
+const PROTOCOL_VERSION = '2025-06-18';
interface JsonRpcRequest {
jsonrpc?: string;
@@ -118,9 +122,17 @@ function providerHealth(): unknown {
const USAGE_RANGES: Record<string, number> = { '24h': 24, '7d': 24 * 7, '30d': 24 * 30 };
function usageSummary(args: Record<string, unknown>): unknown {
- const range = typeof args.range === 'string' && USAGE_RANGES[args.range] ? args.range : '24h';
+ // Object.hasOwn: a prototype key ('constructor', 'toString') is truthy via
+ // the prototype chain but multiplies to NaN below — fall back to 24h.
+ const range = typeof args.range === 'string' && Object.hasOwn(USAGE_RANGES, args.range) ? args.range : '24h';
const db = getDb();
- const since = new Date(Date.now() - USAGE_RANGES[range] * 3600_000).toISOString();
+ // SQLite datetime('now') format (space separator, no ms/Z) — an ISO 'T'
+ // string compares GREATER than every same-day stored row (space < 'T'
+ // lexicographically), which silently dropped the window's boundary day:
+ // "24h" effectively meant "since UTC midnight". Same conversion as
+ // routes/analytics.ts.
+ const since = new Date(Date.now() - USAGE_RANGES[range] * 3600_000)
+ .toISOString().slice(0, 19).replace('T', ' ');
const totals = db.prepare(`
SELECT COALESCE(SUM(total_requests), 0) AS requests,
COALESCE(SUM(success_count), 0) AS successes,
@@ -225,14 +237,23 @@ const TOOLS: Record<string, McpTool> = {
// ── JSON-RPC dispatch ────────────────────────────────────────────────────
+// Returns the JSON-RPC response for a request, or undefined for a
+// notification. JSON-RPC 2.0 defines a notification as a message WITHOUT an
+// `id` member (id:null is a — discouraged — request and gets a response);
+// detecting notifications by the `notifications/` method prefix answered
+// no-id requests and 202'd id-carrying notifications.
function handleRpc(msg: JsonRpcRequest): unknown | undefined {
+ const isNotification = msg.id === undefined;
+ const respond = (response: unknown) => (isNotification ? undefined : response);
const id = msg.id ?? null;
+ return respond(dispatchRpc(msg, id));
+}
+
+function dispatchRpc(msg: JsonRpcRequest, id: number | string | null): unknown {
switch (msg.method) {
case 'initialize': {
- const requested = (msg.params?.protocolVersion as string) ?? '';
- const version = SUPPORTED_PROTOCOL_VERSIONS.has(requested) ? requested : DEFAULT_PROTOCOL_VERSION;
return rpcResult(id, {
- protocolVersion: version,
+ protocolVersion: PROTOCOL_VERSION,
capabilities: { tools: {} },
serverInfo: { name: 'freellmapi', version: '1.0.0' },
instructions: 'FreeLLMAPI gateway introspection: list usable free models (with per-model supported_parameters), check provider/key health, read usage and cache stats, and switch the routing strategy. Inference goes through the OpenAI-compatible /v1 endpoints, not MCP.',
@@ -250,7 +271,10 @@ function handleRpc(msg: JsonRpcRequest): unknown | undefined {
});
case 'tools/call': {
const name = msg.params?.name as string;
- const tool = TOOLS[name];
+ // Object.hasOwn: a bare index lookup resolves prototype members, so
+ // name:"constructor"/"toString" passed the !tool check and died inside
+ // the try as a confusing tool-level error instead of -32602.
+ const tool = typeof name === 'string' && Object.hasOwn(TOOLS, name) ? TOOLS[name] : undefined;
if (!tool) return rpcError(id, -32602, `Unknown tool: ${name}`);
try {
const args = (msg.params?.arguments as Record<string, unknown>) ?? {};
@@ -261,7 +285,9 @@ function handleRpc(msg: JsonRpcRequest): unknown | undefined {
}
}
default:
- if (msg.method?.startsWith('notifications/')) return undefined; // ack silently
+ // Known client notifications (notifications/initialized etc.) land here;
+ // handleRpc drops the response for anything sent without an id, so they
+ // are acked silently without special-casing the method name.
return rpcError(id, -32601, `Method not found: ${msg.method}`);
}
}
@@ -270,7 +296,11 @@ function authenticate(req: Request, res: Response): boolean {
const token = extractApiToken(req);
const unifiedKey = getUnifiedApiKey();
if (!token || !timingSafeStringEqual(token, unifiedKey)) {
- res.status(401).json(rpcError(null, -32001, 'Invalid API key. Authenticate with the unified key as a Bearer token.'));
+ // Echo the request id when the body carries one, so strict JSON-RPC
+ // clients can correlate the auth error with their pending call.
+ const body = req.body;
+ const id = body && typeof body === 'object' && !Array.isArray(body) && body.id !== undefined ? body.id : null;
+ res.status(401).json(rpcError(id, -32001, 'Invalid API key. Authenticate with the unified key as a Bearer token.'));
return false;
}
return true;
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 3fa240b5..36cae521 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -786,8 +786,6 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
);
for await (const chunk of gen) {
- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
const text = streamChunkText(chunk);
if (text.length > 0) sawText = true;
@@ -805,6 +803,16 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
res.write(`data: ${JSON.stringify(frame)}\n\n`);
}
+ // Disconnect before the commit point: the break above fired with no
+ // text seen, which is indistinguishable from an empty completion
+ // below — but it is CLIENT behavior, not a provider failure. Without
+ // this check every Ctrl-C during a reasoning model's TTFB window
+ // benched the healthy model+key for 90s and logged a provider error.
+ if (clientGone && !headerSent && !sawText) {
+ console.log(`[Proxy] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);
+ return 'committed';
+ }
+
if (!sawText) {
// finish_reason 'length' means the model spent the whole output
// budget before any visible text (hidden reasoning) — fail over,
@@ -1251,6 +1259,24 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
options: fusionOptions,
estimatedTokens: estimatedTotal,
});
+ // Structured-output enforcement for fusion (#516 scope gap): the panel/
+ // judge output got no format check, so model:"fusion" could hand back
+ // prose as a "success" for a json_schema request. Fusion has no failover
+ // machinery to hand this to — heal what's healable, otherwise answer
+ // honestly instead of pretending. (Streaming fusion stays unenforced,
+ // same boundary as every other streamed response.)
+ const fusionMsg = (response as any)?.choices?.[0]?.message;
+ if (samplingParams.response_format && fusionMsg && !fusionMsg.tool_calls?.length) {
+ const fusionText = contentToString(fusionMsg.content ?? '');
+ if (fusionText) {
+ const enforced = enforceJsonContent(fusionText);
+ if (!enforced.ok) {
+ res.status(502).json({ error: { message: `fusion produced non-JSON output despite response_format=${samplingParams.response_format.type} — retry, or pin a structured-output-capable model instead of "fusion"`, type: 'server_error' } });
+ return;
+ }
+ if (enforced.healed) fusionMsg.content = enforced.content;
+ }
+ }
res.setHeader('X-Routed-Via', routedVia);
res.json(response);
} catch (err: any) {
@@ -1518,8 +1544,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
);
for await (const chunk of gen) {
- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
const anyChunk = chunk as Record<string, any>;
@@ -1641,6 +1665,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
}
+ // Disconnect before the commit point: nothing usable was (or will
+ // be) delivered, and that is CLIENT behavior, not a provider
+ // failure — do not let it fall through to the empty-completion
+ // throw below, which would bench a healthy model+key for 90s and
+ // log a provider error for every Ctrl-C during a reasoning model's
+ // TTFB window.
+ if (clientGone && !headerSent && heldText.trim().length === 0 && completedCalls.length === 0) {
+ console.log(`[Proxy] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);
+ return 'committed';
+ }
+
const hasText = headerSent || heldText.trim().length > 0;
if (!hasText && completedCalls.length === 0) {
// Nothing usable came out — same failover semantics as the
@@ -1740,25 +1775,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
);
}
- // Structured-output enforcement (#514 follow-up): the client asked for
- // JSON; a model that answered in prose despite the forwarded
- // response_format must not be returned as a "success". Heal the common
- // almost-right shapes (fenced block, prose-wrapped JSON) in place;
- // otherwise fail over. skipBench: the provider is healthy — the MODEL
- // ignored the format — so no cooldown/penalty, just the next candidate.
- if (samplingParams.response_format && respText && (respMsg?.tool_calls?.length ?? 0) === 0) {
- const enforced = enforceJsonContent(respText);
- if (!enforced.ok) {
- throw Object.assign(
- new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),
- { skipBench: true },
- );
- }
- if (enforced.healed && respMsg) {
- respMsg.content = enforced.content;
- }
- }
-
// Inline tool-call dialect rescue (#231 audit): a tool-bearing
// request answered with the call serialized as TEXT (a mid-
// conversation model switch makes the new model imitate the previous
@@ -1783,6 +1799,35 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
}
+ // Structured-output enforcement (#514 follow-up): the client asked for
+ // JSON; a model that answered in prose despite the forwarded
+ // response_format must not be returned as a "success". Heal the common
+ // almost-right shapes (fenced block, prose-wrapped JSON) in place;
+ // otherwise fail over. Deliberately AFTER the dialect rescue (matching
+ // responses.ts): an inline tool-call turn isn't JSON either, and
+ // gating it first burned a failover hop on turns the rescue converts.
+ // skipBench: the provider is healthy — the MODEL misbehaved — so no
+ // cooldown/penalty; skipModelForRequest: a sibling key would misbehave
+ // identically, so rule out the whole model for this request.
+ if (samplingParams.response_format && respText && (respMsg?.tool_calls?.length ?? 0) === 0) {
+ const enforced = enforceJsonContent(respText);
+ if (!enforced.ok) {
+ // finish_reason 'length' = the JSON was CUT OFF by max_tokens, not
+ // ignored — same failover (a terser model may fit the budget), but
+ // an honest error class/trail instead of "ignored response_format".
+ const truncated = result.choices?.[0]?.finish_reason === 'length';
+ throw Object.assign(
+ new Error(truncated
+ ? `truncated JSON from ${route.displayName} (finish_reason=length — raise max_tokens for this ${samplingParams.response_format.type} request)`
+ : `${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),
+ { skipBench: true, skipModelForRequest: true },
+ );
+ }
+ if (enforced.healed && respMsg) {
+ respMsg.content = enforced.content;
+ }
+ }
+
// Usage fallback: providers that omit `usage` used to be logged as 0
// tokens, silently undercounting analytics and the rate-limit ledger.
// Fall back to the same chars/4 estimate the streaming path uses (tool
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 8735c5c7..97656c26 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -503,8 +503,6 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
);
for await (const chunk of gen) {
- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream
// In-band upstream error frame ({"error":...} inside a 200 SSE
// stream — observed live from Groq). Throwing hands it to the catch
@@ -621,6 +619,15 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// been committed yet (the skeleton is lazy), so throwing lets the
// shared loop fail over to the next model on the same SSE connection.
if (msgText.length === 0 && toolAcc.size === 0) {
+ // Disconnect before the commit point: the break above fired with
+ // nothing accumulated — CLIENT behavior, not a provider failure.
+ // Falling through to the empty-completion throw benched a healthy
+ // model+key for 90s on every Ctrl-C during a reasoning model's
+ // TTFB window.
+ if (clientGone && !streamStarted) {
+ console.log(`[Responses] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);
+ return 'committed';
+ }
// finish_reason 'length' = the model spent the whole output budget
// on hidden reasoning before any visible text: fail over, but skip
// the cooldown/penalty (not a provider-health signal).
@@ -739,14 +746,19 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
// Structured-output enforcement — see /chat/completions. Heal fenced or
- // prose-wrapped JSON in place, fail over (skipBench) when the model
- // ignored the requested format outright.
+ // prose-wrapped JSON in place, fail over (skipBench + skipModelForRequest
+ // — a sibling key would misbehave identically) when the model ignored the
+ // requested format; finish_reason 'length' gets an honest "truncated"
+ // class instead, since the JSON was cut off by max_tokens, not ignored.
if (completionOpts.response_format && text && toolCalls.length === 0) {
const enforced = enforceJsonContent(text);
if (!enforced.ok) {
+ const truncated = result.choices[0]?.finish_reason === 'length';
throw Object.assign(
- new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${completionOpts.response_format.type})`),
- { skipBench: true },
+ new Error(truncated
+ ? `truncated JSON from ${route.displayName} (finish_reason=length — raise max_tokens for this ${completionOpts.response_format.type} request)`
+ : `${route.displayName} ignored response_format (returned non-JSON despite ${completionOpts.response_format.type})`),
+ { skipBench: true, skipModelForRequest: true },
);
}
if (enforced.healed) text = enforced.content;
diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.ts
index fc25b1f5..a4f601ee 100644
--- a/server/src/services/fusion.ts
+++ b/server/src/services/fusion.ts
@@ -678,7 +678,15 @@ export async function runFusion(params: {
const cand = resolveFusionCandidate(config.judge!);
return cand ? routePinnedModel(cand.modelDbId, judgeEstimate, skipKeys) : null;
}
- : (skipKeys: Set<string>, skipModels: Set<number>) => routeRequest(judgeEstimate, skipKeys.size ? skipKeys : undefined, undefined, false, false, skipModels.size ? skipModels : undefined);
+ : (skipKeys: Set<string>, skipModels: Set<number>) => routeRequest(
+ judgeEstimate, skipKeys.size ? skipKeys : undefined, undefined, false, false,
+ skipModels.size ? skipModels : undefined, undefined,
+ // The judge writes the final answer the client receives, so a
+ // structured-output request must not land it on a platform whose
+ // policy drops response_format (kilo) — the schema would never even
+ // reach the model. Mirrors the non-fusion routing (#516).
+ options.response_format !== undefined,
+ );
// Stream the judge when the caller wants live tokens (Playground); otherwise
// a single buffered call (plain API clients hitting fusion non-streaming).
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index 5b44262a..2b53be85 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -66,17 +66,31 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
}
}
-export async function checkAllKeys(): Promise<void> {
- const db = getDb();
- const keys = db.prepare('SELECT id, platform FROM api_keys WHERE enabled = 1').all() as { id: number; platform: string }[];
-
- console.log(`[Health] Checking ${keys.length} keys...`);
-
- for (const key of keys) {
- await checkKeyHealth(key.id);
- }
+// Overlap guard: the scheduled 5-minute pass and wake-recovery re-probes can
+// coincide (or SIGCONT spam can queue several) — concurrent full passes
+// multiply provider validate traffic and let two passes each increment the
+// same genuinely-bad key's failureCount, reaching the auto-disable threshold
+// in fewer wall-clock checks than "3 consecutive checks" intends. A second
+// caller joins the in-flight pass instead of starting another.
+let checkAllInFlight: Promise<void> | null = null;
+
+export function checkAllKeys(): Promise<void> {
+ if (checkAllInFlight) return checkAllInFlight;
+ checkAllInFlight = (async () => {
+ const db = getDb();
+ const keys = db.prepare('SELECT id, platform FROM api_keys WHERE enabled = 1').all() as { id: number; platform: string }[];
+
+ console.log(`[Health] Checking ${keys.length} keys...`);
+
+ for (const key of keys) {
+ await checkKeyHealth(key.id);
+ }
- console.log(`[Health] Check complete.`);
+ console.log(`[Health] Check complete.`);
+ })().finally(() => {
+ checkAllInFlight = null;
+ });
+ return checkAllInFlight;
}
let cancelHealthCheck: (() => void) | null = null;
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 16602b9f..fc4e2a59 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -69,6 +69,7 @@ export function summarizeExhaustion(
else if (l.includes('< estimated')) bump('prompt too large for the model');
else if (l.includes('no vision support')) bump('model lacks vision');
else if (l.includes('no tool-calling support')) bump('model lacks tool-calling');
+ else if (l.includes('drops response_format')) bump('platform cannot honor response_format');
else if (/ruled out|already-failed/.test(l)) bump('failed earlier this request');
else if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap/.test(l)) bump('rate-limited or on cooldown');
else bump('unavailable');
@@ -80,6 +81,7 @@ export function summarizeExhaustion(
'prompt too large for the model',
'model lacks vision',
'model lacks tool-calling',
+ 'platform cannot honor response_format',
'failed earlier this request',
'unsupported provider',
'unavailable',