7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix: harden 7/11 features — disconnect misclassification, JSON heal, breaker, MCP, wake flush (#528)
Tashfeen90c84daab798
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.tsindex da69ed2c..bea71005 100644--- a/server/src/__tests__/lib/fallback-loop.test.ts+++ b/server/src/__tests__/lib/fallback-loop.test.ts@@ -163,6 +163,30 @@ describe('recordRetryableFailure skipBench exemption (reasoning truncation)', ()// modelDbId 424242 has no sibling key rows, so the penalty fires.expect(getAllPenalties().some(p => p.modelDbId === route.modelDbId)).toBe(true);});++ it('skipModelForRequest rules out the whole model, not just the key', () => {+ // Format-ignore is MODEL behavior: a sibling key reproduces it exactly, so+ // burning one failover hop per key on the same model was pure waste.+ const route = fakeRoute();+ const state = newFallbackState();+ const err = Object.assign(+ new Error(`${route.displayName} ignored response_format (returned non-JSON despite json_schema)`),+ { skipBench: true, skipModelForRequest: true },+ );+ recordRetryableFailure(route, err, state);++ expect(state.skipModels.has(route.modelDbId)).toBe(true);+ expect(state.skipKeys.has(`fake:fake-model:${route.keyId}`)).toBe(true);+ // skipBench still applies: no cooldown, no penalty.+ const cooldown = getDb().prepare('SELECT 1 FROM rate_limit_cooldowns WHERE platform = ? AND key_id = ?').get('fake', route.keyId);+ expect(cooldown).toBeUndefined();+ expect(getAllPenalties().some(p => p.modelDbId === route.modelDbId)).toBe(false);+ });++ it('classifyAttemptError buckets format-ignore and truncated-JSON as format_ignored', () => {+ expect(classifyAttemptError(new Error('X ignored response_format (returned non-JSON despite json_object)'))).toBe('format_ignored');+ expect(classifyAttemptError(new Error('truncated JSON from X (finish_reason=length — raise max_tokens for this json_schema request)'))).toBe('format_ignored');+ });});describe('exhaustedRetryError attempt trail + auth exhaustion', () => {@@ -399,6 +423,42 @@ describe('runFallbackLoop: circuit-breaker guardrail (max_consecutive_upstream_fexpect(info.attempts).toHaveLength(2);});+ it('skipBench failures (format ignored, hidden reasoning) never trip the breaker', async () => {+ // Three prose answers to a json_schema request say nothing about pool+ // health — candidate four must still get its shot. (#511 x #516)+ const onExhausted = vi.fn();+ let calls = 0;+ const dispatch = vi.fn(async () => {+ calls += 1;+ if (calls <= 3) {+ throw Object.assign(+ new Error(`Fake ${calls} ignored response_format (returned non-JSON despite json_schema)`),+ { skipBench: true },+ );+ }+ return 'done' as const;+ });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 3, dispatch, onExhausted }));++ expect(dispatch).toHaveBeenCalledTimes(4); // 3 skipBench failures did not trip the 3-limit breaker+ expect(onExhausted).not.toHaveBeenCalled();+ });++ it('a breaker trip after client disconnect stops the chain without rendering', async () => {+ const onExhausted = vi.fn();+ let gone = false;+ const dispatch = vi.fn(async () => {+ gone = true; // client hangs up during the (only) attempt+ throw retryable429();+ });++ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, breakerLimit: 1, clientGone: () => gone, dispatch, onExhausted }));++ expect(dispatch).toHaveBeenCalledTimes(1);+ expect(onExhausted).not.toHaveBeenCalled(); // no body to a dead socket+ });+it('a success below the limit ends the request normally (breaker untouched)', async () => {const onExhausted = vi.fn();let calls = 0;diff --git a/server/src/__tests__/lib/guardrails.test.ts b/server/src/__tests__/lib/guardrails.test.tsindex c2c684cc..39dd8883 100644--- a/server/src/__tests__/lib/guardrails.test.ts+++ b/server/src/__tests__/lib/guardrails.test.ts@@ -20,6 +20,7 @@ import {recordBreakerFailure,REQUEST_MAX_TOKENS_BUDGET_SETTING,MAX_CONSECUTIVE_UPSTREAM_FAILS_SETTING,+ TOKEN_BUDGET_OUTPUT_CAP,} from '../../lib/guardrails.js';beforeEach(() => {@@ -95,6 +96,16 @@ describe('applyTokenBudget', () => {expect(applyTokenBudget(5000, undefined)).toEqual({ rejection: null, maxTokens: 3000 });});+ it('clamps the injected max_tokens to the output cap under a generous budget', () => {+ // Regression: budget 100000 minus a 1000-token prompt forwarded+ // max_tokens=99000 verbatim, and providers that validate max_tokens+ // against the model's limits 400'd every candidate in the chain.+ settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '100000');+ expect(applyTokenBudget(1000, undefined)).toEqual({ rejection: null, maxTokens: TOKEN_BUDGET_OUTPUT_CAP });+ // A client-set max_tokens is never rewritten, only validated.+ expect(applyTokenBudget(1000, 50_000)).toEqual({ rejection: null, maxTokens: 50_000 });+ });+it('rejects an absent max_tokens when the input alone fills the budget', () => {settingStore.set(REQUEST_MAX_TOKENS_BUDGET_SETTING, '8000');expect(applyTokenBudget(8000, undefined).rejection).toEqual({ budget: 8000, estimatedTotal: 8000 });diff --git a/server/src/__tests__/lib/structured-output.test.ts b/server/src/__tests__/lib/structured-output.test.tsindex 589d66f9..fbebd8b1 100644--- a/server/src/__tests__/lib/structured-output.test.ts+++ b/server/src/__tests__/lib/structured-output.test.ts@@ -45,4 +45,26 @@ describe('enforceJsonContent', () => {const clean = '{"code":"if (a) { return b; }"}';expect(enforceJsonContent(`prefix ${clean} suffix`)).toEqual({ ok: true, content: clean, healed: true });});++ it('a citation marker in leading prose does not shadow the real object', () => {+ // Regression: first-bracket-wins sliced this to `[1]` — a valid parse of+ // the WRONG value, delivered silently as the structured output.+ const r = enforceJsonContent('According to source [1], here is the JSON: {"city":"Paris"}');+ expect(r).toEqual({ ok: true, content: '{"city":"Paris"}', healed: true });+ });++ it('a bracket token in trailing prose does not shadow the real object', () => {+ const r = enforceJsonContent('{"city":"Paris"} as shown in [2]');+ expect(r).toEqual({ ok: true, content: '{"city":"Paris"}', healed: true });+ });++ it('markdown links around a JSON array still heal to the array', () => {+ const r = enforceJsonContent('See [docs](https://x) first. ["a","b","c"] covers it.');+ expect(r).toEqual({ ok: false }); // first-[ to last-] spans the link — unparseable, honest reject+ });++ it('prose-wrapped array with no other brackets heals', () => {+ const r = enforceJsonContent('Here are your items: [1, 2, 3]');+ expect(r).toEqual({ ok: true, content: '[1, 2, 3]', healed: true });+ });});diff --git a/server/src/__tests__/lib/wake-detect.test.ts b/server/src/__tests__/lib/wake-detect.test.tsindex 050508d5..1582100c 100644--- a/server/src/__tests__/lib/wake-detect.test.ts+++ b/server/src/__tests__/lib/wake-detect.test.ts@@ -40,6 +40,29 @@ describe('wake-detect', () => {expect(onWake.mock.calls[0][0]).toMatchObject({ reason: 'signal', signal: 'SIGUSR2' });});+ it('debounces wake events: one recovery pass per resume, not one per signal', () => {+ // A drift tick and an operator SIGCONT for the SAME wake (or signal spam)+ // used to stack full key re-probes on top of each other.+ const onWake = vi.fn();+ startWakeDetect({ onWake });+ process.emit('SIGUSR2' as any);+ process.emit('SIGCONT' as any);+ process.emit('SIGUSR2' as any);+ expect(onWake).toHaveBeenCalledTimes(1);+ });++ it('stop removes only its OWN signal listeners, not other modules’', () => {+ // removeAllListeners('SIGUSR2') silently unhooked any other registrant+ // (log rotation, heap-dump triggers).+ const foreign = vi.fn();+ process.on('SIGUSR2', foreign);+ startWakeDetect({ onWake: vi.fn() });+ stopWakeDetect();+ process.emit('SIGUSR2' as any);+ expect(foreign).toHaveBeenCalledTimes(1);+ process.removeListener('SIGUSR2', foreign);+ });+it('fires a drift wake when the wall clock jumps past the threshold (host suspend)', () => {vi.useFakeTimers();const onWake = vi.fn();diff --git a/server/src/__tests__/providers/google-params.test.ts b/server/src/__tests__/providers/google-params.test.tsindex 195d54e1..18fd454c 100644--- a/server/src/__tests__/providers/google-params.test.ts+++ b/server/src/__tests__/providers/google-params.test.ts@@ -46,6 +46,27 @@ describe('toGeminiExtendedConfig', () => {expect(cfg).not.toHaveProperty('responseSchema');});+ it('grounding-only pseudo-tools do NOT suppress structured output', () => {+ // google_search converts to a grounding block, not functionDeclarations —+ // raw tools.length was silently dropping responseMimeType for it.+ const cfg = toGeminiExtendedConfig({+ response_format: { type: 'json_object' },+ tools: [{ type: 'function', function: { name: 'google_search', parameters: {} } }],+ });+ expect(cfg.responseMimeType).toBe('application/json');+ });++ it('a grounding tool alongside a real function still suppresses structured output', () => {+ const cfg = toGeminiExtendedConfig({+ response_format: { type: 'json_object' },+ tools: [+ { type: 'function', function: { name: 'google_search', parameters: {} } },+ { type: 'function', function: { name: 'f', parameters: {} } },+ ],+ });+ expect(cfg).not.toHaveProperty('responseMimeType');+ });+it('returns all-undefined fields for empty options (JSON.stringify drops them)', () => {const cfg = toGeminiExtendedConfig(undefined);expect(cfg.topK).toBeUndefined();diff --git a/server/src/__tests__/routes/mcp.test.ts b/server/src/__tests__/routes/mcp.test.tsindex 7d427b0c..a110d5ef 100644--- a/server/src/__tests__/routes/mcp.test.ts+++ b/server/src/__tests__/routes/mcp.test.ts@@ -1,7 +1,7 @@import { describe, it, expect, beforeAll } from 'vitest';import type { Express } from 'express';import { createApp } from '../../app.js';-import { initDb, getUnifiedApiKey } from '../../db/index.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';let app: Express;@@ -41,20 +41,20 @@ describe('MCP server (/mcp, stateless Streamable HTTP)', () => {expect(body.error.code).toBe(-32001);});- it('initialize negotiates a supported protocol version and advertises tools', async () => {+ it('initialize always negotiates 2025-06-18 (batch-free transport) and advertises tools', async () => {+ // Echoing an older requested revision (2025-03-26 allowed batching) while+ // the transport rejects batches promised clients something they never got.const { status, body } = await rpc({jsonrpc: '2.0', id: 1, method: 'initialize',params: { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 't', version: '0' } },});expect(status).toBe(200);- expect(body.result.protocolVersion).toBe('2025-03-26');+ expect(body.result.protocolVersion).toBe('2025-06-18');expect(body.result.capabilities.tools).toBeDefined();expect(body.result.serverInfo.name).toBe('freellmapi');- });- it('falls back to the default protocol version for an unknown one', async () => {- const { body } = await rpc({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '1999-01-01' } });- expect(body.result.protocolVersion).toBe('2025-06-18');+ const unknown = await rpc({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '1999-01-01' } });+ expect(unknown.body.result.protocolVersion).toBe('2025-06-18');});it('accepts notifications with a 202 and no body', async () => {@@ -63,6 +63,67 @@ describe('MCP server (/mcp, stateless Streamable HTTP)', () => {expect(body).toBeNull();});+ it('a notification is any message without an id — never answered, even for regular methods', async () => {+ // JSON-RPC 2.0: no `id` member = notification. Detecting notifications by+ // the method prefix answered no-id pings, which violates the spec.+ const { status, body } = await rpc({ jsonrpc: '2.0', method: 'ping' });+ expect(status).toBe(202);+ expect(body).toBeNull();+ });++ it('echoes id:0 (falsy ids must not collapse to null)', async () => {+ const { body } = await rpc({ jsonrpc: '2.0', id: 0, method: 'ping' });+ expect(body.id).toBe(0);+ expect(body.result).toEqual({});+ });++ it('echoes the request id on auth failures', async () => {+ const { status, body } = await rpc({ jsonrpc: '2.0', id: 42, method: 'tools/list' }, { auth: false });+ expect(status).toBe(401);+ expect(body.id).toBe(42);+ });++ it('prototype-key tool names are unknown tools, not confusing tool errors', async () => {+ for (const name of ['constructor', 'toString', '__proto__', 'hasOwnProperty']) {+ const { body } = await rpc({ jsonrpc: '2.0', id: 10, method: 'tools/call', params: { name } });+ expect(body.error?.code).toBe(-32602);+ }+ });++ it('a prototype-key usage range falls back to 24h instead of throwing', async () => {+ const { body } = await rpc({+ jsonrpc: '2.0', id: 11, method: 'tools/call',+ params: { name: 'usage_summary', arguments: { range: 'constructor' } },+ });+ expect(body.result.isError).toBeUndefined();+ expect(toolResultJson(body).range).toBe('24h');+ });++ it('usage_summary counts rows stored in SQLite datetime format (boundary day included)', async () => {+ // Regression: the ISO 'T' in the since-param sorted GREATER than every+ // same-day SQLite-format row (space < 'T'), so the window's boundary day+ // was silently dropped and "24h" behaved like "since UTC midnight".+ const db = getDb();+ // 23h old: still inside the 24h window but (almost always) on the same+ // UTC date as the since-boundary — exactly the rows the bug dropped.+ const boundaryDayRow = new Date(Date.now() - 23 * 3600_000);+ const sqlite = (d: Date) => d.toISOString().slice(0, 19).replace('T', ' ');+ db.prepare(`INSERT INTO requests (platform, model_id, status, input_tokens, output_tokens, created_at)+ VALUES ('groq', 'test-model', 'success', 100, 50, ?)`).run(sqlite(boundaryDayRow));+ const hour = sqlite(boundaryDayRow).slice(0, 13) + ':00:00';+ db.prepare(`INSERT INTO request_hourly (hour, total_requests, success_count, input_tokens, output_tokens)+ VALUES (?, 1, 1, 100, 50)+ ON CONFLICT(hour) DO UPDATE SET total_requests = total_requests + 1`).run(hour);++ const { body } = await rpc({+ jsonrpc: '2.0', id: 12, method: 'tools/call',+ params: { name: 'usage_summary', arguments: { range: '24h' } },+ });+ const data = toolResultJson(body);+ expect(data.requests).toBeGreaterThanOrEqual(1);+ expect(data.top_models.some((m: any) => m.model_id === 'test-model')).toBe(true);+ });+it('lists the six gateway tools with schemas', async () => {const { body } = await rpc({ jsonrpc: '2.0', id: 2, method: 'tools/list' });const names = body.result.tools.map((t: any) => t.name).sort();diff --git a/server/src/app.ts b/server/src/app.tsindex 3956007b..2494da81 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -104,7 +104,10 @@ export function createApp(config?: Config) {// MCP server (Model Context Protocol over stateless Streamable HTTP):// gateway introspection tools for MCP-speaking agents. Unified-key auth,- // like /v1 — NOT behind the dashboard session gate.+ // like /v1 — NOT behind the dashboard session gate. Same per-IP limiter as+ // /v1 (its own bucket): both surfaces guard the same unified key, so an+ // unauthenticated brute-force must not get a free throttle-less oracle here.+ app.use('/mcp', createProxyRateLimiter(cfg.proxyRateLimitRpm));app.use('/mcp', mcpRouter);// Health checkdiff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 5e3d41d3..41f1b1a3 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -66,6 +66,11 @@ export function isRetryableError(err: any): boolean {// JSON couldn't be healed out of the text) — thrown before any byte// reached the client, so the next candidate can serve it invisibly.|| msg.includes('ignored response_format')+ // The model DID emit JSON but ran out of max_tokens mid-value+ // (finish_reason 'length'). A terser model may fit the same schema in the+ // same budget, so fail over — but the thrower marks the model skipped for+ // this request: retrying the same model would truncate identically.+ || msg.includes('truncated json')|| msg.includes('in-band provider error')|| msg.includes('stream ended unexpectedly')|| msg.includes('stream stalled')diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 369391b5..dd23f61c 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -147,7 +147,11 @@ export function cooldownForError(route: RouteResult, err: any): number {* Callers add the just-failed key to skipKeys via this function (do not pre-add).*/export function recordRetryableFailure(route: RouteResult, err: any, state: FallbackState): void {- if (isModelNotFoundError(err) || isModelAccessForbiddenError(err)) {+ // `skipModelForRequest: true` = the failure is MODEL behavior, not key+ // state (ignored response_format, JSON truncated at max_tokens): a sibling+ // key would reproduce it exactly, so rule out the whole model for this+ // request instead of burning one failover hop per key.+ if (isModelNotFoundError(err) || isModelAccessForbiddenError(err) || err?.skipModelForRequest === true) {state.skipModels.add(route.modelDbId);}state.skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);@@ -224,6 +228,7 @@ export type AttemptErrorClass =| 'forbidden'| 'provider_bad_request'| 'empty_completion'+ | 'format_ignored'| 'timeout'| 'rate_limited'| 'upstream_error'@@ -245,6 +250,7 @@ export function classifyAttemptError(err: any): AttemptErrorClass {if (isProviderBadRequestError(err)) return 'provider_bad_request';const msg = (err?.message ?? '').toLowerCase();if (msg.includes('empty completion')) return 'empty_completion';+ if (msg.includes('ignored response_format') || msg.includes('truncated json')) return 'format_ignored';if (msg.includes('timeout') || msg.includes('stalled') || msg.includes('etimedout') || msg.includes('aborted')) return 'timeout';if (msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') || msg.includes('quota')) return 'rate_limited';const status = typeof err?.status === 'number' ? err.status : 0;@@ -488,6 +494,13 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {const breaker = newBreaker(hooks.breakerLimit);const stopIfBreakerTripped = (): boolean => {if (!recordBreakerFailure(breaker)) return false;+ // Tripped after the client already hung up: stop the chain, but there is+ // no socket to render an exhaustion body to (mirrors the loop-top+ // clientGone check).+ if (hooks.clientGone?.()) {+ console.log(`[FallbackLoop] breaker tripped after client disconnect — stopping without rendering (${attempts.length} failed attempt(s))`);+ return true;+ }hooks.onExhausted(exhaustedRetryError(lastError, maxRetries, { attempts, breakerFails: breaker.consecutive }),{ attempts, timedOut: false },@@ -544,7 +557,12 @@ export async function runFallbackLoop(hooks: FallbackHooks): Promise<void> {recordRetryableFailure(route, err, hooks.state);attempts.push({ platform: route.platform, modelId: route.modelId, keyOrdinal: keyOrdinal(route), errorClass: classifyAttemptError(err) });lastError = err;- if (stopIfBreakerTripped()) return;+ // skipBench failures (format ignored, hidden-reasoning truncation) are+ // model behavior, not provider health — recordRetryableFailure already+ // skips the cooldown/penalty for them, and they must not count toward+ // the "pool looks unhealthy" breaker either: three prose answers to a+ // json_schema request say nothing about whether candidate four is up.+ if (err?.skipBench !== true && stopIfBreakerTripped()) return;continue;}hooks.onFatal(route, err, attempt);diff --git a/server/src/lib/guardrails.ts b/server/src/lib/guardrails.tsindex 7995be6a..64b88bfc 100644--- a/server/src/lib/guardrails.ts+++ b/server/src/lib/guardrails.ts@@ -65,10 +65,21 @@ export interface TokenBudgetRejection {export interface TokenBudgetResult {rejection: TokenBudgetRejection | null;/** The max_tokens to forward upstream: unchanged when the client set one,- * capped to the budget remainder when it didn't. */+ * capped to min(budget remainder, TOKEN_BUDGET_OUTPUT_CAP) when it didn't. */maxTokens: number | undefined;}+// Ceiling on the max_tokens the budget gate INJECTS when the client sent none.+// Without it, a generous budget (say 100000) minus a small prompt forwarded+// max_tokens≈99000 verbatim, and providers that validate max_tokens against+// the model's context/completion limit 400'd a request that previously worked —+// on every candidate in the chain, since the same value rode every retry.+// Routing only reserves a capped ~2000 tokens for output (routingReserveTokens,+// #470), so it never protects against this. A client that genuinely wants a+// bigger completion can always set max_tokens explicitly; the budget then only+// validates, never rewrites.+export const TOKEN_BUDGET_OUTPUT_CAP = 4096;+/*** Apply the per-request token budget to one request. `estimatedInputTokens`* is the surface's existing input estimate (~4 chars/token, images at the@@ -84,7 +95,7 @@ export function applyTokenBudget(estimatedInputTokens: number, maxTokens: numberconst remaining = budget - estimatedInputTokens;// Input alone fills the whole budget: nothing left for output.if (remaining < 1) return { rejection: { budget, estimatedTotal }, maxTokens };- return { rejection: null, maxTokens: remaining };+ return { rejection: null, maxTokens: Math.min(remaining, TOKEN_BUDGET_OUTPUT_CAP) };}return { rejection: null, maxTokens };}diff --git a/server/src/lib/proxy.ts b/server/src/lib/proxy.tsindex 1011c835..d6f360e5 100644--- a/server/src/lib/proxy.ts+++ b/server/src/lib/proxy.ts@@ -400,10 +400,31 @@ export function isProxyActive(): boolean {return _proxyEnabled && !!_proxyUrl;}-/** Force-rebuild the proxy dispatcher on the next request. Called on+/** Force-rebuild the outbound connection pools on the next request. Called on* sleep/wake recovery to drop pooled TCP connections that died while the* host was suspended (undici keeps them warm and would hand a dead socket* to the first post-wake request). */export function flushProxyCache(): void {+ // Outbound-proxy dispatcher (only in play when a proxy URL is configured).cached = null;+ // The default no-proxy path is bare fetch() on Node's GLOBAL undici+ // dispatcher — exactly the pool the headline laptop-lid scenario rides — so+ // nulling the proxy cache alone left the flush a no-op for most+ // deployments. Node keeps that dispatcher in the global symbol registry+ // (getGlobalDispatcher/setGlobalDispatcher read and write the same key), so+ // swap in a fresh instance of its own constructor: new requests get new+ // sockets, in-flight requests keep a reference to the old dispatcher and+ // complete undisturbed. Deliberately NOT `import('undici')`: the built-in+ // fetch uses Node's bundled copy, and the npm package isn't installed in+ // the production image (verified live — the import throws there).+ try {+ const sym = Symbol.for('undici.globalDispatcher.1');+ const current = (globalThis as Record<symbol, unknown>)[sym] as { constructor: new () => unknown } | undefined;+ // Symbol unset = no fetch has run yet, so there are no pooled sockets to drop.+ if (current?.constructor) {+ (globalThis as Record<symbol, unknown>)[sym] = new current.constructor();+ }+ } catch (err: any) {+ console.warn(`[proxy] could not replace the global fetch dispatcher on wake: ${err?.message ?? err}`);+ }}diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.tsindex 717b95dc..0e832ae8 100644--- a/server/src/lib/sampling-params.ts+++ b/server/src/lib/sampling-params.ts@@ -22,6 +22,7 @@// droplist entry is one line to add.import { z } from 'zod';+import type { Platform } from '@freellmapi/shared/types.js';// Every field is `.nullable()` because real clients serialize their whole// request struct and send explicit nulls for unset knobs (#200); null is@@ -118,7 +119,10 @@ export interface PlatformParamPolicy {jsonObjectToSchema?: boolean;}-export const PLATFORM_PARAM_POLICIES: Record<string, PlatformParamPolicy> = {+// Keyed by Platform (not string) so a typo'd platform id fails tsc instead of+// silently no-op'ing the policy; the string-typed accessors below cast at the+// boundary since routes carry platform ids as plain strings.+export const PLATFORM_PARAM_POLICIES: Partial<Record<Platform, PlatformParamPolicy>> = {// Mistral's API is strict (422 on unknown body keys) and names its seed// `random_seed`. It has no top_k/min_p/logit_bias/logprobs equivalents.mistral: {@@ -168,7 +172,7 @@ const ANY_OBJECT_SCHEMA = {*/export function extendedBodyParams(platform: string, options: ExtendedSamplingOptions | undefined): Record<string, unknown> {if (!options) return {};- const policy = PLATFORM_PARAM_POLICIES[platform];+ const policy = PLATFORM_PARAM_POLICIES[platform as Platform];const dropped = new Set<string>(policy?.drop ?? []);const out: Record<string, unknown> = {};for (const key of EXTENDED_SAMPLING_KEYS) {@@ -187,14 +191,14 @@ export function extendedBodyParams(platform: string, options: ExtendedSamplingOp/** True when this platform's policy strips response_format before send — the* router uses it to skip such platforms for structured-output requests. */export function platformDropsResponseFormat(platform: string): boolean {- return PLATFORM_PARAM_POLICIES[platform]?.drop?.includes('response_format') ?? false;+ return PLATFORM_PARAM_POLICIES[platform as Platform]?.drop?.includes('response_format') ?? false;}/** The advertised parameter list for a model on `platform` — the base set* every surface supports, plus tools when the model does, minus the* platform's droplist. */export function supportedParametersFor(platform: string, caps: { tools?: boolean } = {}): string[] {- const policy = PLATFORM_PARAM_POLICIES[platform];+ const policy = PLATFORM_PARAM_POLICIES[platform as Platform];const dropped = new Set<string>(policy?.drop ?? []);const params = ['temperature', 'top_p', 'max_tokens', 'max_completion_tokens', 'stop', 'stream',diff --git a/server/src/lib/structured-output.ts b/server/src/lib/structured-output.tsindex 28b4fcb6..c6707fe4 100644--- a/server/src/lib/structured-output.ts+++ b/server/src/lib/structured-output.ts@@ -33,17 +33,23 @@ function parses(text: string): boolean {}}-/** Slice from the first '{' or '[' to the matching last '}' or ']'. */-function outermostJsonSlice(text: string): string | null {- const firstObj = text.indexOf('{');- const firstArr = text.indexOf('[');- const first = firstObj === -1 ? firstArr : (firstArr === -1 ? firstObj : Math.min(firstObj, firstArr));- if (first === -1) return null;- const open = text[first];- const close = open === '{' ? '}' : ']';- const last = text.lastIndexOf(close);- if (last <= first) return null;- return text.slice(first, last + 1);+/**+ * Candidate slices from the first '{' to the last '}' and from the first '['+ * to the last ']', longest first. Both bracket types must be tried+ * independently: picking whichever opener appears FIRST turned+ * `According to [1], here is the JSON: {"city":"Paris"}` into the citation+ * marker `[1]` — a valid parse of the wrong value, silently delivered as the+ * structured output. Longest-first means an incidental bracket token in prose+ * can never shadow the real payload next to it.+ */+function candidateJsonSlices(text: string): string[] {+ const out: string[] = [];+ for (const [open, close] of [['{', '}'], ['[', ']']] as const) {+ const first = text.indexOf(open);+ const last = text.lastIndexOf(close);+ if (first !== -1 && last > first) out.push(text.slice(first, last + 1));+ }+ return out.sort((a, b) => b.length - a.length);}export function enforceJsonContent(content: string): JsonEnforcement {@@ -60,8 +66,9 @@ export function enforceJsonContent(content: string): JsonEnforcement {}// Leading/trailing prose around one JSON value ("Here is your JSON: {...}").- const slice = outermostJsonSlice(trimmed);- if (slice && parses(slice)) return { ok: true, content: slice, healed: true };+ for (const slice of candidateJsonSlices(trimmed)) {+ if (parses(slice)) return { ok: true, content: slice, healed: true };+ }return { ok: false };}diff --git a/server/src/lib/wake-detect.ts b/server/src/lib/wake-detect.tsindex 56b79be0..a70850b8 100644--- a/server/src/lib/wake-detect.ts+++ b/server/src/lib/wake-detect.ts@@ -27,11 +27,21 @@ export interface WakeEvent {const TICK_MS = 5_000;const DRIFT_THRESHOLD_MS = 30_000;+// One recovery pass per resume: a drift tick and an operator SIGCONT/SIGUSR2+// for the SAME wake (or signal spam) must not stack flushes + full key+// re-probes on top of each other.+const WAKE_DEBOUNCE_MS = 15_000;let hooks: WakeHooks | null = null;let timer: NodeJS.Timeout | null = null;let lastTickAt = 0;+let lastWakeAt = 0;let installed = false;+// Kept so stopWakeDetect can remove exactly the listeners this module added —+// removeAllListeners would silently unhook any OTHER module's SIGUSR2 handler+// (log rotation, heap-dump triggers).+const sigcontHandler = () => handleSignal('SIGCONT');+const sigusr2Handler = () => handleSignal('SIGUSR2');function tick(): void {const now = Date.now();@@ -57,6 +67,9 @@ function handleSignal(name: string): void {// logged and swallowed, sync or async.function invokeHooks(event: WakeEvent): void {if (!hooks) return;+ const now = Date.now();+ if (now - lastWakeAt < WAKE_DEBOUNCE_MS) return;+ lastWakeAt = now;try {Promise.resolve(hooks.onWake(event)).catch((err) => {console.error(`[wake-detect] onWake handler error: ${err?.message ?? err}`);@@ -73,8 +86,8 @@ export function startWakeDetect(h: WakeHooks): void {lastTickAt = Date.now();timer = setTimeout(tick, TICK_MS);if (timer.unref) timer.unref();- process.on('SIGCONT', () => handleSignal('SIGCONT'));- process.on('SIGUSR2', () => handleSignal('SIGUSR2'));+ process.on('SIGCONT', sigcontHandler);+ process.on('SIGUSR2', sigusr2Handler);}export function stopWakeDetect(): void {@@ -84,10 +97,11 @@ export function stopWakeDetect(): void {clearTimeout(timer);timer = null;}- process.removeAllListeners('SIGCONT');- process.removeAllListeners('SIGUSR2');+ process.removeListener('SIGCONT', sigcontHandler);+ process.removeListener('SIGUSR2', sigusr2Handler);hooks = null;lastTickAt = 0;+ lastWakeAt = 0;}export function _resetForTests(): void {diff --git a/server/src/providers/google.ts b/server/src/providers/google.tsindex c6cf7f39..eb94e340 100644--- a/server/src/providers/google.ts+++ b/server/src/providers/google.ts@@ -202,7 +202,12 @@ export function toGeminiExtendedConfig(options?: CompletionOptions): Record<strifrequencyPenalty: options?.frequency_penalty,};const rf = options?.response_format;- const hasTools = (options?.tools?.length ?? 0) > 0;+ // Count only real function declarations, mirroring hasFunctionDeclarations:+ // grounding pseudo-tools (google_search etc.) are converted to a grounding+ // block by toGeminiTools and never conflict with responseMimeType — raw+ // tools.length was silently dropping structured output for grounding-only+ // requests.+ const hasTools = (options?.tools ?? []).some(t => !GROUNDING_TOOL_NAMES.has(t.function.name.toLowerCase()));if (rf && !hasTools) {out.responseMimeType = 'application/json';const schema = rf.type === 'json_schema' ? rf.json_schema?.schema : undefined;@@ -603,20 +608,76 @@ export class GoogleProvider extends BaseProvider {const seenToolCallKeys = new Set<string>();- while (true) {- const { done, value } = await reader.read();- if (done) break;+ try {+ while (true) {+ const { done, value } = await reader.read();+ if (done) break;++ buffer += decoder.decode(value, { stream: true });+ const lines = buffer.split('\n');+ buffer = lines.pop() ?? '';++ for (const line of lines) {+ const trimmed = line.trim();+ if (!trimmed || !trimmed.startsWith('data: ')) continue;+ const raw = trimmed.slice(6);+ if (raw === '[DONE]') {+ if (!emittedFinish) {+ emittedFinish = true;+ yield {+ id,+ object: 'chat.completion.chunk',+ created: Math.floor(Date.now() / 1000),+ model: modelId,+ choices: [{+ index: 0,+ delta: {},+ finish_reason: sawToolCalls ? 'tool_calls' : 'stop',+ }],+ };+ }+ return;+ }++ // Skip malformed SSE frames instead of aborting the whole stream.+ // Matches the defensive parse in openai-compat / cohere / cloudflare:+ // a single corrupt chunk shouldn't take down the rest of the response.+ let chunk: GeminiResponse;+ try {+ chunk = JSON.parse(raw) as GeminiResponse;+ } catch {+ continue;+ }+ const candidate = chunk.candidates?.[0];+ const parts = candidate?.content?.parts ?? [];++ const text = extractText(parts);+ const toolCalls = extractToolCalls(parts).filter(call => {+ const key = `${call.id}:${call.function.name}:${call.function.arguments}`;+ if (seenToolCallKeys.has(key)) return false;+ seenToolCallKeys.add(key);+ return true;+ });- buffer += decoder.decode(value, { stream: true });- const lines = buffer.split('\n');- buffer = lines.pop() ?? '';+ if ((text && text.length > 0) || toolCalls.length > 0) {+ sawToolCalls = sawToolCalls || toolCalls.length > 0;+ yield {+ id,+ object: 'chat.completion.chunk',+ created: Math.floor(Date.now() / 1000),+ model: modelId,+ choices: [{+ index: 0,+ delta: {+ ...(text ? { content: text } : {}),+ ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),+ },+ finish_reason: null,+ }],+ };+ }- for (const line of lines) {- const trimmed = line.trim();- if (!trimmed || !trimmed.startsWith('data: ')) continue;- const raw = trimmed.slice(6);- if (raw === '[DONE]') {- if (!emittedFinish) {+ if (candidate?.finishReason && !emittedFinish) {emittedFinish = true;yield {id,@@ -626,67 +687,21 @@ export class GoogleProvider extends BaseProvider {choices: [{index: 0,delta: {},- finish_reason: sawToolCalls ? 'tool_calls' : 'stop',+ finish_reason: sawToolCalls ? 'tool_calls' : toGeminiFinishReason(candidate.finishReason),}],};+ return;}- return;- }-- // Skip malformed SSE frames instead of aborting the whole stream.- // Matches the defensive parse in openai-compat / cohere / cloudflare:- // a single corrupt chunk shouldn't take down the rest of the response.- let chunk: GeminiResponse;- try {- chunk = JSON.parse(raw) as GeminiResponse;- } catch {- continue;- }- const candidate = chunk.candidates?.[0];- const parts = candidate?.content?.parts ?? [];-- const text = extractText(parts);- const toolCalls = extractToolCalls(parts).filter(call => {- const key = `${call.id}:${call.function.name}:${call.function.arguments}`;- if (seenToolCallKeys.has(key)) return false;- seenToolCallKeys.add(key);- return true;- });-- if ((text && text.length > 0) || toolCalls.length > 0) {- sawToolCalls = sawToolCalls || toolCalls.length > 0;- yield {- id,- object: 'chat.completion.chunk',- created: Math.floor(Date.now() / 1000),- model: modelId,- choices: [{- index: 0,- delta: {- ...(text ? { content: text } : {}),- ...(toolCalls.length > 0 ? { tool_calls: toolCalls } : {}),- },- finish_reason: null,- }],- };- }-- if (candidate?.finishReason && !emittedFinish) {- emittedFinish = true;- yield {- id,- object: 'chat.completion.chunk',- created: Math.floor(Date.now() / 1000),- model: modelId,- choices: [{- index: 0,- delta: {},- finish_reason: sawToolCalls ? 'tool_calls' : toGeminiFinishReason(candidate.finishReason),- }],- };- return;}}+ } finally {+ // Runs on normal completion, on the early returns above, AND when the+ // consumer abandons the generator mid-stream (a client disconnect breaks+ // the route pump's for-await, which calls gen.return() at the yield+ // point). Without it the Gemini generation kept running upstream —+ // quota burning with nobody reading — because this adapter reads the+ // body itself instead of going through readSseStream's shared cleanup.+ reader.cancel().catch(() => { /* upstream already gone */ });}if (!emittedFinish) {diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 92da2945..3abd4ec4 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -369,27 +369,35 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {const body = parsed.data;const requestedModel = body.model ?? 'auto';- const max_tokens = body.max_tokens != null && body.max_tokens > 0 ? body.max_tokens : DEFAULT_MAX_TOKENS;+ // The Anthropic wire format requires max_tokens, but OUR default injection+ // must not change the budget gate's verdict: gating AFTER defaulting made a+ // no-max_tokens request 413 here (input + 1024 over budget) where the chat+ // surface would cap it and serve. Gate on what the client actually sent,+ // then resolve the default against the budget remainder.+ const clientMaxTokens = body.max_tokens != null && body.max_tokens > 0 ? body.max_tokens : undefined;const { temperature, top_p, stream } = body;const { messages, tools, tool_choice, hasImage, wantsTools } = convertRequest(body);- const completionOptions = { temperature, max_tokens, top_p, top_k: body.top_k ?? undefined, tools, tool_choice };const estimatedInputTokens = estimateTokens(messages);const imageCount = messages.reduce((n, m) =>n + (Array.isArray(m.content) ? m.content.filter(b => (b as any)?.type === 'image_url').length : 0), 0);- // Capped output reserve so a large max_tokens can't falsely exclude the model- // pool (#470); input + images count in full.- const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);// Guardrail: per-request token budget (request_max_tokens_budget, default- // off). max_tokens is always set on this surface (Anthropic requires it),- // so a violation can only reject — no capping branch.- const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, max_tokens);+ // off). A client-set max_tokens can only pass or reject; an omitted one is+ // capped to the budget remainder, mirroring /v1/chat/completions.+ const budgetCheck = applyTokenBudget(estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE, clientMaxTokens);if (budgetCheck.rejection) {sendError(res, 413, 'invalid_request_error', tokenBudgetMessage(budgetCheck.rejection));return;}+ const max_tokens = clientMaxTokens+ ?? (budgetCheck.maxTokens != null ? Math.min(DEFAULT_MAX_TOKENS, budgetCheck.maxTokens) : DEFAULT_MAX_TOKENS);++ const completionOptions = { temperature, max_tokens, top_p, top_k: body.top_k ?? undefined, tools, tool_choice };+ // Capped output reserve so a large max_tokens can't falsely exclude the model+ // pool (#470); input + images count in full.+ const estimatedTotal = estimatedInputTokens + imageCount * IMAGE_TOKEN_ESTIMATE + routingReserveTokens(max_tokens);// Resolve the model through the operator's Claude-family map (opus/sonnet/// haiku/default → auto | a pinned catalog model). A concrete catalog id pins@@ -713,6 +721,15 @@ async function streamCompletion(// Nothing usable came out — fail over (message_start was never sent, so the// client never saw this attempt).if (!messageStarted && completedCalls.length === 0) {+ // Disconnect before the commit point: the pump loop broke with nothing+ // accumulated because the CLIENT left, not because the provider failed —+ // benching a healthy model+key for it was wrong. StreamAlreadyStarted+ // maps to 'committed' in the dispatcher: stop, no failover, no+ // exhaustion render (the socket is gone anyway).+ if (ctx.clientGone()) {+ console.log(`[Anthropic] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);+ throw new StreamAlreadyStarted();+ }// finish_reason 'length' = the model spent the whole output budget on// hidden reasoning before any visible text: fail over, but skip the// cooldown/penalty (not a provider-health signal).diff --git a/server/src/routes/mcp.ts b/server/src/routes/mcp.tsindex 41592763..fc1dacff 100644--- a/server/src/routes/mcp.ts+++ b/server/src/routes/mcp.ts@@ -30,8 +30,12 @@ import { getCacheStats } from '../services/cache.js';export const mcpRouter = Router();-const SUPPORTED_PROTOCOL_VERSIONS = new Set(['2025-06-18', '2025-03-26', '2024-11-05']);-const DEFAULT_PROTOCOL_VERSION = '2025-06-18';+// Always negotiated to 2025-06-18: this transport rejects JSON-RPC batches,+// which the 2025-03-26 and 2024-11-05 revisions still allowed — echoing an+// older requested version while enforcing the newer transport rule promised+// clients batching they'd never get. Per the MCP spec the server answers with+// the latest version it supports; clients that can't speak it disconnect.+const PROTOCOL_VERSION = '2025-06-18';interface JsonRpcRequest {jsonrpc?: string;@@ -118,9 +122,17 @@ function providerHealth(): unknown {const USAGE_RANGES: Record<string, number> = { '24h': 24, '7d': 24 * 7, '30d': 24 * 30 };function usageSummary(args: Record<string, unknown>): unknown {- const range = typeof args.range === 'string' && USAGE_RANGES[args.range] ? args.range : '24h';+ // Object.hasOwn: a prototype key ('constructor', 'toString') is truthy via+ // the prototype chain but multiplies to NaN below — fall back to 24h.+ const range = typeof args.range === 'string' && Object.hasOwn(USAGE_RANGES, args.range) ? args.range : '24h';const db = getDb();- const since = new Date(Date.now() - USAGE_RANGES[range] * 3600_000).toISOString();+ // SQLite datetime('now') format (space separator, no ms/Z) — an ISO 'T'+ // string compares GREATER than every same-day stored row (space < 'T'+ // lexicographically), which silently dropped the window's boundary day:+ // "24h" effectively meant "since UTC midnight". Same conversion as+ // routes/analytics.ts.+ const since = new Date(Date.now() - USAGE_RANGES[range] * 3600_000)+ .toISOString().slice(0, 19).replace('T', ' ');const totals = db.prepare(`SELECT COALESCE(SUM(total_requests), 0) AS requests,COALESCE(SUM(success_count), 0) AS successes,@@ -225,14 +237,23 @@ const TOOLS: Record<string, McpTool> = {// ── JSON-RPC dispatch ────────────────────────────────────────────────────+// Returns the JSON-RPC response for a request, or undefined for a+// notification. JSON-RPC 2.0 defines a notification as a message WITHOUT an+// `id` member (id:null is a — discouraged — request and gets a response);+// detecting notifications by the `notifications/` method prefix answered+// no-id requests and 202'd id-carrying notifications.function handleRpc(msg: JsonRpcRequest): unknown | undefined {+ const isNotification = msg.id === undefined;+ const respond = (response: unknown) => (isNotification ? undefined : response);const id = msg.id ?? null;+ return respond(dispatchRpc(msg, id));+}++function dispatchRpc(msg: JsonRpcRequest, id: number | string | null): unknown {switch (msg.method) {case 'initialize': {- const requested = (msg.params?.protocolVersion as string) ?? '';- const version = SUPPORTED_PROTOCOL_VERSIONS.has(requested) ? requested : DEFAULT_PROTOCOL_VERSION;return rpcResult(id, {- protocolVersion: version,+ protocolVersion: PROTOCOL_VERSION,capabilities: { tools: {} },serverInfo: { name: 'freellmapi', version: '1.0.0' },instructions: 'FreeLLMAPI gateway introspection: list usable free models (with per-model supported_parameters), check provider/key health, read usage and cache stats, and switch the routing strategy. Inference goes through the OpenAI-compatible /v1 endpoints, not MCP.',@@ -250,7 +271,10 @@ function handleRpc(msg: JsonRpcRequest): unknown | undefined {});case 'tools/call': {const name = msg.params?.name as string;- const tool = TOOLS[name];+ // Object.hasOwn: a bare index lookup resolves prototype members, so+ // name:"constructor"/"toString" passed the !tool check and died inside+ // the try as a confusing tool-level error instead of -32602.+ const tool = typeof name === 'string' && Object.hasOwn(TOOLS, name) ? TOOLS[name] : undefined;if (!tool) return rpcError(id, -32602, `Unknown tool: ${name}`);try {const args = (msg.params?.arguments as Record<string, unknown>) ?? {};@@ -261,7 +285,9 @@ function handleRpc(msg: JsonRpcRequest): unknown | undefined {}}default:- if (msg.method?.startsWith('notifications/')) return undefined; // ack silently+ // Known client notifications (notifications/initialized etc.) land here;+ // handleRpc drops the response for anything sent without an id, so they+ // are acked silently without special-casing the method name.return rpcError(id, -32601, `Method not found: ${msg.method}`);}}@@ -270,7 +296,11 @@ function authenticate(req: Request, res: Response): boolean {const token = extractApiToken(req);const unifiedKey = getUnifiedApiKey();if (!token || !timingSafeStringEqual(token, unifiedKey)) {- res.status(401).json(rpcError(null, -32001, 'Invalid API key. Authenticate with the unified key as a Bearer token.'));+ // Echo the request id when the body carries one, so strict JSON-RPC+ // clients can correlate the auth error with their pending call.+ const body = req.body;+ const id = body && typeof body === 'object' && !Array.isArray(body) && body.id !== undefined ? body.id : null;+ res.status(401).json(rpcError(id, -32001, 'Invalid API key. Authenticate with the unified key as a Bearer token.'));return false;}return true;diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 3fa240b5..36cae521 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -786,8 +786,6 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {);for await (const chunk of gen) {- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstreamif (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstreamconst text = streamChunkText(chunk);if (text.length > 0) sawText = true;@@ -805,6 +803,16 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {res.write(`data: ${JSON.stringify(frame)}\n\n`);}+ // Disconnect before the commit point: the break above fired with no+ // text seen, which is indistinguishable from an empty completion+ // below — but it is CLIENT behavior, not a provider failure. Without+ // this check every Ctrl-C during a reasoning model's TTFB window+ // benched the healthy model+key for 90s and logged a provider error.+ if (clientGone && !headerSent && !sawText) {+ console.log(`[Proxy] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);+ return 'committed';+ }+if (!sawText) {// finish_reason 'length' means the model spent the whole output// budget before any visible text (hidden reasoning) — fail over,@@ -1251,6 +1259,24 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {options: fusionOptions,estimatedTokens: estimatedTotal,});+ // Structured-output enforcement for fusion (#516 scope gap): the panel/+ // judge output got no format check, so model:"fusion" could hand back+ // prose as a "success" for a json_schema request. Fusion has no failover+ // machinery to hand this to — heal what's healable, otherwise answer+ // honestly instead of pretending. (Streaming fusion stays unenforced,+ // same boundary as every other streamed response.)+ const fusionMsg = (response as any)?.choices?.[0]?.message;+ if (samplingParams.response_format && fusionMsg && !fusionMsg.tool_calls?.length) {+ const fusionText = contentToString(fusionMsg.content ?? '');+ if (fusionText) {+ const enforced = enforceJsonContent(fusionText);+ if (!enforced.ok) {+ res.status(502).json({ error: { message: `fusion produced non-JSON output despite response_format=${samplingParams.response_format.type} — retry, or pin a structured-output-capable model instead of "fusion"`, type: 'server_error' } });+ return;+ }+ if (enforced.healed) fusionMsg.content = enforced.content;+ }+ }res.setHeader('X-Routed-Via', routedVia);res.json(response);} catch (err: any) {@@ -1518,8 +1544,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {);for await (const chunk of gen) {- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstreamif (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstreamconst anyChunk = chunk as Record<string, any>;@@ -1641,6 +1665,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}}+ // Disconnect before the commit point: nothing usable was (or will+ // be) delivered, and that is CLIENT behavior, not a provider+ // failure — do not let it fall through to the empty-completion+ // throw below, which would bench a healthy model+key for 90s and+ // log a provider error for every Ctrl-C during a reasoning model's+ // TTFB window.+ if (clientGone && !headerSent && heldText.trim().length === 0 && completedCalls.length === 0) {+ console.log(`[Proxy] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);+ return 'committed';+ }+const hasText = headerSent || heldText.trim().length > 0;if (!hasText && completedCalls.length === 0) {// Nothing usable came out — same failover semantics as the@@ -1740,25 +1775,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {);}- // Structured-output enforcement (#514 follow-up): the client asked for- // JSON; a model that answered in prose despite the forwarded- // response_format must not be returned as a "success". Heal the common- // almost-right shapes (fenced block, prose-wrapped JSON) in place;- // otherwise fail over. skipBench: the provider is healthy — the MODEL- // ignored the format — so no cooldown/penalty, just the next candidate.- if (samplingParams.response_format && respText && (respMsg?.tool_calls?.length ?? 0) === 0) {- const enforced = enforceJsonContent(respText);- if (!enforced.ok) {- throw Object.assign(- new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),- { skipBench: true },- );- }- if (enforced.healed && respMsg) {- respMsg.content = enforced.content;- }- }-// Inline tool-call dialect rescue (#231 audit): a tool-bearing// request answered with the call serialized as TEXT (a mid-// conversation model switch makes the new model imitate the previous@@ -1783,6 +1799,35 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}}+ // Structured-output enforcement (#514 follow-up): the client asked for+ // JSON; a model that answered in prose despite the forwarded+ // response_format must not be returned as a "success". Heal the common+ // almost-right shapes (fenced block, prose-wrapped JSON) in place;+ // otherwise fail over. Deliberately AFTER the dialect rescue (matching+ // responses.ts): an inline tool-call turn isn't JSON either, and+ // gating it first burned a failover hop on turns the rescue converts.+ // skipBench: the provider is healthy — the MODEL misbehaved — so no+ // cooldown/penalty; skipModelForRequest: a sibling key would misbehave+ // identically, so rule out the whole model for this request.+ if (samplingParams.response_format && respText && (respMsg?.tool_calls?.length ?? 0) === 0) {+ const enforced = enforceJsonContent(respText);+ if (!enforced.ok) {+ // finish_reason 'length' = the JSON was CUT OFF by max_tokens, not+ // ignored — same failover (a terser model may fit the budget), but+ // an honest error class/trail instead of "ignored response_format".+ const truncated = result.choices?.[0]?.finish_reason === 'length';+ throw Object.assign(+ new Error(truncated+ ? `truncated JSON from ${route.displayName} (finish_reason=length — raise max_tokens for this ${samplingParams.response_format.type} request)`+ : `${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),+ { skipBench: true, skipModelForRequest: true },+ );+ }+ if (enforced.healed && respMsg) {+ respMsg.content = enforced.content;+ }+ }+// Usage fallback: providers that omit `usage` used to be logged as 0// tokens, silently undercounting analytics and the rate-limit ledger.// Fall back to the same chars/4 estimate the streaming path uses (tooldiff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 8735c5c7..97656c26 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -503,8 +503,6 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {);for await (const chunk of gen) {- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream- if (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstreamif (clientGone) break; // client hung up: stop pulling; reader.cancel() aborts upstream// In-band upstream error frame ({"error":...} inside a 200 SSE// stream — observed live from Groq). Throwing hands it to the catch@@ -621,6 +619,15 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// been committed yet (the skeleton is lazy), so throwing lets the// shared loop fail over to the next model on the same SSE connection.if (msgText.length === 0 && toolAcc.size === 0) {+ // Disconnect before the commit point: the break above fired with+ // nothing accumulated — CLIENT behavior, not a provider failure.+ // Falling through to the empty-completion throw benched a healthy+ // model+key for 90s on every Ctrl-C during a reasoning model's+ // TTFB window.+ if (clientGone && !streamStarted) {+ console.log(`[Responses] client disconnected before first token from ${route.displayName} — dropping attempt without benching`);+ return 'committed';+ }// finish_reason 'length' = the model spent the whole output budget// on hidden reasoning before any visible text: fail over, but skip// the cooldown/penalty (not a provider-health signal).@@ -739,14 +746,19 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}// Structured-output enforcement — see /chat/completions. Heal fenced or- // prose-wrapped JSON in place, fail over (skipBench) when the model- // ignored the requested format outright.+ // prose-wrapped JSON in place, fail over (skipBench + skipModelForRequest+ // — a sibling key would misbehave identically) when the model ignored the+ // requested format; finish_reason 'length' gets an honest "truncated"+ // class instead, since the JSON was cut off by max_tokens, not ignored.if (completionOpts.response_format && text && toolCalls.length === 0) {const enforced = enforceJsonContent(text);if (!enforced.ok) {+ const truncated = result.choices[0]?.finish_reason === 'length';throw Object.assign(- new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${completionOpts.response_format.type})`),- { skipBench: true },+ new Error(truncated+ ? `truncated JSON from ${route.displayName} (finish_reason=length — raise max_tokens for this ${completionOpts.response_format.type} request)`+ : `${route.displayName} ignored response_format (returned non-JSON despite ${completionOpts.response_format.type})`),+ { skipBench: true, skipModelForRequest: true },);}if (enforced.healed) text = enforced.content;diff --git a/server/src/services/fusion.ts b/server/src/services/fusion.tsindex fc25b1f5..a4f601ee 100644--- a/server/src/services/fusion.ts+++ b/server/src/services/fusion.ts@@ -678,7 +678,15 @@ export async function runFusion(params: {const cand = resolveFusionCandidate(config.judge!);return cand ? routePinnedModel(cand.modelDbId, judgeEstimate, skipKeys) : null;}- : (skipKeys: Set<string>, skipModels: Set<number>) => routeRequest(judgeEstimate, skipKeys.size ? skipKeys : undefined, undefined, false, false, skipModels.size ? skipModels : undefined);+ : (skipKeys: Set<string>, skipModels: Set<number>) => routeRequest(+ judgeEstimate, skipKeys.size ? skipKeys : undefined, undefined, false, false,+ skipModels.size ? skipModels : undefined, undefined,+ // The judge writes the final answer the client receives, so a+ // structured-output request must not land it on a platform whose+ // policy drops response_format (kilo) — the schema would never even+ // reach the model. Mirrors the non-fusion routing (#516).+ options.response_format !== undefined,+ );// Stream the judge when the caller wants live tokens (Playground); otherwise// a single buffered call (plain API clients hitting fusion non-streaming).diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex 5b44262a..2b53be85 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -66,17 +66,31 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {}}-export async function checkAllKeys(): Promise<void> {- const db = getDb();- const keys = db.prepare('SELECT id, platform FROM api_keys WHERE enabled = 1').all() as { id: number; platform: string }[];-- console.log(`[Health] Checking ${keys.length} keys...`);-- for (const key of keys) {- await checkKeyHealth(key.id);- }+// Overlap guard: the scheduled 5-minute pass and wake-recovery re-probes can+// coincide (or SIGCONT spam can queue several) — concurrent full passes+// multiply provider validate traffic and let two passes each increment the+// same genuinely-bad key's failureCount, reaching the auto-disable threshold+// in fewer wall-clock checks than "3 consecutive checks" intends. A second+// caller joins the in-flight pass instead of starting another.+let checkAllInFlight: Promise<void> | null = null;++export function checkAllKeys(): Promise<void> {+ if (checkAllInFlight) return checkAllInFlight;+ checkAllInFlight = (async () => {+ const db = getDb();+ const keys = db.prepare('SELECT id, platform FROM api_keys WHERE enabled = 1').all() as { id: number; platform: string }[];++ console.log(`[Health] Checking ${keys.length} keys...`);++ for (const key of keys) {+ await checkKeyHealth(key.id);+ }- console.log(`[Health] Check complete.`);+ console.log(`[Health] Check complete.`);+ })().finally(() => {+ checkAllInFlight = null;+ });+ return checkAllInFlight;}let cancelHealthCheck: (() => void) | null = null;diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 16602b9f..fc4e2a59 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -69,6 +69,7 @@ export function summarizeExhaustion(else if (l.includes('< estimated')) bump('prompt too large for the model');else if (l.includes('no vision support')) bump('model lacks vision');else if (l.includes('no tool-calling support')) bump('model lacks tool-calling');+ else if (l.includes('drops response_format')) bump('platform cannot honor response_format');else if (/ruled out|already-failed/.test(l)) bump('failed earlier this request');else if (/cooldown|rpm|rpd|tpm|tpd|provider-daily-cap/.test(l)) bump('rate-limited or on cooldown');else bump('unavailable');@@ -80,6 +81,7 @@ export function summarizeExhaustion('prompt too large for the model','model lacks vision','model lacks tool-calling',+ 'platform cannot honor response_format','failed earlier this request','unsupported provider','unavailable',