7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(gateway): structured-output fallback handling - routing filter, JSON healing, format-ignore failover (#516)
Tashfeenc8444a05a2b6
diff --git a/server/src/__tests__/lib/structured-output.test.ts b/server/src/__tests__/lib/structured-output.test.tsnew file mode 100644index 00000000..589d66f9--- /dev/null+++ b/server/src/__tests__/lib/structured-output.test.ts@@ -0,0 +1,48 @@+import { describe, it, expect } from 'vitest';+import { enforceJsonContent } from '../../lib/structured-output.js';++describe('enforceJsonContent', () => {+ it('passes clean JSON through untouched', () => {+ const r = enforceJsonContent('{"city":"Paris","pop":2100000}');+ expect(r).toEqual({ ok: true, content: '{"city":"Paris","pop":2100000}', healed: false });+ });++ it('accepts JSON arrays and scalars (valid json_object outputs vary)', () => {+ expect(enforceJsonContent('[1,2,3]').ok).toBe(true);+ expect(enforceJsonContent(' {"a":1} ')).toMatchObject({ ok: true, healed: true }); // trimmed = healed+ });++ it('heals a ```json fenced block', () => {+ const r = enforceJsonContent('Here you go:\n```json\n{"answer": 42}\n```\nHope that helps!');+ expect(r).toEqual({ ok: true, content: '{"answer": 42}', healed: true });+ });++ it('heals a bare ``` fence', () => {+ const r = enforceJsonContent('```\n{"answer": 42}\n```');+ expect(r).toEqual({ ok: true, content: '{"answer": 42}', healed: true });+ });++ it('heals prose-wrapped JSON ("Here is your JSON: {...}")', () => {+ const r = enforceJsonContent('Sure! Here is the requested JSON: {"name":"Ada","tags":["a","b"]} Let me know if you need more.');+ expect(r).toEqual({ ok: true, content: '{"name":"Ada","tags":["a","b"]}', healed: true });+ });++ it('rejects pure prose', () => {+ expect(enforceJsonContent('The city you asked about is Paris, which has about 2.1M people.')).toEqual({ ok: false });+ });++ it('rejects broken JSON that cannot be healed', () => {+ expect(enforceJsonContent('{"city": "Paris", "pop": }')).toEqual({ ok: false });+ expect(enforceJsonContent('```json\n{"unclosed": true\n```')).toEqual({ ok: false });+ });++ it('rejects empty content', () => {+ expect(enforceJsonContent('')).toEqual({ ok: false });+ expect(enforceJsonContent(' ')).toEqual({ ok: false });+ });++ it('JSON containing braces inside strings survives the slice heuristic', () => {+ const clean = '{"code":"if (a) { return b; }"}';+ expect(enforceJsonContent(`prefix ${clean} suffix`)).toEqual({ ok: true, content: clean, healed: true });+ });+});diff --git a/server/src/__tests__/routes/proxy-cache.test.ts b/server/src/__tests__/routes/proxy-cache.test.tsindex 72766cfa..bfed4aca 100644--- a/server/src/__tests__/routes/proxy-cache.test.ts+++ b/server/src/__tests__/routes/proxy-cache.test.ts@@ -140,7 +140,9 @@ describe('Response cache (proxy integration)', () => {});it('requests differing only in response_format, seed, or stop never collide', async () => {- const counter = mockGroq('knob-sensitive');+ // A JSON string literal: valid JSON, so the response_format variant below+ // survives the structured-output enforcement instead of failing over.+ const counter = mockGroq('"knob-sensitive"');await chat(); // plain → MISS, populates// Same prompt with response_format json_object must be a MISS, not a// replay of the cached plain-text answer.diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 83f48f43..5e3d41d3 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -62,6 +62,10 @@ export function isRetryableError(err: any): boolean {// all thrown before any byte reached the client, so another model can// serve the request invisibly.|| msg.includes('empty completion')+ // The model answered in prose despite a response_format request (and the+ // JSON couldn't be healed out of the text) — thrown before any byte+ // reached the client, so the next candidate can serve it invisibly.+ || msg.includes('ignored response_format')|| msg.includes('in-band provider error')|| msg.includes('stream ended unexpectedly')|| msg.includes('stream stalled')diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.tsindex 6b431ee9..02cce9a9 100644--- a/server/src/lib/sampling-params.ts+++ b/server/src/lib/sampling-params.ts@@ -158,6 +158,12 @@ export function extendedBodyParams(platform: string, options: ExtendedSamplingOpreturn out;}+/** True when this platform's policy strips response_format before send — the+ * router uses it to skip such platforms for structured-output requests. */+export function platformDropsResponseFormat(platform: string): boolean {+ return PLATFORM_PARAM_POLICIES[platform]?.drop?.includes('response_format') ?? false;+}+/** The advertised parameter list for a model on `platform` — the base set* every surface supports, plus tools when the model does, minus the* platform's droplist. */diff --git a/server/src/lib/structured-output.ts b/server/src/lib/structured-output.tsnew file mode 100644index 00000000..28b4fcb6--- /dev/null+++ b/server/src/lib/structured-output.ts@@ -0,0 +1,67 @@+// Structured-output enforcement for the fallback chain (#514 follow-up).+//+// Forwarding response_format is necessary but not sufficient: a free-tier+// model that doesn't understand JSON mode often just answers in prose (or+// wraps the JSON in a markdown fence) and the request LOOKS successful — the+// worst failure mode, because the client asked for machine-readable output+// and got an essay. On non-streaming responses the gateway can check, heal,+// or fail over:+//+// 1. content parses as JSON → pass through untouched+// 2. content wraps JSON in ```fences``` or leading/trailing prose+// → heal: replace content with the JSON+// 3. nothing parseable → retryable failure (skipBench: the+// provider is healthy, the MODEL ignored the format — no cooldown, no+// penalty; the chain just tries the next candidate)+//+// Streaming responses can't be retro-checked (bytes are already on the wire),+// so enforcement is non-stream only — same boundary as the response cache.++export type JsonEnforcement =+ | { ok: true; content: string; healed: boolean }+ | { ok: false };++/** Match a fenced code block (```json ... ``` or bare ``` ... ```). */+const FENCE_RE = /```(?:json)?\s*\n?([\s\S]*?)```/i;++function parses(text: string): boolean {+ try {+ JSON.parse(text);+ return true;+ } catch {+ return false;+ }+}++/** Slice from the first '{' or '[' to the matching last '}' or ']'. */+function outermostJsonSlice(text: string): string | null {+ const firstObj = text.indexOf('{');+ const firstArr = text.indexOf('[');+ const first = firstObj === -1 ? firstArr : (firstArr === -1 ? firstObj : Math.min(firstObj, firstArr));+ if (first === -1) return null;+ const open = text[first];+ const close = open === '{' ? '}' : ']';+ const last = text.lastIndexOf(close);+ if (last <= first) return null;+ return text.slice(first, last + 1);+}++export function enforceJsonContent(content: string): JsonEnforcement {+ const trimmed = content.trim();+ if (trimmed.length === 0) return { ok: false };++ if (parses(trimmed)) return { ok: true, content: trimmed === content ? content : trimmed, healed: trimmed !== content };++ // Fenced block: the most common "almost right" shape free models produce.+ const fence = FENCE_RE.exec(trimmed);+ if (fence) {+ const inner = fence[1].trim();+ if (parses(inner)) return { ok: true, content: inner, healed: true };+ }++ // Leading/trailing prose around one JSON value ("Here is your JSON: {...}").+ const slice = outermostJsonSlice(trimmed);+ if (slice && parses(slice)) return { ok: true, content: slice, healed: true };++ return { ok: false };+}diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 1018ced9..7490fd4c 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -20,6 +20,7 @@ import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKimport { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError } from '../lib/fallback-loop.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';+import { enforceJsonContent } from '../lib/structured-output.js';import type { Platform } from '@freellmapi/shared/types.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from '../services/model-groups.js';@@ -1418,7 +1419,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// model is on record). Turns where injection can't happen — every turn 1, and// sessions that never switched — pay no headroom tax.const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;- return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain);+ return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined);},dispatch: async (route, attempt) => {const modelKey = `${route.platform}:${route.modelId}`;@@ -1723,6 +1724,25 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {);}+ // Structured-output enforcement (#514 follow-up): the client asked for+ // JSON; a model that answered in prose despite the forwarded+ // response_format must not be returned as a "success". Heal the common+ // almost-right shapes (fenced block, prose-wrapped JSON) in place;+ // otherwise fail over. skipBench: the provider is healthy — the MODEL+ // ignored the format — so no cooldown/penalty, just the next candidate.+ if (samplingParams.response_format && respText && (respMsg?.tool_calls?.length ?? 0) === 0) {+ const enforced = enforceJsonContent(respText);+ if (!enforced.ok) {+ throw Object.assign(+ new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),+ { skipBench: true },+ );+ }+ if (enforced.healed && respMsg) {+ respMsg.content = enforced.content;+ }+ }+// Inline tool-call dialect rescue (#231 audit): a tool-bearing// request answered with the call serialized as TEXT (a mid-// conversation model switch makes the new model imitate the previousdiff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 55c75a0e..60d6f0d6 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -26,6 +26,7 @@ import {import { runFallbackLoop, newFallbackState, recordUpstreamSuccess } from '../lib/fallback-loop.js';import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';+import { enforceJsonContent } from '../lib/structured-output.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';@@ -421,7 +422,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {await runFallbackLoop({maxRetries: MAX_RETRIES,state,- route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, false, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined),+ route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, false, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, undefined, completionOpts.response_format !== undefined),dispatch: async (route, attempt) => {traceRouteEvent('Responses', {event: attempt === 0 ? 'start' : 'next',@@ -729,6 +730,20 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {);}+ // Structured-output enforcement — see /chat/completions. Heal fenced or+ // prose-wrapped JSON in place, fail over (skipBench) when the model+ // ignored the requested format outright.+ if (completionOpts.response_format && text && toolCalls.length === 0) {+ const enforced = enforceJsonContent(text);+ if (!enforced.ok) {+ throw Object.assign(+ new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${completionOpts.response_format.type})`),+ { skipBench: true },+ );+ }+ if (enforced.healed) text = enforced.content;+ }+// Usage fallback: a missing provider `usage` block used to record 0// tokens against the rate-limit ledger; promptTokens/completionTokens// above already carry the chars/4 estimate.diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 37329d4b..16602b9f 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -8,6 +8,7 @@ import {speedScore, intelligenceScore, headroomFactor, rateLimitFactor, combineScore,} from './scoring.js';import { parseBudget } from '../lib/budget.js';+import { platformDropsResponseFormat } from '../lib/sampling-params.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from './model-groups.js';import type { BaseProvider } from '../providers/base.js';import type { Platform } from '@freellmapi/shared/types.js';@@ -980,7 +981,7 @@ export function resolveFusionCandidate(modelId: string): FusionCandidate | nullreturn null;}-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[]): RouteResult {+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[], requireStructured = false): RouteResult {const db = getDb();const strategy = getRoutingStrategy();@@ -1042,6 +1043,14 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre// reasoning as vision above.if (requireTools && !entry.supports_tools) { diag.push(`${label}: no tool-calling support`); continue; }+ // Structured-output routing (#514 follow-up): when the request carries a+ // response_format, skip platforms whose param policy can't even receive it+ // (the param would be dropped before send, so the model would answer in+ // prose and burn a failover hop). Platform-level fast path — model-level+ // capability isn't in the catalog; models that accept the param but ignore+ // it are caught by the non-stream JSON enforcement downstream.+ if (requireStructured && platformDropsResponseFormat(entry.platform)) { diag.push(`${label}: platform drops response_format`); continue; }+// Context-aware routing: skip a model whose context window can't hold the// request, so a large prompt never selects a small-context model and burns// a failover hop on a 413 "request too large" (#167). Only enforced when we