Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(gateway): structured-output fallback handling - routing filter, JSON healing, format-ignore failover (#516)

Tashfeenc8444a05a2b6
diff --git a/server/src/__tests__/lib/structured-output.test.ts b/server/src/__tests__/lib/structured-output.test.ts
new file mode 100644
index 00000000..589d66f9
--- /dev/null
+++ b/server/src/__tests__/lib/structured-output.test.ts
@@ -0,0 +1,48 @@
+import { describe, it, expect } from 'vitest';
+import { enforceJsonContent } from '../../lib/structured-output.js';
+
+describe('enforceJsonContent', () => {
+ it('passes clean JSON through untouched', () => {
+ const r = enforceJsonContent('{"city":"Paris","pop":2100000}');
+ expect(r).toEqual({ ok: true, content: '{"city":"Paris","pop":2100000}', healed: false });
+ });
+
+ it('accepts JSON arrays and scalars (valid json_object outputs vary)', () => {
+ expect(enforceJsonContent('[1,2,3]').ok).toBe(true);
+ expect(enforceJsonContent(' {"a":1} ')).toMatchObject({ ok: true, healed: true }); // trimmed = healed
+ });
+
+ it('heals a ```json fenced block', () => {
+ const r = enforceJsonContent('Here you go:\n```json\n{"answer": 42}\n```\nHope that helps!');
+ expect(r).toEqual({ ok: true, content: '{"answer": 42}', healed: true });
+ });
+
+ it('heals a bare ``` fence', () => {
+ const r = enforceJsonContent('```\n{"answer": 42}\n```');
+ expect(r).toEqual({ ok: true, content: '{"answer": 42}', healed: true });
+ });
+
+ it('heals prose-wrapped JSON ("Here is your JSON: {...}")', () => {
+ const r = enforceJsonContent('Sure! Here is the requested JSON: {"name":"Ada","tags":["a","b"]} Let me know if you need more.');
+ expect(r).toEqual({ ok: true, content: '{"name":"Ada","tags":["a","b"]}', healed: true });
+ });
+
+ it('rejects pure prose', () => {
+ expect(enforceJsonContent('The city you asked about is Paris, which has about 2.1M people.')).toEqual({ ok: false });
+ });
+
+ it('rejects broken JSON that cannot be healed', () => {
+ expect(enforceJsonContent('{"city": "Paris", "pop": }')).toEqual({ ok: false });
+ expect(enforceJsonContent('```json\n{"unclosed": true\n```')).toEqual({ ok: false });
+ });
+
+ it('rejects empty content', () => {
+ expect(enforceJsonContent('')).toEqual({ ok: false });
+ expect(enforceJsonContent(' ')).toEqual({ ok: false });
+ });
+
+ it('JSON containing braces inside strings survives the slice heuristic', () => {
+ const clean = '{"code":"if (a) { return b; }"}';
+ expect(enforceJsonContent(`prefix ${clean} suffix`)).toEqual({ ok: true, content: clean, healed: true });
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-cache.test.ts b/server/src/__tests__/routes/proxy-cache.test.ts
index 72766cfa..bfed4aca 100644
--- a/server/src/__tests__/routes/proxy-cache.test.ts
+++ b/server/src/__tests__/routes/proxy-cache.test.ts
@@ -140,7 +140,9 @@ describe('Response cache (proxy integration)', () => {
});
it('requests differing only in response_format, seed, or stop never collide', async () => {
- const counter = mockGroq('knob-sensitive');
+ // A JSON string literal: valid JSON, so the response_format variant below
+ // survives the structured-output enforcement instead of failing over.
+ const counter = mockGroq('"knob-sensitive"');
await chat(); // plain → MISS, populates
// Same prompt with response_format json_object must be a MISS, not a
// replay of the cached plain-text answer.
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 83f48f43..5e3d41d3 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -62,6 +62,10 @@ export function isRetryableError(err: any): boolean {
// all thrown before any byte reached the client, so another model can
// serve the request invisibly.
|| msg.includes('empty completion')
+ // The model answered in prose despite a response_format request (and the
+ // JSON couldn't be healed out of the text) — thrown before any byte
+ // reached the client, so the next candidate can serve it invisibly.
+ || msg.includes('ignored response_format')
|| msg.includes('in-band provider error')
|| msg.includes('stream ended unexpectedly')
|| msg.includes('stream stalled')
diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.ts
index 6b431ee9..02cce9a9 100644
--- a/server/src/lib/sampling-params.ts
+++ b/server/src/lib/sampling-params.ts
@@ -158,6 +158,12 @@ export function extendedBodyParams(platform: string, options: ExtendedSamplingOp
return out;
}
+/** True when this platform's policy strips response_format before send — the
+ * router uses it to skip such platforms for structured-output requests. */
+export function platformDropsResponseFormat(platform: string): boolean {
+ return PLATFORM_PARAM_POLICIES[platform]?.drop?.includes('response_format') ?? false;
+}
+
/** The advertised parameter list for a model on `platform` — the base set
* every surface supports, plus tools when the model does, minus the
* platform's droplist. */
diff --git a/server/src/lib/structured-output.ts b/server/src/lib/structured-output.ts
new file mode 100644
index 00000000..28b4fcb6
--- /dev/null
+++ b/server/src/lib/structured-output.ts
@@ -0,0 +1,67 @@
+// Structured-output enforcement for the fallback chain (#514 follow-up).
+//
+// Forwarding response_format is necessary but not sufficient: a free-tier
+// model that doesn't understand JSON mode often just answers in prose (or
+// wraps the JSON in a markdown fence) and the request LOOKS successful — the
+// worst failure mode, because the client asked for machine-readable output
+// and got an essay. On non-streaming responses the gateway can check, heal,
+// or fail over:
+//
+// 1. content parses as JSON → pass through untouched
+// 2. content wraps JSON in ```fences``` or leading/trailing prose
+// → heal: replace content with the JSON
+// 3. nothing parseable → retryable failure (skipBench: the
+// provider is healthy, the MODEL ignored the format — no cooldown, no
+// penalty; the chain just tries the next candidate)
+//
+// Streaming responses can't be retro-checked (bytes are already on the wire),
+// so enforcement is non-stream only — same boundary as the response cache.
+
+export type JsonEnforcement =
+ | { ok: true; content: string; healed: boolean }
+ | { ok: false };
+
+/** Match a fenced code block (```json ... ``` or bare ``` ... ```). */
+const FENCE_RE = /```(?:json)?\s*\n?([\s\S]*?)```/i;
+
+function parses(text: string): boolean {
+ try {
+ JSON.parse(text);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+/** Slice from the first '{' or '[' to the matching last '}' or ']'. */
+function outermostJsonSlice(text: string): string | null {
+ const firstObj = text.indexOf('{');
+ const firstArr = text.indexOf('[');
+ const first = firstObj === -1 ? firstArr : (firstArr === -1 ? firstObj : Math.min(firstObj, firstArr));
+ if (first === -1) return null;
+ const open = text[first];
+ const close = open === '{' ? '}' : ']';
+ const last = text.lastIndexOf(close);
+ if (last <= first) return null;
+ return text.slice(first, last + 1);
+}
+
+export function enforceJsonContent(content: string): JsonEnforcement {
+ const trimmed = content.trim();
+ if (trimmed.length === 0) return { ok: false };
+
+ if (parses(trimmed)) return { ok: true, content: trimmed === content ? content : trimmed, healed: trimmed !== content };
+
+ // Fenced block: the most common "almost right" shape free models produce.
+ const fence = FENCE_RE.exec(trimmed);
+ if (fence) {
+ const inner = fence[1].trim();
+ if (parses(inner)) return { ok: true, content: inner, healed: true };
+ }
+
+ // Leading/trailing prose around one JSON value ("Here is your JSON: {...}").
+ const slice = outermostJsonSlice(trimmed);
+ if (slice && parses(slice)) return { ok: true, content: slice, healed: true };
+
+ return { ok: false };
+}
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 1018ced9..7490fd4c 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -20,6 +20,7 @@ import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheK
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError } from '../lib/fallback-loop.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { samplingParamSchemaFields, pickSamplingParams, supportedParametersForPlatforms } from '../lib/sampling-params.js';
+import { enforceJsonContent } from '../lib/structured-output.js';
import type { Platform } from '@freellmapi/shared/types.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from '../services/model-groups.js';
@@ -1418,7 +1419,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// model is on record). Turns where injection can't happen — every turn 1, and
// sessions that never switched — pay no headroom tax.
const routingEstimate = handoffPossible ? estimatedTotal + HANDOFF_MAX_TOKENS : estimatedTotal;
- return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain);
+ return routeRequest(routingEstimate, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, hasImage, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, groupChain ?? resolvedChain?.chain, samplingParams.response_format !== undefined);
},
dispatch: async (route, attempt) => {
const modelKey = `${route.platform}:${route.modelId}`;
@@ -1723,6 +1724,25 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
);
}
+ // Structured-output enforcement (#514 follow-up): the client asked for
+ // JSON; a model that answered in prose despite the forwarded
+ // response_format must not be returned as a "success". Heal the common
+ // almost-right shapes (fenced block, prose-wrapped JSON) in place;
+ // otherwise fail over. skipBench: the provider is healthy — the MODEL
+ // ignored the format — so no cooldown/penalty, just the next candidate.
+ if (samplingParams.response_format && respText && (respMsg?.tool_calls?.length ?? 0) === 0) {
+ const enforced = enforceJsonContent(respText);
+ if (!enforced.ok) {
+ throw Object.assign(
+ new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),
+ { skipBench: true },
+ );
+ }
+ if (enforced.healed && respMsg) {
+ respMsg.content = enforced.content;
+ }
+ }
+
// Inline tool-call dialect rescue (#231 audit): a tool-bearing
// request answered with the call serialized as TEXT (a mid-
// conversation model switch makes the new model imitate the previous
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 55c75a0e..60d6f0d6 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -26,6 +26,7 @@ import {
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess } from '../lib/fallback-loop.js';
import { applyTokenBudget, tokenBudgetMessage } from '../lib/guardrails.js';
import { samplingParamSchemaFields, pickSamplingParams, type ResponseFormat } from '../lib/sampling-params.js';
+import { enforceJsonContent } from '../lib/structured-output.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
@@ -421,7 +422,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
await runFallbackLoop({
maxRetries: MAX_RETRIES,
state,
- route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, false, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined),
+ route: () => routeRequest(estimatedTotal, state.skipKeys.size > 0 ? state.skipKeys : undefined, preferredModel, false, wantsTools, state.skipModels.size > 0 ? state.skipModels : undefined, undefined, completionOpts.response_format !== undefined),
dispatch: async (route, attempt) => {
traceRouteEvent('Responses', {
event: attempt === 0 ? 'start' : 'next',
@@ -729,6 +730,20 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
);
}
+ // Structured-output enforcement — see /chat/completions. Heal fenced or
+ // prose-wrapped JSON in place, fail over (skipBench) when the model
+ // ignored the requested format outright.
+ if (completionOpts.response_format && text && toolCalls.length === 0) {
+ const enforced = enforceJsonContent(text);
+ if (!enforced.ok) {
+ throw Object.assign(
+ new Error(`${route.displayName} ignored response_format (returned non-JSON despite ${completionOpts.response_format.type})`),
+ { skipBench: true },
+ );
+ }
+ if (enforced.healed) text = enforced.content;
+ }
+
// Usage fallback: a missing provider `usage` block used to record 0
// tokens against the rate-limit ledger; promptTokens/completionTokens
// above already carry the chars/4 estimate.
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 37329d4b..16602b9f 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -8,6 +8,7 @@ import {
speedScore, intelligenceScore, headroomFactor, rateLimitFactor, combineScore,
} from './scoring.js';
import { parseBudget } from '../lib/budget.js';
+import { platformDropsResponseFormat } from '../lib/sampling-params.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from './model-groups.js';
import type { BaseProvider } from '../providers/base.js';
import type { Platform } from '@freellmapi/shared/types.js';
@@ -980,7 +981,7 @@ export function resolveFusionCandidate(modelId: string): FusionCandidate | null
return null;
}
-export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[]): RouteResult {
+export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, preferredModelDbId?: number, requireVision = false, requireTools = false, skipModels?: Set<number>, prefetchedChain?: ChainRow[], requireStructured = false): RouteResult {
const db = getDb();
const strategy = getRoutingStrategy();
@@ -1042,6 +1043,14 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
// reasoning as vision above.
if (requireTools && !entry.supports_tools) { diag.push(`${label}: no tool-calling support`); continue; }
+ // Structured-output routing (#514 follow-up): when the request carries a
+ // response_format, skip platforms whose param policy can't even receive it
+ // (the param would be dropped before send, so the model would answer in
+ // prose and burn a failover hop). Platform-level fast path — model-level
+ // capability isn't in the catalog; models that accept the param but ignore
+ // it are caught by the non-stream JSON enforcement downstream.
+ if (requireStructured && platformDropsResponseFormat(entry.platform)) { diag.push(`${label}: platform drops response_format`); continue; }
+
// Context-aware routing: skip a model whose context window can't hold the
// request, so a large prompt never selects a small-context model and burns
// a failover hop on a 413 "request too large" (#167). Only enforced when we