Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(proxy): enforce structured-output JSON on streaming responses (#985)

suantea (asuan_suan)8deb4a94c8a8
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
index 107dcb40..46a8e6bd 100644
--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts
+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
@@ -228,6 +228,68 @@ describe('proxy stream turn-integrity', () => {
expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Non-empty'))).toBe(true);
});
+ it('fails over a stream whose model answers in prose despite response_format json_object (#933)', async () => {
+ // The whole point of #933: a streamed structured-output request must not
+ // ship an essay as a "success". The first model ignores the format; the
+ // second honors it. Headers are held in 'json' hold mode, so the first
+ // attempt can still fail over invisibly.
+ const up = mockUpstream([
+ { body: sse(roleChunk, textChunk('Here is your answer in prose form, sorry.'), finishChunk('stop'), '[DONE]') },
+ { body: sse(roleChunk, textChunk('{"city":"Paris"}'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true,
+ messages: [{ role: 'user', content: 'give me json' }],
+ response_format: { type: 'json_object' },
+ });
+ expect(r.status).toBe(200);
+ expect(up.calls()).toBe(2);
+ expect(r.headers.get('x-fallback-attempts')).toBe('1');
+ const fs = frames(r.text);
+ expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('prose form'))).toBe(false);
+ expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('Paris'))).toBe(true);
+ const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];
+ expect(rows[0].status).toBe('error');
+ expect(rows[0].error).toMatch(/ignored response_format/);
+ expect(rows[1].status).toBe('success');
+ });
+
+ it('heals a fenced JSON block streamed for a json_object request (#933)', async () => {
+ // Prose-wrapped/fenced output is the common "almost right" shape: heal it
+ // in place and deliver only the JSON, same as the non-stream path.
+ mockUpstream([
+ { body: sse(roleChunk, textChunk('```json\n{"answer": 42}\n```'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true,
+ messages: [{ role: 'user', content: 'give me json' }],
+ response_format: { type: 'json_object' },
+ });
+ expect(r.status).toBe(200);
+ const fs = frames(r.text);
+ const content = fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('');
+ expect(content).toContain('"answer"');
+ expect(content).not.toContain('```');
+ const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];
+ expect(rows[0].status).toBe('success');
+ });
+
+ it('passes a clean JSON stream through untouched for a json_object request (#933)', async () => {
+ mockUpstream([
+ { body: sse(roleChunk, textChunk('{"ok":true,"items":[1,2,3]}'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true,
+ messages: [{ role: 'user', content: 'give me json' }],
+ response_format: { type: 'json_object' },
+ });
+ expect(r.status).toBe(200);
+ const content = frames(r.text).map(f => f.choices?.[0]?.delta?.content ?? '').join('');
+ expect(content).toBe('{"ok":true,"items":[1,2,3]}');
+ const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];
+ expect(rows[0].status).toBe('success');
+ });
+
it('never leaks raw tool_call deltas riding on role/reasoning chunks (OpenRouter shape)', async () => {
// OpenRouter attaches tool_call fragments to chunks that also carry a
// role or reasoning key. Those must be accumulated, not forwarded raw —
diff --git a/server/src/lib/structured-output.ts b/server/src/lib/structured-output.ts
index c6707fe4..151efe8a 100644
--- a/server/src/lib/structured-output.ts
+++ b/server/src/lib/structured-output.ts
@@ -14,8 +14,11 @@
// provider is healthy, the MODEL ignored the format — no cooldown, no
// penalty; the chain just tries the next candidate)
//
-// Streaming responses can't be retro-checked (bytes are already on the wire),
-// so enforcement is non-stream only — same boundary as the response cache.
+// Streaming responses used to skip enforcement entirely (bytes already on
+// the wire). Since #933 the proxy's stream path buffers structured-output
+// requests in a 'json' hold mode — headers are held until the stream ends,
+// so the same check/heal/fail-over applies there too. The response cache
+// remains non-stream only.
export type JsonEnforcement =
| { ok: true; content: string; healed: boolean }
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 9d27dca9..6665e6f9 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -2067,9 +2067,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
let ttfbMs: number | null = null;
// Hold-window state: 'undecided' until the first text either matches
- // a dialect marker (→ 'dialect': buffer everything, rescue at end) or
- // provably cannot (→ 'passthrough': flush and stream normally).
- let mode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';
+ // a dialect marker (→ 'dialect': buffer everything, rescue at end),
+ // carries a structured-output request (→ 'json': buffer everything,
+ // enforce JSON at end) or provably cannot (→ 'passthrough': flush and
+ // stream normally).
+ let mode: 'undecided' | 'passthrough' | 'dialect' | 'json' = 'undecided';
let heldText = '';
const preamble: unknown[] = []; // role-only chunks held until flush
const toolCallAcc = new Map<number, { id?: string; name: string; args: string }>();
@@ -2231,11 +2233,19 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
heldText += text;
- if (mode === 'dialect') continue;
+ if (mode === 'dialect' || mode === 'json') continue;
const probe = heldText.trimStart();
if (wantsTools && startsWithDialectMarker(probe)) {
mode = 'dialect';
+ } else if (samplingParams.response_format) {
+ // Structured-output request (#933): hold ALL text until the
+ // stream ends, then enforce JSON (mirrors the non-stream check
+ // below). Streaming bytes are already committed once headers
+ // flush, so a model that answers in prose despite the forwarded
+ // response_format must be caught here, before any byte leaves —
+ // the client asked for machine-readable output, not an essay.
+ mode = 'json';
} else if (!wantsTools || !couldBecomeDialectMarker(probe) || probe.length > 256) {
mode = 'passthrough';
flushHeaders();
@@ -2329,6 +2339,28 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
);
}
+ // Structured-output enforcement for streams (#933): the non-stream
+ // path checks JSON before returning; the stream path must too, or a
+ // model that answers in prose despite the forwarded response_format
+ // ships the essay as a "success" — the worst case for a
+ // machine-readable request. json mode held every byte (headers never
+ // flushed), so failing over here is free: skipBench (provider
+ // healthy, the MODEL misbehaved) + skipModelForRequest (a sibling
+ // key would misbehave identically). Mirrors proxy.ts non-stream.
+ if (mode === 'json' && samplingParams.response_format && completedCalls.length === 0) {
+ const enforced = enforceJsonContent(heldText);
+ if (!enforced.ok) {
+ const truncated = upstreamFinish === 'length';
+ throw Object.assign(
+ new Error(truncated
+ ? `truncated JSON from ${route.displayName} (finish_reason=length — raise max_tokens for this ${samplingParams.response_format.type} request)`
+ : `${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),
+ { skipBench: true, skipModelForRequest: true },
+ );
+ }
+ if (enforced.healed) heldText = enforced.content;
+ }
+
flushHeaders();
if (heldText.length > 0) {
writeChunk(mkChunk({ content: heldText }, null));