7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy): enforce structured-output JSON on streaming responses (#985)
suantea (asuan_suan)8deb4a94c8a8
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.tsindex 107dcb40..46a8e6bd 100644--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts@@ -228,6 +228,68 @@ describe('proxy stream turn-integrity', () => {expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Non-empty'))).toBe(true);});+ it('fails over a stream whose model answers in prose despite response_format json_object (#933)', async () => {+ // The whole point of #933: a streamed structured-output request must not+ // ship an essay as a "success". The first model ignores the format; the+ // second honors it. Headers are held in 'json' hold mode, so the first+ // attempt can still fail over invisibly.+ const up = mockUpstream([+ { body: sse(roleChunk, textChunk('Here is your answer in prose form, sorry.'), finishChunk('stop'), '[DONE]') },+ { body: sse(roleChunk, textChunk('{"city":"Paris"}'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true,+ messages: [{ role: 'user', content: 'give me json' }],+ response_format: { type: 'json_object' },+ });+ expect(r.status).toBe(200);+ expect(up.calls()).toBe(2);+ expect(r.headers.get('x-fallback-attempts')).toBe('1');+ const fs = frames(r.text);+ expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('prose form'))).toBe(false);+ expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('Paris'))).toBe(true);+ const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];+ expect(rows[0].status).toBe('error');+ expect(rows[0].error).toMatch(/ignored response_format/);+ expect(rows[1].status).toBe('success');+ });++ it('heals a fenced JSON block streamed for a json_object request (#933)', async () => {+ // Prose-wrapped/fenced output is the common "almost right" shape: heal it+ // in place and deliver only the JSON, same as the non-stream path.+ mockUpstream([+ { body: sse(roleChunk, textChunk('```json\n{"answer": 42}\n```'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true,+ messages: [{ role: 'user', content: 'give me json' }],+ response_format: { type: 'json_object' },+ });+ expect(r.status).toBe(200);+ const fs = frames(r.text);+ const content = fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('');+ expect(content).toContain('"answer"');+ expect(content).not.toContain('```');+ const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];+ expect(rows[0].status).toBe('success');+ });++ it('passes a clean JSON stream through untouched for a json_object request (#933)', async () => {+ mockUpstream([+ { body: sse(roleChunk, textChunk('{"ok":true,"items":[1,2,3]}'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true,+ messages: [{ role: 'user', content: 'give me json' }],+ response_format: { type: 'json_object' },+ });+ expect(r.status).toBe(200);+ const content = frames(r.text).map(f => f.choices?.[0]?.delta?.content ?? '').join('');+ expect(content).toBe('{"ok":true,"items":[1,2,3]}');+ const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];+ expect(rows[0].status).toBe('success');+ });+it('never leaks raw tool_call deltas riding on role/reasoning chunks (OpenRouter shape)', async () => {// OpenRouter attaches tool_call fragments to chunks that also carry a// role or reasoning key. Those must be accumulated, not forwarded raw —diff --git a/server/src/lib/structured-output.ts b/server/src/lib/structured-output.tsindex c6707fe4..151efe8a 100644--- a/server/src/lib/structured-output.ts+++ b/server/src/lib/structured-output.ts@@ -14,8 +14,11 @@// provider is healthy, the MODEL ignored the format — no cooldown, no// penalty; the chain just tries the next candidate)//-// Streaming responses can't be retro-checked (bytes are already on the wire),-// so enforcement is non-stream only — same boundary as the response cache.+// Streaming responses used to skip enforcement entirely (bytes already on+// the wire). Since #933 the proxy's stream path buffers structured-output+// requests in a 'json' hold mode — headers are held until the stream ends,+// so the same check/heal/fail-over applies there too. The response cache+// remains non-stream only.export type JsonEnforcement =| { ok: true; content: string; healed: boolean }diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 9d27dca9..6665e6f9 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -2067,9 +2067,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {let ttfbMs: number | null = null;// Hold-window state: 'undecided' until the first text either matches- // a dialect marker (→ 'dialect': buffer everything, rescue at end) or- // provably cannot (→ 'passthrough': flush and stream normally).- let mode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';+ // a dialect marker (→ 'dialect': buffer everything, rescue at end),+ // carries a structured-output request (→ 'json': buffer everything,+ // enforce JSON at end) or provably cannot (→ 'passthrough': flush and+ // stream normally).+ let mode: 'undecided' | 'passthrough' | 'dialect' | 'json' = 'undecided';let heldText = '';const preamble: unknown[] = []; // role-only chunks held until flushconst toolCallAcc = new Map<number, { id?: string; name: string; args: string }>();@@ -2231,11 +2233,19 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}heldText += text;- if (mode === 'dialect') continue;+ if (mode === 'dialect' || mode === 'json') continue;const probe = heldText.trimStart();if (wantsTools && startsWithDialectMarker(probe)) {mode = 'dialect';+ } else if (samplingParams.response_format) {+ // Structured-output request (#933): hold ALL text until the+ // stream ends, then enforce JSON (mirrors the non-stream check+ // below). Streaming bytes are already committed once headers+ // flush, so a model that answers in prose despite the forwarded+ // response_format must be caught here, before any byte leaves —+ // the client asked for machine-readable output, not an essay.+ mode = 'json';} else if (!wantsTools || !couldBecomeDialectMarker(probe) || probe.length > 256) {mode = 'passthrough';flushHeaders();@@ -2329,6 +2339,28 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {);}+ // Structured-output enforcement for streams (#933): the non-stream+ // path checks JSON before returning; the stream path must too, or a+ // model that answers in prose despite the forwarded response_format+ // ships the essay as a "success" — the worst case for a+ // machine-readable request. json mode held every byte (headers never+ // flushed), so failing over here is free: skipBench (provider+ // healthy, the MODEL misbehaved) + skipModelForRequest (a sibling+ // key would misbehave identically). Mirrors proxy.ts non-stream.+ if (mode === 'json' && samplingParams.response_format && completedCalls.length === 0) {+ const enforced = enforceJsonContent(heldText);+ if (!enforced.ok) {+ const truncated = upstreamFinish === 'length';+ throw Object.assign(+ new Error(truncated+ ? `truncated JSON from ${route.displayName} (finish_reason=length — raise max_tokens for this ${samplingParams.response_format.type} request)`+ : `${route.displayName} ignored response_format (returned non-JSON despite ${samplingParams.response_format.type})`),+ { skipBench: true, skipModelForRequest: true },+ );+ }+ if (enforced.healed) heldText = enforced.content;+ }+flushHeaders();if (heldText.length > 0) {writeChunk(mkChunk({ content: heldText }, null));