7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(proxy): agent turn integrity — stream validation, tool-dialect rescue, sticky-session fixes (#231 audit)
Tashfeen438eaa2389ca
diff --git a/server/src/__tests__/lib/tool-call-rescue.test.ts b/server/src/__tests__/lib/tool-call-rescue.test.tsnew file mode 100644index 00000000..8b7473eb--- /dev/null+++ b/server/src/__tests__/lib/tool-call-rescue.test.ts@@ -0,0 +1,141 @@+import { describe, it, expect } from 'vitest';+import {+ rescueInlineToolCalls,+ startsWithDialectMarker,+ couldBecomeDialectMarker,+ containsDialectMarker,+} from '../../lib/tool-call-rescue.js';++const TOOLS = new Set(['Read', 'Write', 'Bash', 'Grep', 'list_dir']);++describe('inline tool-call dialect rescue', () => {+ describe('Kimi/DeepSeek token dialect', () => {+ it('parses a well-formed functions.NAME:IDX call', () => {+ const text = '<|tool_calls_section_begin|><|tool_call_begin|>functions.Read:0<|tool_call_argument_begin|>{"file_path":"/tmp/x.txt"}<|tool_call_end|><|tool_calls_section_end|>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.detected).toBe(true);+ expect(r.calls).toEqual([{ name: 'Read', arguments: '{"file_path":"/tmp/x.txt"}' }]);+ expect(r.cleanText).toBe('');+ });++ it('parses multiple calls and keeps surrounding prose as cleanText', () => {+ const text = 'Let me check.\n<|tool_call_begin|>functions.Read:0<|tool_call_argument_begin|>{"file_path":"/a"}<|tool_call_end|><|tool_call_begin|>functions.Grep:1<|tool_call_argument_begin|>{"pattern":"TODO"}<|tool_call_end|>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.calls).toHaveLength(2);+ expect(r.calls![1]).toEqual({ name: 'Grep', arguments: '{"pattern":"TODO"}' });+ expect(r.cleanText).toBe('Let me check.');+ });++ it('reports detected-but-unparseable when the id token is degraded (no function name)', () => {+ // The live capture that motivated this module: an opaque id where+ // functions.NAME:IDX should be — there is no way to know the tool.+ const text = '<|tool_calls_section_begin|> <|tool_call_begin|> chatcmpl-tool-bde5fae954a11b1b <|tool_call_argument_begin|> {"file_path": "/private/tmp/demo-project/src"} <|tool_call_end|>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.detected).toBe(true);+ expect(r.calls).toBeNull();+ });++ it('rejects a call naming a tool the request never declared', () => {+ const text = '<|tool_call_begin|>functions.DropTables:0<|tool_call_argument_begin|>{}<|tool_call_end|>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.detected).toBe(true);+ expect(r.calls).toBeNull();+ });+ });++ describe('llama/groq <function=> dialect', () => {+ it('parses <function=NAME{json}</function> (the failed_generation shape)', () => {+ const text = '<function=Bash{"command": "npm run build"}</function>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.detected).toBe(true);+ expect(r.calls).toEqual([{ name: 'Bash', arguments: '{"command": "npm run build"}' }]);+ });++ it('parses the variant with a closing > after the name', () => {+ const text = '<function=Read>{"file_path": "/tmp/a"}</function>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.calls).toEqual([{ name: 'Read', arguments: '{"file_path": "/tmp/a"}' }]);+ });++ it('treats array-shaped arguments as unparseable (live Groq failure shape)', () => {+ const text = '<function=Bash [{"a": "npm run build"}, {"b": "/tmp"}]</function>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.detected).toBe(true);+ expect(r.calls).toBeNull();+ });+ });++ describe('Qwen/Hermes <tool_call> XML dialect', () => {+ it('parses a single block with object arguments', () => {+ const text = '<tool_call>\n{"name": "Write", "arguments": {"file_path": "/tmp/n.txt", "content": "hi"}}\n</tool_call>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.calls).toHaveLength(1);+ expect(r.calls![0].name).toBe('Write');+ expect(JSON.parse(r.calls![0].arguments)).toEqual({ file_path: '/tmp/n.txt', content: 'hi' });+ });++ it('accepts the "parameters" key and multiple blocks', () => {+ const text = '<tool_call>{"name": "Read", "parameters": {"file_path": "/a"}}</tool_call><tool_call>{"name": "Grep", "arguments": {"pattern": "x"}}</tool_call>';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.calls).toHaveLength(2);+ });++ it('treats a truncated unclosed block as unparseable', () => {+ const text = '<tool_call>{"name": "Read", "argum';+ const r = rescueInlineToolCalls(text, TOOLS);+ expect(r.detected).toBe(true);+ expect(r.calls).toBeNull();+ });+ });++ describe('bare / fenced JSON dialect (schema-gated)', () => {+ it('rescues a bare JSON object naming a known tool', () => {+ const r = rescueInlineToolCalls('{"name": "list_dir", "arguments": {"path": "/tmp"}}', TOOLS);+ expect(r.detected).toBe(true);+ expect(r.calls).toEqual([{ name: 'list_dir', arguments: '{"path":"/tmp"}' }]);+ });++ it('rescues a ```json fenced call', () => {+ const r = rescueInlineToolCalls('```json\n{"name": "Bash", "arguments": {"command": "ls"}}\n```', TOOLS);+ expect(r.calls).toHaveLength(1);+ });++ it('does NOT touch ordinary JSON answers that do not name a requested tool', () => {+ const r = rescueInlineToolCalls('{"name": "Tashfeen", "age": 30}', TOOLS);+ expect(r.detected).toBe(false);+ expect(r.cleanText).toBe('{"name": "Tashfeen", "age": 30}');+ });++ it('does NOT touch plain prose', () => {+ const r = rescueInlineToolCalls('The port is 49152, configured in config.json.', TOOLS);+ expect(r.detected).toBe(false);+ });+ });++ describe('stream hold-window helpers', () => {+ it('startsWithDialectMarker matches all dialects, with leading whitespace', () => {+ expect(startsWithDialectMarker('<|tool_calls_section_begin|>x')).toBe(true);+ expect(startsWithDialectMarker(' <|tool_call_begin|>')).toBe(true);+ expect(startsWithDialectMarker('\n<tool_call>{}')).toBe(true);+ expect(startsWithDialectMarker('<function=Bash{')).toBe(true);+ expect(startsWithDialectMarker('Hello!')).toBe(false);+ expect(startsWithDialectMarker('<html>')).toBe(false);+ });++ it('couldBecomeDialectMarker holds strict prefixes and releases divergent text', () => {+ expect(couldBecomeDialectMarker('<')).toBe(true);+ expect(couldBecomeDialectMarker('<|to')).toBe(true);+ expect(couldBecomeDialectMarker('<fun')).toBe(true);+ expect(couldBecomeDialectMarker('<tool_ca')).toBe(true);+ expect(couldBecomeDialectMarker('<div')).toBe(false);+ expect(couldBecomeDialectMarker('Hi')).toBe(false);+ // Full marker present is no longer a *prefix* — startsWith takes over.+ expect(couldBecomeDialectMarker('<tool_call>')).toBe(false);+ });++ it('containsDialectMarker finds markers mid-text', () => {+ expect(containsDialectMarker('prose then <function=Read{"a":1}</function>')).toBe(true);+ expect(containsDialectMarker('plain text')).toBe(false);+ });+ });+});diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.tsnew file mode 100644index 00000000..887e52cb--- /dev/null+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts@@ -0,0 +1,333 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { getStickyModel, setStickyModel } from '../../routes/proxy.js';+import { mintDashboardToken } from '../helpers/auth.js';++// Stream turn-integrity (#231 audit): the proxy must deliver agent-usable+// TURNS, not transport bytes. These tests feed crafted upstream SSE bodies+// through a mocked fetch and assert the failure modes observed live are+// either failed over (before headers) or surfaced honestly (after).++async function request(app: Express, path: string, body: any, extraHeaders: Record<string, string> = {}) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ Authorization: `Bearer ${getUnifiedApiKey()}`,+ ...extraHeaders,+ },+ body: JSON.stringify(body),+ });+ const text = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(text); } catch { /* SSE body */ }+ return { status: res.status, headers: res.headers, text, body: json };+}++/** Parse an SSE response body into JSON frames (excluding [DONE]). */+function frames(text: string): any[] {+ return text.split('\n')+ .filter(l => l.startsWith('data: ') && l.trim() !== 'data: [DONE]')+ .map(l => JSON.parse(l.slice(6)));+}++const sse = (...payloads: (object | string)[]) =>+ payloads.map(p => `data: ${typeof p === 'string' ? p : JSON.stringify(p)}\n\n`).join('');++const roleChunk = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', content: null }, finish_reason: null }] };+const textChunk = (s: string, finish: string | null = null) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { content: s }, finish_reason: finish }] });+const finishChunk = (reason: string) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: {}, finish_reason: reason }] });++const TOOLS = [{ type: 'function', function: { name: 'Read', description: 'read a file', parameters: { type: 'object', properties: { file_path: { type: 'string' } }, required: ['file_path'] } } }];++// Sequential upstream responder: call N gets script[N] (last entry repeats).+function mockUpstream(script: Array<{ body: string; status?: number }>) {+ const origFetch = global.fetch;+ let call = 0;+ const seen: Array<{ model: string }> = [];+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ // Only intercept provider upstreams; the test's own localhost request+ // and anything else goes through.+ if (!/api\.groq\.com|openrouter\.ai|api\.cohere|generativelanguage|integrate\.api\.nvidia|api\.cerebras|api\.mistral|router\.huggingface|api\.sambanova|llm\.chutes|api\.cloudflare|models\.github|open\.bigmodel|api\.llm7|api\.kilo|text\.pollinations|ollama\.com|opencode\.ai/.test(urlStr)) {+ return origFetch(url as any, init);+ }+ const reqBody = JSON.parse(String((init as RequestInit).body));+ seen.push({ model: reqBody.model });+ const step = script[Math.min(call++, script.length - 1)];+ return new Response(step.body, {+ status: step.status ?? 200,+ headers: { 'Content-Type': 'text/event-stream' },+ });+ });+ return { seen, calls: () => call };+}++describe('proxy stream turn-integrity', () => {+ let app: Express;+ let dashToken = '';++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ dashToken = mintDashboardToken();+ });++ beforeEach(async () => {+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare('DELETE FROM requests').run();+ db.prepare('DELETE FROM rate_limit_cooldowns').run();+ db.prepare('DELETE FROM rate_limit_usage').run();+ // One groq key: the seeded chain has several tool-capable groq models, so+ // failover hops between groq models while staying on this mock.+ const { status } = await request(app, '/api/keys',+ { platform: 'groq', key: 'gsk_stream_integrity', label: 't' },+ { Authorization: `Bearer ${dashToken}` });+ expect(status).toBe(201);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ it('fails over when the upstream stream opens with an in-band error frame (Groq tool_use_failed)', async () => {+ const up = mockUpstream([+ { body: sse({ error: { message: "Failed to call a function. Please adjust your prompt.", type: 'invalid_request_error', code: 'tool_use_failed', status_code: 400 } }, '[DONE]') },+ { body: sse(roleChunk, textChunk('All good from the next model.'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, messages: [{ role: 'user', content: 'in-band error failover test' }],+ });+ expect(r.status).toBe(200);+ expect(up.calls()).toBe(2);+ expect(r.headers.get('x-fallback-attempts')).toBe('1');+ const fs = frames(r.text);+ expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('All good'))).toBe(true);+ expect(fs.some(f => f.error)).toBe(false);+ // The dead turn is recorded as an error, not a success.+ const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];+ expect(rows[0].status).toBe('error');+ expect(rows[0].error).toMatch(/in-band provider error/);+ expect(rows[1].status).toBe('success');+ });++ it('synthesizes finish_reason tool_calls and ids for a stream that ends without a terminal reason', async () => {+ // minimax/command-r live shape: valid tool_call deltas, then [DONE] with+ // no finish_reason chunk at all.+ const tcChunk = (frag: object) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { tool_calls: [frag] }, finish_reason: null }] });+ mockUpstream([{+ body: sse(+ roleChunk,+ tcChunk({ index: 0, function: { name: 'Read', arguments: '{"file_pa' } }),+ tcChunk({ index: 0, function: { arguments: 'th":"/tmp/a"}' } }),+ '[DONE]',+ ),+ }]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'finish synthesis test' }],+ });+ expect(r.status).toBe(200);+ const fs = frames(r.text);+ const tcFrame = fs.find(f => f.choices?.[0]?.delta?.tool_calls);+ expect(tcFrame).toBeDefined();+ const call = tcFrame.choices[0].delta.tool_calls[0];+ expect(call.function.name).toBe('Read');+ expect(JSON.parse(call.function.arguments)).toEqual({ file_path: '/tmp/a' });+ expect(call.id).toBe('call_stream_1'); // synthesized — upstream sent none+ const finishes = fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean);+ expect(finishes).toEqual(['tool_calls']);+ });++ it('rescues a streamed inline dialect turn into structured tool_calls', async () => {+ mockUpstream([{+ body: sse(+ roleChunk,+ textChunk('<|tool_calls_sec'),+ textChunk('tion_begin|><|tool_call_begin|>functions.Read:0<|tool_call_argument_begin|>{"file_path":"/a"}<|tool_call_end|><|tool_calls_section_end|>'),+ finishChunk('stop'),+ '[DONE]',+ ),+ }]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'dialect stream rescue test' }],+ });+ expect(r.status).toBe(200);+ const fs = frames(r.text);+ // The raw dialect text never reaches the client...+ expect(fs.some(f => typeof f.choices?.[0]?.delta?.content === 'string' && f.choices[0].delta.content.includes('<|tool_call'))).toBe(false);+ // ...a structured call does, with the correct terminal reason.+ const tcFrame = fs.find(f => f.choices?.[0]?.delta?.tool_calls);+ expect(tcFrame.choices[0].delta.tool_calls[0].function.name).toBe('Read');+ expect(tcFrame.choices[0].delta.tool_calls[0].id).toBe('call_rescued_1');+ expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['tool_calls']);+ });++ it('fails over an unparseable dialect turn (degraded id token) before headers', async () => {+ const up = mockUpstream([+ { body: sse(roleChunk, textChunk('<|tool_call_begin|> chatcmpl-tool-bde5 <|tool_call_argument_begin|> {"file_path": "/a"}'), finishChunk('stop'), '[DONE]') },+ { body: sse(roleChunk, textChunk('Recovered by the next model.'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'unparseable dialect failover test' }],+ });+ expect(r.status).toBe(200);+ expect(up.calls()).toBe(2);+ expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Recovered'))).toBe(true);+ });++ it('fails over an abrupt EOF that happens before any payload', async () => {+ const up = mockUpstream([+ { body: sse(roleChunk) /* EOF: no [DONE], no finish_reason, no payload */ },+ { body: sse(roleChunk, textChunk('Second model answers.'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, messages: [{ role: 'user', content: 'pre-payload truncation test' }],+ });+ expect(r.status).toBe(200);+ expect(up.calls()).toBe(2);+ expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Second model'))).toBe(true);+ const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];+ expect(rows[0].status).toBe('error');+ expect(rows[0].error).toMatch(/stream ended unexpectedly/);+ });++ it('surfaces an honest error frame when truncation happens after payload reached the client', async () => {+ mockUpstream([+ { body: sse(roleChunk, textChunk('Partial ans')) /* EOF mid-generation */ },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, messages: [{ role: 'user', content: 'post-payload truncation test' }],+ });+ expect(r.status).toBe(200);+ const fs = frames(r.text);+ expect(fs.some(f => f.choices?.[0]?.delta?.content === 'Partial ans')).toBe(true);+ expect(fs.some(f => f.error?.type === 'stream_error')).toBe(true);+ const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];+ expect(rows[0].status).toBe('error'); // truncation is never a success+ });++ it('fails over a stream that completes with no content and no tool calls', async () => {+ const up = mockUpstream([+ { body: sse(roleChunk, finishChunk('stop'), '[DONE]') },+ { body: sse(roleChunk, textChunk('Non-empty.'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, messages: [{ role: 'user', content: 'empty stream failover test' }],+ });+ expect(r.status).toBe(200);+ expect(up.calls()).toBe(2);+ expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Non-empty'))).toBe(true);+ });++ it('never leaks raw tool_call deltas riding on role/reasoning chunks (OpenRouter shape)', async () => {+ // OpenRouter attaches tool_call fragments to chunks that also carry a+ // role or reasoning key. Those must be accumulated, not forwarded raw —+ // forwarding both the fragments AND the assembled call duplicates it.+ mockUpstream([{+ body: sse(+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', reasoning: '', tool_calls: [{ index: 0, id: 'or_1', function: { name: 'Read', arguments: '{"file_' } }] }, finish_reason: null }] },+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { reasoning: null, tool_calls: [{ index: 0, function: { arguments: 'path":"/x"}' } }] }, finish_reason: null }] },+ finishChunk('tool_calls'),+ '[DONE]',+ ),+ }]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'delta leak test' }],+ });+ expect(r.status).toBe(200);+ const fs = frames(r.text);+ const tcFrames = fs.filter(f => f.choices?.[0]?.delta?.tool_calls);+ expect(tcFrames).toHaveLength(1); // exactly one complete emission, no raw fragments+ const call = tcFrames[0].choices[0].delta.tool_calls[0];+ expect(call.id).toBe('or_1');+ expect(JSON.parse(call.function.arguments)).toEqual({ file_path: '/x' });+ });++ it('streams ordinary text through unmodified, always ending in a terminal finish_reason', async () => {+ mockUpstream([+ { body: sse(roleChunk, textChunk('Hello'), textChunk(' world'), finishChunk('stop'), '[DONE]') },+ ]);+ const r = await request(app, '/v1/chat/completions', {+ stream: true, messages: [{ role: 'user', content: 'plain passthrough test' }],+ });+ const fs = frames(r.text);+ const text = fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('');+ expect(text).toBe('Hello world');+ expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['stop']);+ expect(r.text.trim().endsWith('data: [DONE]')).toBe(true);+ });++ it('rescues a non-streaming inline dialect answer into structured tool_calls', async () => {+ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (!urlStr.includes('api.groq.com')) return origFetch(url as any, init);+ return new Response(JSON.stringify({+ id: 'r1', object: 'chat.completion', created: 1, model: 'm',+ choices: [{ index: 0, message: { role: 'assistant', content: '<function=Read{"file_path": "/tmp/a"}</function>' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 5, completion_tokens: 9, total_tokens: 14 },+ }), { status: 200, headers: { 'Content-Type': 'application/json' } });+ });+ const r = await request(app, '/v1/chat/completions', {+ stream: false, tools: TOOLS, messages: [{ role: 'user', content: 'non-stream dialect rescue test' }],+ });+ expect(r.status).toBe(200);+ const msg = r.body.choices[0].message;+ expect(msg.tool_calls).toHaveLength(1);+ expect(msg.tool_calls[0].function.name).toBe('Read');+ expect(msg.tool_calls[0].id).toBe('call_rescued_1');+ expect(msg.content).toBeNull();+ expect(r.body.choices[0].finish_reason).toBe('tool_calls');+ });+});++describe('sticky session integrity', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ it('keeps affinity from turn 1 to turn 2 (the old single/multi key gap)', () => {+ const t1 = [+ { role: 'system' as const, content: 'sys' },+ { role: 'user' as const, content: 'sticky turn gap probe' },+ ];+ setStickyModel(t1, 42);+ const t2 = [...t1,+ { role: 'assistant' as const, content: 'answer' },+ { role: 'user' as const, content: 'follow-up' },+ ];+ expect(getStickyModel(t2)).toBe(42);+ });++ it('applies to array-of-blocks content (opencode-style agents)', () => {+ const t1 = [+ { role: 'user' as const, content: [{ type: 'text', text: 'array content sticky probe' }] as any },+ ];+ setStickyModel(t1, 7);+ const t2 = [...t1,+ { role: 'assistant' as const, content: 'ok' },+ { role: 'user' as const, content: [{ type: 'text', text: 'next' }] as any },+ ];+ expect(getStickyModel(t2)).toBe(7);+ });++ it('honors an explicit x-session-id over message hashing', () => {+ const conv1 = [{ role: 'user' as const, content: 'conversation one' }];+ setStickyModel(conv1, 11, 'session-abc');+ const conv2 = [+ { role: 'user' as const, content: 'completely different opener' },+ { role: 'assistant' as const, content: 'hi' },+ { role: 'user' as const, content: 'next' },+ ];+ expect(getStickyModel(conv2, 'session-abc')).toBe(11);+ expect(getStickyModel(conv2)).toBeUndefined();+ });+});diff --git a/server/src/lib/tool-call-rescue.ts b/server/src/lib/tool-call-rescue.tsnew file mode 100644index 00000000..97289b9d--- /dev/null+++ b/server/src/lib/tool-call-rescue.ts@@ -0,0 +1,243 @@+/**+ * Inline tool-call dialect rescue (#231 audit).+ *+ * When a conversation switches models mid-task (failover, sticky miss), the+ * new model often continues the previous model's tool-call style and emits+ * the call as TEXT in its private training dialect instead of a structured+ * `tool_calls` array. The client's agent loop sees prose, treats the turn as+ * a final answer, and dies mid-task — observed live with the OpenAI Agents+ * SDK when Kimi-K2.6 continued a DeepSeek history:+ *+ * <|tool_calls_section_begin|> <|tool_call_begin|> chatcmpl-tool-bde5...+ *+ * This module detects the known dialects and re-parses them into standard+ * OpenAI tool_calls, schema-gated against the request's tool list. A turn+ * that is detected as a dialect but cannot be parsed into a known tool is a+ * DEAD turn — the caller fails over instead of delivering gibberish.+ *+ * Supported dialects:+ * 1. Kimi / DeepSeek token style:+ * <|tool_calls_section_begin|><|tool_call_begin|>functions.NAME:0+ * <|tool_call_argument_begin|>{...}<|tool_call_end|>...+ * 2. Llama / Groq function tags: <function=NAME{...}</function> and+ * <function=NAME>{...}</function>+ * 3. Qwen / Hermes XML: <tool_call>{"name": ..., "arguments": ...}</tool_call>+ * 4. Bare or ```json-fenced single JSON object: {"name": KNOWN, "arguments": {...}}+ * (only rescued when "name" matches a requested tool — bare JSON is a+ * legitimate answer shape, so this one is strictly schema-gated)+ */++export interface RescuedToolCall {+ name: string;+ /** JSON string, exactly like OpenAI's function.arguments */+ arguments: string;+}++export interface RescueResult {+ /** True when the text contains inline tool-call dialect markers. */+ detected: boolean;+ /** Parsed calls; null when detected but unparseable (dead turn). */+ calls: RescuedToolCall[] | null;+ /** Text with the dialect blocks removed (may be ''). */+ cleanText: string;+}++// Markers that begin an inline dialect block. Used both for full-text+// detection and for the streaming hold-window decision in proxy.ts.+const DIALECT_MARKERS = [+ '<|tool_calls_section_begin|>',+ '<|tool_call_begin|>',+ '<tool_call>',+ '<function=',+] as const;++/** Does the (trimmed) text start with a known dialect marker? */+export function startsWithDialectMarker(text: string): boolean {+ const t = text.trimStart();+ return DIALECT_MARKERS.some(m => t.startsWith(m));+}++/**+ * Streaming hold-window helper: could `text` still grow into a dialect+ * marker? True while text is a strict prefix of some marker (e.g. "<|too"),+ * so the stream loop keeps holding; once this and startsWithDialectMarker+ * are both false the text is ordinary prose and can be flushed.+ */+export function couldBecomeDialectMarker(text: string): boolean {+ const t = text.trimStart();+ if (t.length === 0) return true;+ return DIALECT_MARKERS.some(m => m.startsWith(t) && t.length < m.length);+}++/** Anywhere-in-text detection for the non-streaming path. */+export function containsDialectMarker(text: string): boolean {+ return DIALECT_MARKERS.some(m => text.includes(m));+}++/**+ * Extract one balanced JSON object or array starting at text[from] (which+ * must be '{' or '['). Returns the slice and the index after it, or null.+ * String-aware so braces inside JSON strings don't break the balance.+ */+function extractBalancedJson(text: string, from: number): { json: string; end: number } | null {+ const open = text[from];+ if (open !== '{' && open !== '[') return null;+ const close = open === '{' ? '}' : ']';+ let depth = 0;+ let inString = false;+ let escaped = false;+ for (let i = from; i < text.length; i++) {+ const ch = text[i];+ if (inString) {+ if (escaped) escaped = false;+ else if (ch === '\\') escaped = true;+ else if (ch === '"') inString = false;+ continue;+ }+ if (ch === '"') inString = true;+ else if (ch === open) depth++;+ else if (ch === close) {+ depth--;+ if (depth === 0) return { json: text.slice(from, i + 1), end: i + 1 };+ }+ }+ return null;+}++const isKnownTool = (name: string, toolNames: Set<string>): boolean =>+ toolNames.size === 0 || toolNames.has(name);++/** Parse `{"name": ..., "arguments"|"parameters": ...}` into a call. */+function callFromNamedJson(json: string, toolNames: Set<string>): RescuedToolCall | null {+ let obj: unknown;+ try { obj = JSON.parse(json); } catch { return null; }+ if (typeof obj !== 'object' || obj === null) return null;+ const o = obj as Record<string, unknown>;+ const name = typeof o.name === 'string' ? o.name : undefined;+ if (!name || !isKnownTool(name, toolNames)) return null;+ const rawArgs = o.arguments ?? o.parameters ?? {};+ const args = typeof rawArgs === 'string' ? rawArgs : JSON.stringify(rawArgs);+ try { JSON.parse(args); } catch { return null; }+ return { name, arguments: args };+}++/** Dialect 1: Kimi/DeepSeek <|tool_call_begin|> token blocks. */+function parseTokenDialect(text: string, toolNames: Set<string>): { calls: RescuedToolCall[] | null; cleanText: string } {+ const calls: RescuedToolCall[] = [];+ let clean = text;+ // Strip section wrappers first; they carry no information.+ clean = clean.replaceAll('<|tool_calls_section_begin|>', '').replaceAll('<|tool_calls_section_end|>', '');++ const callRe = /<\|tool_call_begin\|>\s*([\s\S]*?)\s*<\|tool_call_argument_begin\|>\s*/g;+ let m: RegExpExecArray | null;+ let parsedAll = true;+ const spans: Array<{ from: number; to: number }> = [];+ while ((m = callRe.exec(clean)) !== null) {+ const idToken = m[1].trim();+ const argStart = m.index + m[0].length;+ const jsonStart = clean.indexOf('{', argStart);+ const extracted = jsonStart === -1 ? null : extractBalancedJson(clean, jsonStart);+ // Function name rides in the id token as `functions.NAME:IDX`. Some+ // models degrade it to an opaque id (observed: `chatcmpl-tool-<hex>`),+ // which leaves no way to know WHICH tool was meant — unparseable.+ const nameMatch = /^functions\.([A-Za-z0-9_.-]+):\d+$/.exec(idToken);+ const name = nameMatch?.[1];+ let argsOk = false;+ if (extracted && name && isKnownTool(name, toolNames)) {+ try { JSON.parse(extracted.json); argsOk = true; } catch { /* fall through */ }+ if (argsOk) calls.push({ name, arguments: extracted.json });+ }+ if (!argsOk) parsedAll = false;+ const endTag = clean.indexOf('<|tool_call_end|>', extracted?.end ?? argStart);+ spans.push({ from: m.index, to: endTag === -1 ? (extracted?.end ?? argStart) : endTag + '<|tool_call_end|>'.length });+ }+ for (const s of [...spans].reverse()) clean = clean.slice(0, s.from) + clean.slice(s.to);+ return { calls: parsedAll && calls.length > 0 ? calls : null, cleanText: clean.trim() };+}++/** Dialect 2: <function=NAME{...}</function> (with or without a '>' after the name). */+function parseFunctionTagDialect(text: string, toolNames: Set<string>): { calls: RescuedToolCall[] | null; cleanText: string } {+ const calls: RescuedToolCall[] = [];+ let clean = text;+ let parsedAll = true;+ const headRe = /<function=([A-Za-z0-9_.-]+)\s*>?\s*/g;+ let m: RegExpExecArray | null;+ const spans: Array<{ from: number; to: number }> = [];+ while ((m = headRe.exec(text)) !== null) {+ const name = m[1];+ const afterHead = m.index + m[0].length;+ const jsonStart = text[afterHead] === '{' || text[afterHead] === '['+ ? afterHead+ : text.indexOf('{', afterHead);+ const extracted = jsonStart === -1 ? null : extractBalancedJson(text, jsonStart);+ let ok = false;+ if (extracted && isKnownTool(name, toolNames) && extracted.json.startsWith('{')) {+ try { JSON.parse(extracted.json); ok = true; } catch { /* fall through */ }+ if (ok) calls.push({ name, arguments: extracted.json });+ }+ if (!ok) parsedAll = false; // array-shaped or invalid args: not a callable shape+ const closeTag = text.indexOf('</function>', extracted?.end ?? m.index + m[0].length);+ spans.push({ from: m.index, to: closeTag === -1 ? (extracted?.end ?? m.index + m[0].length) : closeTag + '</function>'.length });+ }+ for (const s of [...spans].reverse()) clean = clean.slice(0, s.from) + clean.slice(s.to);+ return { calls: parsedAll && calls.length > 0 ? calls : null, cleanText: clean.trim() };+}++/** Dialect 3: <tool_call>{...}</tool_call> XML-JSON blocks. */+function parseXmlDialect(text: string, toolNames: Set<string>): { calls: RescuedToolCall[] | null; cleanText: string } {+ const calls: RescuedToolCall[] = [];+ let parsedAll = true;+ const re = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g;+ let m: RegExpExecArray | null;+ let clean = text;+ const matches: string[] = [];+ while ((m = re.exec(text)) !== null) matches.push(m[1]);+ for (const inner of matches) {+ const call = callFromNamedJson(inner, toolNames);+ if (call) calls.push(call);+ else parsedAll = false;+ }+ clean = clean.replace(re, '');+ // An opening tag with no close (truncated stream) is detected-but-broken.+ if (/<tool_call>/.test(clean)) { parsedAll = false; clean = clean.replace(/<tool_call>[\s\S]*$/, ''); }+ return { calls: parsedAll && calls.length > 0 ? calls : null, cleanText: clean.trim() };+}++/**+ * Rescue inline tool-call dialects out of an assistant text answer.+ *+ * @param text the assistant message content+ * @param toolNames the names of the tools the REQUEST declared; rescued+ * calls must match one (empty set = accept any name,+ * used by tests only)+ */+export function rescueInlineToolCalls(text: string, toolNames: Set<string>): RescueResult {+ if (!text) return { detected: false, calls: null, cleanText: text };++ if (text.includes('<|tool_call_begin|>') || text.includes('<|tool_calls_section_begin|>')) {+ const { calls, cleanText } = parseTokenDialect(text, toolNames);+ return { detected: true, calls, cleanText };+ }+ if (text.includes('<function=')) {+ const { calls, cleanText } = parseFunctionTagDialect(text, toolNames);+ return { detected: true, calls, cleanText };+ }+ if (text.includes('<tool_call>')) {+ const { calls, cleanText } = parseXmlDialect(text, toolNames);+ return { detected: true, calls, cleanText };+ }++ // Dialect 4: the entire answer is one JSON object naming a known tool —+ // either bare or inside a ```json fence. Strictly schema-gated.+ const trimmed = text.trim();+ const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/.exec(trimmed);+ const candidate = (fenced ? fenced[1] : trimmed).trim();+ if (candidate.startsWith('{') && candidate.endsWith('}')) {+ const call = callFromNamedJson(candidate, toolNames);+ // Only treat as dialect when it actually names a requested tool;+ // arbitrary JSON answers must pass through untouched.+ if (call) return { detected: true, calls: [call], cleanText: '' };+ }++ return { detected: false, calls: null, cleanText: text };+}diff --git a/server/src/providers/base.ts b/server/src/providers/base.tsindex 32139b10..2e8d9352 100644--- a/server/src/providers/base.ts+++ b/server/src/providers/base.ts@@ -59,4 +59,74 @@ export abstract class BaseProvider {protected makeId(): string {return `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;}++ /**+ * Shared SSE reader for OpenAI-wire streaming endpoints (#231 audit).+ *+ * Hardened against the upstream failure modes observed live:+ * - Inactivity timeout: fetchWithTimeout's abort timer dies the moment+ * response HEADERS arrive, so a provider that stalls mid-body used to+ * hang the client forever. Each read now has its own deadline.+ * - Abrupt EOF: a stream that ends without `[DONE]` AND without any+ * `finish_reason` is a truncated generation, not a completion. It used+ * to end the generator silently (truncation logged as success); it now+ * throws a retryable error so the proxy can fail over or report it.+ * Providers that skip `[DONE]` but do send a terminal finish_reason+ * (several compat shims) still complete normally.+ *+ * Malformed data lines are skipped, matching previous behavior.+ */+ protected async *readSseStream(+ res: Response,+ inactivityTimeoutMs = 90000,+ ): AsyncGenerator<ChatCompletionChunk> {+ const reader = res.body?.getReader();+ if (!reader) throw new Error('No response body');++ const decoder = new TextDecoder();+ let buffer = '';+ let sawFinishReason = false;++ try {+ while (true) {+ let timer: ReturnType<typeof setTimeout> | undefined;+ const result = await Promise.race([+ reader.read(),+ new Promise<never>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error(`${this.name} stream stalled: no data for ${inactivityTimeoutMs}ms (timeout)`)),+ inactivityTimeoutMs,+ );+ }),+ ]).finally(() => clearTimeout(timer));++ const { done, value } = result;+ if (done) break;++ buffer += decoder.decode(value, { stream: true });+ const lines = buffer.split('\n');+ buffer = lines.pop() ?? '';++ for (const line of lines) {+ const trimmed = line.trim();+ if (!trimmed || !trimmed.startsWith('data: ')) continue;+ const data = trimmed.slice(6);+ if (data === '[DONE]') return;+ try {+ const chunk = JSON.parse(data) as ChatCompletionChunk;+ if (chunk.choices?.some(c => c.finish_reason != null)) sawFinishReason = true;+ yield chunk;+ } catch {+ // Skip malformed chunks+ }+ }+ }+ } finally {+ reader.cancel().catch(() => { /* upstream already gone */ });+ }++ if (!sawFinishReason) {+ throw new Error(`${this.name} stream ended unexpectedly (no [DONE], no finish_reason) — connection reset or truncated upstream`);+ }+ }}diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.tsindex ff748cdf..a80ecf67 100644--- a/server/src/providers/cloudflare.ts+++ b/server/src/providers/cloudflare.ts@@ -99,32 +99,7 @@ export class CloudflareProvider extends BaseProvider {throw new Error(`Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);}- const reader = res.body?.getReader();- if (!reader) throw new Error('No response body');-- const decoder = new TextDecoder();- let buffer = '';-- while (true) {- const { done, value } = await reader.read();- if (done) break;-- buffer += decoder.decode(value, { stream: true });- const lines = buffer.split('\n');- buffer = lines.pop() ?? '';-- for (const line of lines) {- const trimmed = line.trim();- if (!trimmed || !trimmed.startsWith('data: ')) continue;- const data = trimmed.slice(6);- if (data === '[DONE]') return;- try {- yield JSON.parse(data) as ChatCompletionChunk;- } catch {- // Skip malformed chunks- }- }- }+ yield* this.readSseStream(res);}async validateKey(apiKey: string): Promise<boolean> {diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex 8ef0a2c2..5fcb0a4b 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -78,32 +78,7 @@ export class CohereProvider extends BaseProvider {throw new Error(`Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}- const reader = res.body?.getReader();- if (!reader) throw new Error('No response body');-- const decoder = new TextDecoder();- let buffer = '';-- while (true) {- const { done, value } = await reader.read();- if (done) break;-- buffer += decoder.decode(value, { stream: true });- const lines = buffer.split('\n');- buffer = lines.pop() ?? '';-- for (const line of lines) {- const trimmed = line.trim();- if (!trimmed || !trimmed.startsWith('data: ')) continue;- const data = trimmed.slice(6);- if (data === '[DONE]') return;- try {- yield JSON.parse(data) as ChatCompletionChunk;- } catch {- // Skip malformed chunks- }- }- }+ yield* this.readSseStream(res);}async validateKey(apiKey: string): Promise<boolean> {diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex 717ea78f..6cce5c1d 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -126,32 +126,7 @@ export class OpenAICompatProvider extends BaseProvider {throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}- const reader = res.body?.getReader();- if (!reader) throw new Error('No response body');-- const decoder = new TextDecoder();- let buffer = '';-- while (true) {- const { done, value } = await reader.read();- if (done) break;-- buffer += decoder.decode(value, { stream: true });- const lines = buffer.split('\n');- buffer = lines.pop() ?? '';-- for (const line of lines) {- const trimmed = line.trim();- if (!trimmed || !trimmed.startsWith('data: ')) continue;- const data = trimmed.slice(6);- if (data === '[DONE]') return;- try {- yield JSON.parse(data) as ChatCompletionChunk;- } catch {- // Skip malformed chunks- }- }- }+ yield* this.readSseStream(res);}async validateKey(apiKey: string): Promise<boolean> {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 13709592..cbb416e1 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -11,6 +11,7 @@ import { getDb, getUnifiedApiKey } from '../db/index.js';import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';+import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';export const proxyRouter = Router();@@ -59,23 +60,34 @@ export function extractApiToken(req: Request): string | undefined {const stickySessionMap = new Map<string, { modelDbId: number; lastUsed: number }>();const STICKY_TTL_MS = 30 * 60 * 1000; // 30 min session TTL-function getSessionKey(messages: ChatMessage[]): string {- // Use the first user message as session identifier — clients like Hermes- // re-send the full conversation each turn, so the first user message is- // stable across turns. Hash the FULL message (not a 100-char slice) so- // distinct conversations with identical openings don't collide.+function getSessionKey(messages: ChatMessage[], sessionIdHeader?: string): string {+ // Explicit session pinning: clients that manage their own conversation ids+ // (agent harnesses especially) can send X-Session-Id and get exact+ // affinity regardless of how their message history mutates. (#231)+ if (sessionIdHeader) return `hdr:${sessionIdHeader}`;++ // Otherwise the first user message identifies the session — clients re-send+ // the full conversation each turn, so it is stable across turns. Flatten+ // array-of-blocks content before hashing: opencode-style agents send+ // [{type:'text',...}] even for plain text, and the old string-only check+ // silently disabled stickiness for them, re-routing every turn (#231 audit:+ // observed a rank-2 → rank-11 mid-conversation flip). No turn-count suffix:+ // the old ':single'/':multi' split guaranteed a sticky MISS on turn 2,+ // exactly where agents replay the assistant's tool-call dialect and a model+ // switch causes cross-dialect contamination.const firstUser = messages.find(m => m.role === 'user');- if (!firstUser || typeof firstUser.content !== 'string') return '';- const hash = crypto.createHash('sha1').update(firstUser.content).digest('hex');- return `${hash}:${messages.length > 2 ? 'multi' : 'single'}`;+ if (!firstUser) return '';+ const text = contentToString(firstUser.content ?? '');+ if (!text) return '';+ return crypto.createHash('sha1').update(text).digest('hex');}-export function getStickyModel(messages: ChatMessage[]): number | undefined {+export function getStickyModel(messages: ChatMessage[], sessionIdHeader?: string): number | undefined {// Only apply sticky for multi-turn (has assistant messages = continuation)const hasAssistant = messages.some(m => m.role === 'assistant');if (!hasAssistant) return undefined;- const key = getSessionKey(messages);+ const key = getSessionKey(messages, sessionIdHeader);if (!key) return undefined;const entry = stickySessionMap.get(key);@@ -88,8 +100,8 @@ export function getStickyModel(messages: ChatMessage[]): number | undefined {return entry.modelDbId;}-export function setStickyModel(messages: ChatMessage[], modelDbId: number) {- const key = getSessionKey(messages);+export function setStickyModel(messages: ChatMessage[], modelDbId: number, sessionIdHeader?: string) {+ const key = getSessionKey(messages, sessionIdHeader);if (!key) return;stickySessionMap.set(key, { modelDbId, lastUsed: Date.now() });@@ -311,7 +323,15 @@ export function isRetryableError(err: any): boolean {// provider (Kimi K2.6 is on HF + Cloudflare + NVIDIA), so fail over instead// of killing the workflow. Paired with a long cooldown (isPaymentRequiredError)// so we don't re-hammer the broke key every retry.- || isPaymentRequiredError(err);+ || isPaymentRequiredError(err)+ // Dead-turn classes from the stream turn-integrity layer (#231 audit):+ // all thrown before any byte reached the client, so another model can+ // serve the request invisibly.+ || msg.includes('empty completion')+ || msg.includes('in-band provider error')+ || msg.includes('stream ended unexpectedly')+ || msg.includes('stream stalled')+ || msg.includes('unparseable inline tool-call dialect');}// A 402 Payment Required / out-of-credits error. Distinct from a transient 429:@@ -547,6 +567,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {return;}+ // Optional client-managed session affinity (see getSessionKey). Express+ // lower-cases header names; a repeated header arrives as an array — take+ // the first value.+ const rawSessionId = req.headers['x-session-id'];+ const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;+// Explicit `model` field pins routing. If the catalog has no enabled row// matching the requested id, return 400 — silently auto-routing to a// different model would be surprising to OpenAI-compatible clients.@@ -554,7 +580,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {let preferredModel: number | undefined;if (isAutoModel(requestedModel)) {// Explicit "auto" → behave exactly like an omitted model field.- preferredModel = getStickyModel(messages);+ preferredModel = getStickyModel(messages, sessionIdHeader);} else if (requestedModel) {const db = getDb();const enabled = db.prepare('SELECT id FROM models WHERE model_id = ? AND enabled = 1').get(requestedModel) as { id: number } | undefined;@@ -573,7 +599,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {return;}} else {- preferredModel = getStickyModel(messages);+ preferredModel = getStickyModel(messages, sessionIdHeader);}// For analytics: the model id the client pinned, null when auto-routed@@ -611,12 +637,58 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {try {if (stream) {- // Lazy header set: pre-stream errors stay retryable (no headers sent yet);- // mid-stream errors emit an `error` SSE frame so the client sees a real signal- // instead of a silently truncated stream.+ // — Stream turn-integrity (#231 audit) —+ // The old loop forwarded upstream chunks verbatim and called any+ // stream that produced bytes a success. Live failure modes that+ // slipped through: in-band `{"error":...}` frames delivered as dead+ // turns, tool calls with no terminal finish_reason, inline tool-call+ // dialect emitted as text, truncations logged as success. This loop+ // validates the TURN, not the transport:+ // - headers are held until the first real payload, so anything that+ // dies before producing one fails over invisibly;+ // - text that starts with an inline tool-call dialect marker is held+ // and rescued into structured tool_calls (or failed over);+ // - tool_call deltas are buffered, argument-repaired, and emitted as+ // one complete chunk, always followed by finish_reason+ // "tool_calls" — agents never see calls without a terminal reason;+ // - a stream that ends with neither content nor calls is an empty+ // completion and fails over like the non-stream path.let totalOutputTokens = 0;- let streamStarted = false;+ let headerSent = false;let ttfbMs: number | null = null;++ // Hold-window state: 'undecided' until the first text either matches+ // a dialect marker (→ 'dialect': buffer everything, rescue at end) or+ // provably cannot (→ 'passthrough': flush and stream normally).+ let mode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';+ let heldText = '';+ const preamble: unknown[] = []; // role-only chunks held until flush+ const toolCallAcc = new Map<number, { id?: string; name: string; args: string }>();+ let upstreamFinish: string | null = null;+ let usageChunk: unknown = null;+ let lastMeta: { id?: string; model?: string; created?: number } = {};++ const flushHeaders = () => {+ if (headerSent) return;+ ttfbMs = Date.now() - start;+ res.setHeader('Content-Type', 'text/event-stream');+ res.setHeader('Cache-Control', 'no-cache');+ res.setHeader('Connection', 'keep-alive');+ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);+ if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));+ headerSent = true;+ for (const p of preamble) res.write(`data: ${JSON.stringify(p)}\n\n`);+ preamble.length = 0;+ };+ const mkChunk = (delta: Record<string, unknown>, finish: string | null) => ({+ id: lastMeta.id ?? `chatcmpl-${Date.now()}`,+ object: 'chat.completion.chunk',+ created: lastMeta.created ?? Math.floor(Date.now() / 1000),+ model: lastMeta.model ?? route.modelId,+ choices: [{ index: 0, delta, finish_reason: finish }],+ });+ const writeChunk = (c: unknown) => res.write(`data: ${JSON.stringify(c)}\n\n`);+try {const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId,@@ -624,52 +696,153 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {);for await (const chunk of gen) {- if (!streamStarted) {- // Time-to-first-byte: dispatch → first chunk. Feeds the router's- // latency axis (server/src/services/scoring.ts).- ttfbMs = Date.now() - start;- res.setHeader('Content-Type', 'text/event-stream');- res.setHeader('Cache-Control', 'no-cache');- res.setHeader('Connection', 'keep-alive');- res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);- if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));- streamStarted = true;+ const anyChunk = chunk as Record<string, any>;++ // In-band upstream error frame (observed live: Groq emits+ // {"error":{...,"code":"tool_use_failed"}} inside a 200 SSE+ // stream). Before headers: retryable, the next model gets the+ // request. After: surface an error frame instead of pretending+ // the turn succeeded.+ if (anyChunk.error && !anyChunk.choices) {+ const msg = anyChunk.error.message ?? JSON.stringify(anyChunk.error).slice(0, 200);+ if (!headerSent) throw new Error(`in-band provider error from ${route.displayName}: ${msg}`);+ console.error(`[Proxy] In-band error frame from ${route.displayName} mid-stream:`, msg);+ writeChunk({ error: { message: `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(String(msg))}`, type: 'stream_error' } });+ try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, `in-band error frame: ${sanitizeProviderErrorMessage(String(msg))}`, ttfbMs, pinnedModelId);+ return;}- // Coerce array-shaped delta.content to a string before forwarding,- // so spec-conforming clients don't break and tool_calls survive (#166).++ if (anyChunk.id) lastMeta = { id: anyChunk.id, model: anyChunk.model, created: anyChunk.created };++ const choice = anyChunk.choices?.[0];+ if (!choice) {+ // Usage-only frame (stream_options.include_usage) — held and+ // re-emitted after our finish chunk to preserve OpenAI ordering.+ if (anyChunk.usage) usageChunk = anyChunk;+ continue;+ }++ if (choice.finish_reason) upstreamFinish = choice.finish_reason;++ // Buffer tool_call deltas — emitted complete + repaired at end.+ for (const tc of choice.delta?.tool_calls ?? []) {+ const idx = tc.index ?? 0;+ if (!toolCallAcc.has(idx)) toolCallAcc.set(idx, { id: undefined, name: '', args: '' });+ const acc = toolCallAcc.get(idx)!;+ if (tc.id && !acc.id) acc.id = tc.id;+ if (tc.function?.name) acc.name += tc.function.name;+ if (tc.function?.arguments) acc.args += tc.function.arguments;+ }+normalizeOutboundContent(chunk);- const text = streamChunkText(chunk);+ const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';++ if (text.length === 0) {+ // Role preamble / keep-alive: hold until first payload decides+ // the mode, forward afterwards. tool_calls and finish_reason are+ // stripped — both are re-emitted complete at the end (OpenRouter+ // attaches tool_call deltas to chunks that also carry role/+ // reasoning keys; forwarding them raw would duplicate the call).+ if (choice.delta && Object.keys(choice.delta).some(k => k !== 'content' && k !== 'tool_calls' && choice.delta[k] != null)) {+ const cleaned = { ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] };+ if (headerSent) writeChunk(cleaned); else preamble.push(cleaned);+ }+ continue;+ }+totalOutputTokens += Math.ceil(text.length / 4);- res.write(`data: ${JSON.stringify(chunk)}\n\n`);++ if (mode === 'passthrough') {+ writeChunk({ ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] });+ continue;+ }++ heldText += text;+ if (mode === 'dialect') continue;++ const probe = heldText.trimStart();+ if (startsWithDialectMarker(probe)) {+ mode = 'dialect';+ } else if (!couldBecomeDialectMarker(probe) || heldText.length > 256) {+ mode = 'passthrough';+ flushHeaders();+ writeChunk(mkChunk({ content: heldText }, null));+ heldText = '';+ }+ // else: still a strict prefix of a marker — keep holding.+ }++ // — Stream ended cleanly (provider saw [DONE] or a finish_reason) —++ // Assemble buffered tool calls: synthesize missing ids, repair+ // double-encoded arguments against the request's schemas, drop+ // calls whose args still aren't valid JSON.+ const schemas = toolSchemaMap(tools);+ let syntheticStreamIds = 0;+ const completedCalls = [...toolCallAcc.entries()]+ .sort((a, b) => a[0] - b[0])+ .map(([, acc]) => ({+ id: acc.id && acc.id.length > 0 ? acc.id : `call_stream_${++syntheticStreamIds}`,+ type: 'function' as const,+ function: { name: acc.name, arguments: repairToolArguments(acc.args || '{}', schemas.get(acc.name)) },+ }))+ .filter(c => { try { JSON.parse(c.function.arguments); return c.function.name.length > 0; } catch { return false; } });++ // Dialect rescue: the held text is an inline tool call in some+ // model's private syntax. Parse it into structured calls or treat+ // the turn as dead (headers were never sent in dialect mode, so+ // failing over is free).+ if (mode === 'dialect' || (mode === 'undecided' && heldText.length > 0 && containsDialectMarker(heldText))) {+ const rescue = rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)));+ if (rescue.detected) {+ if (!rescue.calls) throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${heldText.slice(0, 120)}`);+ let rescuedIds = 0;+ for (const c of rescue.calls) {+ completedCalls.push({ id: `call_rescued_${++rescuedIds}`, type: 'function', function: { name: c.name, arguments: repairToolArguments(c.arguments, schemas.get(c.name)) } });+ }+ heldText = rescue.cleanText;+ console.log(`[Proxy] Rescued ${rescuedIds} inline tool call(s) from ${route.displayName} into structured tool_calls`);+ }}- if (!streamStarted) {- // Upstream returned zero chunks — an empty completion in stream- // clothing. No headers are out yet, so fail over to the next model- // instead of handing the client a valid-looking empty stream- // (production case: nemotron-3-super returning nothing on large- // contexts while the request logs as success).- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (stream produced no chunks)', null, pinnedModelId);- skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);- setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));- recordRateLimitHit(route.modelDbId);- lastError = new Error(`empty completion from ${route.displayName}`);- continue;+ const hasText = headerSent || heldText.trim().length > 0;+ if (!hasText && completedCalls.length === 0) {+ // Nothing usable came out — same failover semantics as the+ // non-stream empty-completion path. Headers can't have been sent+ // (header flush requires payload), so the client never notices.+ throw new Error(`empty completion from ${route.displayName} (stream produced no content and no tool calls)`);}++ flushHeaders();+ if (heldText.length > 0) {+ writeChunk(mkChunk({ content: heldText }, null));+ totalOutputTokens += Math.ceil(heldText.length / 4);+ }+ if (completedCalls.length > 0) {+ writeChunk(mkChunk({ tool_calls: completedCalls.map((c, i) => ({ index: i, ...c })) }, null));+ totalOutputTokens += Math.ceil(completedCalls.reduce((n, c) => n + c.function.arguments.length, 0) / 4);+ }+ // Terminal finish_reason, ALWAYS present: calls win over a sloppy+ // upstream 'stop'; 'length'/'content_filter' survive for pure-text+ // turns; missing upstream reason is synthesized.+ const finish = completedCalls.length > 0+ ? 'tool_calls'+ : (upstreamFinish && upstreamFinish !== 'tool_calls' ? upstreamFinish : 'stop');+ writeChunk(mkChunk({}, finish));+ if (usageChunk) writeChunk(usageChunk);res.write('data: [DONE]\n\n');res.end();recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);recordSuccess(route.modelDbId);- setStickyModel(messages, route.modelDbId);+ setStickyModel(messages, route.modelDbId, sessionIdHeader);logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);return;} catch (streamErr: any) {- if (streamStarted) {- // Mid-stream error — finish the SSE response cleanly instead of leaving- // the client hanging or letting Express's default handler take over.- // Full upstream message goes to the log; the client sees a generic- // message so we don't leak provider internals into a partial stream.+ if (headerSent) {+ // Mid-stream error after real payload reached the client — finish+ // the SSE response honestly instead of leaving the client hanging.console.error(`[Proxy] Mid-stream error from ${route.displayName}:`, streamErr.message);const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };try { res.write(`data: ${JSON.stringify(payload)}\n\n`); } catch { /* socket gone */ }@@ -677,7 +850,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, sanitizeProviderErrorMessage(streamErr.message), null, pinnedModelId);return;}- // Pre-stream error — bubble to outer retry/502 handler.+ // Headers never sent — bubble to the outer retry handler, which+ // cooldowns this model+key and tries the next one. Covers upstream+ // HTTP errors, in-band error frames, abrupt EOF, stalls, empty+ // completions, and unparseable dialect turns alike.throw streamErr;}} else {@@ -700,10 +876,34 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {continue;}+ // Inline tool-call dialect rescue (#231 audit): a tool-bearing+ // request answered with the call serialized as TEXT (a mid-+ // conversation model switch makes the new model imitate the previous+ // model's private syntax). Re-parse it into structured tool_calls so+ // the client's agent loop keeps working; a detected-but-unparseable+ // dialect is a dead turn and fails over like an empty completion.+ if (wantsTools && respMsg && (respMsg.tool_calls?.length ?? 0) === 0 && respText) {+ const rescue = rescueInlineToolCalls(respText, new Set((tools ?? []).map(t => t.function.name)));+ if (rescue.detected) {+ if (!rescue.calls) {+ throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${respText.slice(0, 120)}`);+ }+ const schemas = toolSchemaMap(tools);+ respMsg.tool_calls = rescue.calls.map((c, i) => ({+ id: `call_rescued_${i + 1}`,+ type: 'function' as const,+ function: { name: c.name, arguments: repairToolArguments(c.arguments, schemas.get(c.name)) },+ }));+ respMsg.content = rescue.cleanText.length > 0 ? rescue.cleanText : null;+ if (result.choices?.[0]) result.choices[0].finish_reason = 'tool_calls';+ console.log(`[Proxy] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName} into structured tool_calls`);+ }+ }+const totalTokens = result.usage?.total_tokens ?? 0;recordTokens(route.platform, route.modelId, route.keyId, totalTokens);recordSuccess(route.modelDbId);- setStickyModel(messages, route.modelDbId);+ setStickyModel(messages, route.modelDbId, sessionIdHeader);res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 790825aa..ce8f7826 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -13,6 +13,7 @@ import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,import { getUnifiedApiKey } from '../db/index.js';import { contentToString } from '../lib/content.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';+import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import {isRetryableError,isPaymentRequiredError,@@ -316,7 +317,10 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {0,);const estimatedTotal = estimatedInputTokens + (reqData.max_output_tokens ?? 1000);- const preferredModel = getStickyModel(messages);+ // Optional client-managed session affinity (mirrors /chat/completions).+ const rawSessionId = req.headers['x-session-id'];+ const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;+ const preferredModel = getStickyModel(messages, sessionIdHeader);// Tool-bearing requests (the normal case for Codex/agent clients on this// endpoint) must stay on models that emit structured tool_calls — a model@@ -376,9 +380,41 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const toolAcc = new Map<number, { outputIndex: number; itemId: string; callId: string; name: string; args: string }>();let totalOutputTokens = 0;+ // Inline-dialect hold window (#231): first text is held until it+ // either matches a tool-call dialect marker (held to the end and+ // rescued into function_call items) or provably cannot (flushed and+ // streamed normally). Mirrors the /chat/completions stream loop.+ let dialectMode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';+ let heldText = '';++ // Open the text output item and stream `text` as its first delta.+ const openTextItem = (text: string) => {+ msgItemId = newId('msg');+ sse('response.output_item.added', {+ output_index: outputIndex,+ item: { id: msgItemId, type: 'message', status: 'in_progress', role: 'assistant', content: [] },+ });+ sse('response.content_part.added', {+ item_id: msgItemId, output_index: outputIndex, content_index: 0,+ part: { type: 'output_text', text: '', annotations: [] },+ });+ if (text) {+ sse('response.output_text.delta', { item_id: msgItemId, output_index: outputIndex, content_index: 0, delta: text });+ msgText += text;+ }+ };+const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, completionOpts);for await (const chunk of gen) {+ // In-band upstream error frame ({"error":...} inside a 200 SSE+ // stream — observed live from Groq). Throw before the lazy header+ // block so a first-frame error keeps streamStarted=false and takes+ // the normal failover path in the catch below.+ const anyChunk = chunk as Record<string, any>;+ if (anyChunk.error && !anyChunk.choices) {+ throw new Error(`in-band provider error from ${route.displayName}: ${anyChunk.error.message ?? 'provider error'}`);+ }// LAZY header set — headers + the response.created/in_progress// skeleton go out only once the provider actually streams a chunk.// Sending them before the provider call (the previous behavior)@@ -405,28 +441,33 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {streamStarted = true;}- const delta = chunk.choices[0]?.delta;+ const delta = chunk.choices?.[0]?.delta;if (!delta) continue;- // Text deltas → output_text events on a single message item.+ // Text deltas → output_text events on a single message item, after+ // the dialect hold window has decided the text is real prose.const text = delta.content ?? '';if (text) {- if (msgItemId === null) {- msgItemId = newId('msg');- sse('response.output_item.added', {- output_index: outputIndex,- item: { id: msgItemId, type: 'message', status: 'in_progress', role: 'assistant', content: [] },- });- sse('response.content_part.added', {- item_id: msgItemId, output_index: outputIndex, content_index: 0,- part: { type: 'output_text', text: '', annotations: [] },+ totalOutputTokens += Math.ceil(text.length / 4);+ if (dialectMode === 'passthrough') {+ if (msgItemId === null) openTextItem('');+ sse('response.output_text.delta', {+ item_id: msgItemId, output_index: 0, content_index: 0, delta: text,});+ msgText += text;+ } else {+ heldText += text;+ if (dialectMode === 'undecided') {+ const probe = heldText.trimStart();+ if (startsWithDialectMarker(probe)) {+ dialectMode = 'dialect';+ } else if (!couldBecomeDialectMarker(probe) || heldText.length > 256) {+ dialectMode = 'passthrough';+ openTextItem(heldText);+ heldText = '';+ }+ }}- sse('response.output_text.delta', {- item_id: msgItemId, output_index: outputIndex, content_index: 0, delta: text,- });- msgText += text;- totalOutputTokens += Math.ceil(text.length / 4);}// Tool-call deltas → function_call item + argument deltas.@@ -459,6 +500,46 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {}}+ // Resolve the dialect hold window now that the full text is known.+ // Held text was never emitted, so a dead dialect turn can still fail+ // over on the same SSE stream (only the skeleton events are out).+ if (heldText.length > 0) {+ const rescue = (dialectMode === 'dialect' || containsDialectMarker(heldText))+ ? rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)))+ : { detected: false as const, calls: null, cleanText: heldText };+ if (rescue.detected && !rescue.calls) {+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, `unparseable inline tool-call dialect: ${heldText.slice(0, 120)}`);+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));+ recordRateLimitHit(route.modelDbId);+ lastError = new Error(`unparseable inline tool-call dialect from ${route.displayName}`);+ continue;+ }+ if (rescue.detected && rescue.calls) {+ // Rescued calls become function_call items, exactly as if the+ // provider had streamed them structurally.+ console.log(`[Responses] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName}`);+ if (rescue.cleanText.length > 0 && msgItemId === null) openTextItem(rescue.cleanText);+ let rescuedIdx = 0;+ for (const c of rescue.calls) {+ const idx = 1000 + rescuedIdx++; // synthetic accumulator keys, past any provider index+ const acc = {+ outputIndex: toolAcc.size + (msgText.length > 0 ? 1 : 0),+ itemId: newId('fc'), callId: newId('call'), name: c.name, args: c.arguments,+ };+ toolAcc.set(idx, acc);+ sse('response.output_item.added', {+ output_index: acc.outputIndex,+ item: { id: acc.itemId, type: 'function_call', status: 'in_progress', call_id: acc.callId, name: acc.name, arguments: '' },+ });+ }+ } else if (msgItemId === null) {+ // Plain short answer that never left the hold window (e.g. "Hi").+ openTextItem(heldText);+ }+ heldText = '';+ }+// Empty completion — the provider returned 200 with no text AND no// tool calls. Seen in production from nemotron-3-super on ~65k-token// contexts: transport-level "success", zero usable output, so the@@ -505,18 +586,35 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);recordSuccess(route.modelDbId);- setStickyModel(messages, route.modelDbId);+ setStickyModel(messages, route.modelDbId, sessionIdHeader);logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);return;} else {const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, completionOpts);const msg = result.choices[0]?.message;- const text = contentToString(msg?.content ?? '');- const toolCalls = (msg?.tool_calls ?? []).map((tc) => ({+ let text = contentToString(msg?.content ?? '');+ let toolCalls = (msg?.tool_calls ?? []).map((tc) => ({...tc,function: { ...tc.function, arguments: repairToolArguments(tc.function.arguments, toolSchemas.get(tc.function.name)) },}));++ // Inline tool-call dialect rescue (#231) — see /chat/completions.+ if (wantsTools && toolCalls.length === 0 && text) {+ const rescue = rescueInlineToolCalls(text, new Set((tools ?? []).map(t => t.function.name)));+ if (rescue.detected) {+ if (!rescue.calls) {+ throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${text.slice(0, 120)}`);+ }+ console.log(`[Responses] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName}`);+ toolCalls = rescue.calls.map((c, i) => ({+ id: `call_rescued_${i + 1}`,+ type: 'function' as const,+ function: { name: c.name, arguments: repairToolArguments(c.arguments, toolSchemas.get(c.name)) },+ }));+ text = rescue.cleanText;+ }+ }const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);@@ -532,7 +630,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);recordSuccess(route.modelDbId);- setStickyModel(messages, route.modelDbId);+ setStickyModel(messages, route.modelDbId, sessionIdHeader);res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));