Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(proxy): agent turn integrity — stream validation, tool-dialect rescue, sticky-session fixes (#231 audit)

Tashfeen438eaa2389ca
diff --git a/server/src/__tests__/lib/tool-call-rescue.test.ts b/server/src/__tests__/lib/tool-call-rescue.test.ts
new file mode 100644
index 00000000..8b7473eb
--- /dev/null
+++ b/server/src/__tests__/lib/tool-call-rescue.test.ts
@@ -0,0 +1,141 @@
+import { describe, it, expect } from 'vitest';
+import {
+ rescueInlineToolCalls,
+ startsWithDialectMarker,
+ couldBecomeDialectMarker,
+ containsDialectMarker,
+} from '../../lib/tool-call-rescue.js';
+
+const TOOLS = new Set(['Read', 'Write', 'Bash', 'Grep', 'list_dir']);
+
+describe('inline tool-call dialect rescue', () => {
+ describe('Kimi/DeepSeek token dialect', () => {
+ it('parses a well-formed functions.NAME:IDX call', () => {
+ const text = '<|tool_calls_section_begin|><|tool_call_begin|>functions.Read:0<|tool_call_argument_begin|>{"file_path":"/tmp/x.txt"}<|tool_call_end|><|tool_calls_section_end|>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.detected).toBe(true);
+ expect(r.calls).toEqual([{ name: 'Read', arguments: '{"file_path":"/tmp/x.txt"}' }]);
+ expect(r.cleanText).toBe('');
+ });
+
+ it('parses multiple calls and keeps surrounding prose as cleanText', () => {
+ const text = 'Let me check.\n<|tool_call_begin|>functions.Read:0<|tool_call_argument_begin|>{"file_path":"/a"}<|tool_call_end|><|tool_call_begin|>functions.Grep:1<|tool_call_argument_begin|>{"pattern":"TODO"}<|tool_call_end|>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.calls).toHaveLength(2);
+ expect(r.calls![1]).toEqual({ name: 'Grep', arguments: '{"pattern":"TODO"}' });
+ expect(r.cleanText).toBe('Let me check.');
+ });
+
+ it('reports detected-but-unparseable when the id token is degraded (no function name)', () => {
+ // The live capture that motivated this module: an opaque id where
+ // functions.NAME:IDX should be — there is no way to know the tool.
+ const text = '<|tool_calls_section_begin|> <|tool_call_begin|> chatcmpl-tool-bde5fae954a11b1b <|tool_call_argument_begin|> {"file_path": "/private/tmp/demo-project/src"} <|tool_call_end|>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.detected).toBe(true);
+ expect(r.calls).toBeNull();
+ });
+
+ it('rejects a call naming a tool the request never declared', () => {
+ const text = '<|tool_call_begin|>functions.DropTables:0<|tool_call_argument_begin|>{}<|tool_call_end|>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.detected).toBe(true);
+ expect(r.calls).toBeNull();
+ });
+ });
+
+ describe('llama/groq <function=> dialect', () => {
+ it('parses <function=NAME{json}</function> (the failed_generation shape)', () => {
+ const text = '<function=Bash{"command": "npm run build"}</function>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.detected).toBe(true);
+ expect(r.calls).toEqual([{ name: 'Bash', arguments: '{"command": "npm run build"}' }]);
+ });
+
+ it('parses the variant with a closing > after the name', () => {
+ const text = '<function=Read>{"file_path": "/tmp/a"}</function>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.calls).toEqual([{ name: 'Read', arguments: '{"file_path": "/tmp/a"}' }]);
+ });
+
+ it('treats array-shaped arguments as unparseable (live Groq failure shape)', () => {
+ const text = '<function=Bash [{"a": "npm run build"}, {"b": "/tmp"}]</function>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.detected).toBe(true);
+ expect(r.calls).toBeNull();
+ });
+ });
+
+ describe('Qwen/Hermes <tool_call> XML dialect', () => {
+ it('parses a single block with object arguments', () => {
+ const text = '<tool_call>\n{"name": "Write", "arguments": {"file_path": "/tmp/n.txt", "content": "hi"}}\n</tool_call>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.calls).toHaveLength(1);
+ expect(r.calls![0].name).toBe('Write');
+ expect(JSON.parse(r.calls![0].arguments)).toEqual({ file_path: '/tmp/n.txt', content: 'hi' });
+ });
+
+ it('accepts the "parameters" key and multiple blocks', () => {
+ const text = '<tool_call>{"name": "Read", "parameters": {"file_path": "/a"}}</tool_call><tool_call>{"name": "Grep", "arguments": {"pattern": "x"}}</tool_call>';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.calls).toHaveLength(2);
+ });
+
+ it('treats a truncated unclosed block as unparseable', () => {
+ const text = '<tool_call>{"name": "Read", "argum';
+ const r = rescueInlineToolCalls(text, TOOLS);
+ expect(r.detected).toBe(true);
+ expect(r.calls).toBeNull();
+ });
+ });
+
+ describe('bare / fenced JSON dialect (schema-gated)', () => {
+ it('rescues a bare JSON object naming a known tool', () => {
+ const r = rescueInlineToolCalls('{"name": "list_dir", "arguments": {"path": "/tmp"}}', TOOLS);
+ expect(r.detected).toBe(true);
+ expect(r.calls).toEqual([{ name: 'list_dir', arguments: '{"path":"/tmp"}' }]);
+ });
+
+ it('rescues a ```json fenced call', () => {
+ const r = rescueInlineToolCalls('```json\n{"name": "Bash", "arguments": {"command": "ls"}}\n```', TOOLS);
+ expect(r.calls).toHaveLength(1);
+ });
+
+ it('does NOT touch ordinary JSON answers that do not name a requested tool', () => {
+ const r = rescueInlineToolCalls('{"name": "Tashfeen", "age": 30}', TOOLS);
+ expect(r.detected).toBe(false);
+ expect(r.cleanText).toBe('{"name": "Tashfeen", "age": 30}');
+ });
+
+ it('does NOT touch plain prose', () => {
+ const r = rescueInlineToolCalls('The port is 49152, configured in config.json.', TOOLS);
+ expect(r.detected).toBe(false);
+ });
+ });
+
+ describe('stream hold-window helpers', () => {
+ it('startsWithDialectMarker matches all dialects, with leading whitespace', () => {
+ expect(startsWithDialectMarker('<|tool_calls_section_begin|>x')).toBe(true);
+ expect(startsWithDialectMarker(' <|tool_call_begin|>')).toBe(true);
+ expect(startsWithDialectMarker('\n<tool_call>{}')).toBe(true);
+ expect(startsWithDialectMarker('<function=Bash{')).toBe(true);
+ expect(startsWithDialectMarker('Hello!')).toBe(false);
+ expect(startsWithDialectMarker('<html>')).toBe(false);
+ });
+
+ it('couldBecomeDialectMarker holds strict prefixes and releases divergent text', () => {
+ expect(couldBecomeDialectMarker('<')).toBe(true);
+ expect(couldBecomeDialectMarker('<|to')).toBe(true);
+ expect(couldBecomeDialectMarker('<fun')).toBe(true);
+ expect(couldBecomeDialectMarker('<tool_ca')).toBe(true);
+ expect(couldBecomeDialectMarker('<div')).toBe(false);
+ expect(couldBecomeDialectMarker('Hi')).toBe(false);
+ // Full marker present is no longer a *prefix* — startsWith takes over.
+ expect(couldBecomeDialectMarker('<tool_call>')).toBe(false);
+ });
+
+ it('containsDialectMarker finds markers mid-text', () => {
+ expect(containsDialectMarker('prose then <function=Read{"a":1}</function>')).toBe(true);
+ expect(containsDialectMarker('plain text')).toBe(false);
+ });
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
new file mode 100644
index 00000000..887e52cb
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
@@ -0,0 +1,333 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+import { getStickyModel, setStickyModel } from '../../routes/proxy.js';
+import { mintDashboardToken } from '../helpers/auth.js';
+
+// Stream turn-integrity (#231 audit): the proxy must deliver agent-usable
+// TURNS, not transport bytes. These tests feed crafted upstream SSE bodies
+// through a mocked fetch and assert the failure modes observed live are
+// either failed over (before headers) or surfaced honestly (after).
+
+async function request(app: Express, path: string, body: any, extraHeaders: Record<string, string> = {}) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ Authorization: `Bearer ${getUnifiedApiKey()}`,
+ ...extraHeaders,
+ },
+ body: JSON.stringify(body),
+ });
+ const text = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(text); } catch { /* SSE body */ }
+ return { status: res.status, headers: res.headers, text, body: json };
+}
+
+/** Parse an SSE response body into JSON frames (excluding [DONE]). */
+function frames(text: string): any[] {
+ return text.split('\n')
+ .filter(l => l.startsWith('data: ') && l.trim() !== 'data: [DONE]')
+ .map(l => JSON.parse(l.slice(6)));
+}
+
+const sse = (...payloads: (object | string)[]) =>
+ payloads.map(p => `data: ${typeof p === 'string' ? p : JSON.stringify(p)}\n\n`).join('');
+
+const roleChunk = { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', content: null }, finish_reason: null }] };
+const textChunk = (s: string, finish: string | null = null) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { content: s }, finish_reason: finish }] });
+const finishChunk = (reason: string) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: {}, finish_reason: reason }] });
+
+const TOOLS = [{ type: 'function', function: { name: 'Read', description: 'read a file', parameters: { type: 'object', properties: { file_path: { type: 'string' } }, required: ['file_path'] } } }];
+
+// Sequential upstream responder: call N gets script[N] (last entry repeats).
+function mockUpstream(script: Array<{ body: string; status?: number }>) {
+ const origFetch = global.fetch;
+ let call = 0;
+ const seen: Array<{ model: string }> = [];
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ // Only intercept provider upstreams; the test's own localhost request
+ // and anything else goes through.
+ if (!/api\.groq\.com|openrouter\.ai|api\.cohere|generativelanguage|integrate\.api\.nvidia|api\.cerebras|api\.mistral|router\.huggingface|api\.sambanova|llm\.chutes|api\.cloudflare|models\.github|open\.bigmodel|api\.llm7|api\.kilo|text\.pollinations|ollama\.com|opencode\.ai/.test(urlStr)) {
+ return origFetch(url as any, init);
+ }
+ const reqBody = JSON.parse(String((init as RequestInit).body));
+ seen.push({ model: reqBody.model });
+ const step = script[Math.min(call++, script.length - 1)];
+ return new Response(step.body, {
+ status: step.status ?? 200,
+ headers: { 'Content-Type': 'text/event-stream' },
+ });
+ });
+ return { seen, calls: () => call };
+}
+
+describe('proxy stream turn-integrity', () => {
+ let app: Express;
+ let dashToken = '';
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ dashToken = mintDashboardToken();
+ });
+
+ beforeEach(async () => {
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM requests').run();
+ db.prepare('DELETE FROM rate_limit_cooldowns').run();
+ db.prepare('DELETE FROM rate_limit_usage').run();
+ // One groq key: the seeded chain has several tool-capable groq models, so
+ // failover hops between groq models while staying on this mock.
+ const { status } = await request(app, '/api/keys',
+ { platform: 'groq', key: 'gsk_stream_integrity', label: 't' },
+ { Authorization: `Bearer ${dashToken}` });
+ expect(status).toBe(201);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('fails over when the upstream stream opens with an in-band error frame (Groq tool_use_failed)', async () => {
+ const up = mockUpstream([
+ { body: sse({ error: { message: "Failed to call a function. Please adjust your prompt.", type: 'invalid_request_error', code: 'tool_use_failed', status_code: 400 } }, '[DONE]') },
+ { body: sse(roleChunk, textChunk('All good from the next model.'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, messages: [{ role: 'user', content: 'in-band error failover test' }],
+ });
+ expect(r.status).toBe(200);
+ expect(up.calls()).toBe(2);
+ expect(r.headers.get('x-fallback-attempts')).toBe('1');
+ const fs = frames(r.text);
+ expect(fs.some(f => f.choices?.[0]?.delta?.content?.includes('All good'))).toBe(true);
+ expect(fs.some(f => f.error)).toBe(false);
+ // The dead turn is recorded as an error, not a success.
+ const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];
+ expect(rows[0].status).toBe('error');
+ expect(rows[0].error).toMatch(/in-band provider error/);
+ expect(rows[1].status).toBe('success');
+ });
+
+ it('synthesizes finish_reason tool_calls and ids for a stream that ends without a terminal reason', async () => {
+ // minimax/command-r live shape: valid tool_call deltas, then [DONE] with
+ // no finish_reason chunk at all.
+ const tcChunk = (frag: object) => ({ id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { tool_calls: [frag] }, finish_reason: null }] });
+ mockUpstream([{
+ body: sse(
+ roleChunk,
+ tcChunk({ index: 0, function: { name: 'Read', arguments: '{"file_pa' } }),
+ tcChunk({ index: 0, function: { arguments: 'th":"/tmp/a"}' } }),
+ '[DONE]',
+ ),
+ }]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'finish synthesis test' }],
+ });
+ expect(r.status).toBe(200);
+ const fs = frames(r.text);
+ const tcFrame = fs.find(f => f.choices?.[0]?.delta?.tool_calls);
+ expect(tcFrame).toBeDefined();
+ const call = tcFrame.choices[0].delta.tool_calls[0];
+ expect(call.function.name).toBe('Read');
+ expect(JSON.parse(call.function.arguments)).toEqual({ file_path: '/tmp/a' });
+ expect(call.id).toBe('call_stream_1'); // synthesized — upstream sent none
+ const finishes = fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean);
+ expect(finishes).toEqual(['tool_calls']);
+ });
+
+ it('rescues a streamed inline dialect turn into structured tool_calls', async () => {
+ mockUpstream([{
+ body: sse(
+ roleChunk,
+ textChunk('<|tool_calls_sec'),
+ textChunk('tion_begin|><|tool_call_begin|>functions.Read:0<|tool_call_argument_begin|>{"file_path":"/a"}<|tool_call_end|><|tool_calls_section_end|>'),
+ finishChunk('stop'),
+ '[DONE]',
+ ),
+ }]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'dialect stream rescue test' }],
+ });
+ expect(r.status).toBe(200);
+ const fs = frames(r.text);
+ // The raw dialect text never reaches the client...
+ expect(fs.some(f => typeof f.choices?.[0]?.delta?.content === 'string' && f.choices[0].delta.content.includes('<|tool_call'))).toBe(false);
+ // ...a structured call does, with the correct terminal reason.
+ const tcFrame = fs.find(f => f.choices?.[0]?.delta?.tool_calls);
+ expect(tcFrame.choices[0].delta.tool_calls[0].function.name).toBe('Read');
+ expect(tcFrame.choices[0].delta.tool_calls[0].id).toBe('call_rescued_1');
+ expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['tool_calls']);
+ });
+
+ it('fails over an unparseable dialect turn (degraded id token) before headers', async () => {
+ const up = mockUpstream([
+ { body: sse(roleChunk, textChunk('<|tool_call_begin|> chatcmpl-tool-bde5 <|tool_call_argument_begin|> {"file_path": "/a"}'), finishChunk('stop'), '[DONE]') },
+ { body: sse(roleChunk, textChunk('Recovered by the next model.'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'unparseable dialect failover test' }],
+ });
+ expect(r.status).toBe(200);
+ expect(up.calls()).toBe(2);
+ expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Recovered'))).toBe(true);
+ });
+
+ it('fails over an abrupt EOF that happens before any payload', async () => {
+ const up = mockUpstream([
+ { body: sse(roleChunk) /* EOF: no [DONE], no finish_reason, no payload */ },
+ { body: sse(roleChunk, textChunk('Second model answers.'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, messages: [{ role: 'user', content: 'pre-payload truncation test' }],
+ });
+ expect(r.status).toBe(200);
+ expect(up.calls()).toBe(2);
+ expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Second model'))).toBe(true);
+ const rows = getDb().prepare("SELECT status, error FROM requests ORDER BY id").all() as any[];
+ expect(rows[0].status).toBe('error');
+ expect(rows[0].error).toMatch(/stream ended unexpectedly/);
+ });
+
+ it('surfaces an honest error frame when truncation happens after payload reached the client', async () => {
+ mockUpstream([
+ { body: sse(roleChunk, textChunk('Partial ans')) /* EOF mid-generation */ },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, messages: [{ role: 'user', content: 'post-payload truncation test' }],
+ });
+ expect(r.status).toBe(200);
+ const fs = frames(r.text);
+ expect(fs.some(f => f.choices?.[0]?.delta?.content === 'Partial ans')).toBe(true);
+ expect(fs.some(f => f.error?.type === 'stream_error')).toBe(true);
+ const rows = getDb().prepare("SELECT status FROM requests ORDER BY id").all() as any[];
+ expect(rows[0].status).toBe('error'); // truncation is never a success
+ });
+
+ it('fails over a stream that completes with no content and no tool calls', async () => {
+ const up = mockUpstream([
+ { body: sse(roleChunk, finishChunk('stop'), '[DONE]') },
+ { body: sse(roleChunk, textChunk('Non-empty.'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, messages: [{ role: 'user', content: 'empty stream failover test' }],
+ });
+ expect(r.status).toBe(200);
+ expect(up.calls()).toBe(2);
+ expect(frames(r.text).some(f => f.choices?.[0]?.delta?.content?.includes('Non-empty'))).toBe(true);
+ });
+
+ it('never leaks raw tool_call deltas riding on role/reasoning chunks (OpenRouter shape)', async () => {
+ // OpenRouter attaches tool_call fragments to chunks that also carry a
+ // role or reasoning key. Those must be accumulated, not forwarded raw —
+ // forwarding both the fragments AND the assembled call duplicates it.
+ mockUpstream([{
+ body: sse(
+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { role: 'assistant', reasoning: '', tool_calls: [{ index: 0, id: 'or_1', function: { name: 'Read', arguments: '{"file_' } }] }, finish_reason: null }] },
+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'm', choices: [{ index: 0, delta: { reasoning: null, tool_calls: [{ index: 0, function: { arguments: 'path":"/x"}' } }] }, finish_reason: null }] },
+ finishChunk('tool_calls'),
+ '[DONE]',
+ ),
+ }]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, tools: TOOLS, messages: [{ role: 'user', content: 'delta leak test' }],
+ });
+ expect(r.status).toBe(200);
+ const fs = frames(r.text);
+ const tcFrames = fs.filter(f => f.choices?.[0]?.delta?.tool_calls);
+ expect(tcFrames).toHaveLength(1); // exactly one complete emission, no raw fragments
+ const call = tcFrames[0].choices[0].delta.tool_calls[0];
+ expect(call.id).toBe('or_1');
+ expect(JSON.parse(call.function.arguments)).toEqual({ file_path: '/x' });
+ });
+
+ it('streams ordinary text through unmodified, always ending in a terminal finish_reason', async () => {
+ mockUpstream([
+ { body: sse(roleChunk, textChunk('Hello'), textChunk(' world'), finishChunk('stop'), '[DONE]') },
+ ]);
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true, messages: [{ role: 'user', content: 'plain passthrough test' }],
+ });
+ const fs = frames(r.text);
+ const text = fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('');
+ expect(text).toBe('Hello world');
+ expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['stop']);
+ expect(r.text.trim().endsWith('data: [DONE]')).toBe(true);
+ });
+
+ it('rescues a non-streaming inline dialect answer into structured tool_calls', async () => {
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (!urlStr.includes('api.groq.com')) return origFetch(url as any, init);
+ return new Response(JSON.stringify({
+ id: 'r1', object: 'chat.completion', created: 1, model: 'm',
+ choices: [{ index: 0, message: { role: 'assistant', content: '<function=Read{"file_path": "/tmp/a"}</function>' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 5, completion_tokens: 9, total_tokens: 14 },
+ }), { status: 200, headers: { 'Content-Type': 'application/json' } });
+ });
+ const r = await request(app, '/v1/chat/completions', {
+ stream: false, tools: TOOLS, messages: [{ role: 'user', content: 'non-stream dialect rescue test' }],
+ });
+ expect(r.status).toBe(200);
+ const msg = r.body.choices[0].message;
+ expect(msg.tool_calls).toHaveLength(1);
+ expect(msg.tool_calls[0].function.name).toBe('Read');
+ expect(msg.tool_calls[0].id).toBe('call_rescued_1');
+ expect(msg.content).toBeNull();
+ expect(r.body.choices[0].finish_reason).toBe('tool_calls');
+ });
+});
+
+describe('sticky session integrity', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ it('keeps affinity from turn 1 to turn 2 (the old single/multi key gap)', () => {
+ const t1 = [
+ { role: 'system' as const, content: 'sys' },
+ { role: 'user' as const, content: 'sticky turn gap probe' },
+ ];
+ setStickyModel(t1, 42);
+ const t2 = [...t1,
+ { role: 'assistant' as const, content: 'answer' },
+ { role: 'user' as const, content: 'follow-up' },
+ ];
+ expect(getStickyModel(t2)).toBe(42);
+ });
+
+ it('applies to array-of-blocks content (opencode-style agents)', () => {
+ const t1 = [
+ { role: 'user' as const, content: [{ type: 'text', text: 'array content sticky probe' }] as any },
+ ];
+ setStickyModel(t1, 7);
+ const t2 = [...t1,
+ { role: 'assistant' as const, content: 'ok' },
+ { role: 'user' as const, content: [{ type: 'text', text: 'next' }] as any },
+ ];
+ expect(getStickyModel(t2)).toBe(7);
+ });
+
+ it('honors an explicit x-session-id over message hashing', () => {
+ const conv1 = [{ role: 'user' as const, content: 'conversation one' }];
+ setStickyModel(conv1, 11, 'session-abc');
+ const conv2 = [
+ { role: 'user' as const, content: 'completely different opener' },
+ { role: 'assistant' as const, content: 'hi' },
+ { role: 'user' as const, content: 'next' },
+ ];
+ expect(getStickyModel(conv2, 'session-abc')).toBe(11);
+ expect(getStickyModel(conv2)).toBeUndefined();
+ });
+});
diff --git a/server/src/lib/tool-call-rescue.ts b/server/src/lib/tool-call-rescue.ts
new file mode 100644
index 00000000..97289b9d
--- /dev/null
+++ b/server/src/lib/tool-call-rescue.ts
@@ -0,0 +1,243 @@
+/**
+ * Inline tool-call dialect rescue (#231 audit).
+ *
+ * When a conversation switches models mid-task (failover, sticky miss), the
+ * new model often continues the previous model's tool-call style and emits
+ * the call as TEXT in its private training dialect instead of a structured
+ * `tool_calls` array. The client's agent loop sees prose, treats the turn as
+ * a final answer, and dies mid-task — observed live with the OpenAI Agents
+ * SDK when Kimi-K2.6 continued a DeepSeek history:
+ *
+ * <|tool_calls_section_begin|> <|tool_call_begin|> chatcmpl-tool-bde5...
+ *
+ * This module detects the known dialects and re-parses them into standard
+ * OpenAI tool_calls, schema-gated against the request's tool list. A turn
+ * that is detected as a dialect but cannot be parsed into a known tool is a
+ * DEAD turn — the caller fails over instead of delivering gibberish.
+ *
+ * Supported dialects:
+ * 1. Kimi / DeepSeek token style:
+ * <|tool_calls_section_begin|><|tool_call_begin|>functions.NAME:0
+ * <|tool_call_argument_begin|>{...}<|tool_call_end|>...
+ * 2. Llama / Groq function tags: <function=NAME{...}</function> and
+ * <function=NAME>{...}</function>
+ * 3. Qwen / Hermes XML: <tool_call>{"name": ..., "arguments": ...}</tool_call>
+ * 4. Bare or ```json-fenced single JSON object: {"name": KNOWN, "arguments": {...}}
+ * (only rescued when "name" matches a requested tool — bare JSON is a
+ * legitimate answer shape, so this one is strictly schema-gated)
+ */
+
+export interface RescuedToolCall {
+ name: string;
+ /** JSON string, exactly like OpenAI's function.arguments */
+ arguments: string;
+}
+
+export interface RescueResult {
+ /** True when the text contains inline tool-call dialect markers. */
+ detected: boolean;
+ /** Parsed calls; null when detected but unparseable (dead turn). */
+ calls: RescuedToolCall[] | null;
+ /** Text with the dialect blocks removed (may be ''). */
+ cleanText: string;
+}
+
+// Markers that begin an inline dialect block. Used both for full-text
+// detection and for the streaming hold-window decision in proxy.ts.
+const DIALECT_MARKERS = [
+ '<|tool_calls_section_begin|>',
+ '<|tool_call_begin|>',
+ '<tool_call>',
+ '<function=',
+] as const;
+
+/** Does the (trimmed) text start with a known dialect marker? */
+export function startsWithDialectMarker(text: string): boolean {
+ const t = text.trimStart();
+ return DIALECT_MARKERS.some(m => t.startsWith(m));
+}
+
+/**
+ * Streaming hold-window helper: could `text` still grow into a dialect
+ * marker? True while text is a strict prefix of some marker (e.g. "<|too"),
+ * so the stream loop keeps holding; once this and startsWithDialectMarker
+ * are both false the text is ordinary prose and can be flushed.
+ */
+export function couldBecomeDialectMarker(text: string): boolean {
+ const t = text.trimStart();
+ if (t.length === 0) return true;
+ return DIALECT_MARKERS.some(m => m.startsWith(t) && t.length < m.length);
+}
+
+/** Anywhere-in-text detection for the non-streaming path. */
+export function containsDialectMarker(text: string): boolean {
+ return DIALECT_MARKERS.some(m => text.includes(m));
+}
+
+/**
+ * Extract one balanced JSON object or array starting at text[from] (which
+ * must be '{' or '['). Returns the slice and the index after it, or null.
+ * String-aware so braces inside JSON strings don't break the balance.
+ */
+function extractBalancedJson(text: string, from: number): { json: string; end: number } | null {
+ const open = text[from];
+ if (open !== '{' && open !== '[') return null;
+ const close = open === '{' ? '}' : ']';
+ let depth = 0;
+ let inString = false;
+ let escaped = false;
+ for (let i = from; i < text.length; i++) {
+ const ch = text[i];
+ if (inString) {
+ if (escaped) escaped = false;
+ else if (ch === '\\') escaped = true;
+ else if (ch === '"') inString = false;
+ continue;
+ }
+ if (ch === '"') inString = true;
+ else if (ch === open) depth++;
+ else if (ch === close) {
+ depth--;
+ if (depth === 0) return { json: text.slice(from, i + 1), end: i + 1 };
+ }
+ }
+ return null;
+}
+
+const isKnownTool = (name: string, toolNames: Set<string>): boolean =>
+ toolNames.size === 0 || toolNames.has(name);
+
+/** Parse `{"name": ..., "arguments"|"parameters": ...}` into a call. */
+function callFromNamedJson(json: string, toolNames: Set<string>): RescuedToolCall | null {
+ let obj: unknown;
+ try { obj = JSON.parse(json); } catch { return null; }
+ if (typeof obj !== 'object' || obj === null) return null;
+ const o = obj as Record<string, unknown>;
+ const name = typeof o.name === 'string' ? o.name : undefined;
+ if (!name || !isKnownTool(name, toolNames)) return null;
+ const rawArgs = o.arguments ?? o.parameters ?? {};
+ const args = typeof rawArgs === 'string' ? rawArgs : JSON.stringify(rawArgs);
+ try { JSON.parse(args); } catch { return null; }
+ return { name, arguments: args };
+}
+
+/** Dialect 1: Kimi/DeepSeek <|tool_call_begin|> token blocks. */
+function parseTokenDialect(text: string, toolNames: Set<string>): { calls: RescuedToolCall[] | null; cleanText: string } {
+ const calls: RescuedToolCall[] = [];
+ let clean = text;
+ // Strip section wrappers first; they carry no information.
+ clean = clean.replaceAll('<|tool_calls_section_begin|>', '').replaceAll('<|tool_calls_section_end|>', '');
+
+ const callRe = /<\|tool_call_begin\|>\s*([\s\S]*?)\s*<\|tool_call_argument_begin\|>\s*/g;
+ let m: RegExpExecArray | null;
+ let parsedAll = true;
+ const spans: Array<{ from: number; to: number }> = [];
+ while ((m = callRe.exec(clean)) !== null) {
+ const idToken = m[1].trim();
+ const argStart = m.index + m[0].length;
+ const jsonStart = clean.indexOf('{', argStart);
+ const extracted = jsonStart === -1 ? null : extractBalancedJson(clean, jsonStart);
+ // Function name rides in the id token as `functions.NAME:IDX`. Some
+ // models degrade it to an opaque id (observed: `chatcmpl-tool-<hex>`),
+ // which leaves no way to know WHICH tool was meant — unparseable.
+ const nameMatch = /^functions\.([A-Za-z0-9_.-]+):\d+$/.exec(idToken);
+ const name = nameMatch?.[1];
+ let argsOk = false;
+ if (extracted && name && isKnownTool(name, toolNames)) {
+ try { JSON.parse(extracted.json); argsOk = true; } catch { /* fall through */ }
+ if (argsOk) calls.push({ name, arguments: extracted.json });
+ }
+ if (!argsOk) parsedAll = false;
+ const endTag = clean.indexOf('<|tool_call_end|>', extracted?.end ?? argStart);
+ spans.push({ from: m.index, to: endTag === -1 ? (extracted?.end ?? argStart) : endTag + '<|tool_call_end|>'.length });
+ }
+ for (const s of [...spans].reverse()) clean = clean.slice(0, s.from) + clean.slice(s.to);
+ return { calls: parsedAll && calls.length > 0 ? calls : null, cleanText: clean.trim() };
+}
+
+/** Dialect 2: <function=NAME{...}</function> (with or without a '>' after the name). */
+function parseFunctionTagDialect(text: string, toolNames: Set<string>): { calls: RescuedToolCall[] | null; cleanText: string } {
+ const calls: RescuedToolCall[] = [];
+ let clean = text;
+ let parsedAll = true;
+ const headRe = /<function=([A-Za-z0-9_.-]+)\s*>?\s*/g;
+ let m: RegExpExecArray | null;
+ const spans: Array<{ from: number; to: number }> = [];
+ while ((m = headRe.exec(text)) !== null) {
+ const name = m[1];
+ const afterHead = m.index + m[0].length;
+ const jsonStart = text[afterHead] === '{' || text[afterHead] === '['
+ ? afterHead
+ : text.indexOf('{', afterHead);
+ const extracted = jsonStart === -1 ? null : extractBalancedJson(text, jsonStart);
+ let ok = false;
+ if (extracted && isKnownTool(name, toolNames) && extracted.json.startsWith('{')) {
+ try { JSON.parse(extracted.json); ok = true; } catch { /* fall through */ }
+ if (ok) calls.push({ name, arguments: extracted.json });
+ }
+ if (!ok) parsedAll = false; // array-shaped or invalid args: not a callable shape
+ const closeTag = text.indexOf('</function>', extracted?.end ?? m.index + m[0].length);
+ spans.push({ from: m.index, to: closeTag === -1 ? (extracted?.end ?? m.index + m[0].length) : closeTag + '</function>'.length });
+ }
+ for (const s of [...spans].reverse()) clean = clean.slice(0, s.from) + clean.slice(s.to);
+ return { calls: parsedAll && calls.length > 0 ? calls : null, cleanText: clean.trim() };
+}
+
+/** Dialect 3: <tool_call>{...}</tool_call> XML-JSON blocks. */
+function parseXmlDialect(text: string, toolNames: Set<string>): { calls: RescuedToolCall[] | null; cleanText: string } {
+ const calls: RescuedToolCall[] = [];
+ let parsedAll = true;
+ const re = /<tool_call>\s*([\s\S]*?)\s*<\/tool_call>/g;
+ let m: RegExpExecArray | null;
+ let clean = text;
+ const matches: string[] = [];
+ while ((m = re.exec(text)) !== null) matches.push(m[1]);
+ for (const inner of matches) {
+ const call = callFromNamedJson(inner, toolNames);
+ if (call) calls.push(call);
+ else parsedAll = false;
+ }
+ clean = clean.replace(re, '');
+ // An opening tag with no close (truncated stream) is detected-but-broken.
+ if (/<tool_call>/.test(clean)) { parsedAll = false; clean = clean.replace(/<tool_call>[\s\S]*$/, ''); }
+ return { calls: parsedAll && calls.length > 0 ? calls : null, cleanText: clean.trim() };
+}
+
+/**
+ * Rescue inline tool-call dialects out of an assistant text answer.
+ *
+ * @param text the assistant message content
+ * @param toolNames the names of the tools the REQUEST declared; rescued
+ * calls must match one (empty set = accept any name,
+ * used by tests only)
+ */
+export function rescueInlineToolCalls(text: string, toolNames: Set<string>): RescueResult {
+ if (!text) return { detected: false, calls: null, cleanText: text };
+
+ if (text.includes('<|tool_call_begin|>') || text.includes('<|tool_calls_section_begin|>')) {
+ const { calls, cleanText } = parseTokenDialect(text, toolNames);
+ return { detected: true, calls, cleanText };
+ }
+ if (text.includes('<function=')) {
+ const { calls, cleanText } = parseFunctionTagDialect(text, toolNames);
+ return { detected: true, calls, cleanText };
+ }
+ if (text.includes('<tool_call>')) {
+ const { calls, cleanText } = parseXmlDialect(text, toolNames);
+ return { detected: true, calls, cleanText };
+ }
+
+ // Dialect 4: the entire answer is one JSON object naming a known tool —
+ // either bare or inside a ```json fence. Strictly schema-gated.
+ const trimmed = text.trim();
+ const fenced = /^```(?:json)?\s*([\s\S]*?)\s*```$/.exec(trimmed);
+ const candidate = (fenced ? fenced[1] : trimmed).trim();
+ if (candidate.startsWith('{') && candidate.endsWith('}')) {
+ const call = callFromNamedJson(candidate, toolNames);
+ // Only treat as dialect when it actually names a requested tool;
+ // arbitrary JSON answers must pass through untouched.
+ if (call) return { detected: true, calls: [call], cleanText: '' };
+ }
+
+ return { detected: false, calls: null, cleanText: text };
+}
diff --git a/server/src/providers/base.ts b/server/src/providers/base.ts
index 32139b10..2e8d9352 100644
--- a/server/src/providers/base.ts
+++ b/server/src/providers/base.ts
@@ -59,4 +59,74 @@ export abstract class BaseProvider {
protected makeId(): string {
return `chatcmpl-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
}
+
+ /**
+ * Shared SSE reader for OpenAI-wire streaming endpoints (#231 audit).
+ *
+ * Hardened against the upstream failure modes observed live:
+ * - Inactivity timeout: fetchWithTimeout's abort timer dies the moment
+ * response HEADERS arrive, so a provider that stalls mid-body used to
+ * hang the client forever. Each read now has its own deadline.
+ * - Abrupt EOF: a stream that ends without `[DONE]` AND without any
+ * `finish_reason` is a truncated generation, not a completion. It used
+ * to end the generator silently (truncation logged as success); it now
+ * throws a retryable error so the proxy can fail over or report it.
+ * Providers that skip `[DONE]` but do send a terminal finish_reason
+ * (several compat shims) still complete normally.
+ *
+ * Malformed data lines are skipped, matching previous behavior.
+ */
+ protected async *readSseStream(
+ res: Response,
+ inactivityTimeoutMs = 90000,
+ ): AsyncGenerator<ChatCompletionChunk> {
+ const reader = res.body?.getReader();
+ if (!reader) throw new Error('No response body');
+
+ const decoder = new TextDecoder();
+ let buffer = '';
+ let sawFinishReason = false;
+
+ try {
+ while (true) {
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const result = await Promise.race([
+ reader.read(),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error(`${this.name} stream stalled: no data for ${inactivityTimeoutMs}ms (timeout)`)),
+ inactivityTimeoutMs,
+ );
+ }),
+ ]).finally(() => clearTimeout(timer));
+
+ const { done, value } = result;
+ if (done) break;
+
+ buffer += decoder.decode(value, { stream: true });
+ const lines = buffer.split('\n');
+ buffer = lines.pop() ?? '';
+
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (!trimmed || !trimmed.startsWith('data: ')) continue;
+ const data = trimmed.slice(6);
+ if (data === '[DONE]') return;
+ try {
+ const chunk = JSON.parse(data) as ChatCompletionChunk;
+ if (chunk.choices?.some(c => c.finish_reason != null)) sawFinishReason = true;
+ yield chunk;
+ } catch {
+ // Skip malformed chunks
+ }
+ }
+ }
+ } finally {
+ reader.cancel().catch(() => { /* upstream already gone */ });
+ }
+
+ if (!sawFinishReason) {
+ throw new Error(`${this.name} stream ended unexpectedly (no [DONE], no finish_reason) — connection reset or truncated upstream`);
+ }
+ }
}
diff --git a/server/src/providers/cloudflare.ts b/server/src/providers/cloudflare.ts
index ff748cdf..a80ecf67 100644
--- a/server/src/providers/cloudflare.ts
+++ b/server/src/providers/cloudflare.ts
@@ -99,32 +99,7 @@ export class CloudflareProvider extends BaseProvider {
throw new Error(`Cloudflare API error ${res.status}: ${(err as any).error?.message ?? (err as any).errors?.[0]?.message ?? res.statusText}`);
}
- const reader = res.body?.getReader();
- if (!reader) throw new Error('No response body');
-
- const decoder = new TextDecoder();
- let buffer = '';
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
-
- buffer += decoder.decode(value, { stream: true });
- const lines = buffer.split('\n');
- buffer = lines.pop() ?? '';
-
- for (const line of lines) {
- const trimmed = line.trim();
- if (!trimmed || !trimmed.startsWith('data: ')) continue;
- const data = trimmed.slice(6);
- if (data === '[DONE]') return;
- try {
- yield JSON.parse(data) as ChatCompletionChunk;
- } catch {
- // Skip malformed chunks
- }
- }
- }
+ yield* this.readSseStream(res);
}
async validateKey(apiKey: string): Promise<boolean> {
diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.ts
index 8ef0a2c2..5fcb0a4b 100644
--- a/server/src/providers/cohere.ts
+++ b/server/src/providers/cohere.ts
@@ -78,32 +78,7 @@ export class CohereProvider extends BaseProvider {
throw new Error(`Cohere API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
- const reader = res.body?.getReader();
- if (!reader) throw new Error('No response body');
-
- const decoder = new TextDecoder();
- let buffer = '';
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
-
- buffer += decoder.decode(value, { stream: true });
- const lines = buffer.split('\n');
- buffer = lines.pop() ?? '';
-
- for (const line of lines) {
- const trimmed = line.trim();
- if (!trimmed || !trimmed.startsWith('data: ')) continue;
- const data = trimmed.slice(6);
- if (data === '[DONE]') return;
- try {
- yield JSON.parse(data) as ChatCompletionChunk;
- } catch {
- // Skip malformed chunks
- }
- }
- }
+ yield* this.readSseStream(res);
}
async validateKey(apiKey: string): Promise<boolean> {
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 717ea78f..6cce5c1d 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -126,32 +126,7 @@ export class OpenAICompatProvider extends BaseProvider {
throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
- const reader = res.body?.getReader();
- if (!reader) throw new Error('No response body');
-
- const decoder = new TextDecoder();
- let buffer = '';
-
- while (true) {
- const { done, value } = await reader.read();
- if (done) break;
-
- buffer += decoder.decode(value, { stream: true });
- const lines = buffer.split('\n');
- buffer = lines.pop() ?? '';
-
- for (const line of lines) {
- const trimmed = line.trim();
- if (!trimmed || !trimmed.startsWith('data: ')) continue;
- const data = trimmed.slice(6);
- if (data === '[DONE]') return;
- try {
- yield JSON.parse(data) as ChatCompletionChunk;
- } catch {
- // Skip malformed chunks
- }
- }
- }
+ yield* this.readSseStream(res);
}
async validateKey(apiKey: string): Promise<boolean> {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 13709592..cbb416e1 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -11,6 +11,7 @@ import { getDb, getUnifiedApiKey } from '../db/index.js';
import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
+import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
export const proxyRouter = Router();
@@ -59,23 +60,34 @@ export function extractApiToken(req: Request): string | undefined {
const stickySessionMap = new Map<string, { modelDbId: number; lastUsed: number }>();
const STICKY_TTL_MS = 30 * 60 * 1000; // 30 min session TTL
-function getSessionKey(messages: ChatMessage[]): string {
- // Use the first user message as session identifier — clients like Hermes
- // re-send the full conversation each turn, so the first user message is
- // stable across turns. Hash the FULL message (not a 100-char slice) so
- // distinct conversations with identical openings don't collide.
+function getSessionKey(messages: ChatMessage[], sessionIdHeader?: string): string {
+ // Explicit session pinning: clients that manage their own conversation ids
+ // (agent harnesses especially) can send X-Session-Id and get exact
+ // affinity regardless of how their message history mutates. (#231)
+ if (sessionIdHeader) return `hdr:${sessionIdHeader}`;
+
+ // Otherwise the first user message identifies the session — clients re-send
+ // the full conversation each turn, so it is stable across turns. Flatten
+ // array-of-blocks content before hashing: opencode-style agents send
+ // [{type:'text',...}] even for plain text, and the old string-only check
+ // silently disabled stickiness for them, re-routing every turn (#231 audit:
+ // observed a rank-2 → rank-11 mid-conversation flip). No turn-count suffix:
+ // the old ':single'/':multi' split guaranteed a sticky MISS on turn 2,
+ // exactly where agents replay the assistant's tool-call dialect and a model
+ // switch causes cross-dialect contamination.
const firstUser = messages.find(m => m.role === 'user');
- if (!firstUser || typeof firstUser.content !== 'string') return '';
- const hash = crypto.createHash('sha1').update(firstUser.content).digest('hex');
- return `${hash}:${messages.length > 2 ? 'multi' : 'single'}`;
+ if (!firstUser) return '';
+ const text = contentToString(firstUser.content ?? '');
+ if (!text) return '';
+ return crypto.createHash('sha1').update(text).digest('hex');
}
-export function getStickyModel(messages: ChatMessage[]): number | undefined {
+export function getStickyModel(messages: ChatMessage[], sessionIdHeader?: string): number | undefined {
// Only apply sticky for multi-turn (has assistant messages = continuation)
const hasAssistant = messages.some(m => m.role === 'assistant');
if (!hasAssistant) return undefined;
- const key = getSessionKey(messages);
+ const key = getSessionKey(messages, sessionIdHeader);
if (!key) return undefined;
const entry = stickySessionMap.get(key);
@@ -88,8 +100,8 @@ export function getStickyModel(messages: ChatMessage[]): number | undefined {
return entry.modelDbId;
}
-export function setStickyModel(messages: ChatMessage[], modelDbId: number) {
- const key = getSessionKey(messages);
+export function setStickyModel(messages: ChatMessage[], modelDbId: number, sessionIdHeader?: string) {
+ const key = getSessionKey(messages, sessionIdHeader);
if (!key) return;
stickySessionMap.set(key, { modelDbId, lastUsed: Date.now() });
@@ -311,7 +323,15 @@ export function isRetryableError(err: any): boolean {
// provider (Kimi K2.6 is on HF + Cloudflare + NVIDIA), so fail over instead
// of killing the workflow. Paired with a long cooldown (isPaymentRequiredError)
// so we don't re-hammer the broke key every retry.
- || isPaymentRequiredError(err);
+ || isPaymentRequiredError(err)
+ // Dead-turn classes from the stream turn-integrity layer (#231 audit):
+ // all thrown before any byte reached the client, so another model can
+ // serve the request invisibly.
+ || msg.includes('empty completion')
+ || msg.includes('in-band provider error')
+ || msg.includes('stream ended unexpectedly')
+ || msg.includes('stream stalled')
+ || msg.includes('unparseable inline tool-call dialect');
}
// A 402 Payment Required / out-of-credits error. Distinct from a transient 429:
@@ -547,6 +567,12 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
return;
}
+ // Optional client-managed session affinity (see getSessionKey). Express
+ // lower-cases header names; a repeated header arrives as an array — take
+ // the first value.
+ const rawSessionId = req.headers['x-session-id'];
+ const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;
+
// Explicit `model` field pins routing. If the catalog has no enabled row
// matching the requested id, return 400 — silently auto-routing to a
// different model would be surprising to OpenAI-compatible clients.
@@ -554,7 +580,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
let preferredModel: number | undefined;
if (isAutoModel(requestedModel)) {
// Explicit "auto" → behave exactly like an omitted model field.
- preferredModel = getStickyModel(messages);
+ preferredModel = getStickyModel(messages, sessionIdHeader);
} else if (requestedModel) {
const db = getDb();
const enabled = db.prepare('SELECT id FROM models WHERE model_id = ? AND enabled = 1').get(requestedModel) as { id: number } | undefined;
@@ -573,7 +599,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
return;
}
} else {
- preferredModel = getStickyModel(messages);
+ preferredModel = getStickyModel(messages, sessionIdHeader);
}
// For analytics: the model id the client pinned, null when auto-routed
@@ -611,12 +637,58 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
try {
if (stream) {
- // Lazy header set: pre-stream errors stay retryable (no headers sent yet);
- // mid-stream errors emit an `error` SSE frame so the client sees a real signal
- // instead of a silently truncated stream.
+ // — Stream turn-integrity (#231 audit) —
+ // The old loop forwarded upstream chunks verbatim and called any
+ // stream that produced bytes a success. Live failure modes that
+ // slipped through: in-band `{"error":...}` frames delivered as dead
+ // turns, tool calls with no terminal finish_reason, inline tool-call
+ // dialect emitted as text, truncations logged as success. This loop
+ // validates the TURN, not the transport:
+ // - headers are held until the first real payload, so anything that
+ // dies before producing one fails over invisibly;
+ // - text that starts with an inline tool-call dialect marker is held
+ // and rescued into structured tool_calls (or failed over);
+ // - tool_call deltas are buffered, argument-repaired, and emitted as
+ // one complete chunk, always followed by finish_reason
+ // "tool_calls" — agents never see calls without a terminal reason;
+ // - a stream that ends with neither content nor calls is an empty
+ // completion and fails over like the non-stream path.
let totalOutputTokens = 0;
- let streamStarted = false;
+ let headerSent = false;
let ttfbMs: number | null = null;
+
+ // Hold-window state: 'undecided' until the first text either matches
+ // a dialect marker (→ 'dialect': buffer everything, rescue at end) or
+ // provably cannot (→ 'passthrough': flush and stream normally).
+ let mode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';
+ let heldText = '';
+ const preamble: unknown[] = []; // role-only chunks held until flush
+ const toolCallAcc = new Map<number, { id?: string; name: string; args: string }>();
+ let upstreamFinish: string | null = null;
+ let usageChunk: unknown = null;
+ let lastMeta: { id?: string; model?: string; created?: number } = {};
+
+ const flushHeaders = () => {
+ if (headerSent) return;
+ ttfbMs = Date.now() - start;
+ res.setHeader('Content-Type', 'text/event-stream');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.setHeader('Connection', 'keep-alive');
+ res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
+ if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
+ headerSent = true;
+ for (const p of preamble) res.write(`data: ${JSON.stringify(p)}\n\n`);
+ preamble.length = 0;
+ };
+ const mkChunk = (delta: Record<string, unknown>, finish: string | null) => ({
+ id: lastMeta.id ?? `chatcmpl-${Date.now()}`,
+ object: 'chat.completion.chunk',
+ created: lastMeta.created ?? Math.floor(Date.now() / 1000),
+ model: lastMeta.model ?? route.modelId,
+ choices: [{ index: 0, delta, finish_reason: finish }],
+ });
+ const writeChunk = (c: unknown) => res.write(`data: ${JSON.stringify(c)}\n\n`);
+
try {
const gen = route.provider.streamChatCompletion(
route.apiKey, messages, route.modelId,
@@ -624,52 +696,153 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
);
for await (const chunk of gen) {
- if (!streamStarted) {
- // Time-to-first-byte: dispatch → first chunk. Feeds the router's
- // latency axis (server/src/services/scoring.ts).
- ttfbMs = Date.now() - start;
- res.setHeader('Content-Type', 'text/event-stream');
- res.setHeader('Cache-Control', 'no-cache');
- res.setHeader('Connection', 'keep-alive');
- res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
- if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
- streamStarted = true;
+ const anyChunk = chunk as Record<string, any>;
+
+ // In-band upstream error frame (observed live: Groq emits
+ // {"error":{...,"code":"tool_use_failed"}} inside a 200 SSE
+ // stream). Before headers: retryable, the next model gets the
+ // request. After: surface an error frame instead of pretending
+ // the turn succeeded.
+ if (anyChunk.error && !anyChunk.choices) {
+ const msg = anyChunk.error.message ?? JSON.stringify(anyChunk.error).slice(0, 200);
+ if (!headerSent) throw new Error(`in-band provider error from ${route.displayName}: ${msg}`);
+ console.error(`[Proxy] In-band error frame from ${route.displayName} mid-stream:`, msg);
+ writeChunk({ error: { message: `Provider error (${route.displayName}): ${sanitizeProviderErrorMessage(String(msg))}`, type: 'stream_error' } });
+ try { res.write('data: [DONE]\n\n'); res.end(); } catch { /* socket gone */ }
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, `in-band error frame: ${sanitizeProviderErrorMessage(String(msg))}`, ttfbMs, pinnedModelId);
+ return;
}
- // Coerce array-shaped delta.content to a string before forwarding,
- // so spec-conforming clients don't break and tool_calls survive (#166).
+
+ if (anyChunk.id) lastMeta = { id: anyChunk.id, model: anyChunk.model, created: anyChunk.created };
+
+ const choice = anyChunk.choices?.[0];
+ if (!choice) {
+ // Usage-only frame (stream_options.include_usage) — held and
+ // re-emitted after our finish chunk to preserve OpenAI ordering.
+ if (anyChunk.usage) usageChunk = anyChunk;
+ continue;
+ }
+
+ if (choice.finish_reason) upstreamFinish = choice.finish_reason;
+
+ // Buffer tool_call deltas — emitted complete + repaired at end.
+ for (const tc of choice.delta?.tool_calls ?? []) {
+ const idx = tc.index ?? 0;
+ if (!toolCallAcc.has(idx)) toolCallAcc.set(idx, { id: undefined, name: '', args: '' });
+ const acc = toolCallAcc.get(idx)!;
+ if (tc.id && !acc.id) acc.id = tc.id;
+ if (tc.function?.name) acc.name += tc.function.name;
+ if (tc.function?.arguments) acc.args += tc.function.arguments;
+ }
+
normalizeOutboundContent(chunk);
- const text = streamChunkText(chunk);
+ const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';
+
+ if (text.length === 0) {
+ // Role preamble / keep-alive: hold until first payload decides
+ // the mode, forward afterwards. tool_calls and finish_reason are
+ // stripped — both are re-emitted complete at the end (OpenRouter
+ // attaches tool_call deltas to chunks that also carry role/
+ // reasoning keys; forwarding them raw would duplicate the call).
+ if (choice.delta && Object.keys(choice.delta).some(k => k !== 'content' && k !== 'tool_calls' && choice.delta[k] != null)) {
+ const cleaned = { ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] };
+ if (headerSent) writeChunk(cleaned); else preamble.push(cleaned);
+ }
+ continue;
+ }
+
totalOutputTokens += Math.ceil(text.length / 4);
- res.write(`data: ${JSON.stringify(chunk)}\n\n`);
+
+ if (mode === 'passthrough') {
+ writeChunk({ ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] });
+ continue;
+ }
+
+ heldText += text;
+ if (mode === 'dialect') continue;
+
+ const probe = heldText.trimStart();
+ if (startsWithDialectMarker(probe)) {
+ mode = 'dialect';
+ } else if (!couldBecomeDialectMarker(probe) || heldText.length > 256) {
+ mode = 'passthrough';
+ flushHeaders();
+ writeChunk(mkChunk({ content: heldText }, null));
+ heldText = '';
+ }
+ // else: still a strict prefix of a marker — keep holding.
+ }
+
+ // — Stream ended cleanly (provider saw [DONE] or a finish_reason) —
+
+ // Assemble buffered tool calls: synthesize missing ids, repair
+ // double-encoded arguments against the request's schemas, drop
+ // calls whose args still aren't valid JSON.
+ const schemas = toolSchemaMap(tools);
+ let syntheticStreamIds = 0;
+ const completedCalls = [...toolCallAcc.entries()]
+ .sort((a, b) => a[0] - b[0])
+ .map(([, acc]) => ({
+ id: acc.id && acc.id.length > 0 ? acc.id : `call_stream_${++syntheticStreamIds}`,
+ type: 'function' as const,
+ function: { name: acc.name, arguments: repairToolArguments(acc.args || '{}', schemas.get(acc.name)) },
+ }))
+ .filter(c => { try { JSON.parse(c.function.arguments); return c.function.name.length > 0; } catch { return false; } });
+
+ // Dialect rescue: the held text is an inline tool call in some
+ // model's private syntax. Parse it into structured calls or treat
+ // the turn as dead (headers were never sent in dialect mode, so
+ // failing over is free).
+ if (mode === 'dialect' || (mode === 'undecided' && heldText.length > 0 && containsDialectMarker(heldText))) {
+ const rescue = rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)));
+ if (rescue.detected) {
+ if (!rescue.calls) throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${heldText.slice(0, 120)}`);
+ let rescuedIds = 0;
+ for (const c of rescue.calls) {
+ completedCalls.push({ id: `call_rescued_${++rescuedIds}`, type: 'function', function: { name: c.name, arguments: repairToolArguments(c.arguments, schemas.get(c.name)) } });
+ }
+ heldText = rescue.cleanText;
+ console.log(`[Proxy] Rescued ${rescuedIds} inline tool call(s) from ${route.displayName} into structured tool_calls`);
+ }
}
- if (!streamStarted) {
- // Upstream returned zero chunks — an empty completion in stream
- // clothing. No headers are out yet, so fail over to the next model
- // instead of handing the client a valid-looking empty stream
- // (production case: nemotron-3-super returning nothing on large
- // contexts while the request logs as success).
- logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, 'empty completion (stream produced no chunks)', null, pinnedModelId);
- skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
- setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
- recordRateLimitHit(route.modelDbId);
- lastError = new Error(`empty completion from ${route.displayName}`);
- continue;
+ const hasText = headerSent || heldText.trim().length > 0;
+ if (!hasText && completedCalls.length === 0) {
+ // Nothing usable came out — same failover semantics as the
+ // non-stream empty-completion path. Headers can't have been sent
+ // (header flush requires payload), so the client never notices.
+ throw new Error(`empty completion from ${route.displayName} (stream produced no content and no tool calls)`);
}
+
+ flushHeaders();
+ if (heldText.length > 0) {
+ writeChunk(mkChunk({ content: heldText }, null));
+ totalOutputTokens += Math.ceil(heldText.length / 4);
+ }
+ if (completedCalls.length > 0) {
+ writeChunk(mkChunk({ tool_calls: completedCalls.map((c, i) => ({ index: i, ...c })) }, null));
+ totalOutputTokens += Math.ceil(completedCalls.reduce((n, c) => n + c.function.arguments.length, 0) / 4);
+ }
+ // Terminal finish_reason, ALWAYS present: calls win over a sloppy
+ // upstream 'stop'; 'length'/'content_filter' survive for pure-text
+ // turns; missing upstream reason is synthesized.
+ const finish = completedCalls.length > 0
+ ? 'tool_calls'
+ : (upstreamFinish && upstreamFinish !== 'tool_calls' ? upstreamFinish : 'stop');
+ writeChunk(mkChunk({}, finish));
+ if (usageChunk) writeChunk(usageChunk);
res.write('data: [DONE]\n\n');
res.end();
recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
recordSuccess(route.modelDbId);
- setStickyModel(messages, route.modelDbId);
+ setStickyModel(messages, route.modelDbId, sessionIdHeader);
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null, ttfbMs, pinnedModelId);
return;
} catch (streamErr: any) {
- if (streamStarted) {
- // Mid-stream error — finish the SSE response cleanly instead of leaving
- // the client hanging or letting Express's default handler take over.
- // Full upstream message goes to the log; the client sees a generic
- // message so we don't leak provider internals into a partial stream.
+ if (headerSent) {
+ // Mid-stream error after real payload reached the client — finish
+ // the SSE response honestly instead of leaving the client hanging.
console.error(`[Proxy] Mid-stream error from ${route.displayName}:`, streamErr.message);
const payload = { error: { message: `Provider error (${route.displayName}): stream interrupted`, type: 'stream_error' } };
try { res.write(`data: ${JSON.stringify(payload)}\n\n`); } catch { /* socket gone */ }
@@ -677,7 +850,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, totalOutputTokens, Date.now() - start, sanitizeProviderErrorMessage(streamErr.message), null, pinnedModelId);
return;
}
- // Pre-stream error — bubble to outer retry/502 handler.
+ // Headers never sent — bubble to the outer retry handler, which
+ // cooldowns this model+key and tries the next one. Covers upstream
+ // HTTP errors, in-band error frames, abrupt EOF, stalls, empty
+ // completions, and unparseable dialect turns alike.
throw streamErr;
}
} else {
@@ -700,10 +876,34 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
continue;
}
+ // Inline tool-call dialect rescue (#231 audit): a tool-bearing
+ // request answered with the call serialized as TEXT (a mid-
+ // conversation model switch makes the new model imitate the previous
+ // model's private syntax). Re-parse it into structured tool_calls so
+ // the client's agent loop keeps working; a detected-but-unparseable
+ // dialect is a dead turn and fails over like an empty completion.
+ if (wantsTools && respMsg && (respMsg.tool_calls?.length ?? 0) === 0 && respText) {
+ const rescue = rescueInlineToolCalls(respText, new Set((tools ?? []).map(t => t.function.name)));
+ if (rescue.detected) {
+ if (!rescue.calls) {
+ throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${respText.slice(0, 120)}`);
+ }
+ const schemas = toolSchemaMap(tools);
+ respMsg.tool_calls = rescue.calls.map((c, i) => ({
+ id: `call_rescued_${i + 1}`,
+ type: 'function' as const,
+ function: { name: c.name, arguments: repairToolArguments(c.arguments, schemas.get(c.name)) },
+ }));
+ respMsg.content = rescue.cleanText.length > 0 ? rescue.cleanText : null;
+ if (result.choices?.[0]) result.choices[0].finish_reason = 'tool_calls';
+ console.log(`[Proxy] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName} into structured tool_calls`);
+ }
+ }
+
const totalTokens = result.usage?.total_tokens ?? 0;
recordTokens(route.platform, route.modelId, route.keyId, totalTokens);
recordSuccess(route.modelDbId);
- setStickyModel(messages, route.modelDbId);
+ setStickyModel(messages, route.modelDbId, sessionIdHeader);
res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 790825aa..ce8f7826 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -13,6 +13,7 @@ import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit,
import { getUnifiedApiKey } from '../db/index.js';
import { contentToString } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
+import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import {
isRetryableError,
isPaymentRequiredError,
@@ -316,7 +317,10 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
0,
);
const estimatedTotal = estimatedInputTokens + (reqData.max_output_tokens ?? 1000);
- const preferredModel = getStickyModel(messages);
+ // Optional client-managed session affinity (mirrors /chat/completions).
+ const rawSessionId = req.headers['x-session-id'];
+ const sessionIdHeader = Array.isArray(rawSessionId) ? rawSessionId[0] : rawSessionId;
+ const preferredModel = getStickyModel(messages, sessionIdHeader);
// Tool-bearing requests (the normal case for Codex/agent clients on this
// endpoint) must stay on models that emit structured tool_calls — a model
@@ -376,9 +380,41 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
const toolAcc = new Map<number, { outputIndex: number; itemId: string; callId: string; name: string; args: string }>();
let totalOutputTokens = 0;
+ // Inline-dialect hold window (#231): first text is held until it
+ // either matches a tool-call dialect marker (held to the end and
+ // rescued into function_call items) or provably cannot (flushed and
+ // streamed normally). Mirrors the /chat/completions stream loop.
+ let dialectMode: 'undecided' | 'passthrough' | 'dialect' = 'undecided';
+ let heldText = '';
+
+ // Open the text output item and stream `text` as its first delta.
+ const openTextItem = (text: string) => {
+ msgItemId = newId('msg');
+ sse('response.output_item.added', {
+ output_index: outputIndex,
+ item: { id: msgItemId, type: 'message', status: 'in_progress', role: 'assistant', content: [] },
+ });
+ sse('response.content_part.added', {
+ item_id: msgItemId, output_index: outputIndex, content_index: 0,
+ part: { type: 'output_text', text: '', annotations: [] },
+ });
+ if (text) {
+ sse('response.output_text.delta', { item_id: msgItemId, output_index: outputIndex, content_index: 0, delta: text });
+ msgText += text;
+ }
+ };
+
const gen = route.provider.streamChatCompletion(route.apiKey, messages, route.modelId, completionOpts);
for await (const chunk of gen) {
+ // In-band upstream error frame ({"error":...} inside a 200 SSE
+ // stream — observed live from Groq). Throw before the lazy header
+ // block so a first-frame error keeps streamStarted=false and takes
+ // the normal failover path in the catch below.
+ const anyChunk = chunk as Record<string, any>;
+ if (anyChunk.error && !anyChunk.choices) {
+ throw new Error(`in-band provider error from ${route.displayName}: ${anyChunk.error.message ?? 'provider error'}`);
+ }
// LAZY header set — headers + the response.created/in_progress
// skeleton go out only once the provider actually streams a chunk.
// Sending them before the provider call (the previous behavior)
@@ -405,28 +441,33 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
streamStarted = true;
}
- const delta = chunk.choices[0]?.delta;
+ const delta = chunk.choices?.[0]?.delta;
if (!delta) continue;
- // Text deltas → output_text events on a single message item.
+ // Text deltas → output_text events on a single message item, after
+ // the dialect hold window has decided the text is real prose.
const text = delta.content ?? '';
if (text) {
- if (msgItemId === null) {
- msgItemId = newId('msg');
- sse('response.output_item.added', {
- output_index: outputIndex,
- item: { id: msgItemId, type: 'message', status: 'in_progress', role: 'assistant', content: [] },
- });
- sse('response.content_part.added', {
- item_id: msgItemId, output_index: outputIndex, content_index: 0,
- part: { type: 'output_text', text: '', annotations: [] },
+ totalOutputTokens += Math.ceil(text.length / 4);
+ if (dialectMode === 'passthrough') {
+ if (msgItemId === null) openTextItem('');
+ sse('response.output_text.delta', {
+ item_id: msgItemId, output_index: 0, content_index: 0, delta: text,
});
+ msgText += text;
+ } else {
+ heldText += text;
+ if (dialectMode === 'undecided') {
+ const probe = heldText.trimStart();
+ if (startsWithDialectMarker(probe)) {
+ dialectMode = 'dialect';
+ } else if (!couldBecomeDialectMarker(probe) || heldText.length > 256) {
+ dialectMode = 'passthrough';
+ openTextItem(heldText);
+ heldText = '';
+ }
+ }
}
- sse('response.output_text.delta', {
- item_id: msgItemId, output_index: outputIndex, content_index: 0, delta: text,
- });
- msgText += text;
- totalOutputTokens += Math.ceil(text.length / 4);
}
// Tool-call deltas → function_call item + argument deltas.
@@ -459,6 +500,46 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
}
}
+ // Resolve the dialect hold window now that the full text is known.
+ // Held text was never emitted, so a dead dialect turn can still fail
+ // over on the same SSE stream (only the skeleton events are out).
+ if (heldText.length > 0) {
+ const rescue = (dialectMode === 'dialect' || containsDialectMarker(heldText))
+ ? rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)))
+ : { detected: false as const, calls: null, cleanText: heldText };
+ if (rescue.detected && !rescue.calls) {
+ logRequest(route.platform, route.modelId, route.keyId, 'error', estimatedInputTokens, 0, Date.now() - start, `unparseable inline tool-call dialect: ${heldText.slice(0, 120)}`);
+ skipKeys.add(`${route.platform}:${route.modelId}:${route.keyId}`);
+ setCooldown(route.platform, route.modelId, route.keyId, getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
+ recordRateLimitHit(route.modelDbId);
+ lastError = new Error(`unparseable inline tool-call dialect from ${route.displayName}`);
+ continue;
+ }
+ if (rescue.detected && rescue.calls) {
+ // Rescued calls become function_call items, exactly as if the
+ // provider had streamed them structurally.
+ console.log(`[Responses] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName}`);
+ if (rescue.cleanText.length > 0 && msgItemId === null) openTextItem(rescue.cleanText);
+ let rescuedIdx = 0;
+ for (const c of rescue.calls) {
+ const idx = 1000 + rescuedIdx++; // synthetic accumulator keys, past any provider index
+ const acc = {
+ outputIndex: toolAcc.size + (msgText.length > 0 ? 1 : 0),
+ itemId: newId('fc'), callId: newId('call'), name: c.name, args: c.arguments,
+ };
+ toolAcc.set(idx, acc);
+ sse('response.output_item.added', {
+ output_index: acc.outputIndex,
+ item: { id: acc.itemId, type: 'function_call', status: 'in_progress', call_id: acc.callId, name: acc.name, arguments: '' },
+ });
+ }
+ } else if (msgItemId === null) {
+ // Plain short answer that never left the hold window (e.g. "Hi").
+ openTextItem(heldText);
+ }
+ heldText = '';
+ }
+
// Empty completion — the provider returned 200 with no text AND no
// tool calls. Seen in production from nemotron-3-super on ~65k-token
// contexts: transport-level "success", zero usable output, so the
@@ -505,18 +586,35 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
recordTokens(route.platform, route.modelId, route.keyId, estimatedInputTokens + totalOutputTokens);
recordSuccess(route.modelDbId);
- setStickyModel(messages, route.modelDbId);
+ setStickyModel(messages, route.modelDbId, sessionIdHeader);
logRequest(route.platform, route.modelId, route.keyId, 'success', estimatedInputTokens, totalOutputTokens, Date.now() - start, null);
return;
} else {
const result = await route.provider.chatCompletion(route.apiKey, messages, route.modelId, completionOpts);
const msg = result.choices[0]?.message;
- const text = contentToString(msg?.content ?? '');
- const toolCalls = (msg?.tool_calls ?? []).map((tc) => ({
+ let text = contentToString(msg?.content ?? '');
+ let toolCalls = (msg?.tool_calls ?? []).map((tc) => ({
...tc,
function: { ...tc.function, arguments: repairToolArguments(tc.function.arguments, toolSchemas.get(tc.function.name)) },
}));
+
+ // Inline tool-call dialect rescue (#231) — see /chat/completions.
+ if (wantsTools && toolCalls.length === 0 && text) {
+ const rescue = rescueInlineToolCalls(text, new Set((tools ?? []).map(t => t.function.name)));
+ if (rescue.detected) {
+ if (!rescue.calls) {
+ throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${text.slice(0, 120)}`);
+ }
+ console.log(`[Responses] Rescued ${rescue.calls.length} inline tool call(s) from ${route.displayName}`);
+ toolCalls = rescue.calls.map((c, i) => ({
+ id: `call_rescued_${i + 1}`,
+ type: 'function' as const,
+ function: { name: c.name, arguments: repairToolArguments(c.arguments, toolSchemas.get(c.name)) },
+ }));
+ text = rescue.cleanText;
+ }
+ }
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);
@@ -532,7 +630,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
recordTokens(route.platform, route.modelId, route.keyId, result.usage?.total_tokens ?? 0);
recordSuccess(route.modelDbId);
- setStickyModel(messages, route.modelDbId);
+ setStickyModel(messages, route.modelDbId, sessionIdHeader);
res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);
if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));