Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(tools): skip the dialect rescue when no tools were declared, and validate rescued calls (#854)

OhOkThisIsFinef7665b3c08d6
diff --git a/server/src/__tests__/lib/tool-call-rescue.test.ts b/server/src/__tests__/lib/tool-call-rescue.test.ts
index 8b7473eb..3404e9f1 100644
--- a/server/src/__tests__/lib/tool-call-rescue.test.ts
+++ b/server/src/__tests__/lib/tool-call-rescue.test.ts
@@ -138,4 +138,33 @@ describe('inline tool-call dialect rescue', () => {
expect(containsDialectMarker('plain text')).toBe(false);
});
});
+
+ // A turn that declared no tools has nothing callable, so no name can be
+ // "known". The gate used to read `toolNames.size === 0 || has(name)`, which
+ // meant the empty set matched EVERY name: prose that merely looked like a
+ // dialect was reconstructed into tool_calls the caller never asked for and
+ // cannot interpret.
+ describe('an empty tool set makes nothing callable', () => {
+ const EMPTY = new Set<string>();
+
+ it('does not synthesize a call from the function-tag dialect', () => {
+ const out = rescueInlineToolCalls('<function=Bash{"command":"rm -rf /"}</function>', EMPTY);
+ expect(out.calls).toBeNull();
+ });
+
+ it('does not synthesize a call from the Kimi/DeepSeek token dialect', () => {
+ const text = '<|tool_call_begin|>functions.Bash:0<|tool_call_argument_begin|>{"command":"ls"}<|tool_call_end|>';
+ expect(rescueInlineToolCalls(text, EMPTY).calls).toBeNull();
+ });
+
+ it('does not synthesize a call from the Qwen/Hermes XML dialect', () => {
+ const text = '<tool_call>{"name":"Bash","arguments":{"command":"ls"}}</tool_call>';
+ expect(rescueInlineToolCalls(text, EMPTY).calls).toBeNull();
+ });
+
+ it('still rescues the same text when the tool WAS declared', () => {
+ const out = rescueInlineToolCalls('<function=Bash{"command":"ls"}</function>', TOOLS);
+ expect(out.calls).toEqual([{ name: 'Bash', arguments: '{"command":"ls"}' }]);
+ });
+ });
});
diff --git a/server/src/__tests__/routes/rescue-wants-tools.test.ts b/server/src/__tests__/routes/rescue-wants-tools.test.ts
new file mode 100644
index 00000000..1036fc09
--- /dev/null
+++ b/server/src/__tests__/routes/rescue-wants-tools.test.ts
@@ -0,0 +1,200 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import type { Express } from 'express';
+
+// Two defects on the streaming tool-call rescue path, both fixed together
+// because they are the same block of code:
+//
+// 1. The rescue ran on turns that declared NO tools. Every non-streaming path
+// (proxy.ts, anthropic.ts, responses.ts) and the shared inbound-chat lane
+// already gated on `wantsTools`, and openai-compat's failed-generation
+// rescue guards `toolNames.size === 0` explicitly — the three STREAMING
+// paths did not. An ordinary answer that happened to contain `<function=…>`
+// was stripped out of the prose and re-emitted as a fabricated tool_calls
+// entry the client never asked for and cannot interpret.
+//
+// 2. The opt-in schema verdict ran BEFORE the rescue appended its calls, so
+// the calls reconstructed from prose — the ones most likely to be malformed
+// — were the only ones the validator never saw.
+//
+// Same scripted-provider mock pattern as tool-validate-surfaces.test.ts.
+const chatCompletion = vi.fn();
+const streamChatCompletion = vi.fn();
+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;
+
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ getProvider: () => fakeProvider,
+ resolveProvider: () => fakeProvider,
+ };
+});
+
+const { createApp } = await import('../../app.js');
+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { setRoutingStrategy } = await import('../../services/router.js');
+
+async function post(app: Express, path: string, body: any, headers: Record<string, string>) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', ...headers },
+ body: JSON.stringify(body),
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch { /* SSE, or an error page */ }
+ return { status: res.status, body: json, raw };
+}
+
+/** Collect the concatenated `content` and any tool_calls out of an SSE body. */
+function readStream(raw: string): { content: string; toolCalls: any[] } {
+ let content = '';
+ const toolCalls: any[] = [];
+ for (const line of raw.split('\n')) {
+ if (!line.startsWith('data: ') || line.includes('[DONE]')) continue;
+ let chunk: any;
+ try { chunk = JSON.parse(line.slice(6)); } catch { continue; }
+ const delta = chunk.choices?.[0]?.delta;
+ if (!delta) continue;
+ if (typeof delta.content === 'string') content += delta.content;
+ if (delta.tool_calls) toolCalls.push(...delta.tool_calls);
+ }
+ return { content, toolCalls };
+}
+
+/** A model that answers in prose containing a function-tag dialect fragment. */
+function streamTextTurn(text: string) {
+ return async function* () {
+ yield { choices: [{ delta: { content: text } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+ };
+}
+
+const PARAMS = {
+ type: 'object',
+ properties: { command: { type: 'string' } },
+ required: ['command'],
+};
+const BASH_TOOL = { type: 'function', function: { name: 'Bash', parameters: PARAMS } };
+
+// A complete, parseable dialect call. With tools declared it is a legitimate
+// rescue; with none declared it must stay prose.
+const DIALECT = '<function=Bash{"command":"ls"}</function>';
+// Parseable as a call, but the arguments violate the declared schema
+// (`command` is required and absent).
+const DIALECT_INVALID = '<function=Bash{"nope":"ls"}</function>';
+
+describe('streaming tool-call rescue respects whether the request wanted tools', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+
+ setRoutingStrategy('priority');
+ const { encrypted, iv, authTag } = encrypt('test-key');
+ getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)
+ `).run(encrypted, iv, authTag);
+ });
+
+ beforeEach(() => {
+ chatCompletion.mockReset();
+ streamChatCompletion.mockReset();
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ delete process.env.VALIDATE_TOOL_ARGUMENTS;
+ });
+
+ afterEach(() => {
+ delete process.env.VALIDATE_TOOL_ARGUMENTS;
+ });
+
+ const bearer = () => ({ Authorization: `Bearer ${key}` });
+
+ describe('no tools declared', () => {
+ it('delivers dialect-looking prose as content instead of fabricating a tool call', async () => {
+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT));
+
+ const { status, raw } = await post(app, '/v1/chat/completions', {
+ model: 'auto', stream: true, messages: [{ role: 'user', content: 'how do I list files?' }],
+ }, bearer());
+
+ expect(status).toBe(200);
+ const { content, toolCalls } = readStream(raw);
+ expect(toolCalls).toEqual([]);
+ expect(content).toContain('<function=Bash');
+ });
+
+ it('does not kill the turn when the dialect is unparseable', async () => {
+ // An opaque id leaves no way to know which tool was meant, so WITH tools
+ // this is a dead turn that fails over. With none, it is just prose.
+ const opaque = '<|tool_call_begin|>chatcmpl-tool-abc<|tool_call_argument_begin|>{"a":1}<|tool_call_end|>';
+ streamChatCompletion.mockImplementation(streamTextTurn(opaque));
+
+ const { status, raw } = await post(app, '/v1/chat/completions', {
+ model: 'auto', stream: true, messages: [{ role: 'user', content: 'hi' }],
+ }, bearer());
+
+ expect(status).toBe(200);
+ const { content, toolCalls } = readStream(raw);
+ expect(toolCalls).toEqual([]);
+ expect(content).toContain('<|tool_call_begin|>');
+ });
+ });
+
+ describe('tools declared', () => {
+ it('still rescues the same text into a structured tool call', async () => {
+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT));
+
+ const { status, raw } = await post(app, '/v1/chat/completions', {
+ model: 'auto', stream: true, tools: [BASH_TOOL],
+ messages: [{ role: 'user', content: 'list files' }],
+ }, bearer());
+
+ expect(status).toBe(200);
+ const { toolCalls } = readStream(raw);
+ expect(toolCalls).toHaveLength(1);
+ expect(toolCalls[0].function.name).toBe('Bash');
+ expect(JSON.parse(toolCalls[0].function.arguments)).toEqual({ command: 'ls' });
+ });
+
+ it('delivers a schema-invalid rescued call while validation is off', async () => {
+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT_INVALID));
+
+ const { status, raw } = await post(app, '/v1/chat/completions', {
+ model: 'auto', stream: true, tools: [BASH_TOOL],
+ messages: [{ role: 'user', content: 'list files' }],
+ }, bearer());
+
+ expect(status).toBe(200);
+ const { toolCalls } = readStream(raw);
+ expect(toolCalls).toHaveLength(1);
+ expect(JSON.parse(toolCalls[0].function.arguments)).toEqual({ nope: 'ls' });
+ });
+
+ // The ordering half: the verdict is taken after the rescue, so a rescued
+ // call gets the same scrutiny as one the provider emitted structurally.
+ // Running first exempted precisely the reconstructed calls.
+ it('fails a schema-invalid RESCUED call over when validation is on', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT_INVALID));
+
+ const { status } = await post(app, '/v1/chat/completions', {
+ model: 'auto', stream: true, tools: [BASH_TOOL],
+ messages: [{ role: 'user', content: 'list files' }],
+ }, bearer());
+
+ // Every candidate answers the same way, so the ladder exhausts rather
+ // than delivering arguments the tool cannot accept.
+ expect(status).toBeGreaterThanOrEqual(400);
+ });
+ });
+});
diff --git a/server/src/lib/tool-call-rescue.ts b/server/src/lib/tool-call-rescue.ts
index 97289b9d..06c26ef4 100644
--- a/server/src/lib/tool-call-rescue.ts
+++ b/server/src/lib/tool-call-rescue.ts
@@ -104,8 +104,11 @@ function extractBalancedJson(text: string, from: number): { json: string; end: n
return null;
}
+// An empty set means the request declared no tools, so NOTHING is callable.
+// This used to read `toolNames.size === 0 || …`, which inverted that: a turn
+// with no tools opened the gate for every name the dialect parsers found.
const isKnownTool = (name: string, toolNames: Set<string>): boolean =>
- toolNames.size === 0 || toolNames.has(name);
+ toolNames.has(name);
/** Parse `{"name": ..., "arguments"|"parameters": ...}` into a call. */
function callFromNamedJson(json: string, toolNames: Set<string>): RescuedToolCall | null {
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 3a4b4adf..39cf9677 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -851,7 +851,7 @@ async function streamCompletion(
// committed). A rescued dialect becomes tool_use blocks; leftover clean text
// is emitted as a text block first.
if (heldText.length > 0) {
- const rescue = (dialectMode === 'dialect' || containsDialectMarker(heldText))
+ const rescue = ((ctx.tools?.length ?? 0) > 0 && (dialectMode === 'dialect' || containsDialectMarker(heldText)))
? rescueInlineToolCalls(heldText, new Set((ctx.tools ?? []).map(t => t.function.name)))
: { detected: false as const, calls: null, cleanText: heldText };
if (rescue.detected && !rescue.calls) {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 0817dfc3..ea45b610 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -1995,9 +1995,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
if (mode === 'dialect') continue;
const probe = heldText.trimStart();
- if (startsWithDialectMarker(probe)) {
+ if (wantsTools && startsWithDialectMarker(probe)) {
mode = 'dialect';
- } else if (!couldBecomeDialectMarker(probe) || probe.length > 256) {
+ } else if (!wantsTools || !couldBecomeDialectMarker(probe) || probe.length > 256) {
mode = 'passthrough';
flushHeaders();
writeChunk(mkChunk({ content: heldText }, null));
@@ -2022,25 +2022,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}))
.filter(c => { try { JSON.parse(c.function.arguments); return c.function.name.length > 0; } catch { return false; } });
- // Opt-in schema verdict. `!headerSent` is the whole licence to throw
- // here: the commit point is held until the first meaningful content,
- // so the common tool-call turn (no prose before the call) has sent no
- // bytes yet and can still fail over invisibly. A turn that already
- // flushed prose is past the point of no return — the catch below
- // would have to tear the SSE stream down with a `stream_error`, which
- // is strictly worse for the client than forwarding a tool call the
- // schema dislikes. Off-by-default or not, this check must never turn
- // a served answer into a broken one.
- if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {
- const invalid = invalidToolCallReasons(completedCalls, schemas);
- if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
- }
-
// Dialect rescue: the held text is an inline tool call in some
// model's private syntax. Parse it into structured calls or treat
// the turn as dead (headers were never sent in dialect mode, so
// failing over is free).
- if (mode === 'dialect' || (mode === 'undecided' && heldText.length > 0 && containsDialectMarker(heldText))) {
+ if (wantsTools && (mode === 'dialect' || (mode === 'undecided' && heldText.length > 0 && containsDialectMarker(heldText)))) {
const rescue = rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)));
if (rescue.detected) {
if (!rescue.calls) throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${heldText.slice(0, 120)}`);
@@ -2053,6 +2039,23 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
}
+ // Opt-in schema verdict, taken AFTER the rescue so it covers the
+ // calls the rescue reconstructed from prose — those are the ones
+ // most likely to be malformed, and running first exempted exactly
+ // them. `!headerSent` is the whole licence to throw here: the commit
+ // point is held until the first meaningful content, so the common
+ // tool-call turn (no prose before the call) has sent no bytes yet and
+ // can still fail over invisibly. A turn that already flushed prose is
+ // past the point of no return — the catch below would have to tear
+ // the SSE stream down with a `stream_error`, which is strictly worse
+ // for the client than forwarding a tool call the schema dislikes.
+ // Off-by-default or not, this check must never turn a served answer
+ // into a broken one.
+ if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {
+ const invalid = invalidToolCallReasons(completedCalls, schemas);
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
+
// Disconnect before the commit point: nothing usable was (or will
// be) delivered, and that is CLIENT behavior, not a provider
// failure — do not let it fall through to the empty-completion
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index 5565c251..83c18b43 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -861,7 +861,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// Held text was never emitted, so a dead dialect turn can still fail
// over on the same SSE stream (nothing has been committed yet).
if (heldText.length > 0) {
- const rescue = (dialectMode === 'dialect' || containsDialectMarker(heldText))
+ const rescue = (wantsTools && (dialectMode === 'dialect' || containsDialectMarker(heldText)))
? rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)))
: { detected: false as const, calls: null, cleanText: heldText };
if (rescue.detected && !rescue.calls) {