7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(tools): skip the dialect rescue when no tools were declared, and validate rescued calls (#854)
OhOkThisIsFinef7665b3c08d6
diff --git a/server/src/__tests__/lib/tool-call-rescue.test.ts b/server/src/__tests__/lib/tool-call-rescue.test.tsindex 8b7473eb..3404e9f1 100644--- a/server/src/__tests__/lib/tool-call-rescue.test.ts+++ b/server/src/__tests__/lib/tool-call-rescue.test.ts@@ -138,4 +138,33 @@ describe('inline tool-call dialect rescue', () => {expect(containsDialectMarker('plain text')).toBe(false);});});++ // A turn that declared no tools has nothing callable, so no name can be+ // "known". The gate used to read `toolNames.size === 0 || has(name)`, which+ // meant the empty set matched EVERY name: prose that merely looked like a+ // dialect was reconstructed into tool_calls the caller never asked for and+ // cannot interpret.+ describe('an empty tool set makes nothing callable', () => {+ const EMPTY = new Set<string>();++ it('does not synthesize a call from the function-tag dialect', () => {+ const out = rescueInlineToolCalls('<function=Bash{"command":"rm -rf /"}</function>', EMPTY);+ expect(out.calls).toBeNull();+ });++ it('does not synthesize a call from the Kimi/DeepSeek token dialect', () => {+ const text = '<|tool_call_begin|>functions.Bash:0<|tool_call_argument_begin|>{"command":"ls"}<|tool_call_end|>';+ expect(rescueInlineToolCalls(text, EMPTY).calls).toBeNull();+ });++ it('does not synthesize a call from the Qwen/Hermes XML dialect', () => {+ const text = '<tool_call>{"name":"Bash","arguments":{"command":"ls"}}</tool_call>';+ expect(rescueInlineToolCalls(text, EMPTY).calls).toBeNull();+ });++ it('still rescues the same text when the tool WAS declared', () => {+ const out = rescueInlineToolCalls('<function=Bash{"command":"ls"}</function>', TOOLS);+ expect(out.calls).toEqual([{ name: 'Bash', arguments: '{"command":"ls"}' }]);+ });+ });});diff --git a/server/src/__tests__/routes/rescue-wants-tools.test.ts b/server/src/__tests__/routes/rescue-wants-tools.test.tsnew file mode 100644index 00000000..1036fc09--- /dev/null+++ b/server/src/__tests__/routes/rescue-wants-tools.test.ts@@ -0,0 +1,200 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';+import type { Express } from 'express';++// Two defects on the streaming tool-call rescue path, both fixed together+// because they are the same block of code:+//+// 1. The rescue ran on turns that declared NO tools. Every non-streaming path+// (proxy.ts, anthropic.ts, responses.ts) and the shared inbound-chat lane+// already gated on `wantsTools`, and openai-compat's failed-generation+// rescue guards `toolNames.size === 0` explicitly — the three STREAMING+// paths did not. An ordinary answer that happened to contain `<function=…>`+// was stripped out of the prose and re-emitted as a fabricated tool_calls+// entry the client never asked for and cannot interpret.+//+// 2. The opt-in schema verdict ran BEFORE the rescue appended its calls, so+// the calls reconstructed from prose — the ones most likely to be malformed+// — were the only ones the validator never saw.+//+// Same scripted-provider mock pattern as tool-validate-surfaces.test.ts.+const chatCompletion = vi.fn();+const streamChatCompletion = vi.fn();+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;++vi.mock('../../providers/index.js', async (importOriginal) => {+ const actual = await importOriginal() as any;+ return {+ ...actual,+ getProvider: () => fakeProvider,+ resolveProvider: () => fakeProvider,+ };+});++const { createApp } = await import('../../app.js');+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const { setRoutingStrategy } = await import('../../services/router.js');++async function post(app: Express, path: string, body: any, headers: Record<string, string>) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', ...headers },+ body: JSON.stringify(body),+ });+ const raw = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(raw); } catch { /* SSE, or an error page */ }+ return { status: res.status, body: json, raw };+}++/** Collect the concatenated `content` and any tool_calls out of an SSE body. */+function readStream(raw: string): { content: string; toolCalls: any[] } {+ let content = '';+ const toolCalls: any[] = [];+ for (const line of raw.split('\n')) {+ if (!line.startsWith('data: ') || line.includes('[DONE]')) continue;+ let chunk: any;+ try { chunk = JSON.parse(line.slice(6)); } catch { continue; }+ const delta = chunk.choices?.[0]?.delta;+ if (!delta) continue;+ if (typeof delta.content === 'string') content += delta.content;+ if (delta.tool_calls) toolCalls.push(...delta.tool_calls);+ }+ return { content, toolCalls };+}++/** A model that answers in prose containing a function-tag dialect fragment. */+function streamTextTurn(text: string) {+ return async function* () {+ yield { choices: [{ delta: { content: text } }] };+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };+ };+}++const PARAMS = {+ type: 'object',+ properties: { command: { type: 'string' } },+ required: ['command'],+};+const BASH_TOOL = { type: 'function', function: { name: 'Bash', parameters: PARAMS } };++// A complete, parseable dialect call. With tools declared it is a legitimate+// rescue; with none declared it must stay prose.+const DIALECT = '<function=Bash{"command":"ls"}</function>';+// Parseable as a call, but the arguments violate the declared schema+// (`command` is required and absent).+const DIALECT_INVALID = '<function=Bash{"nope":"ls"}</function>';++describe('streaming tool-call rescue respects whether the request wanted tools', () => {+ let app: Express;+ let key: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ key = getUnifiedApiKey();++ setRoutingStrategy('priority');+ const { encrypted, iv, authTag } = encrypt('test-key');+ getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)+ `).run(encrypted, iv, authTag);+ });++ beforeEach(() => {+ chatCompletion.mockReset();+ streamChatCompletion.mockReset();+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ delete process.env.VALIDATE_TOOL_ARGUMENTS;+ });++ afterEach(() => {+ delete process.env.VALIDATE_TOOL_ARGUMENTS;+ });++ const bearer = () => ({ Authorization: `Bearer ${key}` });++ describe('no tools declared', () => {+ it('delivers dialect-looking prose as content instead of fabricating a tool call', async () => {+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT));++ const { status, raw } = await post(app, '/v1/chat/completions', {+ model: 'auto', stream: true, messages: [{ role: 'user', content: 'how do I list files?' }],+ }, bearer());++ expect(status).toBe(200);+ const { content, toolCalls } = readStream(raw);+ expect(toolCalls).toEqual([]);+ expect(content).toContain('<function=Bash');+ });++ it('does not kill the turn when the dialect is unparseable', async () => {+ // An opaque id leaves no way to know which tool was meant, so WITH tools+ // this is a dead turn that fails over. With none, it is just prose.+ const opaque = '<|tool_call_begin|>chatcmpl-tool-abc<|tool_call_argument_begin|>{"a":1}<|tool_call_end|>';+ streamChatCompletion.mockImplementation(streamTextTurn(opaque));++ const { status, raw } = await post(app, '/v1/chat/completions', {+ model: 'auto', stream: true, messages: [{ role: 'user', content: 'hi' }],+ }, bearer());++ expect(status).toBe(200);+ const { content, toolCalls } = readStream(raw);+ expect(toolCalls).toEqual([]);+ expect(content).toContain('<|tool_call_begin|>');+ });+ });++ describe('tools declared', () => {+ it('still rescues the same text into a structured tool call', async () => {+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT));++ const { status, raw } = await post(app, '/v1/chat/completions', {+ model: 'auto', stream: true, tools: [BASH_TOOL],+ messages: [{ role: 'user', content: 'list files' }],+ }, bearer());++ expect(status).toBe(200);+ const { toolCalls } = readStream(raw);+ expect(toolCalls).toHaveLength(1);+ expect(toolCalls[0].function.name).toBe('Bash');+ expect(JSON.parse(toolCalls[0].function.arguments)).toEqual({ command: 'ls' });+ });++ it('delivers a schema-invalid rescued call while validation is off', async () => {+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT_INVALID));++ const { status, raw } = await post(app, '/v1/chat/completions', {+ model: 'auto', stream: true, tools: [BASH_TOOL],+ messages: [{ role: 'user', content: 'list files' }],+ }, bearer());++ expect(status).toBe(200);+ const { toolCalls } = readStream(raw);+ expect(toolCalls).toHaveLength(1);+ expect(JSON.parse(toolCalls[0].function.arguments)).toEqual({ nope: 'ls' });+ });++ // The ordering half: the verdict is taken after the rescue, so a rescued+ // call gets the same scrutiny as one the provider emitted structurally.+ // Running first exempted precisely the reconstructed calls.+ it('fails a schema-invalid RESCUED call over when validation is on', async () => {+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';+ streamChatCompletion.mockImplementation(streamTextTurn(DIALECT_INVALID));++ const { status } = await post(app, '/v1/chat/completions', {+ model: 'auto', stream: true, tools: [BASH_TOOL],+ messages: [{ role: 'user', content: 'list files' }],+ }, bearer());++ // Every candidate answers the same way, so the ladder exhausts rather+ // than delivering arguments the tool cannot accept.+ expect(status).toBeGreaterThanOrEqual(400);+ });+ });+});diff --git a/server/src/lib/tool-call-rescue.ts b/server/src/lib/tool-call-rescue.tsindex 97289b9d..06c26ef4 100644--- a/server/src/lib/tool-call-rescue.ts+++ b/server/src/lib/tool-call-rescue.ts@@ -104,8 +104,11 @@ function extractBalancedJson(text: string, from: number): { json: string; end: nreturn null;}+// An empty set means the request declared no tools, so NOTHING is callable.+// This used to read `toolNames.size === 0 || …`, which inverted that: a turn+// with no tools opened the gate for every name the dialect parsers found.const isKnownTool = (name: string, toolNames: Set<string>): boolean =>- toolNames.size === 0 || toolNames.has(name);+ toolNames.has(name);/** Parse `{"name": ..., "arguments"|"parameters": ...}` into a call. */function callFromNamedJson(json: string, toolNames: Set<string>): RescuedToolCall | null {diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 3a4b4adf..39cf9677 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -851,7 +851,7 @@ async function streamCompletion(// committed). A rescued dialect becomes tool_use blocks; leftover clean text// is emitted as a text block first.if (heldText.length > 0) {- const rescue = (dialectMode === 'dialect' || containsDialectMarker(heldText))+ const rescue = ((ctx.tools?.length ?? 0) > 0 && (dialectMode === 'dialect' || containsDialectMarker(heldText)))? rescueInlineToolCalls(heldText, new Set((ctx.tools ?? []).map(t => t.function.name))): { detected: false as const, calls: null, cleanText: heldText };if (rescue.detected && !rescue.calls) {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 0817dfc3..ea45b610 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -1995,9 +1995,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {if (mode === 'dialect') continue;const probe = heldText.trimStart();- if (startsWithDialectMarker(probe)) {+ if (wantsTools && startsWithDialectMarker(probe)) {mode = 'dialect';- } else if (!couldBecomeDialectMarker(probe) || probe.length > 256) {+ } else if (!wantsTools || !couldBecomeDialectMarker(probe) || probe.length > 256) {mode = 'passthrough';flushHeaders();writeChunk(mkChunk({ content: heldText }, null));@@ -2022,25 +2022,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {})).filter(c => { try { JSON.parse(c.function.arguments); return c.function.name.length > 0; } catch { return false; } });- // Opt-in schema verdict. `!headerSent` is the whole licence to throw- // here: the commit point is held until the first meaningful content,- // so the common tool-call turn (no prose before the call) has sent no- // bytes yet and can still fail over invisibly. A turn that already- // flushed prose is past the point of no return — the catch below- // would have to tear the SSE stream down with a `stream_error`, which- // is strictly worse for the client than forwarding a tool call the- // schema dislikes. Off-by-default or not, this check must never turn- // a served answer into a broken one.- if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {- const invalid = invalidToolCallReasons(completedCalls, schemas);- if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);- }-// Dialect rescue: the held text is an inline tool call in some// model's private syntax. Parse it into structured calls or treat// the turn as dead (headers were never sent in dialect mode, so// failing over is free).- if (mode === 'dialect' || (mode === 'undecided' && heldText.length > 0 && containsDialectMarker(heldText))) {+ if (wantsTools && (mode === 'dialect' || (mode === 'undecided' && heldText.length > 0 && containsDialectMarker(heldText)))) {const rescue = rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name)));if (rescue.detected) {if (!rescue.calls) throw new Error(`unparseable inline tool-call dialect from ${route.displayName}: ${heldText.slice(0, 120)}`);@@ -2053,6 +2039,23 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}}+ // Opt-in schema verdict, taken AFTER the rescue so it covers the+ // calls the rescue reconstructed from prose — those are the ones+ // most likely to be malformed, and running first exempted exactly+ // them. `!headerSent` is the whole licence to throw here: the commit+ // point is held until the first meaningful content, so the common+ // tool-call turn (no prose before the call) has sent no bytes yet and+ // can still fail over invisibly. A turn that already flushed prose is+ // past the point of no return — the catch below would have to tear+ // the SSE stream down with a `stream_error`, which is strictly worse+ // for the client than forwarding a tool call the schema dislikes.+ // Off-by-default or not, this check must never turn a served answer+ // into a broken one.+ if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {+ const invalid = invalidToolCallReasons(completedCalls, schemas);+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);+ }+// Disconnect before the commit point: nothing usable was (or will// be) delivered, and that is CLIENT behavior, not a provider// failure — do not let it fall through to the empty-completiondiff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex 5565c251..83c18b43 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -861,7 +861,7 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {// Held text was never emitted, so a dead dialect turn can still fail// over on the same SSE stream (nothing has been committed yet).if (heldText.length > 0) {- const rescue = (dialectMode === 'dialect' || containsDialectMarker(heldText))+ const rescue = (wantsTools && (dialectMode === 'dialect' || containsDialectMarker(heldText)))? rescueInlineToolCalls(heldText, new Set((tools ?? []).map(t => t.function.name))): { detected: false as const, calls: null, cleanText: heldText };if (rescue.detected && !rescue.calls) {