Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(tools): extend the tool-argument schema verdict to the remaining surfaces (#821)

OhOkThisIsFine9161cfba3ec4
diff --git a/.env.example b/.env.example
index 79b6ea98..988bd0fb 100644
--- a/.env.example
+++ b/.env.example
@@ -119,6 +119,11 @@ PORT=3001
# no indication anything went wrong. A different model usually gets the same
# call right. Off by default because a false positive costs a failover hop —
# turn it on, watch X-Fallback-Trail for `invalid_tool_arguments`, then decide.
+# Applies wherever the turn can still fail over — every non-streaming surface,
+# and the streams that buffer tool calls to the end (/v1/chat/completions,
+# /v1/messages, the Gemini and Ollama emulations). /v1/responses streaming is
+# the exception: it commits on the first tool-call delta, before the arguments
+# are complete, so a verdict there could only break a stream in flight.
# Can also be set at runtime via the validate_tool_arguments settings key,
# which takes precedence.
# VALIDATE_TOOL_ARGUMENTS=1
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.ts
index 6457e439..0a2d3cf1 100644
--- a/server/src/__tests__/providers/openai-compat.test.ts
+++ b/server/src/__tests__/providers/openai-compat.test.ts
@@ -624,6 +624,44 @@ describe('OpenAICompatProvider - platform instances', () => {
expect(chunks.some(c => c.choices[0].finish_reason === 'tool_calls')).toBe(true);
});
+ // The rescue turns a provider 400 into a success. If what it recovered does
+ // not satisfy the tool's schema, that success is one the client cannot use —
+ // so with the opt-in verdict on, decline the rescue and let the original
+ // error propagate, exactly as it did before #264.
+ it('declines the rescue when the recovered arguments violate the schema', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ try {
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: false, status: 400, statusText: 'Bad Request',
+ json: () => Promise.resolve({
+ ...failBody,
+ error: { ...failBody.error, failed_generation: '<function=read={"nope": "sample.txt"}</function>' },
+ }),
+ } as any);
+
+ await expect(
+ provider.chatCompletion('key', [{ role: 'user', content: 'read sample.txt' }], 'llama-3.3-70b-versatile', { tools, tool_choice: 'auto' }),
+ ).rejects.toThrow(/API error 400/);
+ } finally {
+ delete process.env.VALIDATE_TOOL_ARGUMENTS;
+ }
+ });
+
+ it('still rescues schema-valid arguments while the verdict is on', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ try {
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: false, status: 400, statusText: 'Bad Request',
+ json: () => Promise.resolve(failBody),
+ } as any);
+
+ const r = await provider.chatCompletion('key', [{ role: 'user', content: 'read sample.txt' }], 'llama-3.3-70b-versatile', { tools, tool_choice: 'auto' });
+ expect(JSON.parse(r.choices[0].message.tool_calls![0].function.arguments)).toEqual({ file_path: 'sample.txt' });
+ } finally {
+ delete process.env.VALIDATE_TOOL_ARGUMENTS;
+ }
+ });
+
it('still throws when there is no failed_generation to rescue', async () => {
vi.spyOn(global, 'fetch').mockResolvedValueOnce({
ok: false, status: 400, statusText: 'Bad Request',
diff --git a/server/src/__tests__/routes/tool-validate-surfaces.test.ts b/server/src/__tests__/routes/tool-validate-surfaces.test.ts
new file mode 100644
index 00000000..c2c9c34f
--- /dev/null
+++ b/server/src/__tests__/routes/tool-validate-surfaces.test.ts
@@ -0,0 +1,257 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import type { Express } from 'express';
+
+// Follow-up to the opt-in tool-argument schema verdict: it shipped wired at
+// /v1/chat/completions (both paths) and Anthropic non-streaming only. These
+// tests pin the remaining surfaces — /v1/responses non-streaming, the Anthropic
+// stream, and the shared inbound-chat lane behind the Gemini/Ollama emulations —
+// plus the one surface that deliberately stays unwired, so the gap is a pinned
+// decision rather than something to rediscover.
+//
+// Same scripted-provider mock pattern as responses-tool-args-repair.test.ts.
+const chatCompletion = vi.fn();
+const streamChatCompletion = vi.fn();
+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;
+
+vi.mock('../../providers/index.js', async (importOriginal) => {
+ const actual = await importOriginal() as any;
+ return {
+ ...actual,
+ getProvider: () => fakeProvider,
+ resolveProvider: () => fakeProvider,
+ };
+});
+
+const { createApp } = await import('../../app.js');
+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');
+const { encrypt } = await import('../../lib/crypto.js');
+const { setRoutingStrategy } = await import('../../services/router.js');
+
+async function post(app: Express, path: string, body: any, headers: Record<string, string>) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', ...headers },
+ body: JSON.stringify(body),
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch { /* SSE, or an error page */ }
+ return { status: res.status, body: json, raw };
+}
+
+// `path` is required and absent. Nothing tool-args can repair — the value is
+// not a mis-encoded anything, it is simply not the argument the tool declared.
+const INVALID_ARGS = JSON.stringify({ nope: 'README.md' });
+const VALID_ARGS = JSON.stringify({ path: 'README.md' });
+
+const PARAMS = {
+ type: 'object',
+ properties: { path: { type: 'string' } },
+ required: ['path'],
+};
+
+const RESPONSES_TOOL = { type: 'function', name: 'read_file', parameters: PARAMS };
+const ANTHROPIC_TOOL = { name: 'read_file', input_schema: PARAMS };
+const GEMINI_TOOL = { functionDeclarations: [{ name: 'read_file', parameters: PARAMS }] };
+
+function nonStreamToolTurn(args: string) {
+ return {
+ choices: [{
+ index: 0,
+ message: { role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'read_file', arguments: args } }] },
+ finish_reason: 'tool_calls',
+ }],
+ usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },
+ };
+}
+
+function streamToolTurn(args: string) {
+ return async function* () {
+ yield { choices: [{ delta: { tool_calls: [{ index: 0, id: 'call_1', function: { name: 'read_file', arguments: args } }] } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'tool_calls' }] };
+ };
+}
+
+describe('tool-argument schema verdict — remaining surfaces', () => {
+ let app: Express;
+ let key: string;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ key = getUnifiedApiKey();
+
+ setRoutingStrategy('priority');
+ const { encrypted, iv, authTag } = encrypt('test-key');
+ getDb().prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)
+ `).run(encrypted, iv, authTag);
+ });
+
+ beforeEach(() => {
+ chatCompletion.mockReset();
+ streamChatCompletion.mockReset();
+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();
+ delete process.env.VALIDATE_TOOL_ARGUMENTS;
+ });
+
+ afterEach(() => {
+ delete process.env.VALIDATE_TOOL_ARGUMENTS;
+ });
+
+ const bearer = () => ({ Authorization: `Bearer ${key}` });
+
+ describe('/v1/responses, non-streaming', () => {
+ it('delivers a schema-invalid call untouched while validation is off', async () => {
+ chatCompletion.mockResolvedValue(nonStreamToolTurn(INVALID_ARGS));
+
+ const { status, body } = await post(app, '/v1/responses', {
+ input: 'read the readme', tools: [RESPONSES_TOOL],
+ }, bearer());
+
+ expect(status).toBe(200);
+ const fc = body.output.find((o: any) => o.type === 'function_call');
+ expect(JSON.parse(fc.arguments)).toEqual({ nope: 'README.md' });
+ });
+
+ it('fails the turn over instead of delivering it when validation is on', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ chatCompletion.mockResolvedValue(nonStreamToolTurn(INVALID_ARGS));
+
+ const { status, body } = await post(app, '/v1/responses', {
+ input: 'read the readme', tools: [RESPONSES_TOOL],
+ }, bearer());
+
+ // Every candidate answers the same way, so the ladder exhausts rather
+ // than handing the client a call it cannot use.
+ expect(status).not.toBe(200);
+ expect(body?.output).toBeUndefined();
+ expect(chatCompletion).toHaveBeenCalled();
+ });
+
+ it('leaves a schema-valid call alone when validation is on', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ chatCompletion.mockResolvedValue(nonStreamToolTurn(VALID_ARGS));
+
+ const { status, body } = await post(app, '/v1/responses', {
+ input: 'read the readme', tools: [RESPONSES_TOOL],
+ }, bearer());
+
+ expect(status).toBe(200);
+ const fc = body.output.find((o: any) => o.type === 'function_call');
+ expect(JSON.parse(fc.arguments)).toEqual({ path: 'README.md' });
+ });
+ });
+
+ describe('/v1/responses, streaming', () => {
+ // Deliberately NOT wired — this route commits on the first tool-call delta,
+ // long before the arguments are complete. A verdict could only turn a
+ // delivered call into a response.failed on a stream the client is already
+ // reading. Pinned so the asymmetry with the non-streaming path above reads
+ // as a decision rather than an oversight.
+ it('still delivers a schema-invalid call, because it has already committed', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ streamChatCompletion.mockImplementation(streamToolTurn(INVALID_ARGS));
+
+ const { status, raw } = await post(app, '/v1/responses', {
+ input: 'read the readme', stream: true, tools: [RESPONSES_TOOL],
+ }, bearer());
+
+ expect(status).toBe(200);
+ expect(raw).toContain('response.function_call_arguments.done');
+ expect(raw).toContain('nope');
+ });
+ });
+
+ describe('/v1/messages, streaming', () => {
+ it('fails a tool-only turn over before message_start when validation is on', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ streamChatCompletion.mockImplementation(streamToolTurn(INVALID_ARGS));
+
+ const { raw } = await post(app, '/v1/messages', {
+ model: 'auto', max_tokens: 64, stream: true,
+ messages: [{ role: 'user', content: 'read the readme' }],
+ tools: [ANTHROPIC_TOOL],
+ }, { 'x-api-key': key });
+
+ // Tool calls are buffered to the end of the stream here, so nothing was
+ // committed: no tool_use block reaches the client.
+ expect(raw).not.toContain('tool_use');
+ expect(raw).not.toContain('nope');
+ });
+
+ it('delivers a schema-valid tool-only turn unchanged', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ streamChatCompletion.mockImplementation(streamToolTurn(VALID_ARGS));
+
+ const { status, raw } = await post(app, '/v1/messages', {
+ model: 'auto', max_tokens: 64, stream: true,
+ messages: [{ role: 'user', content: 'read the readme' }],
+ tools: [ANTHROPIC_TOOL],
+ }, { 'x-api-key': key });
+
+ expect(status).toBe(200);
+ expect(raw).toContain('tool_use');
+ expect(raw).toContain('README.md');
+ });
+ });
+
+ describe('inbound-chat lane (native Gemini surface)', () => {
+ it('non-stream: fails over instead of returning an invalid functionCall', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ chatCompletion.mockResolvedValue(nonStreamToolTurn(INVALID_ARGS));
+
+ const { status, body } = await post(app, '/v1beta/models/gemini-2.5-flash:generateContent', {
+ contents: [{ role: 'user', parts: [{ text: 'read the readme' }] }],
+ tools: [GEMINI_TOOL],
+ }, { 'x-goog-api-key': key });
+
+ expect(status).not.toBe(200);
+ expect(body?.candidates).toBeUndefined();
+ });
+
+ it('non-stream: delivers the same turn when validation is off', async () => {
+ chatCompletion.mockResolvedValue(nonStreamToolTurn(INVALID_ARGS));
+
+ const { status, body } = await post(app, '/v1beta/models/gemini-2.5-flash:generateContent', {
+ contents: [{ role: 'user', parts: [{ text: 'read the readme' }] }],
+ tools: [GEMINI_TOOL],
+ }, { 'x-goog-api-key': key });
+
+ expect(status).toBe(200);
+ expect(body.candidates[0].content.parts[0].functionCall.args).toEqual({ nope: 'README.md' });
+ });
+
+ it('stream: a tool-only turn is buffered, so it can still fail over', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ streamChatCompletion.mockImplementation(streamToolTurn(INVALID_ARGS));
+
+ const { raw } = await post(app, '/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse', {
+ contents: [{ role: 'user', parts: [{ text: 'read the readme' }] }],
+ tools: [GEMINI_TOOL],
+ }, { 'x-goog-api-key': key });
+
+ expect(raw).not.toContain('functionCall');
+ expect(raw).not.toContain('nope');
+ });
+
+ it('stream: a schema-valid tool-only turn is delivered unchanged', async () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ streamChatCompletion.mockImplementation(streamToolTurn(VALID_ARGS));
+
+ const { status, raw } = await post(app, '/v1beta/models/gemini-2.5-flash:streamGenerateContent?alt=sse', {
+ contents: [{ role: 'user', parts: [{ text: 'read the readme' }] }],
+ tools: [GEMINI_TOOL],
+ }, { 'x-goog-api-key': key });
+
+ expect(status).toBe(200);
+ expect(raw).toContain('functionCall');
+ expect(raw).toContain('README.md');
+ });
+ });
+});
diff --git a/server/src/lib/inbound-chat.ts b/server/src/lib/inbound-chat.ts
index 329a3ade..4e85dde1 100644
--- a/server/src/lib/inbound-chat.ts
+++ b/server/src/lib/inbound-chat.ts
@@ -32,6 +32,7 @@ import { routedViaValue } from './header-value.js';
import { applyTokenBudget, tokenBudgetMessage } from './guardrails.js';
import { contentToString } from './content.js';
import { repairToolArguments, toolSchemaMap } from './tool-args.js';
+import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from './tool-validate.js';
import {
containsDialectMarker,
couldBecomeDialectMarker,
@@ -282,6 +283,12 @@ export async function runInboundChat(
schemas.get(call.function.name),
);
}
+ // Opt-in schema verdict on what the repair could not fix. Nothing has
+ // been written yet on this path, so the failover hop is invisible.
+ if (isToolArgumentValidationEnabled() && toolCalls.length > 0) {
+ const invalid = invalidToolCallReasons(toolCalls, schemas);
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
const completionTokens = result.usage?.completion_tokens
?? Math.ceil((text.length + reasoning.length + toolCalls.reduce(
@@ -428,6 +435,15 @@ export async function runInboundChat(
},
...(call.thoughtSignature ? { thought_signature: call.thoughtSignature } : {}),
}));
+ // Opt-in schema verdict, before the tool calls are committed. Tool
+ // calls are buffered to the end of the stream on this path, so a
+ // tool-only turn has sent nothing yet and can still fail over. A turn
+ // that already streamed text is past its commit point — leave it
+ // alone rather than tearing down a stream the client is reading.
+ if (isToolArgumentValidationEnabled() && toolCalls.length > 0 && !committed) {
+ const invalid = invalidToolCallReasons(toolCalls, schemas);
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
if (toolCalls.length) {
commit();
wire.sendToolCalls?.(res, route, toolCalls);
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 5fabb5f6..08b1c173 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -10,6 +10,7 @@ import { extendedBodyParams, resolveMaxTokens } from '../lib/sampling-params.js'
import { rescueInlineToolCalls } from '../lib/tool-call-rescue.js';
import { extractThinkFromMessage } from '../lib/think-tags.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
+import { invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
import { recordQuotaObservationsFromResponse, type QuotaObservationContext } from '../services/provider-quota.js';
import { providerTimeoutMs } from '../lib/provider-timeout.js';
import { isAbortLikeError } from '../lib/error-classify.js';
@@ -83,11 +84,21 @@ export class OpenAICompatProvider extends BaseProvider {
const rescue = rescueInlineToolCalls(failed, toolNames);
if (!rescue.detected || !rescue.calls?.length) return null;
const schemas = toolSchemaMap(options?.tools);
- return rescue.calls.map((c, i) => ({
+ const rescued = rescue.calls.map((c, i) => ({
id: `call_rescued_${i + 1}`,
type: 'function' as const,
function: { name: c.name, arguments: repairToolArguments(c.arguments, schemas.get(c.name)) },
}));
+ // Opt-in schema verdict. A rescue that produces schema-invalid arguments
+ // has turned the provider's 400 into a "success" the client cannot use, so
+ // decline it instead: the original upstream error propagates and the loop
+ // fails over exactly as it did before the rescue existed. Declining is the
+ // right shape here rather than throwing our own error — we are inside the
+ // provider's error path already.
+ if (isToolArgumentValidationEnabled() && invalidToolCallReasons(rescued, schemas).length > 0) {
+ return null;
+ }
+ return rescued;
}
/** Extract the useful text from an upstream error body. Most providers put it
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index f1575407..f061a7f3 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -15,6 +15,7 @@ import { resolveAuth, prependSystemPrompt } from '../lib/system-prompt.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from '../services/model-groups.js';
import { contentToString } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
+import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import {
extractApiToken,
@@ -761,6 +762,17 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
// Codex hard-rejects the call ("invalid type: string, expected a
// sequence"). Clients consume the *.done events / final response for
// arguments, so repairing here covers the streamed path too.
+ //
+ // No schema verdict here, deliberately. This surface commits on the
+ // FIRST tool-call delta (`commit()` above, where the function_call
+ // item is opened) — the arguments are not complete until this point,
+ // which is long after the skeleton and the item.added events have
+ // left. A verdict here could only turn a delivered-but-invalid call
+ // into a `response.failed` on a stream the client is already reading,
+ // which is strictly worse. Wiring it would mean holding the
+ // function_call item until its arguments finish, the way
+ // /chat/completions buffers — a change to this route's commit point,
+ // not to validation.
const finalToolCalls: ChatToolCall[] = [];
for (const acc of toolAcc.values()) {
const repairedArgs = repairToolArguments(acc.args, toolSchemas.get(acc.name));
@@ -851,6 +863,14 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
text = rescue.cleanText;
}
}
+
+ // Opt-in schema verdict on what the repair could not fix. Thrown before
+ // anything is written, so the failover hop is invisible to the client.
+ if (isToolArgumentValidationEnabled() && toolCalls.length > 0) {
+ const invalid = invalidToolCallReasons(toolCalls, toolSchemas);
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
+
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
// #764: include reasoning tokens (message.reasoning_content / reasoning)
// in the chars/4 estimate so thinking models aren't undercounted when the