Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(tools): opt-in schema verdict on tool arguments, failing over when invalid (#802)

OhOkThisIsFine0ed6dba2617d
diff --git a/.env.example b/.env.example
index 618b352e..31881102 100644
--- a/.env.example
+++ b/.env.example
@@ -112,6 +112,17 @@ PORT=3001
# expose_fallback_detail_header settings key, which takes precedence.
# FALLBACK_DETAIL_HEADER=1
+# Check tool-call arguments against the schema the caller declared, and fail
+# over to another model when they violate it (default: off). lib/tool-args.ts
+# already repairs what it can prove is repairable; this catches what is left,
+# which today reaches Anthropic clients as a tool_use block with `input: {}` and
+# no indication anything went wrong. A different model usually gets the same
+# call right. Off by default because a false positive costs a failover hop —
+# turn it on, watch X-Fallback-Trail for `invalid_tool_arguments`, then decide.
+# Can also be set at runtime via the validate_tool_arguments settings key,
+# which takes precedence.
+# VALIDATE_TOOL_ARGUMENTS=1
+
# Request guardrails, both OFF (0) by default. Also runtime-tunable via the
# request_max_tokens_budget / max_consecutive_upstream_fails settings keys
# (dashboard API PUT /api/settings/guardrails), which take precedence.
diff --git a/package-lock.json b/package-lock.json
index b3b01b3f..6e1e6422 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -22,6 +22,7 @@
"cli": {
"name": "freellmapi",
"version": "0.3.0",
+ "license": "MIT",
"dependencies": {
"jsonc-parser": "^3.3.1"
},
@@ -12217,6 +12218,7 @@
"version": "0.2.1",
"dependencies": {
"@freellmapi/shared": "*",
+ "ajv": "^8.20.0",
"compression": "^1.8.0",
"cors": "^2.8.5",
"dotenv": "^17.4.2",
@@ -12256,6 +12258,28 @@
"undici-types": "~6.21.0"
}
},
+ "server/node_modules/ajv": {
+ "version": "8.20.0",
+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",
+ "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==",
+ "license": "MIT",
+ "dependencies": {
+ "fast-deep-equal": "^3.1.3",
+ "fast-uri": "^3.0.1",
+ "json-schema-traverse": "^1.0.0",
+ "require-from-string": "^2.0.2"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/epoberezkin"
+ }
+ },
+ "server/node_modules/json-schema-traverse": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
+ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
+ "license": "MIT"
+ },
"server/node_modules/typescript": {
"version": "5.9.3",
"resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",
diff --git a/server/package.json b/server/package.json
index ca88bbc5..e0cf1711 100644
--- a/server/package.json
+++ b/server/package.json
@@ -19,6 +19,7 @@
},
"dependencies": {
"@freellmapi/shared": "*",
+ "ajv": "^8.20.0",
"compression": "^1.8.0",
"cors": "^2.8.5",
"dotenv": "^17.4.2",
diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.ts
index 5491b64d..b3f2c129 100644
--- a/server/src/__tests__/lib/fallback-loop.test.ts
+++ b/server/src/__tests__/lib/fallback-loop.test.ts
@@ -39,6 +39,7 @@ import {
isProviderLevelError,
newClientAbortError,
} from '../../lib/error-classify.js';
+import { invalidToolArgumentsError } from '../../lib/tool-validate.js';
import { getAllPenalties } from '../../services/router.js';
import type { RouteResult } from '../../services/router.js';
@@ -598,6 +599,25 @@ describe('isProviderLevelError (#788: the PROVIDER is sick, not this key)', () =
// A vanished client says nothing about provider health.
expect(isProviderLevelError(newClientAbortError())).toBe(false);
});
+
+ it('never condemns a platform for MODEL behavior, however the message reads', () => {
+ // These messages quote caller- and model-supplied text: a tool called
+ // `set_timeout`, or an Ajv complaint about the instance path `/timeout`,
+ // puts a timeout marker in the sentence without a timeout having happened.
+ // `skipModelForRequest` is the structured truth and outranks the text.
+ expect(isProviderLevelError(invalidToolArgumentsError(
+ 'alpha alpha-big',
+ ['set_timeout: /timeout must be number'],
+ ))).toBe(false);
+ expect(isProviderLevelError(invalidToolArgumentsError(
+ 'alpha alpha-big',
+ ['run_query: /mode must be equal to one of the allowed values (degraded)'],
+ ))).toBe(false);
+ expect(isProviderLevelError(Object.assign(
+ new Error('alpha alpha-big ignored response_format (returned non-JSON despite json_object)'),
+ { skipBench: true, skipModelForRequest: true },
+ ))).toBe(false);
+ });
});
describe('runFallbackLoop: a provider-level failure skips the whole platform (#788)', () => {
@@ -662,6 +682,27 @@ describe('runFallbackLoop: a provider-level failure skips the whole platform (#7
expect(state.skipPlatforms.size).toBe(0);
});
+ it('invalid tool arguments rule out the MODEL, never the platform', async () => {
+ // The provider served the turn perfectly; the model wrote a bad call. The
+ // sibling key would write the same one (skipModelForRequest), but alpha's
+ // OTHER model is still a fine next hop — skipping the platform here would
+ // strand it, and the tool name in the message carries a timeout marker
+ // precisely to prove the text does not drive the decision.
+ const state = newFallbackState();
+ const dispatch = vi.fn()
+ .mockRejectedValueOnce(invalidToolArgumentsError('alpha alpha-big', ['set_timeout: /timeout must be number']))
+ .mockResolvedValueOnce('done');
+
+ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, state, route: miniRouter(state), dispatch }));
+
+ expect(dispatch).toHaveBeenCalledTimes(2);
+ const second = dispatch.mock.calls[1][0] as RouteResult;
+ expect(second.platform).toBe('alpha');
+ expect(second.modelId).toBe('alpha-small'); // not the sibling key of alpha-big
+ expect(state.skipPlatforms.size).toBe(0);
+ expect(state.skipModels.has(788_001)).toBe(true);
+ });
+
it('a timeout with no HTTP status rules the platform out too', async () => {
const state = newFallbackState();
const dispatch = vi.fn()
diff --git a/server/src/__tests__/lib/tool-validate.test.ts b/server/src/__tests__/lib/tool-validate.test.ts
new file mode 100644
index 00000000..4065467d
--- /dev/null
+++ b/server/src/__tests__/lib/tool-validate.test.ts
@@ -0,0 +1,175 @@
+import { describe, it, expect, beforeAll, afterEach } from 'vitest';
+import {
+ TOOL_ARGUMENT_VALIDATION_SETTING,
+ invalidToolArgumentsError,
+ invalidToolCallReasons,
+ isToolArgumentValidationEnabled,
+ stableStringify,
+ validateToolArguments,
+} from '../../lib/tool-validate.js';
+import { isRetryableError } from '../../lib/error-classify.js';
+import { classifyAttemptError } from '../../lib/fallback-loop.js';
+import { initDb, getDb, setSetting } from '../../db/index.js';
+
+const WEATHER_SCHEMA = {
+ type: 'object',
+ properties: { city: { type: 'string' }, days: { type: 'number' } },
+ required: ['city'],
+};
+
+const call = (name: string, args: string) => ({ function: { name, arguments: args } });
+
+describe('validateToolArguments', () => {
+ it('accepts arguments that satisfy the schema', () => {
+ expect(validateToolArguments('get_weather', '{"city":"Berlin"}', WEATHER_SCHEMA)).toEqual({ ok: true });
+ });
+
+ it('reports a missing required property', () => {
+ const verdict = validateToolArguments('get_weather', '{"days":3}', WEATHER_SCHEMA);
+ expect(verdict.ok).toBe(false);
+ if (!verdict.ok) expect(verdict.reason).toContain('city');
+ });
+
+ it('reports a wrong scalar type', () => {
+ const verdict = validateToolArguments('get_weather', '{"city":"Berlin","days":"three"}', WEATHER_SCHEMA);
+ expect(verdict.ok).toBe(false);
+ if (!verdict.ok) expect(verdict.reason).toContain('days');
+ });
+
+ it('names the tool in the reason, so a multi-call turn is diagnosable', () => {
+ const verdict = validateToolArguments('get_weather', '{}', WEATHER_SCHEMA);
+ if (!verdict.ok) expect(verdict.reason.startsWith('get_weather:')).toBe(true);
+ });
+});
+
+// Every branch where we cannot form an opinion must pass, because the cost of a
+// verdict is a failover hop.
+describe('validateToolArguments fails open', () => {
+ it('with no schema', () => {
+ expect(validateToolArguments('t', '{"anything":1}', undefined)).toEqual({ ok: true });
+ expect(validateToolArguments('t', '{"anything":1}', null)).toEqual({ ok: true });
+ });
+
+ it('with a schema Ajv cannot compile', () => {
+ expect(validateToolArguments('t', '{}', { type: 'not-a-real-type' })).toEqual({ ok: true });
+ });
+
+ it('with an unrecognised $schema dialect, which is stripped before compiling', () => {
+ const schema = { $schema: 'https://example.invalid/draft-99', ...WEATHER_SCHEMA };
+ expect(validateToolArguments('t', '{"city":"Berlin"}', schema)).toEqual({ ok: true });
+ // Still a real check afterwards, not a blanket pass.
+ expect(validateToolArguments('t', '{}', schema).ok).toBe(false);
+ });
+
+ it('with arguments that are not JSON at all', () => {
+ // tool-args deliberately passes unparseable arguments through untouched;
+ // reporting them is someone else's job.
+ expect(validateToolArguments('t', '{not json', WEATHER_SCHEMA)).toEqual({ ok: true });
+ expect(validateToolArguments('t', '', WEATHER_SCHEMA)).toEqual({ ok: true });
+ });
+
+ it('with arguments that are not an object', () => {
+ expect(validateToolArguments('t', '[1,2]', WEATHER_SCHEMA)).toEqual({ ok: true });
+ expect(validateToolArguments('t', '"a string"', WEATHER_SCHEMA)).toEqual({ ok: true });
+ });
+
+ it('tolerates vendor extensions rather than rejecting the schema', () => {
+ const schema = { ...WEATHER_SCHEMA, 'x-vendor-hint': 'something', additionalProperties: false };
+ expect(validateToolArguments('t', '{"city":"Berlin"}', schema)).toEqual({ ok: true });
+ });
+});
+
+describe('stableStringify', () => {
+ it('is order-independent, so key order does not fork the compile cache', () => {
+ expect(stableStringify({ a: 1, b: 2 })).toBe(stableStringify({ b: 2, a: 1 }));
+ });
+
+ it('still distinguishes different schemas', () => {
+ expect(stableStringify({ a: 1 })).not.toBe(stableStringify({ a: 2 }));
+ });
+
+ it('handles nesting and arrays', () => {
+ expect(stableStringify({ x: [{ b: 1, a: 2 }] })).toBe(stableStringify({ x: [{ a: 2, b: 1 }] }));
+ });
+});
+
+describe('invalidToolCallReasons', () => {
+ const schemas = new Map<string, unknown>([['get_weather', WEATHER_SCHEMA]]);
+
+ it('is empty for a well-formed turn', () => {
+ expect(invalidToolCallReasons([call('get_weather', '{"city":"Berlin"}')], schemas)).toEqual([]);
+ });
+
+ it('collects one reason per bad call', () => {
+ const reasons = invalidToolCallReasons(
+ [call('get_weather', '{}'), call('get_weather', '{"city":1}')],
+ schemas,
+ );
+ expect(reasons).toHaveLength(2);
+ });
+
+ it('ignores a tool the caller declared no schema for', () => {
+ expect(invalidToolCallReasons([call('unknown_tool', '{"whatever":1}')], schemas)).toEqual([]);
+ });
+
+ it('is empty for no calls at all', () => {
+ expect(invalidToolCallReasons(undefined, schemas)).toEqual([]);
+ expect(invalidToolCallReasons([], schemas)).toEqual([]);
+ });
+});
+
+describe('invalidToolArgumentsError', () => {
+ const err = invalidToolArgumentsError('Groq llama-3.3-70b', ['get_weather: /city must be string']);
+
+ it('is recognised as retryable, so the loop fails over', () => {
+ expect(isRetryableError(err)).toBe(true);
+ });
+
+ it('classifies honestly in the trail rather than as a bare error', () => {
+ expect(classifyAttemptError(err)).toBe('invalid_tool_arguments');
+ });
+
+ it('spares the provider: the model misbehaved, not the endpoint', () => {
+ // skipBench -> no cooldown or score penalty; skipModelForRequest -> a
+ // sibling key would misbehave identically.
+ expect((err as any).skipBench).toBe(true);
+ expect((err as any).skipModelForRequest).toBe(true);
+ });
+
+ it('names the model and the reason', () => {
+ expect(err.message).toContain('Groq llama-3.3-70b');
+ expect(err.message).toContain('/city must be string');
+ });
+});
+
+describe('isToolArgumentValidationEnabled', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ afterEach(() => {
+ delete process.env.VALIDATE_TOOL_ARGUMENTS;
+ getDb().prepare('DELETE FROM settings WHERE key = ?').run(TOOL_ARGUMENT_VALIDATION_SETTING);
+ });
+
+ it('is off by default — a false positive costs a failover hop', () => {
+ expect(isToolArgumentValidationEnabled()).toBe(false);
+ });
+
+ it('honours the settings-table value', () => {
+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');
+ expect(isToolArgumentValidationEnabled()).toBe(true);
+ });
+
+ it('falls back to the env var when nothing is stored', () => {
+ process.env.VALIDATE_TOOL_ARGUMENTS = 'true';
+ expect(isToolArgumentValidationEnabled()).toBe(true);
+ });
+
+ it('lets the settings table win over the env var', () => {
+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '0');
+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';
+ expect(isToolArgumentValidationEnabled()).toBe(false);
+ });
+});
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index 9ef37a04..25833815 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -96,7 +96,13 @@ export function isRetryableError(err: any): boolean {
// First-byte timeout (#584): the grace budget expired before ANY byte
// reached the client, so the next candidate can serve it invisibly.
|| msg.includes('no first byte')
- || msg.includes('unparseable inline tool-call dialect');
+ || msg.includes('unparseable inline tool-call dialect')
+ // The model emitted a tool call whose arguments violate the schema the
+ // caller declared (opt-in check, lib/tool-validate.ts). Thrown before any
+ // byte reached the client, and a different model usually gets the same
+ // call right — the thrower marks the model skipped for this request, since
+ // a sibling key would misbehave identically.
+ || msg.includes('invalid tool arguments');
}
// A genuine provider QUOTA signal: a structured 429 or rate-limit/quota wording.
@@ -256,6 +262,14 @@ export function isProviderDegradedError(err: any): boolean {
// unavailable model would each condemn a healthy platform for the whole request.
// A message check therefore only runs when there is no status to trust.
export function isProviderLevelError(err: any): boolean {
+ // A failure the thrower explicitly scoped to the MODEL (`skipModelForRequest`
+ // — an ignored response_format, invalid tool arguments) is never provider
+ // health, and its message quotes caller- and model-supplied text: a tool
+ // named `set_timeout`, or an Ajv complaint about an instance path `/timeout`,
+ // would otherwise trip the substring checks below and condemn a healthy
+ // platform for the whole request. The structured marker is authoritative;
+ // the text is not.
+ if (err?.skipModelForRequest === true) return false;
const status = typeof err?.status === 'number' ? err.status : 0;
if (status >= 500) return true;
// A DEGRADED hosted deployment (NVIDIA NIM, #522) is provider health wearing
diff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.ts
index 7aad94ac..d407a663 100644
--- a/server/src/lib/fallback-loop.ts
+++ b/server/src/lib/fallback-loop.ts
@@ -406,6 +406,7 @@ export type AttemptErrorClass =
| 'provider_bad_request'
| 'empty_completion'
| 'format_ignored'
+ | 'invalid_tool_arguments'
| 'timeout'
| 'rate_limited'
| 'upstream_error'
@@ -435,6 +436,9 @@ export function classifyAttemptError(err: any): AttemptErrorClass {
const msg = (err?.message ?? '').toLowerCase();
if (msg.includes('empty completion')) return 'empty_completion';
if (msg.includes('ignored response_format') || msg.includes('truncated json')) return 'format_ignored';
+ // Before the generic classes so the trail names the real cause rather than
+ // booking a schema violation as a bare 'error'.
+ if (msg.includes('invalid tool arguments')) return 'invalid_tool_arguments';
if (isTimeoutErrorText(msg)) return 'timeout';
if (msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') || msg.includes('quota')) return 'rate_limited';
const status = typeof err?.status === 'number' ? err.status : 0;
diff --git a/server/src/lib/tool-validate.ts b/server/src/lib/tool-validate.ts
new file mode 100644
index 00000000..7f0f424e
--- /dev/null
+++ b/server/src/lib/tool-validate.ts
@@ -0,0 +1,179 @@
+import Ajv2020 from 'ajv/dist/2020.js';
+import type { ValidateFunction } from 'ajv';
+import { getSetting } from '../db/index.js';
+
+// A verdict on whether a tool call's arguments actually satisfy the schema the
+// caller declared for that tool.
+//
+// lib/tool-args.ts repairs what it can prove is repairable, and deliberately
+// invents nothing. What it cannot do is say "this is still wrong" — so a call
+// that violates its schema flows on unchallenged, and the Anthropic surface
+// turns it into `input: {}` (routes/anthropic.ts parseToolInput does
+// `try { JSON.parse } catch { return {} }`). The client then gets a tool_use
+// block with no arguments and no explanation, which reads to an agent as the
+// model having chosen to call the tool with nothing.
+//
+// Free-tier models are exactly the population that emits malformed tool calls,
+// and a different model usually gets it right, so the useful response is the
+// one the loop already has for a dead turn: throw before anything is committed
+// and let failover try the next candidate.
+//
+// OFF by default. A false positive here costs a failover hop, and schemas in
+// the wild are strange enough that the honest default is to look before
+// leaping — enable it, watch the trail, then decide.
+
+/** 2020-12, because that is what zod-to-json-schema and Pydantic v2 emit; a
+ * draft-07-only Ajv throws on their `$schema`. `strict: false` keeps an
+ * unknown keyword from being fatal — plenty of real schemas carry vendor
+ * extensions, and rejecting the schema would fail calls that are fine. */
+const ajv = new (Ajv2020 as unknown as typeof import('ajv').default)({
+ strict: false,
+ allErrors: true,
+ validateFormats: false,
+});
+
+export const TOOL_ARGUMENT_VALIDATION_SETTING = 'validate_tool_arguments';
+
+export function isToolArgumentValidationEnabled(): boolean {
+ let stored: string | undefined;
+ try {
+ stored = getSetting(TOOL_ARGUMENT_VALIDATION_SETTING);
+ } catch {
+ stored = undefined; // DB not ready — never throw on the proxy hot path.
+ }
+ for (const raw of [stored, process.env.VALIDATE_TOOL_ARGUMENTS]) {
+ if (raw === undefined || raw.trim() === '') continue;
+ const value = raw.trim().toLowerCase();
+ return value === '1' || value === 'true';
+ }
+ return false;
+}
+
+/**
+ * Order-independent serialisation, so two schemas that differ only in key order
+ * share one compiled validator.
+ */
+export function stableStringify(value: unknown): string {
+ if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+ const entries = Object.entries(value as Record<string, unknown>)
+ .filter(([, v]) => v !== undefined)
+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))
+ .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`);
+ return `{${entries.join(',')}}`;
+}
+
+/** Compiled validators, keyed by the schema's stable form. `null` is cached
+ * too: a schema Ajv refuses to compile must not be recompiled per request. */
+const compiled = new Map<string, ValidateFunction | null>();
+/** The stable string is the expensive half, and tool schemas are the same
+ * object for every call in a request, so memoise per object identity. */
+const stableKeys = new WeakMap<object, string>();
+
+function keyFor(schema: object): string {
+ const cached = stableKeys.get(schema);
+ if (cached !== undefined) return cached;
+ const key = stableStringify(schema);
+ stableKeys.set(schema, key);
+ return key;
+}
+
+function compiledFor(schema: object): ValidateFunction | null {
+ const key = keyFor(schema);
+ const hit = compiled.get(key);
+ if (hit !== undefined) return hit;
+
+ let fn: ValidateFunction | null = null;
+ try {
+ // Drop `$schema`: a declared dialect Ajv does not recognise is fatal at
+ // compile time, and the dialect is not what we are checking.
+ const { $schema, ...rest } = schema as Record<string, unknown>;
+ void $schema;
+ fn = ajv.compile(rest);
+ } catch {
+ fn = null; // Unusable schema — see validateToolArguments, which fails OPEN.
+ }
+ compiled.set(key, fn);
+ return fn;
+}
+
+export type ToolArgumentVerdict =
+ | { ok: true }
+ | { ok: false; reason: string };
+
+/**
+ * Check `argsJson` against `schema`.
+ *
+ * Fails **open** in every case where we cannot form an opinion: no schema, a
+ * schema Ajv will not compile, or arguments that are not JSON at all (the last
+ * is someone else's error to report, and tool-args deliberately passes
+ * unparseable arguments through untouched). Only a definite schema violation
+ * returns a verdict, because the consequence of a verdict is a failover hop.
+ */
+export function validateToolArguments(
+ toolName: string,
+ argsJson: string,
+ schema: unknown,
+): ToolArgumentVerdict {
+ if (!schema || typeof schema !== 'object') return { ok: true };
+
+ const validate = compiledFor(schema as object);
+ if (!validate) return { ok: true };
+
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(argsJson);
+ } catch {
+ return { ok: true };
+ }
+ // A tool call's arguments are an object by definition; anything else is a
+ // shape problem the schema will report on its own terms if it cares.
+ if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return { ok: true };
+
+ if (validate(parsed)) return { ok: true };
+
+ const detail = (validate.errors ?? [])
+ .slice(0, 3)
+ .map(e => `${e.instancePath || '/'} ${e.message ?? 'is invalid'}`)
+ .join('; ');
+ return { ok: false, reason: `${toolName}: ${detail || 'does not match its schema'}` };
+}
+
+/**
+ * The error thrown when a tool call cannot be served. The message carries the
+ * marker `invalid tool arguments`, which `isRetryableError` recognises, so the
+ * loop fails over exactly as it does for a dead dialect turn.
+ *
+ * `skipBench` because the PROVIDER is healthy — the model misbehaved, so no
+ * cooldown or score penalty. `skipModelForRequest` because a sibling key would
+ * misbehave identically, so the whole model is ruled out for this request.
+ */
+export function invalidToolArgumentsError(displayName: string, reasons: string[]): Error {
+ const detail = reasons.slice(0, 3).join('; ');
+ return Object.assign(
+ new Error(`invalid tool arguments from ${displayName} (${detail})`),
+ { skipBench: true, skipModelForRequest: true },
+ );
+}
+
+/**
+ * Verdicts for a whole turn's tool calls. Returns the reasons for those that
+ * violate their schema; an empty array means the turn is servable.
+ */
+export function invalidToolCallReasons(
+ calls: Array<{ function?: { name?: string; arguments?: string } }> | undefined,
+ schemas: Map<string, unknown>,
+): string[] {
+ if (!calls || calls.length === 0) return [];
+ const reasons: string[] = [];
+ for (const call of calls) {
+ const name = call.function?.name;
+ const args = call.function?.arguments;
+ if (!name || typeof args !== 'string') continue;
+ const schema = schemas.get(name);
+ if (schema === undefined) continue; // Tool the caller never declared a schema for.
+ const verdict = validateToolArguments(name, args, schema);
+ if (!verdict.ok) reasons.push(verdict.reason);
+ }
+ return reasons;
+}
diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.ts
index 6a477e06..3a4b4adf 100644
--- a/server/src/routes/anthropic.ts
+++ b/server/src/routes/anthropic.ts
@@ -13,6 +13,7 @@ import { routeRequest, resolveStickyPreference, routingReserveTokens, type Route
import { getSetting, getUnifiedApiKey } from '../db/index.js';
import { contentToString } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
+import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropic-documents.js';
@@ -617,6 +618,14 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {
tc.function.arguments = repairToolArguments(tc.function.arguments, schemas.get(tc.function.name));
}
}
+ // Opt-in schema verdict on what the repair could not fix. This surface
+ // is where the silent failure was worst: parseToolInput turns
+ // unparseable arguments into `input: {}`, so the client sees a tool_use
+ // block with nothing in it and no indication anything went wrong.
+ if (isToolArgumentValidationEnabled()) {
+ const invalid = invalidToolCallReasons(respToolCalls, schemas);
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
}
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
@@ -863,6 +872,20 @@ async function streamCompletion(
heldText = '';
}
+ // Opt-in schema verdict, same rule as the non-streaming surface above and
+ // as /chat/completions: this is the surface the silent failure hurt most,
+ // and Claude Code streams. Placed after the dialect rescue so a rescued
+ // call is judged too, and gated on `!messageStarted` — once message_start
+ // has gone out there is no failing over, and tearing the SSE stream down
+ // would be worse for the client than a tool_use the schema dislikes.
+ if (isToolArgumentValidationEnabled() && !messageStarted && completedCalls.length > 0) {
+ const invalid = invalidToolCallReasons(
+ completedCalls.map(c => ({ function: { name: c.name, arguments: c.arguments } })),
+ schemas,
+ );
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
+
// Nothing usable came out — fail over (message_start was never sent, so the
// client never saw this attempt).
if (!messageStarted && completedCalls.length === 0) {
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index a944a880..94c35bcd 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -12,6 +12,7 @@ import { getDb } from '../db/index.js';
import { resolveAuth, prependSystemPrompt, type ResolvedAuth } from '../lib/system-prompt.js';
import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
+import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';
@@ -2018,6 +2019,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}))
.filter(c => { try { JSON.parse(c.function.arguments); return c.function.name.length > 0; } catch { return false; } });
+ // Opt-in schema verdict. `!headerSent` is the whole licence to throw
+ // here: the commit point is held until the first meaningful content,
+ // so the common tool-call turn (no prose before the call) has sent no
+ // bytes yet and can still fail over invisibly. A turn that already
+ // flushed prose is past the point of no return — the catch below
+ // would have to tear the SSE stream down with a `stream_error`, which
+ // is strictly worse for the client than forwarding a tool call the
+ // schema dislikes. Off-by-default or not, this check must never turn
+ // a served answer into a broken one.
+ if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {
+ const invalid = invalidToolCallReasons(completedCalls, schemas);
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
+
// Dialect rescue: the held text is an inline tool call in some
// model's private syntax. Parse it into structured calls or treat
// the turn as dead (headers were never sent in dialect mode, so
@@ -2221,11 +2236,38 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
}
+ // Repair double-encoded tool arguments against the request's tool
+ // schemas (e.g. GLM emitting an array parameter as a JSON string),
+ // so strict clients don't reject the call. Schema-gated — a true
+ // string parameter is never touched. See lib/tool-args.ts.
+ //
+ // Deliberately BEFORE the success bookkeeping below: the opt-in schema
+ // verdict that follows can still fail this attempt over, and crediting
+ // recordUpstreamSuccess / rememberReasoning / setStickyModel to an
+ // attempt we are about to discard would bill a model that never served
+ // the turn and pin the session to it for the next one.
+ if (respMsg?.tool_calls?.length) {
+ const schemas = toolSchemaMap(tools);
+ for (const tc of respMsg.tool_calls) {
+ if (tc?.function?.arguments != null) {
+ tc.function.arguments = repairToolArguments(tc.function.arguments, schemas.get(tc.function.name));
+ }
+ }
+ // Whatever the repair could not fix is still broken. Opt-in, and
+ // thrown before anything is written, so failover is invisible.
+ if (isToolArgumentValidationEnabled()) {
+ const invalid = invalidToolCallReasons(respMsg.tool_calls, schemas);
+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);
+ }
+ }
+
// Usage fallback: providers that omit `usage` used to be logged as 0
// tokens, silently undercounting analytics and the rate-limit ledger.
// Fall back to the same chars/4 estimate the streaming path uses (tool
- // arguments included, mirroring the stream accounting; reasoning tokens
- // included too, so thinking models aren't undercounted — #764).
+ // arguments included, mirroring the stream accounting; counted after
+ // the repair above, so it measures the bytes actually sent, and
+ // reasoning tokens included too, so thinking models aren't
+ // undercounted — #764).
const respToolArgChars = (respMsg?.tool_calls ?? []).reduce((n, tc) => n + (tc?.function?.arguments?.length ?? 0), 0);
const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;
const completionTokens = result.usage?.completion_tokens
@@ -2249,18 +2291,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
setFallbackHeaders(res, attempt, attemptLog);
- // Repair double-encoded tool arguments against the request's tool
- // schemas (e.g. GLM emitting an array parameter as a JSON string),
- // so strict clients don't reject the call. Schema-gated — a true
- // string parameter is never touched. See lib/tool-args.ts.
- if (respMsg?.tool_calls?.length) {
- const schemas = toolSchemaMap(tools);
- for (const tc of respMsg.tool_calls) {
- if (tc?.function?.arguments != null) {
- tc.function.arguments = repairToolArguments(tc.function.arguments, schemas.get(tc.function.name));
- }
- }
- }
// Normalize array-shaped message.content to a string on the way out (#166).
const outboundBody = sanitizeResponse(normalizeOutboundContent(result));
res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');