7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(tools): opt-in schema verdict on tool arguments, failing over when invalid (#802)
OhOkThisIsFine0ed6dba2617d
diff --git a/.env.example b/.env.exampleindex 618b352e..31881102 100644--- a/.env.example+++ b/.env.example@@ -112,6 +112,17 @@ PORT=3001# expose_fallback_detail_header settings key, which takes precedence.# FALLBACK_DETAIL_HEADER=1+# Check tool-call arguments against the schema the caller declared, and fail+# over to another model when they violate it (default: off). lib/tool-args.ts+# already repairs what it can prove is repairable; this catches what is left,+# which today reaches Anthropic clients as a tool_use block with `input: {}` and+# no indication anything went wrong. A different model usually gets the same+# call right. Off by default because a false positive costs a failover hop —+# turn it on, watch X-Fallback-Trail for `invalid_tool_arguments`, then decide.+# Can also be set at runtime via the validate_tool_arguments settings key,+# which takes precedence.+# VALIDATE_TOOL_ARGUMENTS=1+# Request guardrails, both OFF (0) by default. Also runtime-tunable via the# request_max_tokens_budget / max_consecutive_upstream_fails settings keys# (dashboard API PUT /api/settings/guardrails), which take precedence.diff --git a/package-lock.json b/package-lock.jsonindex b3b01b3f..6e1e6422 100644--- a/package-lock.json+++ b/package-lock.json@@ -22,6 +22,7 @@"cli": {"name": "freellmapi","version": "0.3.0",+ "license": "MIT","dependencies": {"jsonc-parser": "^3.3.1"},@@ -12217,6 +12218,7 @@"version": "0.2.1","dependencies": {"@freellmapi/shared": "*",+ "ajv": "^8.20.0","compression": "^1.8.0","cors": "^2.8.5","dotenv": "^17.4.2",@@ -12256,6 +12258,28 @@"undici-types": "~6.21.0"}},+ "server/node_modules/ajv": {+ "version": "8.20.0",+ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",+ "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==",+ "license": "MIT",+ "dependencies": {+ "fast-deep-equal": "^3.1.3",+ "fast-uri": "^3.0.1",+ "json-schema-traverse": "^1.0.0",+ "require-from-string": "^2.0.2"+ },+ "funding": {+ "type": "github",+ "url": "https://github.com/sponsors/epoberezkin"+ }+ },+ "server/node_modules/json-schema-traverse": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",+ "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",+ "license": "MIT"+ },"server/node_modules/typescript": {"version": "5.9.3","resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz",diff --git a/server/package.json b/server/package.jsonindex ca88bbc5..e0cf1711 100644--- a/server/package.json+++ b/server/package.json@@ -19,6 +19,7 @@},"dependencies": {"@freellmapi/shared": "*",+ "ajv": "^8.20.0","compression": "^1.8.0","cors": "^2.8.5","dotenv": "^17.4.2",diff --git a/server/src/__tests__/lib/fallback-loop.test.ts b/server/src/__tests__/lib/fallback-loop.test.tsindex 5491b64d..b3f2c129 100644--- a/server/src/__tests__/lib/fallback-loop.test.ts+++ b/server/src/__tests__/lib/fallback-loop.test.ts@@ -39,6 +39,7 @@ import {isProviderLevelError,newClientAbortError,} from '../../lib/error-classify.js';+import { invalidToolArgumentsError } from '../../lib/tool-validate.js';import { getAllPenalties } from '../../services/router.js';import type { RouteResult } from '../../services/router.js';@@ -598,6 +599,25 @@ describe('isProviderLevelError (#788: the PROVIDER is sick, not this key)', () =// A vanished client says nothing about provider health.expect(isProviderLevelError(newClientAbortError())).toBe(false);});++ it('never condemns a platform for MODEL behavior, however the message reads', () => {+ // These messages quote caller- and model-supplied text: a tool called+ // `set_timeout`, or an Ajv complaint about the instance path `/timeout`,+ // puts a timeout marker in the sentence without a timeout having happened.+ // `skipModelForRequest` is the structured truth and outranks the text.+ expect(isProviderLevelError(invalidToolArgumentsError(+ 'alpha alpha-big',+ ['set_timeout: /timeout must be number'],+ ))).toBe(false);+ expect(isProviderLevelError(invalidToolArgumentsError(+ 'alpha alpha-big',+ ['run_query: /mode must be equal to one of the allowed values (degraded)'],+ ))).toBe(false);+ expect(isProviderLevelError(Object.assign(+ new Error('alpha alpha-big ignored response_format (returned non-JSON despite json_object)'),+ { skipBench: true, skipModelForRequest: true },+ ))).toBe(false);+ });});describe('runFallbackLoop: a provider-level failure skips the whole platform (#788)', () => {@@ -662,6 +682,27 @@ describe('runFallbackLoop: a provider-level failure skips the whole platform (#7expect(state.skipPlatforms.size).toBe(0);});+ it('invalid tool arguments rule out the MODEL, never the platform', async () => {+ // The provider served the turn perfectly; the model wrote a bad call. The+ // sibling key would write the same one (skipModelForRequest), but alpha's+ // OTHER model is still a fine next hop — skipping the platform here would+ // strand it, and the tool name in the message carries a timeout marker+ // precisely to prove the text does not drive the decision.+ const state = newFallbackState();+ const dispatch = vi.fn()+ .mockRejectedValueOnce(invalidToolArgumentsError('alpha alpha-big', ['set_timeout: /timeout must be number']))+ .mockResolvedValueOnce('done');++ await runFallbackLoop(hooksSkeleton({ maxRetries: 20, state, route: miniRouter(state), dispatch }));++ expect(dispatch).toHaveBeenCalledTimes(2);+ const second = dispatch.mock.calls[1][0] as RouteResult;+ expect(second.platform).toBe('alpha');+ expect(second.modelId).toBe('alpha-small'); // not the sibling key of alpha-big+ expect(state.skipPlatforms.size).toBe(0);+ expect(state.skipModels.has(788_001)).toBe(true);+ });+it('a timeout with no HTTP status rules the platform out too', async () => {const state = newFallbackState();const dispatch = vi.fn()diff --git a/server/src/__tests__/lib/tool-validate.test.ts b/server/src/__tests__/lib/tool-validate.test.tsnew file mode 100644index 00000000..4065467d--- /dev/null+++ b/server/src/__tests__/lib/tool-validate.test.ts@@ -0,0 +1,175 @@+import { describe, it, expect, beforeAll, afterEach } from 'vitest';+import {+ TOOL_ARGUMENT_VALIDATION_SETTING,+ invalidToolArgumentsError,+ invalidToolCallReasons,+ isToolArgumentValidationEnabled,+ stableStringify,+ validateToolArguments,+} from '../../lib/tool-validate.js';+import { isRetryableError } from '../../lib/error-classify.js';+import { classifyAttemptError } from '../../lib/fallback-loop.js';+import { initDb, getDb, setSetting } from '../../db/index.js';++const WEATHER_SCHEMA = {+ type: 'object',+ properties: { city: { type: 'string' }, days: { type: 'number' } },+ required: ['city'],+};++const call = (name: string, args: string) => ({ function: { name, arguments: args } });++describe('validateToolArguments', () => {+ it('accepts arguments that satisfy the schema', () => {+ expect(validateToolArguments('get_weather', '{"city":"Berlin"}', WEATHER_SCHEMA)).toEqual({ ok: true });+ });++ it('reports a missing required property', () => {+ const verdict = validateToolArguments('get_weather', '{"days":3}', WEATHER_SCHEMA);+ expect(verdict.ok).toBe(false);+ if (!verdict.ok) expect(verdict.reason).toContain('city');+ });++ it('reports a wrong scalar type', () => {+ const verdict = validateToolArguments('get_weather', '{"city":"Berlin","days":"three"}', WEATHER_SCHEMA);+ expect(verdict.ok).toBe(false);+ if (!verdict.ok) expect(verdict.reason).toContain('days');+ });++ it('names the tool in the reason, so a multi-call turn is diagnosable', () => {+ const verdict = validateToolArguments('get_weather', '{}', WEATHER_SCHEMA);+ if (!verdict.ok) expect(verdict.reason.startsWith('get_weather:')).toBe(true);+ });+});++// Every branch where we cannot form an opinion must pass, because the cost of a+// verdict is a failover hop.+describe('validateToolArguments fails open', () => {+ it('with no schema', () => {+ expect(validateToolArguments('t', '{"anything":1}', undefined)).toEqual({ ok: true });+ expect(validateToolArguments('t', '{"anything":1}', null)).toEqual({ ok: true });+ });++ it('with a schema Ajv cannot compile', () => {+ expect(validateToolArguments('t', '{}', { type: 'not-a-real-type' })).toEqual({ ok: true });+ });++ it('with an unrecognised $schema dialect, which is stripped before compiling', () => {+ const schema = { $schema: 'https://example.invalid/draft-99', ...WEATHER_SCHEMA };+ expect(validateToolArguments('t', '{"city":"Berlin"}', schema)).toEqual({ ok: true });+ // Still a real check afterwards, not a blanket pass.+ expect(validateToolArguments('t', '{}', schema).ok).toBe(false);+ });++ it('with arguments that are not JSON at all', () => {+ // tool-args deliberately passes unparseable arguments through untouched;+ // reporting them is someone else's job.+ expect(validateToolArguments('t', '{not json', WEATHER_SCHEMA)).toEqual({ ok: true });+ expect(validateToolArguments('t', '', WEATHER_SCHEMA)).toEqual({ ok: true });+ });++ it('with arguments that are not an object', () => {+ expect(validateToolArguments('t', '[1,2]', WEATHER_SCHEMA)).toEqual({ ok: true });+ expect(validateToolArguments('t', '"a string"', WEATHER_SCHEMA)).toEqual({ ok: true });+ });++ it('tolerates vendor extensions rather than rejecting the schema', () => {+ const schema = { ...WEATHER_SCHEMA, 'x-vendor-hint': 'something', additionalProperties: false };+ expect(validateToolArguments('t', '{"city":"Berlin"}', schema)).toEqual({ ok: true });+ });+});++describe('stableStringify', () => {+ it('is order-independent, so key order does not fork the compile cache', () => {+ expect(stableStringify({ a: 1, b: 2 })).toBe(stableStringify({ b: 2, a: 1 }));+ });++ it('still distinguishes different schemas', () => {+ expect(stableStringify({ a: 1 })).not.toBe(stableStringify({ a: 2 }));+ });++ it('handles nesting and arrays', () => {+ expect(stableStringify({ x: [{ b: 1, a: 2 }] })).toBe(stableStringify({ x: [{ a: 2, b: 1 }] }));+ });+});++describe('invalidToolCallReasons', () => {+ const schemas = new Map<string, unknown>([['get_weather', WEATHER_SCHEMA]]);++ it('is empty for a well-formed turn', () => {+ expect(invalidToolCallReasons([call('get_weather', '{"city":"Berlin"}')], schemas)).toEqual([]);+ });++ it('collects one reason per bad call', () => {+ const reasons = invalidToolCallReasons(+ [call('get_weather', '{}'), call('get_weather', '{"city":1}')],+ schemas,+ );+ expect(reasons).toHaveLength(2);+ });++ it('ignores a tool the caller declared no schema for', () => {+ expect(invalidToolCallReasons([call('unknown_tool', '{"whatever":1}')], schemas)).toEqual([]);+ });++ it('is empty for no calls at all', () => {+ expect(invalidToolCallReasons(undefined, schemas)).toEqual([]);+ expect(invalidToolCallReasons([], schemas)).toEqual([]);+ });+});++describe('invalidToolArgumentsError', () => {+ const err = invalidToolArgumentsError('Groq llama-3.3-70b', ['get_weather: /city must be string']);++ it('is recognised as retryable, so the loop fails over', () => {+ expect(isRetryableError(err)).toBe(true);+ });++ it('classifies honestly in the trail rather than as a bare error', () => {+ expect(classifyAttemptError(err)).toBe('invalid_tool_arguments');+ });++ it('spares the provider: the model misbehaved, not the endpoint', () => {+ // skipBench -> no cooldown or score penalty; skipModelForRequest -> a+ // sibling key would misbehave identically.+ expect((err as any).skipBench).toBe(true);+ expect((err as any).skipModelForRequest).toBe(true);+ });++ it('names the model and the reason', () => {+ expect(err.message).toContain('Groq llama-3.3-70b');+ expect(err.message).toContain('/city must be string');+ });+});++describe('isToolArgumentValidationEnabled', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ afterEach(() => {+ delete process.env.VALIDATE_TOOL_ARGUMENTS;+ getDb().prepare('DELETE FROM settings WHERE key = ?').run(TOOL_ARGUMENT_VALIDATION_SETTING);+ });++ it('is off by default — a false positive costs a failover hop', () => {+ expect(isToolArgumentValidationEnabled()).toBe(false);+ });++ it('honours the settings-table value', () => {+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '1');+ expect(isToolArgumentValidationEnabled()).toBe(true);+ });++ it('falls back to the env var when nothing is stored', () => {+ process.env.VALIDATE_TOOL_ARGUMENTS = 'true';+ expect(isToolArgumentValidationEnabled()).toBe(true);+ });++ it('lets the settings table win over the env var', () => {+ setSetting(TOOL_ARGUMENT_VALIDATION_SETTING, '0');+ process.env.VALIDATE_TOOL_ARGUMENTS = '1';+ expect(isToolArgumentValidationEnabled()).toBe(false);+ });+});diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.tsindex 9ef37a04..25833815 100644--- a/server/src/lib/error-classify.ts+++ b/server/src/lib/error-classify.ts@@ -96,7 +96,13 @@ export function isRetryableError(err: any): boolean {// First-byte timeout (#584): the grace budget expired before ANY byte// reached the client, so the next candidate can serve it invisibly.|| msg.includes('no first byte')- || msg.includes('unparseable inline tool-call dialect');+ || msg.includes('unparseable inline tool-call dialect')+ // The model emitted a tool call whose arguments violate the schema the+ // caller declared (opt-in check, lib/tool-validate.ts). Thrown before any+ // byte reached the client, and a different model usually gets the same+ // call right — the thrower marks the model skipped for this request, since+ // a sibling key would misbehave identically.+ || msg.includes('invalid tool arguments');}// A genuine provider QUOTA signal: a structured 429 or rate-limit/quota wording.@@ -256,6 +262,14 @@ export function isProviderDegradedError(err: any): boolean {// unavailable model would each condemn a healthy platform for the whole request.// A message check therefore only runs when there is no status to trust.export function isProviderLevelError(err: any): boolean {+ // A failure the thrower explicitly scoped to the MODEL (`skipModelForRequest`+ // — an ignored response_format, invalid tool arguments) is never provider+ // health, and its message quotes caller- and model-supplied text: a tool+ // named `set_timeout`, or an Ajv complaint about an instance path `/timeout`,+ // would otherwise trip the substring checks below and condemn a healthy+ // platform for the whole request. The structured marker is authoritative;+ // the text is not.+ if (err?.skipModelForRequest === true) return false;const status = typeof err?.status === 'number' ? err.status : 0;if (status >= 500) return true;// A DEGRADED hosted deployment (NVIDIA NIM, #522) is provider health wearingdiff --git a/server/src/lib/fallback-loop.ts b/server/src/lib/fallback-loop.tsindex 7aad94ac..d407a663 100644--- a/server/src/lib/fallback-loop.ts+++ b/server/src/lib/fallback-loop.ts@@ -406,6 +406,7 @@ export type AttemptErrorClass =| 'provider_bad_request'| 'empty_completion'| 'format_ignored'+ | 'invalid_tool_arguments'| 'timeout'| 'rate_limited'| 'upstream_error'@@ -435,6 +436,9 @@ export function classifyAttemptError(err: any): AttemptErrorClass {const msg = (err?.message ?? '').toLowerCase();if (msg.includes('empty completion')) return 'empty_completion';if (msg.includes('ignored response_format') || msg.includes('truncated json')) return 'format_ignored';+ // Before the generic classes so the trail names the real cause rather than+ // booking a schema violation as a bare 'error'.+ if (msg.includes('invalid tool arguments')) return 'invalid_tool_arguments';if (isTimeoutErrorText(msg)) return 'timeout';if (msg.includes('429') || msg.includes('rate limit') || msg.includes('too many requests') || msg.includes('quota')) return 'rate_limited';const status = typeof err?.status === 'number' ? err.status : 0;diff --git a/server/src/lib/tool-validate.ts b/server/src/lib/tool-validate.tsnew file mode 100644index 00000000..7f0f424e--- /dev/null+++ b/server/src/lib/tool-validate.ts@@ -0,0 +1,179 @@+import Ajv2020 from 'ajv/dist/2020.js';+import type { ValidateFunction } from 'ajv';+import { getSetting } from '../db/index.js';++// A verdict on whether a tool call's arguments actually satisfy the schema the+// caller declared for that tool.+//+// lib/tool-args.ts repairs what it can prove is repairable, and deliberately+// invents nothing. What it cannot do is say "this is still wrong" — so a call+// that violates its schema flows on unchallenged, and the Anthropic surface+// turns it into `input: {}` (routes/anthropic.ts parseToolInput does+// `try { JSON.parse } catch { return {} }`). The client then gets a tool_use+// block with no arguments and no explanation, which reads to an agent as the+// model having chosen to call the tool with nothing.+//+// Free-tier models are exactly the population that emits malformed tool calls,+// and a different model usually gets it right, so the useful response is the+// one the loop already has for a dead turn: throw before anything is committed+// and let failover try the next candidate.+//+// OFF by default. A false positive here costs a failover hop, and schemas in+// the wild are strange enough that the honest default is to look before+// leaping — enable it, watch the trail, then decide.++/** 2020-12, because that is what zod-to-json-schema and Pydantic v2 emit; a+ * draft-07-only Ajv throws on their `$schema`. `strict: false` keeps an+ * unknown keyword from being fatal — plenty of real schemas carry vendor+ * extensions, and rejecting the schema would fail calls that are fine. */+const ajv = new (Ajv2020 as unknown as typeof import('ajv').default)({+ strict: false,+ allErrors: true,+ validateFormats: false,+});++export const TOOL_ARGUMENT_VALIDATION_SETTING = 'validate_tool_arguments';++export function isToolArgumentValidationEnabled(): boolean {+ let stored: string | undefined;+ try {+ stored = getSetting(TOOL_ARGUMENT_VALIDATION_SETTING);+ } catch {+ stored = undefined; // DB not ready — never throw on the proxy hot path.+ }+ for (const raw of [stored, process.env.VALIDATE_TOOL_ARGUMENTS]) {+ if (raw === undefined || raw.trim() === '') continue;+ const value = raw.trim().toLowerCase();+ return value === '1' || value === 'true';+ }+ return false;+}++/**+ * Order-independent serialisation, so two schemas that differ only in key order+ * share one compiled validator.+ */+export function stableStringify(value: unknown): string {+ if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;+ const entries = Object.entries(value as Record<string, unknown>)+ .filter(([, v]) => v !== undefined)+ .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0))+ .map(([k, v]) => `${JSON.stringify(k)}:${stableStringify(v)}`);+ return `{${entries.join(',')}}`;+}++/** Compiled validators, keyed by the schema's stable form. `null` is cached+ * too: a schema Ajv refuses to compile must not be recompiled per request. */+const compiled = new Map<string, ValidateFunction | null>();+/** The stable string is the expensive half, and tool schemas are the same+ * object for every call in a request, so memoise per object identity. */+const stableKeys = new WeakMap<object, string>();++function keyFor(schema: object): string {+ const cached = stableKeys.get(schema);+ if (cached !== undefined) return cached;+ const key = stableStringify(schema);+ stableKeys.set(schema, key);+ return key;+}++function compiledFor(schema: object): ValidateFunction | null {+ const key = keyFor(schema);+ const hit = compiled.get(key);+ if (hit !== undefined) return hit;++ let fn: ValidateFunction | null = null;+ try {+ // Drop `$schema`: a declared dialect Ajv does not recognise is fatal at+ // compile time, and the dialect is not what we are checking.+ const { $schema, ...rest } = schema as Record<string, unknown>;+ void $schema;+ fn = ajv.compile(rest);+ } catch {+ fn = null; // Unusable schema — see validateToolArguments, which fails OPEN.+ }+ compiled.set(key, fn);+ return fn;+}++export type ToolArgumentVerdict =+ | { ok: true }+ | { ok: false; reason: string };++/**+ * Check `argsJson` against `schema`.+ *+ * Fails **open** in every case where we cannot form an opinion: no schema, a+ * schema Ajv will not compile, or arguments that are not JSON at all (the last+ * is someone else's error to report, and tool-args deliberately passes+ * unparseable arguments through untouched). Only a definite schema violation+ * returns a verdict, because the consequence of a verdict is a failover hop.+ */+export function validateToolArguments(+ toolName: string,+ argsJson: string,+ schema: unknown,+): ToolArgumentVerdict {+ if (!schema || typeof schema !== 'object') return { ok: true };++ const validate = compiledFor(schema as object);+ if (!validate) return { ok: true };++ let parsed: unknown;+ try {+ parsed = JSON.parse(argsJson);+ } catch {+ return { ok: true };+ }+ // A tool call's arguments are an object by definition; anything else is a+ // shape problem the schema will report on its own terms if it cares.+ if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) return { ok: true };++ if (validate(parsed)) return { ok: true };++ const detail = (validate.errors ?? [])+ .slice(0, 3)+ .map(e => `${e.instancePath || '/'} ${e.message ?? 'is invalid'}`)+ .join('; ');+ return { ok: false, reason: `${toolName}: ${detail || 'does not match its schema'}` };+}++/**+ * The error thrown when a tool call cannot be served. The message carries the+ * marker `invalid tool arguments`, which `isRetryableError` recognises, so the+ * loop fails over exactly as it does for a dead dialect turn.+ *+ * `skipBench` because the PROVIDER is healthy — the model misbehaved, so no+ * cooldown or score penalty. `skipModelForRequest` because a sibling key would+ * misbehave identically, so the whole model is ruled out for this request.+ */+export function invalidToolArgumentsError(displayName: string, reasons: string[]): Error {+ const detail = reasons.slice(0, 3).join('; ');+ return Object.assign(+ new Error(`invalid tool arguments from ${displayName} (${detail})`),+ { skipBench: true, skipModelForRequest: true },+ );+}++/**+ * Verdicts for a whole turn's tool calls. Returns the reasons for those that+ * violate their schema; an empty array means the turn is servable.+ */+export function invalidToolCallReasons(+ calls: Array<{ function?: { name?: string; arguments?: string } }> | undefined,+ schemas: Map<string, unknown>,+): string[] {+ if (!calls || calls.length === 0) return [];+ const reasons: string[] = [];+ for (const call of calls) {+ const name = call.function?.name;+ const args = call.function?.arguments;+ if (!name || typeof args !== 'string') continue;+ const schema = schemas.get(name);+ if (schema === undefined) continue; // Tool the caller never declared a schema for.+ const verdict = validateToolArguments(name, args, schema);+ if (!verdict.ok) reasons.push(verdict.reason);+ }+ return reasons;+}diff --git a/server/src/routes/anthropic.ts b/server/src/routes/anthropic.tsindex 6a477e06..3a4b4adf 100644--- a/server/src/routes/anthropic.ts+++ b/server/src/routes/anthropic.ts@@ -13,6 +13,7 @@ import { routeRequest, resolveStickyPreference, routingReserveTokens, type Routeimport { getSetting, getUnifiedApiKey } from '../db/index.js';import { contentToString } from '../lib/content.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';+import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { convertDocumentBlock, documentRejectionMessage } from '../lib/anthropic-documents.js';@@ -617,6 +618,14 @@ anthropicRouter.post('/messages', async (req: Request, res: Response) => {tc.function.arguments = repairToolArguments(tc.function.arguments, schemas.get(tc.function.name));}}+ // Opt-in schema verdict on what the repair could not fix. This surface+ // is where the silent failure was worst: parseToolInput turns+ // unparseable arguments into `input: {}`, so the client sees a tool_use+ // block with nothing in it and no indication anything went wrong.+ if (isToolArgumentValidationEnabled()) {+ const invalid = invalidToolCallReasons(respToolCalls, schemas);+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);+ }}const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;@@ -863,6 +872,20 @@ async function streamCompletion(heldText = '';}+ // Opt-in schema verdict, same rule as the non-streaming surface above and+ // as /chat/completions: this is the surface the silent failure hurt most,+ // and Claude Code streams. Placed after the dialect rescue so a rescued+ // call is judged too, and gated on `!messageStarted` — once message_start+ // has gone out there is no failing over, and tearing the SSE stream down+ // would be worse for the client than a tool_use the schema dislikes.+ if (isToolArgumentValidationEnabled() && !messageStarted && completedCalls.length > 0) {+ const invalid = invalidToolCallReasons(+ completedCalls.map(c => ({ function: { name: c.name, arguments: c.arguments } })),+ schemas,+ );+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);+ }+// Nothing usable came out — fail over (message_start was never sent, so the// client never saw this attempt).if (!messageStarted && completedCalls.length === 0) {diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex a944a880..94c35bcd 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -12,6 +12,7 @@ import { getDb } from '../db/index.js';import { resolveAuth, prependSystemPrompt, type ResolvedAuth } from '../lib/system-prompt.js';import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../lib/content.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';+import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandoff, recordSuccessfulModel, hasPriorModel, HANDOFF_MAX_TOKENS } from '../services/context-handoff.js';@@ -2018,6 +2019,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {})).filter(c => { try { JSON.parse(c.function.arguments); return c.function.name.length > 0; } catch { return false; } });+ // Opt-in schema verdict. `!headerSent` is the whole licence to throw+ // here: the commit point is held until the first meaningful content,+ // so the common tool-call turn (no prose before the call) has sent no+ // bytes yet and can still fail over invisibly. A turn that already+ // flushed prose is past the point of no return — the catch below+ // would have to tear the SSE stream down with a `stream_error`, which+ // is strictly worse for the client than forwarding a tool call the+ // schema dislikes. Off-by-default or not, this check must never turn+ // a served answer into a broken one.+ if (isToolArgumentValidationEnabled() && !headerSent && completedCalls.length > 0) {+ const invalid = invalidToolCallReasons(completedCalls, schemas);+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);+ }+// Dialect rescue: the held text is an inline tool call in some// model's private syntax. Parse it into structured calls or treat// the turn as dead (headers were never sent in dialect mode, so@@ -2221,11 +2236,38 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}}+ // Repair double-encoded tool arguments against the request's tool+ // schemas (e.g. GLM emitting an array parameter as a JSON string),+ // so strict clients don't reject the call. Schema-gated — a true+ // string parameter is never touched. See lib/tool-args.ts.+ //+ // Deliberately BEFORE the success bookkeeping below: the opt-in schema+ // verdict that follows can still fail this attempt over, and crediting+ // recordUpstreamSuccess / rememberReasoning / setStickyModel to an+ // attempt we are about to discard would bill a model that never served+ // the turn and pin the session to it for the next one.+ if (respMsg?.tool_calls?.length) {+ const schemas = toolSchemaMap(tools);+ for (const tc of respMsg.tool_calls) {+ if (tc?.function?.arguments != null) {+ tc.function.arguments = repairToolArguments(tc.function.arguments, schemas.get(tc.function.name));+ }+ }+ // Whatever the repair could not fix is still broken. Opt-in, and+ // thrown before anything is written, so failover is invisible.+ if (isToolArgumentValidationEnabled()) {+ const invalid = invalidToolCallReasons(respMsg.tool_calls, schemas);+ if (invalid.length > 0) throw invalidToolArgumentsError(route.displayName, invalid);+ }+ }+// Usage fallback: providers that omit `usage` used to be logged as 0// tokens, silently undercounting analytics and the rate-limit ledger.// Fall back to the same chars/4 estimate the streaming path uses (tool- // arguments included, mirroring the stream accounting; reasoning tokens- // included too, so thinking models aren't undercounted — #764).+ // arguments included, mirroring the stream accounting; counted after+ // the repair above, so it measures the bytes actually sent, and+ // reasoning tokens included too, so thinking models aren't+ // undercounted — #764).const respToolArgChars = (respMsg?.tool_calls ?? []).reduce((n, tc) => n + (tc?.function?.arguments?.length ?? 0), 0);const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;const completionTokens = result.usage?.completion_tokens@@ -2249,18 +2291,6 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));setFallbackHeaders(res, attempt, attemptLog);- // Repair double-encoded tool arguments against the request's tool- // schemas (e.g. GLM emitting an array parameter as a JSON string),- // so strict clients don't reject the call. Schema-gated — a true- // string parameter is never touched. See lib/tool-args.ts.- if (respMsg?.tool_calls?.length) {- const schemas = toolSchemaMap(tools);- for (const tc of respMsg.tool_calls) {- if (tc?.function?.arguments != null) {- tc.function.arguments = repairToolArguments(tc.function.arguments, schemas.get(tc.function.name));- }- }- }// Normalize array-shaped message.content to a string on the way out (#166).const outboundBody = sanitizeResponse(normalizeOutboundContent(result));res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');