7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(proxy): schema-aware repair of double-encoded tool-call arguments (#193)
Tashfeenda808d395b3a
diff --git a/server/src/__tests__/lib/tool-args.test.ts b/server/src/__tests__/lib/tool-args.test.tsnew file mode 100644index 00000000..1c92ee89--- /dev/null+++ b/server/src/__tests__/lib/tool-args.test.ts@@ -0,0 +1,84 @@+import { describe, it, expect } from 'vitest';+import { repairToolArguments, toolSchemaMap } from '../../lib/tool-args.js';++const PLAN_SCHEMA = {+ type: 'object',+ properties: {+ explanation: { type: 'string' },+ plan: { type: 'array' },+ config: { type: 'object' },+ },+};++describe('repairToolArguments', () => {+ it('decodes an array parameter that arrived as a JSON string (the Codex update_plan case)', () => {+ const broken = JSON.stringify({+ explanation: 'next steps',+ plan: '[{"step": "Review design", "status": "in_progress"}, {"step": "QA", "status": "pending"}]',+ });+ const repaired = JSON.parse(repairToolArguments(broken, PLAN_SCHEMA));+ expect(Array.isArray(repaired.plan)).toBe(true);+ expect(repaired.plan).toHaveLength(2);+ expect(repaired.plan[0].step).toBe('Review design');+ expect(repaired.explanation).toBe('next steps');+ });++ it('decodes an object parameter that arrived as a JSON string', () => {+ const broken = JSON.stringify({ config: '{"retries": 3}' });+ const repaired = JSON.parse(repairToolArguments(broken, PLAN_SCHEMA));+ expect(repaired.config).toEqual({ retries: 3 });+ });++ it('NEVER touches a parameter whose schema type is string, even if it looks like JSON', () => {+ const args = JSON.stringify({ explanation: '["this is literal text the user wants"]' });+ expect(repairToolArguments(args, PLAN_SCHEMA)).toBe(args);+ });++ it('leaves a string alone when it does not parse to the schema type', () => {+ // schema wants array, string parses to an object → mismatch, untouched+ const args = JSON.stringify({ plan: '{"not": "an array"}' });+ expect(repairToolArguments(args, PLAN_SCHEMA)).toBe(args);+ });++ it('leaves non-JSON strings alone', () => {+ const args = JSON.stringify({ plan: 'just do the thing' });+ expect(repairToolArguments(args, PLAN_SCHEMA)).toBe(args);+ });++ it('unwraps whole-arguments double encoding without needing a schema', () => {+ const broken = JSON.stringify(JSON.stringify({ city: 'Berlin' }));+ expect(JSON.parse(repairToolArguments(broken))).toEqual({ city: 'Berlin' });+ });++ it('returns unparseable arguments untouched', () => {+ expect(repairToolArguments('{not json', PLAN_SCHEMA)).toBe('{not json');+ expect(repairToolArguments('', PLAN_SCHEMA)).toBe('');+ });++ it('is a no-op on already-correct arguments', () => {+ const good = JSON.stringify({ plan: [{ step: 'a' }], explanation: 'x' });+ expect(repairToolArguments(good, PLAN_SCHEMA)).toBe(good);+ });++ it('does nothing schema-specific without a schema (beyond whole-args unwrap)', () => {+ const args = JSON.stringify({ plan: '[{"step":"a"}]' });+ expect(repairToolArguments(args)).toBe(args);+ });+});++describe('toolSchemaMap', () => {+ it('maps function tools by name and skips non-function/unnamed entries', () => {+ const map = toolSchemaMap([+ { type: 'function', function: { name: 'f1', parameters: { type: 'object' } } },+ { type: 'function', function: { name: 'f2' } },+ { type: 'web_search' } as any,+ ]);+ expect(map.get('f1')).toEqual({ type: 'object' });+ expect(map.has('f2')).toBe(false);+ expect(map.size).toBe(1);+ });++ it('handles undefined tools', () => {+ expect(toolSchemaMap(undefined).size).toBe(0);+ });+});diff --git a/server/src/__tests__/routes/responses-tool-args-repair.test.ts b/server/src/__tests__/routes/responses-tool-args-repair.test.tsnew file mode 100644index 00000000..8eeb7893--- /dev/null+++ b/server/src/__tests__/routes/responses-tool-args-repair.test.ts@@ -0,0 +1,135 @@+import { describe, it, expect, beforeAll, beforeEach, vi } from 'vitest';+import type { Express } from 'express';++// Same scripted-provider mock pattern as proxy-empty-completion.test.ts.+const chatCompletion = vi.fn();+const streamChatCompletion = vi.fn();+const fakeProvider = { name: 'fake', chatCompletion, streamChatCompletion } as any;++vi.mock('../../providers/index.js', async (importOriginal) => {+ const actual = await importOriginal() as any;+ return {+ ...actual,+ getProvider: () => fakeProvider,+ resolveProvider: () => fakeProvider,+ };+});++const { createApp } = await import('../../app.js');+const { initDb, getDb, getUnifiedApiKey } = await import('../../db/index.js');+const { encrypt } = await import('../../lib/crypto.js');+const { setRoutingStrategy } = await import('../../services/router.js');++async function post(app: Express, path: string, body: any, key: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${key}` },+ body: JSON.stringify(body),+ });+ const raw = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(raw); } catch {}+ return { status: res.status, body: json, raw };+}++// The exact production failure: GLM emitted update_plan's `plan` array as a+// JSON-encoded STRING; Codex rejected it ("invalid type: string, expected a+// sequence") and the agent turn died at its status-update call.+const BROKEN_ARGS = JSON.stringify({+ explanation: 'plan update',+ plan: '[{"step": "Review design", "status": "in_progress"}]',+});++const UPDATE_PLAN_TOOL = {+ type: 'function',+ name: 'update_plan',+ parameters: {+ type: 'object',+ properties: { explanation: { type: 'string' }, plan: { type: 'array' } },+ },+};++async function* brokenToolCallStream() {+ yield { choices: [{ delta: { tool_calls: [{ index: 0, id: 'call_1', function: { name: 'update_plan', arguments: BROKEN_ARGS } }] } }] };+}++describe('Tool-argument repair on /v1/responses (double-encoded nested JSON)', () => {+ let app: Express;+ let key: string;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ key = getUnifiedApiKey();++ const db = getDb();+ setRoutingStrategy('priority');+ const { encrypted, iv, authTag } = encrypt('test-key');+ db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'test', ?, ?, ?, 'healthy', 1)+ `).run(encrypted, iv, authTag);+ });++ beforeEach(() => {+ chatCompletion.mockReset();+ streamChatCompletion.mockReset();+ getDb().prepare('DELETE FROM rate_limit_cooldowns').run();+ });++ it('stream: repairs the arguments in function_call_arguments.done and the final response', async () => {+ streamChatCompletion.mockReturnValueOnce(brokenToolCallStream());++ const { status, raw } = await post(app, '/v1/responses', {+ input: 'update the plan',+ stream: true,+ tools: [UPDATE_PLAN_TOOL],+ }, key);++ expect(status).toBe(200);+ const doneLine = raw.split('\n').find((l) => l.startsWith('data:') && l.includes('response.function_call_arguments.done'));+ expect(doneLine).toBeDefined();+ const done = JSON.parse(doneLine!.slice('data: '.length));+ const repaired = JSON.parse(done.arguments);+ expect(Array.isArray(repaired.plan)).toBe(true);+ expect(repaired.plan[0].step).toBe('Review design');+ });++ it('non-stream: repairs the arguments in the function_call output item', async () => {+ chatCompletion.mockResolvedValueOnce({+ choices: [{ message: { role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'update_plan', arguments: BROKEN_ARGS } }] } }],+ usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },+ });++ const { status, body } = await post(app, '/v1/responses', {+ input: 'update the plan',+ tools: [UPDATE_PLAN_TOOL],+ }, key);++ expect(status).toBe(200);+ const fc = body.output.find((o: any) => o.type === 'function_call');+ expect(fc).toBeDefined();+ const repaired = JSON.parse(fc.arguments);+ expect(Array.isArray(repaired.plan)).toBe(true);+ });++ it('non-stream /v1/chat/completions: repairs tool_calls in the message', async () => {+ chatCompletion.mockResolvedValueOnce({+ choices: [{ message: { role: 'assistant', content: null, tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'update_plan', arguments: BROKEN_ARGS } }] } }],+ usage: { prompt_tokens: 10, completion_tokens: 20, total_tokens: 30 },+ });++ const { status, body } = await post(app, '/v1/chat/completions', {+ messages: [{ role: 'user', content: 'update the plan' }],+ tools: [{ type: 'function', function: { name: 'update_plan', parameters: UPDATE_PLAN_TOOL.parameters } }],+ }, key);++ expect(status).toBe(200);+ const repaired = JSON.parse(body.choices[0].message.tool_calls[0].function.arguments);+ expect(Array.isArray(repaired.plan)).toBe(true);+ });+});diff --git a/server/src/lib/tool-args.ts b/server/src/lib/tool-args.tsnew file mode 100644index 00000000..e1124ab3--- /dev/null+++ b/server/src/lib/tool-args.ts@@ -0,0 +1,99 @@+// Schema-aware repair of double-encoded tool-call arguments.+//+// Several free-tier models (GLM family prominently) emit NESTED JSON inside+// tool arguments as a string: `{"plan": "[{\"step\":...}]"}` instead of+// `{"plan": [{"step":...}]}`. Strict clients reject the call — observed in+// production as Codex `failed to parse function arguments: invalid type:+// string ..., expected a sequence`, which killed the agent turn right at its+// status-update call. The gateway has the request's tool schemas, so it can+// repair this principled-ly: only when the schema says a parameter is an+// array/object AND the string value parses to exactly that JSON type. A+// parameter whose schema says "string" is never touched, even if it looks+// like JSON.+//+// Also handles whole-arguments double encoding (the arguments field itself+// being a JSON-encoded string of a JSON object), which needs no schema.++interface JsonSchemaish {+ type?: string;+ properties?: Record<string, JsonSchemaish>;+}++/**+ * Repair a tool call's `arguments` JSON string against the tool's parameter+ * schema. Returns the original string untouched whenever anything doesn't+ * parse or doesn't match — this must never corrupt a valid call.+ */+export function repairToolArguments(args: string, paramSchema?: JsonSchemaish): string {+ let parsed: unknown;+ try {+ parsed = JSON.parse(args);+ } catch {+ return args;+ }++ let changed = false;++ // Whole-arguments double encoding: `"{\"a\":1}"` parses to a string that is+ // itself JSON of an object. Unwrap one level.+ if (typeof parsed === 'string') {+ try {+ const inner = JSON.parse(parsed);+ if (inner !== null && typeof inner === 'object' && !Array.isArray(inner)) {+ parsed = inner;+ changed = true;+ } else {+ return args;+ }+ } catch {+ return args;+ }+ }++ if (parsed === null || typeof parsed !== 'object' || Array.isArray(parsed)) {+ return changed ? JSON.stringify(parsed) : args;+ }++ const props = paramSchema?.properties;+ if (props) {+ const obj = parsed as Record<string, unknown>;+ for (const [key, value] of Object.entries(obj)) {+ if (typeof value !== 'string') continue;+ const want = props[key]?.type;+ if (want !== 'array' && want !== 'object') continue;+ const trimmed = value.trim();+ if (!(trimmed.startsWith('[') || trimmed.startsWith('{'))) continue;+ try {+ const inner = JSON.parse(trimmed);+ const isMatch = want === 'array'+ ? Array.isArray(inner)+ : inner !== null && typeof inner === 'object' && !Array.isArray(inner);+ if (isMatch) {+ obj[key] = inner;+ changed = true;+ }+ } catch {+ // Not actually JSON — leave the string alone.+ }+ }+ }++ return changed ? JSON.stringify(parsed) : args;+}++/**+ * Build a tool-name → parameter-schema map from an OpenAI-style tools array+ * (chat-completions shape: {type:'function', function:{name, parameters}}).+ */+export function toolSchemaMap(+ tools?: Array<{ type?: string; function?: { name?: string; parameters?: unknown } }>,+): Map<string, JsonSchemaish> {+ const map = new Map<string, JsonSchemaish>();+ for (const t of tools ?? []) {+ const name = t.function?.name;+ if (t.type === 'function' && name && t.function?.parameters && typeof t.function.parameters === 'object') {+ map.set(name, t.function.parameters as JsonSchemaish);+ }+ }+ return map;+}diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 5b89eda7..69009f8c 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -8,6 +8,7 @@ import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit }import { pruneRequestAnalytics } from '../services/request-retention.js';import { getDb, getUnifiedApiKey } from '../db/index.js';import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';+import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';export const proxyRouter = Router();@@ -586,6 +587,18 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.setHeader('X-Routed-Via', `${route.platform}/${route.modelId}`);if (attempt > 0) res.setHeader('X-Fallback-Attempts', String(attempt));+ // Repair double-encoded tool arguments against the request's tool+ // schemas (e.g. GLM emitting an array parameter as a JSON string),+ // so strict clients don't reject the call. Schema-gated — a true+ // string parameter is never touched. See lib/tool-args.ts.+ if (respMsg?.tool_calls?.length) {+ const schemas = toolSchemaMap(tools);+ for (const tc of respMsg.tool_calls) {+ if (tc?.function?.arguments != null) {+ tc.function.arguments = repairToolArguments(tc.function.arguments, schemas.get(tc.function.name));+ }+ }+ }// Normalize array-shaped message.content to a string on the way out (#166).res.json(normalizeOutboundContent(result));diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex edbe7dc7..fc8c7cec 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -12,6 +12,7 @@ import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledToolsModel,import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit } from '../services/ratelimit.js';import { getUnifiedApiKey } from '../db/index.js';import { contentToString } from '../lib/content.js';+import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import {isRetryableError,timingSafeStringEqual,@@ -296,6 +297,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const stream = reqData.stream ?? false;const messages = toChatMessages(reqData);const tools = toChatTools(reqData.tools);+ // name → parameter schema, for repairing double-encoded tool arguments on+ // the way back out (see lib/tool-args.ts).+ const toolSchemas = toolSchemaMap(tools);const tool_choice = toChatToolChoice(reqData.tool_choice);const completionOpts = {temperature: reqData.temperature ?? undefined,@@ -477,12 +481,18 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {sse('response.content_part.done', { item_id: msgItemId, output_index: 0, content_index: 0, part: { type: 'output_text', text: msgText, annotations: [] } });sse('response.output_item.done', { output_index: 0, item: { id: msgItemId, type: 'message', status: 'completed', role: 'assistant', content: [{ type: 'output_text', text: msgText, annotations: [] }] } });}- // Finalize tool-call items.+ // Finalize tool-call items. Arguments are repaired against the tool's+ // parameter schema at this point (after the full string accumulated):+ // models like GLM double-encode nested arrays/objects as strings, and+ // Codex hard-rejects the call ("invalid type: string, expected a+ // sequence"). Clients consume the *.done events / final response for+ // arguments, so repairing here covers the streamed path too.const finalToolCalls: ChatToolCall[] = [];for (const acc of toolAcc.values()) {- sse('response.function_call_arguments.done', { item_id: acc.itemId, output_index: acc.outputIndex, arguments: acc.args });- sse('response.output_item.done', { output_index: acc.outputIndex, item: { id: acc.itemId, type: 'function_call', status: 'completed', call_id: acc.callId, name: acc.name, arguments: acc.args } });- finalToolCalls.push({ id: acc.callId, type: 'function', function: { name: acc.name, arguments: acc.args } });+ const repairedArgs = repairToolArguments(acc.args, toolSchemas.get(acc.name));+ sse('response.function_call_arguments.done', { item_id: acc.itemId, output_index: acc.outputIndex, arguments: repairedArgs });+ sse('response.output_item.done', { output_index: acc.outputIndex, item: { id: acc.itemId, type: 'function_call', status: 'completed', call_id: acc.callId, name: acc.name, arguments: repairedArgs } });+ finalToolCalls.push({ id: acc.callId, type: 'function', function: { name: acc.name, arguments: repairedArgs } });}const finalResponse = buildResponseObject({@@ -502,7 +512,10 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {const msg = result.choices[0]?.message;const text = contentToString(msg?.content ?? '');- const toolCalls = msg?.tool_calls ?? [];+ const toolCalls = (msg?.tool_calls ?? []).map((tc) => ({+ ...tc,+ function: { ...tc.function, arguments: repairToolArguments(tc.function.arguments, toolSchemas.get(tc.function.name)) },+ }));const promptTokens = result.usage?.prompt_tokens ?? estimatedInputTokens;const completionTokens = result.usage?.completion_tokens ?? Math.ceil(text.length / 4);