Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat: tool-call/limit/robustness hardening adapted from community forks (#357)

Tashfeen20cb82fc981e
diff --git a/README.md b/README.md
index 19552459..cf883418 100644
--- a/README.md
+++ b/README.md
@@ -571,6 +571,11 @@ PRs should include a test, keep the existing test suite green, and match the `.e
<a href="https://github.com/mhmdkrmabd"><img src="https://images.weserv.nl/?url=github.com/mhmdkrmabd.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@mhmdkrmabd" /></a>
<a href="https://github.com/DemeulemeesterxMaxime"><img src="https://images.weserv.nl/?url=github.com/DemeulemeesterxMaxime.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@DemeulemeesterxMaxime" /></a>
<a href="https://github.com/HoodBlah"><img src="https://images.weserv.nl/?url=github.com/HoodBlah.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@HoodBlah" /></a>
+<a href="https://github.com/SeanPedersen"><img src="https://images.weserv.nl/?url=github.com/SeanPedersen.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@SeanPedersen" /></a>
+<a href="https://github.com/andersmmg"><img src="https://images.weserv.nl/?url=github.com/andersmmg.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@andersmmg" /></a>
+<a href="https://github.com/chirag127"><img src="https://images.weserv.nl/?url=github.com/chirag127.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@chirag127" /></a>
+<a href="https://github.com/jasnoorgill"><img src="https://images.weserv.nl/?url=github.com/jasnoorgill.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@jasnoorgill" /></a>
+<a href="https://github.com/allababbot"><img src="https://images.weserv.nl/?url=github.com/allababbot.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@allababbot" /></a>
## Terms of Service review
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.ts
index 082bfb3d..d2ff2d6e 100644
--- a/server/src/__tests__/lib/content.test.ts
+++ b/server/src/__tests__/lib/content.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent } from '../../lib/content.js';
+import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../../lib/content.js';
describe('contentToString', () => {
it('passes strings through', () => {
@@ -124,3 +124,52 @@ describe('normalizeOutboundContent (#166)', () => {
expect(() => normalizeOutboundContent({} as unknown)).not.toThrow();
});
});
+
+describe('sanitizeResponse', () => {
+ it('defaults a missing choice finish_reason to null', () => {
+ const result = { choices: [{ index: 0, message: { role: 'assistant', content: 'hi' } }] };
+ const out = sanitizeResponse(result) as any;
+ expect(out.choices[0].finish_reason).toBeNull();
+ });
+
+ it('leaves an existing finish_reason untouched', () => {
+ const result = { choices: [{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'hi' } }] };
+ expect((sanitizeResponse(result) as any).choices[0].finish_reason).toBe('stop');
+ });
+
+ it('deletes a literal tool_calls: null on message and delta (the #200 mirror)', () => {
+ const nonStream = { choices: [{ message: { role: 'assistant', content: 'hi', tool_calls: null } }] };
+ const out1 = sanitizeResponse(nonStream) as any;
+ expect('tool_calls' in out1.choices[0].message).toBe(false);
+
+ const stream = { choices: [{ delta: { content: 'hi', tool_calls: null } }] };
+ const out2 = sanitizeResponse(stream) as any;
+ expect('tool_calls' in out2.choices[0].delta).toBe(false);
+ });
+
+ it('preserves a real tool_calls array', () => {
+ const result = { choices: [{ message: { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'f', arguments: '{}' } }] } }] };
+ const out = sanitizeResponse(result) as any;
+ expect(out.choices[0].message.tool_calls).toHaveLength(1);
+ expect(out.choices[0].message.tool_calls[0].id).toBe('c1');
+ });
+
+ it('coerces a non-string model to a string', () => {
+ expect((sanitizeResponse({ model: 123, choices: [] }) as any).model).toBe('123');
+ expect((sanitizeResponse({ model: 'gpt', choices: [] }) as any).model).toBe('gpt');
+ expect((sanitizeResponse({ choices: [] }) as any).model).toBeUndefined();
+ });
+
+ it('does not touch content or reasoning_content', () => {
+ const result = { choices: [{ message: { role: 'assistant', content: 'answer', reasoning_content: 'thinking' } }] };
+ const out = sanitizeResponse(result) as any;
+ expect(out.choices[0].message.content).toBe('answer');
+ expect(out.choices[0].message.reasoning_content).toBe('thinking');
+ });
+
+ it('tolerates frames with no choices / non-objects', () => {
+ expect(() => sanitizeResponse({ usage: { prompt_tokens: 1 } })).not.toThrow();
+ expect(() => sanitizeResponse(null as unknown)).not.toThrow();
+ expect(() => sanitizeResponse({} as unknown)).not.toThrow();
+ });
+});
diff --git a/server/src/__tests__/lib/process-safety-net.test.ts b/server/src/__tests__/lib/process-safety-net.test.ts
new file mode 100644
index 00000000..2d5b2092
--- /dev/null
+++ b/server/src/__tests__/lib/process-safety-net.test.ts
@@ -0,0 +1,67 @@
+import { describe, it, expect, vi } from 'vitest';
+import {
+ isTransportError,
+ classifyProcessError,
+ handleProcessError,
+} from '../../lib/process-safety-net.js';
+
+describe('isTransportError', () => {
+ it('matches Node socket error codes', () => {
+ expect(isTransportError(Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }))).toBe(true);
+ expect(isTransportError(Object.assign(new Error('x'), { code: 'ETIMEDOUT' }))).toBe(true);
+ expect(isTransportError(Object.assign(new Error('x'), { code: 'EPIPE' }))).toBe(true);
+ });
+
+ it('matches undici codes nested under err.cause (the late-error shape)', () => {
+ const err = Object.assign(new TypeError('fetch failed'), {
+ cause: Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' }),
+ });
+ expect(isTransportError(err)).toBe(true);
+ });
+
+ it('matches by message hint when no code is present', () => {
+ expect(isTransportError(new Error('terminated'))).toBe(true);
+ expect(isTransportError(new TypeError('fetch failed'))).toBe(true);
+ expect(isTransportError(new Error('socket hang up'))).toBe(true);
+ });
+
+ it('does NOT match genuine programming bugs', () => {
+ expect(isTransportError(new TypeError("Cannot read properties of undefined (reading 'x')"))).toBe(false);
+ expect(isTransportError(Object.assign(new Error('assert'), { code: 'ERR_ASSERTION' }))).toBe(false);
+ expect(isTransportError(new RangeError('Maximum call stack size exceeded'))).toBe(false);
+ });
+
+ it('handles null / undefined / cyclic causes safely', () => {
+ expect(isTransportError(null)).toBe(false);
+ expect(isTransportError(undefined)).toBe(false);
+ const a: any = new Error('loop');
+ a.cause = a; // cycle
+ expect(isTransportError(a)).toBe(false);
+ });
+});
+
+describe('classifyProcessError', () => {
+ it('swallows transport errors and is fatal for everything else', () => {
+ expect(classifyProcessError(Object.assign(new Error('x'), { code: 'ECONNRESET' }))).toBe('swallow');
+ expect(classifyProcessError(new Error('real bug'))).toBe('fatal');
+ });
+});
+
+describe('handleProcessError', () => {
+ it('swallows a transport error without exiting', () => {
+ const exit = vi.fn();
+ const log = vi.fn();
+ const decision = handleProcessError('uncaughtException', Object.assign(new Error('x'), { code: 'ECONNRESET' }), { exit, log });
+ expect(decision).toBe('swallow');
+ expect(exit).not.toHaveBeenCalled();
+ expect(log).toHaveBeenCalledOnce();
+ });
+
+ it('exits(1) on a genuine bug, preserving fail-fast', () => {
+ const exit = vi.fn();
+ const log = vi.fn();
+ const decision = handleProcessError('unhandledRejection', new TypeError('boom'), { exit, log });
+ expect(decision).toBe('fatal');
+ expect(exit).toHaveBeenCalledExactlyOnceWith(1);
+ });
+});
diff --git a/server/src/__tests__/lib/tool-args.test.ts b/server/src/__tests__/lib/tool-args.test.ts
index 1c92ee89..7475fd82 100644
--- a/server/src/__tests__/lib/tool-args.test.ts
+++ b/server/src/__tests__/lib/tool-args.test.ts
@@ -1,5 +1,5 @@
import { describe, it, expect } from 'vitest';
-import { repairToolArguments, toolSchemaMap } from '../../lib/tool-args.js';
+import { repairToolArguments, toolSchemaMap, stripSchemaKeys } from '../../lib/tool-args.js';
const PLAN_SCHEMA = {
type: 'object',
@@ -82,3 +82,70 @@ describe('toolSchemaMap', () => {
expect(toolSchemaMap(undefined).size).toBe(0);
});
});
+
+describe('stripSchemaKeys', () => {
+ const keys = new Set(['additionalProperties', '$schema']);
+
+ it('removes the listed keys at the top level', () => {
+ const input = {
+ $schema: 'https://json-schema.org/draft/2020-12/schema',
+ type: 'object',
+ additionalProperties: false,
+ properties: { city: { type: 'string' } },
+ };
+ expect(stripSchemaKeys(input, keys)).toEqual({
+ type: 'object',
+ properties: { city: { type: 'string' } },
+ });
+ });
+
+ it('removes the listed keys recursively in nested properties and arrays', () => {
+ const input = {
+ type: 'object',
+ additionalProperties: true,
+ properties: {
+ nested: {
+ type: 'object',
+ additionalProperties: false,
+ properties: { n: { type: 'number' } },
+ },
+ list: {
+ type: 'array',
+ items: { type: 'object', additionalProperties: false, properties: {} },
+ },
+ },
+ anyOf: [{ type: 'string', additionalProperties: false }],
+ };
+ expect(stripSchemaKeys(input, keys)).toEqual({
+ type: 'object',
+ properties: {
+ nested: { type: 'object', properties: { n: { type: 'number' } } },
+ list: { type: 'array', items: { type: 'object', properties: {} } },
+ },
+ anyOf: [{ type: 'string' }],
+ });
+ });
+
+ it('does not mutate the input (chain-shared schema safety)', () => {
+ const input = { type: 'object', additionalProperties: false, properties: {} };
+ const copy = JSON.parse(JSON.stringify(input));
+ stripSchemaKeys(input, keys);
+ expect(input).toEqual(copy);
+ });
+
+ it('passes non-object values through unchanged', () => {
+ expect(stripSchemaKeys('hi', keys)).toBe('hi');
+ expect(stripSchemaKeys(42, keys)).toBe(42);
+ expect(stripSchemaKeys(null, keys)).toBe(null);
+ expect(stripSchemaKeys(undefined, keys)).toBe(undefined);
+ });
+
+ it('keeps a property literally named like a stripped key only when it is a value, not a key', () => {
+ // A property whose *value* mentions additionalProperties is untouched;
+ // only object KEYS named additionalProperties are removed.
+ const input = { type: 'object', properties: { additionalProperties: { type: 'boolean' } } };
+ // Here `additionalProperties` is a property NAME nested under `properties`,
+ // so it is a schema key and gets stripped — documenting the known limitation.
+ expect(stripSchemaKeys(input, keys)).toEqual({ type: 'object', properties: {} });
+ });
+});
diff --git a/server/src/__tests__/providers/cohere.test.ts b/server/src/__tests__/providers/cohere.test.ts
index d97654ea..e1793c50 100644
--- a/server/src/__tests__/providers/cohere.test.ts
+++ b/server/src/__tests__/providers/cohere.test.ts
@@ -59,6 +59,65 @@ describe('CohereProvider', () => {
expect(result._routed_via?.platform).toBe('cohere');
});
+ it('strips additionalProperties / $schema from tool parameters before sending', async () => {
+ let capturedBody: any = null;
+ vi.spyOn(global, 'fetch').mockImplementationOnce(async (_url, init) => {
+ capturedBody = JSON.parse((init as any).body);
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'c', object: 'chat.completion', created: 1, model: 'command-a',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ }),
+ } as any;
+ });
+
+ await provider.chatCompletion('k', [{ role: 'user', content: 'Hi' }], 'command-a-03-2025', {
+ tools: [{
+ type: 'function',
+ function: {
+ name: 'get_weather',
+ parameters: {
+ $schema: 'https://json-schema.org/draft/2020-12/schema',
+ type: 'object',
+ additionalProperties: false,
+ properties: {
+ city: { type: 'string' },
+ opts: { type: 'object', additionalProperties: false, properties: {} },
+ },
+ },
+ },
+ }],
+ });
+
+ const params = capturedBody.tools[0].function.parameters;
+ expect(params.additionalProperties).toBeUndefined();
+ expect(params.$schema).toBeUndefined();
+ expect(params.properties.opts.additionalProperties).toBeUndefined();
+ // Real schema content is preserved.
+ expect(params.type).toBe('object');
+ expect(params.properties.city).toEqual({ type: 'string' });
+ });
+
+ it('passes through requests with no tools unchanged', async () => {
+ let capturedBody: any = null;
+ vi.spyOn(global, 'fetch').mockImplementationOnce(async (_url, init) => {
+ capturedBody = JSON.parse((init as any).body);
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'c', object: 'chat.completion', created: 1, model: 'command-a',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+ }),
+ } as any;
+ });
+
+ await provider.chatCompletion('k', [{ role: 'user', content: 'Hi' }], 'command-a-03-2025');
+ expect(capturedBody.tools).toBeUndefined();
+ });
+
it('should validate key', async () => {
vi.spyOn(global, 'fetch').mockResolvedValueOnce({ ok: true } as any);
expect(await provider.validateKey('valid')).toBe(true);
diff --git a/server/src/__tests__/services/learn-limits.test.ts b/server/src/__tests__/services/learn-limits.test.ts
new file mode 100644
index 00000000..7c25c574
--- /dev/null
+++ b/server/src/__tests__/services/learn-limits.test.ts
@@ -0,0 +1,89 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import { parseProviderLimit, learnLimitFromError } from '../../services/ratelimit.js';
+
+describe('parseProviderLimit', () => {
+ it('parses a Groq 413 TPM body', () => {
+ const msg = 'Groq API error 413: Request too large for model `llama-4-scout` in organization org_x service tier `on_demand` on tokens per minute (TPM): Limit 30000, Requested 33476. Please try again later.';
+ expect(parseProviderLimit(msg)).toEqual({ kind: 'tpm', limit: 30000 });
+ });
+
+ it('parses RPM / RPD / TPD axes', () => {
+ expect(parseProviderLimit('on requests per minute (RPM): Limit 30, Requested 31')).toEqual({ kind: 'rpm', limit: 30 });
+ expect(parseProviderLimit('on requests per day (RPD): Limit 1000, Used 1000')).toEqual({ kind: 'rpd', limit: 1000 });
+ expect(parseProviderLimit('on tokens per day (TPD): Limit 500000')).toEqual({ kind: 'tpd', limit: 500000 });
+ });
+
+ it('strips thousands separators in the limit', () => {
+ expect(parseProviderLimit('tokens per minute (TPM): Limit 1,000,000, Requested 2,000,000')).toEqual({ kind: 'tpm', limit: 1_000_000 });
+ });
+
+ it('prefers the token axis when both tokens and requests are mentioned', () => {
+ expect(parseProviderLimit('requests per minute and tokens per minute (TPM): Limit 8000')).toEqual({ kind: 'tpm', limit: 8000 });
+ });
+
+ it('returns null without a confident axis even when a Limit is present', () => {
+ expect(parseProviderLimit('Some limit: 9999 was exceeded')).toBeNull();
+ });
+
+ it('returns null when there is no numeric limit', () => {
+ expect(parseProviderLimit('tokens per minute (TPM) exceeded, slow down')).toBeNull();
+ });
+
+ it('returns null for empty / non-limit errors', () => {
+ expect(parseProviderLimit('')).toBeNull();
+ expect(parseProviderLimit(undefined)).toBeNull();
+ expect(parseProviderLimit('API error 500: internal server error')).toBeNull();
+ });
+});
+
+describe('learnLimitFromError (persistence)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ });
+
+ function insertModel(fields: { tpm?: number | null; rpm?: number | null } = {}): number {
+ const info = getDb().prepare(
+ `INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, tpm_limit, rpm_limit)
+ VALUES ('groq', ?, 'M', 50, 50, ?, ?)`,
+ ).run(`m-${Math.random().toString(36).slice(2)}`, fields.tpm ?? null, fields.rpm ?? null);
+ return info.lastInsertRowid as number;
+ }
+ const tpmCol = (id: number) => (getDb().prepare('SELECT tpm_limit FROM models WHERE id = ?').get(id) as { tpm_limit: number | null }).tpm_limit;
+
+ it('fills a NULL limit from a learned ceiling', () => {
+ const id = insertModel({ tpm: null });
+ const r = learnLimitFromError(id, { message: 'tokens per minute (TPM): Limit 6000, Requested 9000' });
+ expect(r).toEqual({ kind: 'tpm', limit: 6000 });
+ expect(tpmCol(id)).toBe(6000);
+ });
+
+ it('lowers an existing limit that was too high', () => {
+ const id = insertModel({ tpm: 40000 });
+ const r = learnLimitFromError(id, { message: 'tokens per minute (TPM): Limit 30000, Requested 33000' });
+ expect(r).toEqual({ kind: 'tpm', limit: 30000 });
+ expect(tpmCol(id)).toBe(30000);
+ });
+
+ it('never RAISES an already-lower limit', () => {
+ const id = insertModel({ tpm: 6000 });
+ const r = learnLimitFromError(id, { message: 'tokens per minute (TPM): Limit 30000, Requested 33000' });
+ expect(r).toBeNull();
+ expect(tpmCol(id)).toBe(6000);
+ });
+
+ it('is a no-op for an error with no parseable limit', () => {
+ const id = insertModel({ tpm: 12345 });
+ const r = learnLimitFromError(id, { message: 'API error 500: internal server error' });
+ expect(r).toBeNull();
+ expect(tpmCol(id)).toBe(12345);
+ });
+
+ it('writes to the correct column per axis (RPM)', () => {
+ const id = insertModel({ rpm: null });
+ learnLimitFromError(id, { message: 'requests per minute (RPM): Limit 20, Requested 21' });
+ const rpm = (getDb().prepare('SELECT rpm_limit FROM models WHERE id = ?').get(id) as { rpm_limit: number | null }).rpm_limit;
+ expect(rpm).toBe(20);
+ });
+});
diff --git a/server/src/index.ts b/server/src/index.ts
index 1f41acde..8d32fb96 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -4,6 +4,7 @@ import { initDb, getSetting } from './db/index.js';
import { startHealthChecker } from './services/health.js';
import { applyProxyUrl, applyProxyEnabled, applyProxyBypass } from './lib/proxy.js';
import { startCatalogSync } from './services/catalog-sync.js';
+import { installProcessSafetyNet } from './lib/process-safety-net.js';
const PORT = process.env.PORT ?? 3001;
// Dual-stack ('::') by default so the dashboard is reachable over both IPv4
@@ -12,6 +13,10 @@ const PORT = process.env.PORT ?? 3001;
const HOST = process.env.HOST ?? '::';
async function main() {
+ // Install first so a late provider socket reset (undici HTTP/2 error with no
+ // listener) can't take the proxy down. Genuine bugs still exit 1.
+ installProcessSafetyNet();
+
initDb();
// Load the persisted proxy settings from the DB (env var wins if set).
diff --git a/server/src/lib/content.ts b/server/src/lib/content.ts
index 56951486..50e1f1b0 100644
--- a/server/src/lib/content.ts
+++ b/server/src/lib/content.ts
@@ -57,6 +57,34 @@ export function messageHasImage(messages: ChatMessage[]): boolean {
return messages.some((m) => contentHasImage(m.content));
}
+// Harden the OUTBOUND envelope so strict OpenAI clients don't choke on the
+// shape variations free-tier providers emit. Complements normalizeOutboundContent
+// (which fixes array content) by fixing the *frame* fields, not the content:
+// - `model` coerced to a string (some providers send null/number).
+// - `choices[].finish_reason` defaulted to null when absent — the field must
+// exist per spec; Rust agents (the #200 class of compat bugs) branch on it.
+// - a literal `tool_calls: null` on a message/delta is DELETED — OpenAI omits
+// the field entirely when there are no calls, and `null` breaks clients that
+// test for its presence (this is the response-side mirror of the #200 fix).
+// Content, real tool_calls arrays, reasoning_content, and every other field are
+// left untouched. Mutates and returns the same object (frames are parsed fresh
+// per SSE line, so in-place mutation is safe), matching normalizeOutboundContent.
+export function sanitizeResponse<T>(payload: T): T {
+ const p = payload as { model?: unknown; choices?: unknown };
+ if (!p || typeof p !== 'object') return payload;
+ if (p.model != null && typeof p.model !== 'string') p.model = String(p.model);
+ if (Array.isArray(p.choices)) {
+ for (const choice of p.choices) {
+ if (!choice || typeof choice !== 'object') continue;
+ const c = choice as { finish_reason?: unknown; message?: { tool_calls?: unknown }; delta?: { tool_calls?: unknown } };
+ if (c.finish_reason === undefined) c.finish_reason = null;
+ if (c.message && typeof c.message === 'object' && c.message.tool_calls === null) delete c.message.tool_calls;
+ if (c.delta && typeof c.delta === 'object' && c.delta.tool_calls === null) delete c.delta.tool_calls;
+ }
+ }
+ return payload;
+}
+
// Normalize the OUTBOUND (provider → client) shape so we honor the OpenAI
// contract on the response path the same way `contentToString` does on the
// request path. Per spec, `choices[].delta.content` (streaming) and
diff --git a/server/src/lib/process-safety-net.ts b/server/src/lib/process-safety-net.ts
new file mode 100644
index 00000000..b8a54029
--- /dev/null
+++ b/server/src/lib/process-safety-net.ts
@@ -0,0 +1,105 @@
+// Process-level safety net for late network transport errors (fork-validated:
+// jasnoorgill's safe-fetch / process-safety-net).
+//
+// The failure this guards against: undici resolves `fetch()` and we move on, but
+// the underlying HTTP/2 / TLS socket is later reset by a CDN edge (CloudFront,
+// Cloudflare) AFTER the response was handed off. undici emits that late error on
+// a stream with no listener, Node escalates it to an `uncaughtException`, and —
+// with no handler installed — the whole proxy exits 1. For a gateway whose entire
+// job is uptime in front of flaky free-tier providers, a third party closing a
+// socket must never take the process down.
+//
+// Design: swallow ONLY a tight allowlist of transport-error codes/messages, and
+// preserve Node's default fail-fast (exit 1) for everything else, so genuine
+// bugs still surface loudly. The classifier is a pure function so it can be
+// unit-tested without registering global handlers.
+
+const TRANSPORT_ERROR_CODES = new Set([
+ // Node socket-level codes
+ 'ECONNRESET', 'ECONNREFUSED', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE',
+ 'EAI_AGAIN', 'ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'EADDRNOTAVAIL',
+ // undici codes (the late-error culprits)
+ 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT',
+ 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_ABORTED',
+]);
+
+const TRANSPORT_MESSAGE_HINTS = [
+ 'fetch failed', 'other side closed', 'socket hang up', 'terminated',
+ 'premature close', 'econnreset', 'request aborted',
+];
+
+// undici wraps the real socket error in `err.cause`; sometimes nested further.
+// Walk the cause chain (bounded, cycle-safe) collecting codes and messages.
+function walkErrorChain(err: unknown): Array<{ code?: string; message?: string }> {
+ const out: Array<{ code?: string; message?: string }> = [];
+ let cur: any = err;
+ const seen = new Set<unknown>();
+ for (let depth = 0; cur && typeof cur === 'object' && depth < 6; depth++) {
+ if (seen.has(cur)) break;
+ seen.add(cur);
+ out.push({ code: typeof cur.code === 'string' ? cur.code : undefined, message: typeof cur.message === 'string' ? cur.message : undefined });
+ cur = cur.cause;
+ }
+ return out;
+}
+
+/** Pure: true when the error is a recoverable network transport failure that
+ * should not crash the process (vs. a programming bug, which should). */
+export function isTransportError(err: unknown): boolean {
+ if (err == null) return false;
+ const links = walkErrorChain(err);
+ for (const { code } of links) {
+ if (code && TRANSPORT_ERROR_CODES.has(code)) return true;
+ }
+ const joined = links.map(l => l.message ?? '').join(' | ').toLowerCase();
+ return TRANSPORT_MESSAGE_HINTS.some(h => joined.includes(h));
+}
+
+export type ProcessErrorDecision = 'swallow' | 'fatal';
+
+/** Pure: swallow transport errors, treat everything else as fatal. */
+export function classifyProcessError(err: unknown): ProcessErrorDecision {
+ return isTransportError(err) ? 'swallow' : 'fatal';
+}
+
+function describeError(err: unknown): string {
+ const links = walkErrorChain(err);
+ const code = links.find(l => l.code)?.code;
+ const message = links.find(l => l.message)?.message;
+ return code ? `${code} (${message ?? 'no message'})` : (message ?? String(err));
+}
+
+export interface SafetyNetHooks {
+ log?: (...args: unknown[]) => void;
+ exit?: (code: number) => void;
+}
+
+/** Decide and act on a process-level error. Returns the decision so callers and
+ * tests can assert behavior. `fatal` exits 1 (preserving Node's default
+ * fail-fast); `swallow` logs and lets the process continue. */
+export function handleProcessError(
+ kind: 'uncaughtException' | 'unhandledRejection',
+ err: unknown,
+ hooks: SafetyNetHooks = {},
+): ProcessErrorDecision {
+ const log = hooks.log ?? console.error;
+ const decision = classifyProcessError(err);
+ if (decision === 'swallow') {
+ log(`[safety-net] swallowed transient ${kind}: ${describeError(err)}`);
+ return 'swallow';
+ }
+ log(`[safety-net] fatal ${kind}:`, err);
+ (hooks.exit ?? process.exit)(1);
+ return 'fatal';
+}
+
+let installed = false;
+
+/** Install the global handlers once. Idempotent. Call as early as possible at
+ * boot (before the server starts taking traffic). */
+export function installProcessSafetyNet(hooks: SafetyNetHooks = {}): void {
+ if (installed) return;
+ installed = true;
+ process.on('uncaughtException', (err) => handleProcessError('uncaughtException', err, hooks));
+ process.on('unhandledRejection', (reason) => handleProcessError('unhandledRejection', reason, hooks));
+}
diff --git a/server/src/lib/tool-args.ts b/server/src/lib/tool-args.ts
index e1124ab3..0ed55105 100644
--- a/server/src/lib/tool-args.ts
+++ b/server/src/lib/tool-args.ts
@@ -81,6 +81,32 @@ export function repairToolArguments(args: string, paramSchema?: JsonSchemaish):
return changed ? JSON.stringify(parsed) : args;
}
+/**
+ * Recursively remove the given keys from a JSON-Schema-ish value. Used to drop
+ * fields a provider's tool-schema validator rejects with a 400 even though they
+ * carry no meaning for the call — Cohere's compat endpoint, for instance, 400s
+ * on `additionalProperties` (and `$schema`), which strict clients like opencode
+ * and continue.dev routinely emit. Returns a NEW value; never mutates the input
+ * (tools are shared across the fallback chain, so an in-place strip on one
+ * provider would corrupt the schema the next provider sees). Non-object values
+ * pass through unchanged. This is the provider-agnostic sibling of google.ts's
+ * `sanitizeForGemini`, which strips a much larger Gemini-specific key set.
+ */
+export function stripSchemaKeys<T>(schema: T, keys: Set<string>): T {
+ if (Array.isArray(schema)) {
+ return schema.map((s) => stripSchemaKeys(s, keys)) as unknown as T;
+ }
+ if (schema && typeof schema === 'object') {
+ const out: Record<string, unknown> = {};
+ for (const [k, v] of Object.entries(schema as Record<string, unknown>)) {
+ if (keys.has(k)) continue;
+ out[k] = stripSchemaKeys(v, keys);
+ }
+ return out as unknown as T;
+ }
+ return schema;
+}
+
/**
* Build a tool-name → parameter-schema map from an OpenAI-style tools array
* (chat-completions shape: {type:'function', function:{name, parameters}}).
diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.ts
index 73737c9d..b5903d06 100644
--- a/server/src/providers/cohere.ts
+++ b/server/src/providers/cohere.ts
@@ -2,12 +2,31 @@ import type {
ChatMessage,
ChatCompletionResponse,
ChatCompletionChunk,
+ ChatToolDefinition,
} from '@freellmapi/shared/types.js';
import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';
import { flattenMessageContent } from '../lib/content.js';
+import { stripSchemaKeys } from '../lib/tool-args.js';
const API_BASE = 'https://api.cohere.ai/compatibility/v1';
+// Cohere's compat-endpoint tool-schema validator rejects a couple of JSON-Schema
+// keywords that strict clients (opencode, continue.dev) send by default, 400-ing
+// the whole request and silently killing tool calls. They carry no meaning for
+// the provider, so strip them before sending. Mirrors google.ts's sanitizeForGemini
+// but scoped to the keys Cohere actually rejects, since its endpoint is otherwise
+// OpenAI-compatible. (Multi-fork-validated: SeanPedersen, andersmmg, chirag127.)
+const COHERE_UNSUPPORTED_SCHEMA_KEYS = new Set(['additionalProperties', '$schema']);
+
+function sanitizeCohereTools(tools?: ChatToolDefinition[]): ChatToolDefinition[] | undefined {
+ if (!tools || tools.length === 0) return tools;
+ return tools.map((t) =>
+ t.function?.parameters
+ ? { ...t, function: { ...t.function, parameters: stripSchemaKeys(t.function.parameters, COHERE_UNSUPPORTED_SCHEMA_KEYS) } }
+ : t,
+ );
+}
+
export class CohereProvider extends BaseProvider {
readonly platform = 'cohere' as const;
readonly name = 'Cohere';
@@ -24,7 +43,7 @@ export class CohereProvider extends BaseProvider {
temperature: options?.temperature,
max_tokens: options?.max_tokens,
top_p: options?.top_p,
- tools: options?.tools,
+ tools: sanitizeCohereTools(options?.tools),
tool_choice: options?.tool_choice,
};
@@ -59,7 +78,7 @@ export class CohereProvider extends BaseProvider {
temperature: options?.temperature,
max_tokens: options?.max_tokens,
top_p: options?.top_p,
- tools: options?.tools,
+ tools: sanitizeCohereTools(options?.tools),
tool_choice: options?.tool_choice,
stream: true,
};
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 5b59c086..744f62f5 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -4,10 +4,10 @@ import type { Request, Response } from 'express';
import { z } from 'zod';
import type { ChatMessage, ModelListRow } from '@freellmapi/shared/types.js';
import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';
-import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS } from '../services/ratelimit.js';
+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';
import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';
import { getDb, getUnifiedApiKey } from '../db/index.js';
-import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';
+import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';
import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';
@@ -1042,6 +1042,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
normalizeOutboundContent(chunk);
+ sanitizeResponse(chunk);
const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';
if (text.length === 0) {
@@ -1262,7 +1263,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
}
// Normalize array-shaped message.content to a string on the way out (#166).
- res.json(normalizeOutboundContent(result));
+ res.json(sanitizeResponse(normalizeOutboundContent(result)));
traceRouteEvent('Proxy', {
event: 'ok',
@@ -1321,6 +1322,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}, err.retryAfterMs),
);
recordRateLimitHit(route.modelDbId);
+ // Self-correct the catalog: if the provider reported its real ceiling in
+ // the error body (e.g. a Groq 413 "tokens per minute (TPM): Limit 30000"),
+ // persist it so the next request's pre-check fails over BEFORE the 413
+ // instead of re-discovering it. No-op for errors with no parseable limit.
+ learnLimitFromError(route.modelDbId, err);
lastError = err;
continue;
}
diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts
index baa05613..c8936d74 100644
--- a/server/src/routes/responses.ts
+++ b/server/src/routes/responses.ts
@@ -9,7 +9,7 @@ import type {
ChatToolChoice,
} from '@freellmapi/shared/types.js';
import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledToolsModel, type RouteResult } from '../services/router.js';
-import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS } from '../services/ratelimit.js';
+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';
import { getUnifiedApiKey } from '../db/index.js';
import { contentToString } from '../lib/content.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
@@ -736,6 +736,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {
? PAYMENT_REQUIRED_COOLDOWN_MS
: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));
recordRateLimitHit(route.modelDbId);
+ // Learn a provider-reported ceiling (e.g. a 413 TPM limit) so the next
+ // request's pre-check fails over before the 413. Mirrors the chat path.
+ learnLimitFromError(route.modelDbId, err);
lastError = err;
continue;
}
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index 2f9540d6..9eb9e5e8 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -434,3 +434,68 @@ export function getRateLimitStatus(
tpm: { used: tokenCount(platform, modelId, keyId, MINUTE, now), limit: limits.tpm },
};
}
+
+// ── Learning real provider limits from error bodies (self-correcting catalog) ──
+// Free-tier limits drift and our seeded catalog is frequently wrong or null.
+// When a provider rejects a request with its real limit in the body — Groq 413:
+// "...on tokens per minute (TPM): Limit 30000, Requested 33476" — we can learn
+// that ceiling and persist it so the canUseTokens / canMakeRequest pre-checks
+// stop us BEFORE the next 413 instead of re-discovering it on every request.
+// (Fork-validated: andersmmg's updateLimitsFromError parses the same Groq shape.)
+
+export type LearnedLimitKind = 'tpm' | 'tpd' | 'rpm' | 'rpd';
+export interface LearnedLimit { kind: LearnedLimitKind; limit: number; }
+
+// Order matters: check the per-DAY axes before per-MINUTE so "tokens per day"
+// isn't shadowed by the "tpm" word-boundary alternative, and tokens before
+// requests so a body mentioning both lands on the more specific token ceiling.
+const LIMIT_AXIS_PATTERNS: Array<{ kind: LearnedLimitKind; re: RegExp }> = [
+ { kind: 'tpd', re: /tokens?\s*per\s*day|\btpd\b/i },
+ { kind: 'tpm', re: /tokens?\s*per\s*min(?:ute)?|\btpm\b/i },
+ { kind: 'rpd', re: /requests?\s*per\s*day|\brpd\b/i },
+ { kind: 'rpm', re: /requests?\s*per\s*min(?:ute)?|\brpm\b/i },
+];
+
+/**
+ * Pure parser: pull a provider-reported ceiling out of an error message. Returns
+ * null unless BOTH a numeric "Limit N" and a confident axis (TPM/TPD/RPM/RPD)
+ * are present — guessing the axis would write the wrong column and mis-route
+ * every future request, so we refuse to guess.
+ */
+export function parseProviderLimit(message: string | undefined | null): LearnedLimit | null {
+ if (!message) return null;
+ const m = message.match(/\blimit[:\s]+([\d,]+)/i);
+ if (!m) return null;
+ const limit = Number(m[1]!.replace(/,/g, ''));
+ if (!Number.isFinite(limit) || limit <= 0) return null;
+ for (const { kind, re } of LIMIT_AXIS_PATTERNS) {
+ if (re.test(message)) return { kind, limit };
+ }
+ return null;
+}
+
+// Whitelisted column names — the only values ever interpolated into the UPDATE
+// below, so there is no injection surface despite the template literal.
+const LIMIT_COLUMN: Record<LearnedLimitKind, string> = {
+ tpm: 'tpm_limit', tpd: 'tpd_limit', rpm: 'rpm_limit', rpd: 'rpd_limit',
+};
+
+/**
+ * Persist a provider-reported limit onto the model row, but ONLY when it makes
+ * us more conservative: fill a NULL (unknown) limit, or LOWER an existing one
+ * that was too high. Never raises a limit — hitting a ceiling means our pre-check
+ * already let too much through, so the true limit is at or below what we used.
+ * Returns the learned limit when a row was actually changed, else null.
+ * DB-guarded (no-op when the DB is unavailable), like the rest of this module.
+ */
+export function learnLimitFromError(modelDbId: number, err: { message?: string }): LearnedLimit | null {
+ const parsed = parseProviderLimit(err?.message);
+ if (!parsed) return null;
+ const col = LIMIT_COLUMN[parsed.kind];
+ const result = withDb(db =>
+ db.prepare(
+ `UPDATE models SET ${col} = ? WHERE id = ? AND (${col} IS NULL OR ${col} > ?)`,
+ ).run(parsed.limit, modelDbId, parsed.limit),
+ );
+ return result && result.changes > 0 ? parsed : null;
+}