7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat: tool-call/limit/robustness hardening adapted from community forks (#357)
Tashfeen20cb82fc981e
diff --git a/README.md b/README.mdindex 19552459..cf883418 100644--- a/README.md+++ b/README.md@@ -571,6 +571,11 @@ PRs should include a test, keep the existing test suite green, and match the `.e<a href="https://github.com/mhmdkrmabd"><img src="https://images.weserv.nl/?url=github.com/mhmdkrmabd.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@mhmdkrmabd" /></a><a href="https://github.com/DemeulemeesterxMaxime"><img src="https://images.weserv.nl/?url=github.com/DemeulemeesterxMaxime.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@DemeulemeesterxMaxime" /></a><a href="https://github.com/HoodBlah"><img src="https://images.weserv.nl/?url=github.com/HoodBlah.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@HoodBlah" /></a>+<a href="https://github.com/SeanPedersen"><img src="https://images.weserv.nl/?url=github.com/SeanPedersen.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@SeanPedersen" /></a>+<a href="https://github.com/andersmmg"><img src="https://images.weserv.nl/?url=github.com/andersmmg.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@andersmmg" /></a>+<a href="https://github.com/chirag127"><img src="https://images.weserv.nl/?url=github.com/chirag127.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@chirag127" /></a>+<a href="https://github.com/jasnoorgill"><img src="https://images.weserv.nl/?url=github.com/jasnoorgill.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@jasnoorgill" /></a>+<a href="https://github.com/allababbot"><img src="https://images.weserv.nl/?url=github.com/allababbot.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@allababbot" /></a>## Terms of Service reviewdiff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.tsindex 082bfb3d..d2ff2d6e 100644--- a/server/src/__tests__/lib/content.test.ts+++ b/server/src/__tests__/lib/content.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest';-import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent } from '../../lib/content.js';+import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../../lib/content.js';describe('contentToString', () => {it('passes strings through', () => {@@ -124,3 +124,52 @@ describe('normalizeOutboundContent (#166)', () => {expect(() => normalizeOutboundContent({} as unknown)).not.toThrow();});});++describe('sanitizeResponse', () => {+ it('defaults a missing choice finish_reason to null', () => {+ const result = { choices: [{ index: 0, message: { role: 'assistant', content: 'hi' } }] };+ const out = sanitizeResponse(result) as any;+ expect(out.choices[0].finish_reason).toBeNull();+ });++ it('leaves an existing finish_reason untouched', () => {+ const result = { choices: [{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'hi' } }] };+ expect((sanitizeResponse(result) as any).choices[0].finish_reason).toBe('stop');+ });++ it('deletes a literal tool_calls: null on message and delta (the #200 mirror)', () => {+ const nonStream = { choices: [{ message: { role: 'assistant', content: 'hi', tool_calls: null } }] };+ const out1 = sanitizeResponse(nonStream) as any;+ expect('tool_calls' in out1.choices[0].message).toBe(false);++ const stream = { choices: [{ delta: { content: 'hi', tool_calls: null } }] };+ const out2 = sanitizeResponse(stream) as any;+ expect('tool_calls' in out2.choices[0].delta).toBe(false);+ });++ it('preserves a real tool_calls array', () => {+ const result = { choices: [{ message: { role: 'assistant', content: null, tool_calls: [{ id: 'c1', type: 'function', function: { name: 'f', arguments: '{}' } }] } }] };+ const out = sanitizeResponse(result) as any;+ expect(out.choices[0].message.tool_calls).toHaveLength(1);+ expect(out.choices[0].message.tool_calls[0].id).toBe('c1');+ });++ it('coerces a non-string model to a string', () => {+ expect((sanitizeResponse({ model: 123, choices: [] }) as any).model).toBe('123');+ expect((sanitizeResponse({ model: 'gpt', choices: [] }) as any).model).toBe('gpt');+ expect((sanitizeResponse({ choices: [] }) as any).model).toBeUndefined();+ });++ it('does not touch content or reasoning_content', () => {+ const result = { choices: [{ message: { role: 'assistant', content: 'answer', reasoning_content: 'thinking' } }] };+ const out = sanitizeResponse(result) as any;+ expect(out.choices[0].message.content).toBe('answer');+ expect(out.choices[0].message.reasoning_content).toBe('thinking');+ });++ it('tolerates frames with no choices / non-objects', () => {+ expect(() => sanitizeResponse({ usage: { prompt_tokens: 1 } })).not.toThrow();+ expect(() => sanitizeResponse(null as unknown)).not.toThrow();+ expect(() => sanitizeResponse({} as unknown)).not.toThrow();+ });+});diff --git a/server/src/__tests__/lib/process-safety-net.test.ts b/server/src/__tests__/lib/process-safety-net.test.tsnew file mode 100644index 00000000..2d5b2092--- /dev/null+++ b/server/src/__tests__/lib/process-safety-net.test.ts@@ -0,0 +1,67 @@+import { describe, it, expect, vi } from 'vitest';+import {+ isTransportError,+ classifyProcessError,+ handleProcessError,+} from '../../lib/process-safety-net.js';++describe('isTransportError', () => {+ it('matches Node socket error codes', () => {+ expect(isTransportError(Object.assign(new Error('read ECONNRESET'), { code: 'ECONNRESET' }))).toBe(true);+ expect(isTransportError(Object.assign(new Error('x'), { code: 'ETIMEDOUT' }))).toBe(true);+ expect(isTransportError(Object.assign(new Error('x'), { code: 'EPIPE' }))).toBe(true);+ });++ it('matches undici codes nested under err.cause (the late-error shape)', () => {+ const err = Object.assign(new TypeError('fetch failed'), {+ cause: Object.assign(new Error('other side closed'), { code: 'UND_ERR_SOCKET' }),+ });+ expect(isTransportError(err)).toBe(true);+ });++ it('matches by message hint when no code is present', () => {+ expect(isTransportError(new Error('terminated'))).toBe(true);+ expect(isTransportError(new TypeError('fetch failed'))).toBe(true);+ expect(isTransportError(new Error('socket hang up'))).toBe(true);+ });++ it('does NOT match genuine programming bugs', () => {+ expect(isTransportError(new TypeError("Cannot read properties of undefined (reading 'x')"))).toBe(false);+ expect(isTransportError(Object.assign(new Error('assert'), { code: 'ERR_ASSERTION' }))).toBe(false);+ expect(isTransportError(new RangeError('Maximum call stack size exceeded'))).toBe(false);+ });++ it('handles null / undefined / cyclic causes safely', () => {+ expect(isTransportError(null)).toBe(false);+ expect(isTransportError(undefined)).toBe(false);+ const a: any = new Error('loop');+ a.cause = a; // cycle+ expect(isTransportError(a)).toBe(false);+ });+});++describe('classifyProcessError', () => {+ it('swallows transport errors and is fatal for everything else', () => {+ expect(classifyProcessError(Object.assign(new Error('x'), { code: 'ECONNRESET' }))).toBe('swallow');+ expect(classifyProcessError(new Error('real bug'))).toBe('fatal');+ });+});++describe('handleProcessError', () => {+ it('swallows a transport error without exiting', () => {+ const exit = vi.fn();+ const log = vi.fn();+ const decision = handleProcessError('uncaughtException', Object.assign(new Error('x'), { code: 'ECONNRESET' }), { exit, log });+ expect(decision).toBe('swallow');+ expect(exit).not.toHaveBeenCalled();+ expect(log).toHaveBeenCalledOnce();+ });++ it('exits(1) on a genuine bug, preserving fail-fast', () => {+ const exit = vi.fn();+ const log = vi.fn();+ const decision = handleProcessError('unhandledRejection', new TypeError('boom'), { exit, log });+ expect(decision).toBe('fatal');+ expect(exit).toHaveBeenCalledExactlyOnceWith(1);+ });+});diff --git a/server/src/__tests__/lib/tool-args.test.ts b/server/src/__tests__/lib/tool-args.test.tsindex 1c92ee89..7475fd82 100644--- a/server/src/__tests__/lib/tool-args.test.ts+++ b/server/src/__tests__/lib/tool-args.test.ts@@ -1,5 +1,5 @@import { describe, it, expect } from 'vitest';-import { repairToolArguments, toolSchemaMap } from '../../lib/tool-args.js';+import { repairToolArguments, toolSchemaMap, stripSchemaKeys } from '../../lib/tool-args.js';const PLAN_SCHEMA = {type: 'object',@@ -82,3 +82,70 @@ describe('toolSchemaMap', () => {expect(toolSchemaMap(undefined).size).toBe(0);});});++describe('stripSchemaKeys', () => {+ const keys = new Set(['additionalProperties', '$schema']);++ it('removes the listed keys at the top level', () => {+ const input = {+ $schema: 'https://json-schema.org/draft/2020-12/schema',+ type: 'object',+ additionalProperties: false,+ properties: { city: { type: 'string' } },+ };+ expect(stripSchemaKeys(input, keys)).toEqual({+ type: 'object',+ properties: { city: { type: 'string' } },+ });+ });++ it('removes the listed keys recursively in nested properties and arrays', () => {+ const input = {+ type: 'object',+ additionalProperties: true,+ properties: {+ nested: {+ type: 'object',+ additionalProperties: false,+ properties: { n: { type: 'number' } },+ },+ list: {+ type: 'array',+ items: { type: 'object', additionalProperties: false, properties: {} },+ },+ },+ anyOf: [{ type: 'string', additionalProperties: false }],+ };+ expect(stripSchemaKeys(input, keys)).toEqual({+ type: 'object',+ properties: {+ nested: { type: 'object', properties: { n: { type: 'number' } } },+ list: { type: 'array', items: { type: 'object', properties: {} } },+ },+ anyOf: [{ type: 'string' }],+ });+ });++ it('does not mutate the input (chain-shared schema safety)', () => {+ const input = { type: 'object', additionalProperties: false, properties: {} };+ const copy = JSON.parse(JSON.stringify(input));+ stripSchemaKeys(input, keys);+ expect(input).toEqual(copy);+ });++ it('passes non-object values through unchanged', () => {+ expect(stripSchemaKeys('hi', keys)).toBe('hi');+ expect(stripSchemaKeys(42, keys)).toBe(42);+ expect(stripSchemaKeys(null, keys)).toBe(null);+ expect(stripSchemaKeys(undefined, keys)).toBe(undefined);+ });++ it('keeps a property literally named like a stripped key only when it is a value, not a key', () => {+ // A property whose *value* mentions additionalProperties is untouched;+ // only object KEYS named additionalProperties are removed.+ const input = { type: 'object', properties: { additionalProperties: { type: 'boolean' } } };+ // Here `additionalProperties` is a property NAME nested under `properties`,+ // so it is a schema key and gets stripped — documenting the known limitation.+ expect(stripSchemaKeys(input, keys)).toEqual({ type: 'object', properties: {} });+ });+});diff --git a/server/src/__tests__/providers/cohere.test.ts b/server/src/__tests__/providers/cohere.test.tsindex d97654ea..e1793c50 100644--- a/server/src/__tests__/providers/cohere.test.ts+++ b/server/src/__tests__/providers/cohere.test.ts@@ -59,6 +59,65 @@ describe('CohereProvider', () => {expect(result._routed_via?.platform).toBe('cohere');});+ it('strips additionalProperties / $schema from tool parameters before sending', async () => {+ let capturedBody: any = null;+ vi.spyOn(global, 'fetch').mockImplementationOnce(async (_url, init) => {+ capturedBody = JSON.parse((init as any).body);+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'c', object: 'chat.completion', created: 1, model: 'command-a',+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }),+ } as any;+ });++ await provider.chatCompletion('k', [{ role: 'user', content: 'Hi' }], 'command-a-03-2025', {+ tools: [{+ type: 'function',+ function: {+ name: 'get_weather',+ parameters: {+ $schema: 'https://json-schema.org/draft/2020-12/schema',+ type: 'object',+ additionalProperties: false,+ properties: {+ city: { type: 'string' },+ opts: { type: 'object', additionalProperties: false, properties: {} },+ },+ },+ },+ }],+ });++ const params = capturedBody.tools[0].function.parameters;+ expect(params.additionalProperties).toBeUndefined();+ expect(params.$schema).toBeUndefined();+ expect(params.properties.opts.additionalProperties).toBeUndefined();+ // Real schema content is preserved.+ expect(params.type).toBe('object');+ expect(params.properties.city).toEqual({ type: 'string' });+ });++ it('passes through requests with no tools unchanged', async () => {+ let capturedBody: any = null;+ vi.spyOn(global, 'fetch').mockImplementationOnce(async (_url, init) => {+ capturedBody = JSON.parse((init as any).body);+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'c', object: 'chat.completion', created: 1, model: 'command-a',+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },+ }),+ } as any;+ });++ await provider.chatCompletion('k', [{ role: 'user', content: 'Hi' }], 'command-a-03-2025');+ expect(capturedBody.tools).toBeUndefined();+ });+it('should validate key', async () => {vi.spyOn(global, 'fetch').mockResolvedValueOnce({ ok: true } as any);expect(await provider.validateKey('valid')).toBe(true);diff --git a/server/src/__tests__/services/learn-limits.test.ts b/server/src/__tests__/services/learn-limits.test.tsnew file mode 100644index 00000000..7c25c574--- /dev/null+++ b/server/src/__tests__/services/learn-limits.test.ts@@ -0,0 +1,89 @@+import { describe, it, expect, beforeAll } from 'vitest';+import { initDb, getDb } from '../../db/index.js';+import { parseProviderLimit, learnLimitFromError } from '../../services/ratelimit.js';++describe('parseProviderLimit', () => {+ it('parses a Groq 413 TPM body', () => {+ const msg = 'Groq API error 413: Request too large for model `llama-4-scout` in organization org_x service tier `on_demand` on tokens per minute (TPM): Limit 30000, Requested 33476. Please try again later.';+ expect(parseProviderLimit(msg)).toEqual({ kind: 'tpm', limit: 30000 });+ });++ it('parses RPM / RPD / TPD axes', () => {+ expect(parseProviderLimit('on requests per minute (RPM): Limit 30, Requested 31')).toEqual({ kind: 'rpm', limit: 30 });+ expect(parseProviderLimit('on requests per day (RPD): Limit 1000, Used 1000')).toEqual({ kind: 'rpd', limit: 1000 });+ expect(parseProviderLimit('on tokens per day (TPD): Limit 500000')).toEqual({ kind: 'tpd', limit: 500000 });+ });++ it('strips thousands separators in the limit', () => {+ expect(parseProviderLimit('tokens per minute (TPM): Limit 1,000,000, Requested 2,000,000')).toEqual({ kind: 'tpm', limit: 1_000_000 });+ });++ it('prefers the token axis when both tokens and requests are mentioned', () => {+ expect(parseProviderLimit('requests per minute and tokens per minute (TPM): Limit 8000')).toEqual({ kind: 'tpm', limit: 8000 });+ });++ it('returns null without a confident axis even when a Limit is present', () => {+ expect(parseProviderLimit('Some limit: 9999 was exceeded')).toBeNull();+ });++ it('returns null when there is no numeric limit', () => {+ expect(parseProviderLimit('tokens per minute (TPM) exceeded, slow down')).toBeNull();+ });++ it('returns null for empty / non-limit errors', () => {+ expect(parseProviderLimit('')).toBeNull();+ expect(parseProviderLimit(undefined)).toBeNull();+ expect(parseProviderLimit('API error 500: internal server error')).toBeNull();+ });+});++describe('learnLimitFromError (persistence)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ });++ function insertModel(fields: { tpm?: number | null; rpm?: number | null } = {}): number {+ const info = getDb().prepare(+ `INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, tpm_limit, rpm_limit)+ VALUES ('groq', ?, 'M', 50, 50, ?, ?)`,+ ).run(`m-${Math.random().toString(36).slice(2)}`, fields.tpm ?? null, fields.rpm ?? null);+ return info.lastInsertRowid as number;+ }+ const tpmCol = (id: number) => (getDb().prepare('SELECT tpm_limit FROM models WHERE id = ?').get(id) as { tpm_limit: number | null }).tpm_limit;++ it('fills a NULL limit from a learned ceiling', () => {+ const id = insertModel({ tpm: null });+ const r = learnLimitFromError(id, { message: 'tokens per minute (TPM): Limit 6000, Requested 9000' });+ expect(r).toEqual({ kind: 'tpm', limit: 6000 });+ expect(tpmCol(id)).toBe(6000);+ });++ it('lowers an existing limit that was too high', () => {+ const id = insertModel({ tpm: 40000 });+ const r = learnLimitFromError(id, { message: 'tokens per minute (TPM): Limit 30000, Requested 33000' });+ expect(r).toEqual({ kind: 'tpm', limit: 30000 });+ expect(tpmCol(id)).toBe(30000);+ });++ it('never RAISES an already-lower limit', () => {+ const id = insertModel({ tpm: 6000 });+ const r = learnLimitFromError(id, { message: 'tokens per minute (TPM): Limit 30000, Requested 33000' });+ expect(r).toBeNull();+ expect(tpmCol(id)).toBe(6000);+ });++ it('is a no-op for an error with no parseable limit', () => {+ const id = insertModel({ tpm: 12345 });+ const r = learnLimitFromError(id, { message: 'API error 500: internal server error' });+ expect(r).toBeNull();+ expect(tpmCol(id)).toBe(12345);+ });++ it('writes to the correct column per axis (RPM)', () => {+ const id = insertModel({ rpm: null });+ learnLimitFromError(id, { message: 'requests per minute (RPM): Limit 20, Requested 21' });+ const rpm = (getDb().prepare('SELECT rpm_limit FROM models WHERE id = ?').get(id) as { rpm_limit: number | null }).rpm_limit;+ expect(rpm).toBe(20);+ });+});diff --git a/server/src/index.ts b/server/src/index.tsindex 1f41acde..8d32fb96 100644--- a/server/src/index.ts+++ b/server/src/index.ts@@ -4,6 +4,7 @@ import { initDb, getSetting } from './db/index.js';import { startHealthChecker } from './services/health.js';import { applyProxyUrl, applyProxyEnabled, applyProxyBypass } from './lib/proxy.js';import { startCatalogSync } from './services/catalog-sync.js';+import { installProcessSafetyNet } from './lib/process-safety-net.js';const PORT = process.env.PORT ?? 3001;// Dual-stack ('::') by default so the dashboard is reachable over both IPv4@@ -12,6 +13,10 @@ const PORT = process.env.PORT ?? 3001;const HOST = process.env.HOST ?? '::';async function main() {+ // Install first so a late provider socket reset (undici HTTP/2 error with no+ // listener) can't take the proxy down. Genuine bugs still exit 1.+ installProcessSafetyNet();+initDb();// Load the persisted proxy settings from the DB (env var wins if set).diff --git a/server/src/lib/content.ts b/server/src/lib/content.tsindex 56951486..50e1f1b0 100644--- a/server/src/lib/content.ts+++ b/server/src/lib/content.ts@@ -57,6 +57,34 @@ export function messageHasImage(messages: ChatMessage[]): boolean {return messages.some((m) => contentHasImage(m.content));}+// Harden the OUTBOUND envelope so strict OpenAI clients don't choke on the+// shape variations free-tier providers emit. Complements normalizeOutboundContent+// (which fixes array content) by fixing the *frame* fields, not the content:+// - `model` coerced to a string (some providers send null/number).+// - `choices[].finish_reason` defaulted to null when absent — the field must+// exist per spec; Rust agents (the #200 class of compat bugs) branch on it.+// - a literal `tool_calls: null` on a message/delta is DELETED — OpenAI omits+// the field entirely when there are no calls, and `null` breaks clients that+// test for its presence (this is the response-side mirror of the #200 fix).+// Content, real tool_calls arrays, reasoning_content, and every other field are+// left untouched. Mutates and returns the same object (frames are parsed fresh+// per SSE line, so in-place mutation is safe), matching normalizeOutboundContent.+export function sanitizeResponse<T>(payload: T): T {+ const p = payload as { model?: unknown; choices?: unknown };+ if (!p || typeof p !== 'object') return payload;+ if (p.model != null && typeof p.model !== 'string') p.model = String(p.model);+ if (Array.isArray(p.choices)) {+ for (const choice of p.choices) {+ if (!choice || typeof choice !== 'object') continue;+ const c = choice as { finish_reason?: unknown; message?: { tool_calls?: unknown }; delta?: { tool_calls?: unknown } };+ if (c.finish_reason === undefined) c.finish_reason = null;+ if (c.message && typeof c.message === 'object' && c.message.tool_calls === null) delete c.message.tool_calls;+ if (c.delta && typeof c.delta === 'object' && c.delta.tool_calls === null) delete c.delta.tool_calls;+ }+ }+ return payload;+}+// Normalize the OUTBOUND (provider → client) shape so we honor the OpenAI// contract on the response path the same way `contentToString` does on the// request path. Per spec, `choices[].delta.content` (streaming) anddiff --git a/server/src/lib/process-safety-net.ts b/server/src/lib/process-safety-net.tsnew file mode 100644index 00000000..b8a54029--- /dev/null+++ b/server/src/lib/process-safety-net.ts@@ -0,0 +1,105 @@+// Process-level safety net for late network transport errors (fork-validated:+// jasnoorgill's safe-fetch / process-safety-net).+//+// The failure this guards against: undici resolves `fetch()` and we move on, but+// the underlying HTTP/2 / TLS socket is later reset by a CDN edge (CloudFront,+// Cloudflare) AFTER the response was handed off. undici emits that late error on+// a stream with no listener, Node escalates it to an `uncaughtException`, and —+// with no handler installed — the whole proxy exits 1. For a gateway whose entire+// job is uptime in front of flaky free-tier providers, a third party closing a+// socket must never take the process down.+//+// Design: swallow ONLY a tight allowlist of transport-error codes/messages, and+// preserve Node's default fail-fast (exit 1) for everything else, so genuine+// bugs still surface loudly. The classifier is a pure function so it can be+// unit-tested without registering global handlers.++const TRANSPORT_ERROR_CODES = new Set([+ // Node socket-level codes+ 'ECONNRESET', 'ECONNREFUSED', 'ECONNABORTED', 'ETIMEDOUT', 'EPIPE',+ 'EAI_AGAIN', 'ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'EADDRNOTAVAIL',+ // undici codes (the late-error culprits)+ 'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT',+ 'UND_ERR_BODY_TIMEOUT', 'UND_ERR_ABORTED',+]);++const TRANSPORT_MESSAGE_HINTS = [+ 'fetch failed', 'other side closed', 'socket hang up', 'terminated',+ 'premature close', 'econnreset', 'request aborted',+];++// undici wraps the real socket error in `err.cause`; sometimes nested further.+// Walk the cause chain (bounded, cycle-safe) collecting codes and messages.+function walkErrorChain(err: unknown): Array<{ code?: string; message?: string }> {+ const out: Array<{ code?: string; message?: string }> = [];+ let cur: any = err;+ const seen = new Set<unknown>();+ for (let depth = 0; cur && typeof cur === 'object' && depth < 6; depth++) {+ if (seen.has(cur)) break;+ seen.add(cur);+ out.push({ code: typeof cur.code === 'string' ? cur.code : undefined, message: typeof cur.message === 'string' ? cur.message : undefined });+ cur = cur.cause;+ }+ return out;+}++/** Pure: true when the error is a recoverable network transport failure that+ * should not crash the process (vs. a programming bug, which should). */+export function isTransportError(err: unknown): boolean {+ if (err == null) return false;+ const links = walkErrorChain(err);+ for (const { code } of links) {+ if (code && TRANSPORT_ERROR_CODES.has(code)) return true;+ }+ const joined = links.map(l => l.message ?? '').join(' | ').toLowerCase();+ return TRANSPORT_MESSAGE_HINTS.some(h => joined.includes(h));+}++export type ProcessErrorDecision = 'swallow' | 'fatal';++/** Pure: swallow transport errors, treat everything else as fatal. */+export function classifyProcessError(err: unknown): ProcessErrorDecision {+ return isTransportError(err) ? 'swallow' : 'fatal';+}++function describeError(err: unknown): string {+ const links = walkErrorChain(err);+ const code = links.find(l => l.code)?.code;+ const message = links.find(l => l.message)?.message;+ return code ? `${code} (${message ?? 'no message'})` : (message ?? String(err));+}++export interface SafetyNetHooks {+ log?: (...args: unknown[]) => void;+ exit?: (code: number) => void;+}++/** Decide and act on a process-level error. Returns the decision so callers and+ * tests can assert behavior. `fatal` exits 1 (preserving Node's default+ * fail-fast); `swallow` logs and lets the process continue. */+export function handleProcessError(+ kind: 'uncaughtException' | 'unhandledRejection',+ err: unknown,+ hooks: SafetyNetHooks = {},+): ProcessErrorDecision {+ const log = hooks.log ?? console.error;+ const decision = classifyProcessError(err);+ if (decision === 'swallow') {+ log(`[safety-net] swallowed transient ${kind}: ${describeError(err)}`);+ return 'swallow';+ }+ log(`[safety-net] fatal ${kind}:`, err);+ (hooks.exit ?? process.exit)(1);+ return 'fatal';+}++let installed = false;++/** Install the global handlers once. Idempotent. Call as early as possible at+ * boot (before the server starts taking traffic). */+export function installProcessSafetyNet(hooks: SafetyNetHooks = {}): void {+ if (installed) return;+ installed = true;+ process.on('uncaughtException', (err) => handleProcessError('uncaughtException', err, hooks));+ process.on('unhandledRejection', (reason) => handleProcessError('unhandledRejection', reason, hooks));+}diff --git a/server/src/lib/tool-args.ts b/server/src/lib/tool-args.tsindex e1124ab3..0ed55105 100644--- a/server/src/lib/tool-args.ts+++ b/server/src/lib/tool-args.ts@@ -81,6 +81,32 @@ export function repairToolArguments(args: string, paramSchema?: JsonSchemaish):return changed ? JSON.stringify(parsed) : args;}+/**+ * Recursively remove the given keys from a JSON-Schema-ish value. Used to drop+ * fields a provider's tool-schema validator rejects with a 400 even though they+ * carry no meaning for the call — Cohere's compat endpoint, for instance, 400s+ * on `additionalProperties` (and `$schema`), which strict clients like opencode+ * and continue.dev routinely emit. Returns a NEW value; never mutates the input+ * (tools are shared across the fallback chain, so an in-place strip on one+ * provider would corrupt the schema the next provider sees). Non-object values+ * pass through unchanged. This is the provider-agnostic sibling of google.ts's+ * `sanitizeForGemini`, which strips a much larger Gemini-specific key set.+ */+export function stripSchemaKeys<T>(schema: T, keys: Set<string>): T {+ if (Array.isArray(schema)) {+ return schema.map((s) => stripSchemaKeys(s, keys)) as unknown as T;+ }+ if (schema && typeof schema === 'object') {+ const out: Record<string, unknown> = {};+ for (const [k, v] of Object.entries(schema as Record<string, unknown>)) {+ if (keys.has(k)) continue;+ out[k] = stripSchemaKeys(v, keys);+ }+ return out as unknown as T;+ }+ return schema;+}+/*** Build a tool-name → parameter-schema map from an OpenAI-style tools array* (chat-completions shape: {type:'function', function:{name, parameters}}).diff --git a/server/src/providers/cohere.ts b/server/src/providers/cohere.tsindex 73737c9d..b5903d06 100644--- a/server/src/providers/cohere.ts+++ b/server/src/providers/cohere.ts@@ -2,12 +2,31 @@ import type {ChatMessage,ChatCompletionResponse,ChatCompletionChunk,+ ChatToolDefinition,} from '@freellmapi/shared/types.js';import { BaseProvider, providerHttpError, type CompletionOptions } from './base.js';import { flattenMessageContent } from '../lib/content.js';+import { stripSchemaKeys } from '../lib/tool-args.js';const API_BASE = 'https://api.cohere.ai/compatibility/v1';+// Cohere's compat-endpoint tool-schema validator rejects a couple of JSON-Schema+// keywords that strict clients (opencode, continue.dev) send by default, 400-ing+// the whole request and silently killing tool calls. They carry no meaning for+// the provider, so strip them before sending. Mirrors google.ts's sanitizeForGemini+// but scoped to the keys Cohere actually rejects, since its endpoint is otherwise+// OpenAI-compatible. (Multi-fork-validated: SeanPedersen, andersmmg, chirag127.)+const COHERE_UNSUPPORTED_SCHEMA_KEYS = new Set(['additionalProperties', '$schema']);++function sanitizeCohereTools(tools?: ChatToolDefinition[]): ChatToolDefinition[] | undefined {+ if (!tools || tools.length === 0) return tools;+ return tools.map((t) =>+ t.function?.parameters+ ? { ...t, function: { ...t.function, parameters: stripSchemaKeys(t.function.parameters, COHERE_UNSUPPORTED_SCHEMA_KEYS) } }+ : t,+ );+}+export class CohereProvider extends BaseProvider {readonly platform = 'cohere' as const;readonly name = 'Cohere';@@ -24,7 +43,7 @@ export class CohereProvider extends BaseProvider {temperature: options?.temperature,max_tokens: options?.max_tokens,top_p: options?.top_p,- tools: options?.tools,+ tools: sanitizeCohereTools(options?.tools),tool_choice: options?.tool_choice,};@@ -59,7 +78,7 @@ export class CohereProvider extends BaseProvider {temperature: options?.temperature,max_tokens: options?.max_tokens,top_p: options?.top_p,- tools: options?.tools,+ tools: sanitizeCohereTools(options?.tools),tool_choice: options?.tool_choice,stream: true,};diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 5b59c086..744f62f5 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -4,10 +4,10 @@ import type { Request, Response } from 'express';import { z } from 'zod';import type { ChatMessage, ModelListRow } from '@freellmapi/shared/types.js';import { routeRequest, resolveRoutingChain, resolveModelGroupCandidates, recordRateLimitHit, recordSuccess, hasEnabledVisionModel, hasEnabledToolsModel, type RouteResult, type ResolvedChain, type ChainRow } from '../services/router.js';-import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS } from '../services/ratelimit.js';+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, MODEL_FORBIDDEN_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';import { runEmbeddings, EmbeddingsError } from '../services/embeddings.js';import { getDb, getUnifiedApiKey } from '../db/index.js';-import { contentToString, messageHasImage, normalizeOutboundContent } from '../lib/content.js';+import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../lib/content.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';import { sanitizeProviderErrorMessage } from '../lib/error-redaction.js';import { rescueInlineToolCalls, startsWithDialectMarker, couldBecomeDialectMarker, containsDialectMarker } from '../lib/tool-call-rescue.js';@@ -1042,6 +1042,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}normalizeOutboundContent(chunk);+ sanitizeResponse(chunk);const text = typeof choice.delta?.content === 'string' ? choice.delta.content : '';if (text.length === 0) {@@ -1262,7 +1263,7 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}}// Normalize array-shaped message.content to a string on the way out (#166).- res.json(normalizeOutboundContent(result));+ res.json(sanitizeResponse(normalizeOutboundContent(result)));traceRouteEvent('Proxy', {event: 'ok',@@ -1321,6 +1322,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}, err.retryAfterMs),);recordRateLimitHit(route.modelDbId);+ // Self-correct the catalog: if the provider reported its real ceiling in+ // the error body (e.g. a Groq 413 "tokens per minute (TPM): Limit 30000"),+ // persist it so the next request's pre-check fails over BEFORE the 413+ // instead of re-discovering it. No-op for errors with no parseable limit.+ learnLimitFromError(route.modelDbId, err);lastError = err;continue;}diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.tsindex baa05613..c8936d74 100644--- a/server/src/routes/responses.ts+++ b/server/src/routes/responses.ts@@ -9,7 +9,7 @@ import type {ChatToolChoice,} from '@freellmapi/shared/types.js';import { routeRequest, recordRateLimitHit, recordSuccess, hasEnabledToolsModel, type RouteResult } from '../services/router.js';-import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS } from '../services/ratelimit.js';+import { recordRequest, recordTokens, setCooldown, getCooldownDurationForLimit, PAYMENT_REQUIRED_COOLDOWN_MS, learnLimitFromError } from '../services/ratelimit.js';import { getUnifiedApiKey } from '../db/index.js';import { contentToString } from '../lib/content.js';import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';@@ -736,6 +736,9 @@ responsesRouter.post('/responses', async (req: Request, res: Response) => {? PAYMENT_REQUIRED_COOLDOWN_MS: getCooldownDurationForLimit(route.platform, route.modelId, route.keyId, { rpd: route.rpdLimit, tpd: route.tpdLimit }));recordRateLimitHit(route.modelDbId);+ // Learn a provider-reported ceiling (e.g. a 413 TPM limit) so the next+ // request's pre-check fails over before the 413. Mirrors the chat path.+ learnLimitFromError(route.modelDbId, err);lastError = err;continue;}diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.tsindex 2f9540d6..9eb9e5e8 100644--- a/server/src/services/ratelimit.ts+++ b/server/src/services/ratelimit.ts@@ -434,3 +434,68 @@ export function getRateLimitStatus(tpm: { used: tokenCount(platform, modelId, keyId, MINUTE, now), limit: limits.tpm },};}++// ── Learning real provider limits from error bodies (self-correcting catalog) ──+// Free-tier limits drift and our seeded catalog is frequently wrong or null.+// When a provider rejects a request with its real limit in the body — Groq 413:+// "...on tokens per minute (TPM): Limit 30000, Requested 33476" — we can learn+// that ceiling and persist it so the canUseTokens / canMakeRequest pre-checks+// stop us BEFORE the next 413 instead of re-discovering it on every request.+// (Fork-validated: andersmmg's updateLimitsFromError parses the same Groq shape.)++export type LearnedLimitKind = 'tpm' | 'tpd' | 'rpm' | 'rpd';+export interface LearnedLimit { kind: LearnedLimitKind; limit: number; }++// Order matters: check the per-DAY axes before per-MINUTE so "tokens per day"+// isn't shadowed by the "tpm" word-boundary alternative, and tokens before+// requests so a body mentioning both lands on the more specific token ceiling.+const LIMIT_AXIS_PATTERNS: Array<{ kind: LearnedLimitKind; re: RegExp }> = [+ { kind: 'tpd', re: /tokens?\s*per\s*day|\btpd\b/i },+ { kind: 'tpm', re: /tokens?\s*per\s*min(?:ute)?|\btpm\b/i },+ { kind: 'rpd', re: /requests?\s*per\s*day|\brpd\b/i },+ { kind: 'rpm', re: /requests?\s*per\s*min(?:ute)?|\brpm\b/i },+];++/**+ * Pure parser: pull a provider-reported ceiling out of an error message. Returns+ * null unless BOTH a numeric "Limit N" and a confident axis (TPM/TPD/RPM/RPD)+ * are present — guessing the axis would write the wrong column and mis-route+ * every future request, so we refuse to guess.+ */+export function parseProviderLimit(message: string | undefined | null): LearnedLimit | null {+ if (!message) return null;+ const m = message.match(/\blimit[:\s]+([\d,]+)/i);+ if (!m) return null;+ const limit = Number(m[1]!.replace(/,/g, ''));+ if (!Number.isFinite(limit) || limit <= 0) return null;+ for (const { kind, re } of LIMIT_AXIS_PATTERNS) {+ if (re.test(message)) return { kind, limit };+ }+ return null;+}++// Whitelisted column names — the only values ever interpolated into the UPDATE+// below, so there is no injection surface despite the template literal.+const LIMIT_COLUMN: Record<LearnedLimitKind, string> = {+ tpm: 'tpm_limit', tpd: 'tpd_limit', rpm: 'rpm_limit', rpd: 'rpd_limit',+};++/**+ * Persist a provider-reported limit onto the model row, but ONLY when it makes+ * us more conservative: fill a NULL (unknown) limit, or LOWER an existing one+ * that was too high. Never raises a limit — hitting a ceiling means our pre-check+ * already let too much through, so the true limit is at or below what we used.+ * Returns the learned limit when a row was actually changed, else null.+ * DB-guarded (no-op when the DB is unavailable), like the rest of this module.+ */+export function learnLimitFromError(modelDbId: number, err: { message?: string }): LearnedLimit | null {+ const parsed = parseProviderLimit(err?.message);+ if (!parsed) return null;+ const col = LIMIT_COLUMN[parsed.kind];+ const result = withDb(db =>+ db.prepare(+ `UPDATE models SET ${col} = ? WHERE id = ? AND (${col} IS NULL OR ${col} > ?)`,+ ).run(parsed.limit, modelDbId, parsed.limit),+ );+ return result && result.changes > 0 ? parsed : null;+}