Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Fix a batch of routing, streaming, and deployment quick wins (#941)

Tashfeena9895bc5576f
diff --git a/Dockerfile b/Dockerfile
index 904324c5..27c5dea1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -62,7 +62,14 @@ ENV FREELLMAPI_COMMIT_SHA=${FREELLMAPI_COMMIT_SHA}
USER node
EXPOSE 3001
-VOLUME ["/app/server/data"]
+
+# No VOLUME for /app/server/data on purpose. Persistence is the deployment's
+# job — docker-compose.yml maps the named `freellmapi-data` volume there, and a
+# plain `docker run` takes -v. Declaring it here instead creates an ANONYMOUS
+# volume on every container that doesn't override it: PaaS runtimes that build
+# from the Dockerfile (Railway, Render, Coolify, Dokploy, CapRover) then either
+# refuse the image or silently hand each redeploy a fresh empty volume, and the
+# declaration also shadows a bind mount made at the same path.
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
CMD node -e "fetch('http://127.0.0.1:' + (process.env.PORT || 3001) + '/api/ping').then((res) => { if (!res.ok) process.exit(1); }).catch(() => process.exit(1));"
diff --git a/server/src/__tests__/lib/content.test.ts b/server/src/__tests__/lib/content.test.ts
index 8b875642..ede11d52 100644
--- a/server/src/__tests__/lib/content.test.ts
+++ b/server/src/__tests__/lib/content.test.ts
@@ -1,5 +1,6 @@
import { describe, it, expect } from 'vitest';
-import { contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages } from '../../lib/content.js';
+import { GITHUB_MAX_INPUT_TOKENS, contentToString, flattenMessageContent, messageHasImage, normalizeOutboundContent, sanitizeResponse, stripImagesFromMessages, truncateMessagesForGithub } from '../../lib/content.js';
+import type { ChatMessage } from '@freellmapi/shared/types.js';
describe('contentToString', () => {
it('passes strings through', () => {
@@ -202,3 +203,70 @@ describe('sanitizeResponse', () => {
expect(() => sanitizeResponse({} as unknown)).not.toThrow();
});
});
+
+describe('truncateMessagesForGithub', () => {
+ const msg = (role: ChatMessage['role'], content: string): ChatMessage => ({ role, content });
+ // ~4000 tokens each under the chars/4 estimate the gateway uses everywhere.
+ const big = (c: string) => c.repeat(16000);
+
+ it('returns the same array when the request already fits', () => {
+ const messages = [msg('system', 'be brief'), msg('user', 'hi')];
+ expect(truncateMessagesForGithub(messages)).toBe(messages);
+ });
+
+ it('leaves an empty list alone', () => {
+ const messages: ChatMessage[] = [];
+ expect(truncateMessagesForGithub(messages)).toBe(messages);
+ });
+
+ it('drops the oldest turns, keeping the system prompt and the newest context', () => {
+ const messages = [
+ msg('system', 'be brief'),
+ msg('user', big('a')),
+ msg('assistant', big('b')),
+ msg('user', big('c')),
+ msg('user', 'the actual question'),
+ ];
+ const out = truncateMessagesForGithub(messages);
+ expect(out).not.toBe(messages);
+ expect(out[0]).toBe(messages[0]);
+ expect(out[out.length - 1].content).toBe('the actual question');
+ // Only one of the ~4000-token turns fits alongside the newest question.
+ expect(out.map(m => m.content)).not.toContain(big('a'));
+ expect(out.map(m => m.content)).toContain(big('c'));
+ const total = out.reduce((sum, m) => sum + Math.ceil(String(m.content).length / 4), 0);
+ expect(total).toBeLessThanOrEqual(GITHUB_MAX_INPUT_TOKENS);
+ });
+
+ it('keeps the system prompt verbatim wherever it sits in the list', () => {
+ const system = msg('system', 'be brief');
+ const messages = [msg('user', big('a')), system, msg('user', big('b'))];
+ const out = truncateMessagesForGithub(messages);
+ expect(out[0]).toBe(system);
+ });
+
+ it('truncates a single oversized message instead of sending nothing', () => {
+ const out = truncateMessagesForGithub([msg('user', big('y').repeat(3))]);
+ expect(out).toHaveLength(1);
+ expect(String(out[0].content).length).toBe(GITHUB_MAX_INPUT_TOKENS * 4);
+ });
+
+ it('reserves the system prompt budget when truncating the newest message', () => {
+ const system = msg('system', 'x'.repeat(400)); // 100 tokens
+ const out = truncateMessagesForGithub([system, msg('user', big('z').repeat(3))]);
+ expect(out[0]).toBe(system);
+ expect(String(out[1].content).length).toBe((GITHUB_MAX_INPUT_TOKENS - 100) * 4);
+ });
+
+ it('honours a caller-supplied budget', () => {
+ const messages = [msg('user', 'one'), msg('user', 'two'), msg('user', 'the newest')];
+ const out = truncateMessagesForGithub(messages, 3); // 12 chars of room
+ expect(out.map(m => m.content)).toEqual(['the newest']);
+ });
+
+ it('never rewrites multimodal content, only drops whole messages', () => {
+ const image: ChatMessage = { role: 'user', content: [{ type: 'image_url', image_url: { url: 'data:x' } }] } as ChatMessage;
+ const out = truncateMessagesForGithub([msg('user', big('a')), image], 1);
+ expect(out).toEqual([image]);
+ });
+});
diff --git a/server/src/__tests__/lib/crypto.test.ts b/server/src/__tests__/lib/crypto.test.ts
index 121ddcfd..31518a6a 100644
--- a/server/src/__tests__/lib/crypto.test.ts
+++ b/server/src/__tests__/lib/crypto.test.ts
@@ -41,8 +41,34 @@ describe('Crypto', () => {
expect(maskKey('gsk_test1234567890abcdef')).toBe('gsk_...cdef');
});
- it('should mask short keys', () => {
- expect(maskKey('abcd')).toBe('****abcd');
+ // Regression: the old short-key branch was '****' + key.slice(-4), which
+ // echoed a 4-character key back IN FULL and revealed 4 of 5 characters of a
+ // 5-character key. Short keys are real (proxy credentials, self-hosted
+ // Ollama tokens), and the mask is display-only — it must never be enough to
+ // reconstruct the secret.
+ it('should reveal nothing at all for keys shorter than 5 characters', () => {
+ expect(maskKey('abcd')).toBe('****');
+ expect(maskKey('abc')).toBe('****');
+ expect(maskKey('a')).toBe('****');
+ expect(maskKey('')).toBe('****');
+ });
+
+ it('should reveal at most the last two characters for 5-8 character keys', () => {
+ expect(maskKey('abcde')).toBe('****de');
+ expect(maskKey('abcdef')).toBe('****ef');
+ expect(maskKey('abcdefgh')).toBe('****gh');
+ });
+
+ it('should keep the prefix+suffix form once a key is long enough', () => {
+ expect(maskKey('abcdefghi')).toBe('abcd...fghi');
+ });
+
+ it('should never emit more than a third of a short key', () => {
+ for (const key of ['a', 'ab', 'abc', 'abcd', 'abcde', 'abcdef', 'abcdefg', 'abcdefgh']) {
+ const revealed = maskKey(key).replace(/[*.]/g, '');
+ expect(revealed.length).toBeLessThanOrEqual(2);
+ expect(maskKey(key)).not.toBe(key);
+ }
});
});
});
diff --git a/server/src/__tests__/lib/db-backup.test.ts b/server/src/__tests__/lib/db-backup.test.ts
index 6341cc02..f76d901e 100644
--- a/server/src/__tests__/lib/db-backup.test.ts
+++ b/server/src/__tests__/lib/db-backup.test.ts
@@ -2,14 +2,15 @@ import fs from 'fs';
import os from 'os';
import path from 'path';
import Database from 'better-sqlite3';
-import { afterEach, describe, expect, it } from 'vitest';
-import { backupDbNow, restoreDbBackupIfNeeded } from '../../lib/db-backup.js';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { backupDbNow, parseHuggingFaceTarget, restoreDbBackupIfNeeded } from '../../lib/db-backup.js';
import { aclEntries } from '../helpers/acl.js';
const ORIGINAL_BACKUP_PATH = process.env.FREEAPI_DB_BACKUP_PATH;
const ORIGINAL_BACKUP_TARGET = process.env.FREEAPI_DB_BACKUP_TARGET;
const ORIGINAL_BACKUP_URL = process.env.FREEAPI_DB_BACKUP_URL;
const ORIGINAL_BACKUP_KEY = process.env.FREEAPI_DB_BACKUP_KEY;
+const ORIGINAL_BACKUP_TOKEN = process.env.FREEAPI_DB_BACKUP_TOKEN;
const ORIGINAL_ENCRYPTION_KEY = process.env.ENCRYPTION_KEY;
function restoreEnv() {
@@ -21,6 +22,8 @@ function restoreEnv() {
else process.env.FREEAPI_DB_BACKUP_URL = ORIGINAL_BACKUP_URL;
if (ORIGINAL_BACKUP_KEY === undefined) delete process.env.FREEAPI_DB_BACKUP_KEY;
else process.env.FREEAPI_DB_BACKUP_KEY = ORIGINAL_BACKUP_KEY;
+ if (ORIGINAL_BACKUP_TOKEN === undefined) delete process.env.FREEAPI_DB_BACKUP_TOKEN;
+ else process.env.FREEAPI_DB_BACKUP_TOKEN = ORIGINAL_BACKUP_TOKEN;
if (ORIGINAL_ENCRYPTION_KEY === undefined) delete process.env.ENCRYPTION_KEY;
else process.env.ENCRYPTION_KEY = ORIGINAL_ENCRYPTION_KEY;
}
@@ -128,3 +131,176 @@ describe('backup file permissions', () => {
expect(entries).toHaveLength(3);
});
});
+
+describe('parseHuggingFaceTarget', () => {
+ it('maps a dataset download URL onto the commit API', () => {
+ expect(parseHuggingFaceTarget('https://huggingface.co/datasets/acme/state/resolve/main/backups/freeapi.db.enc')).toEqual({
+ commitUrl: 'https://huggingface.co/api/datasets/acme/state/commit/main',
+ filePath: 'backups/freeapi.db.enc',
+ });
+ });
+
+ it('treats the prefix-less repo path as a model repo', () => {
+ expect(parseHuggingFaceTarget('https://huggingface.co/acme/state/resolve/main/freeapi.db.enc')).toEqual({
+ commitUrl: 'https://huggingface.co/api/models/acme/state/commit/main',
+ filePath: 'freeapi.db.enc',
+ });
+ });
+
+ it('maps a space download URL', () => {
+ expect(parseHuggingFaceTarget('https://huggingface.co/spaces/acme/app/resolve/main/data/freeapi.db.enc')?.commitUrl)
+ .toBe('https://huggingface.co/api/spaces/acme/app/commit/main');
+ });
+
+ it('leaves every other target alone', () => {
+ expect(parseHuggingFaceTarget('https://example.com/acme/state/resolve/main/freeapi.db.enc')).toBeNull();
+ // A HF URL that is not a /resolve/ download has no commit route to derive.
+ expect(parseHuggingFaceTarget('https://huggingface.co/acme/state')).toBeNull();
+ expect(parseHuggingFaceTarget('/var/backups/freeapi.db.enc')).toBeNull();
+ });
+});
+
+describe('Hugging Face backup target', () => {
+ // The /resolve/ URL a HF repo serves downloads from is read-only: the PUT this
+ // used to send was answered 404 and the blob never landed, so an ephemeral
+ // host restored nothing on its next cold start. Uploads go through the commit
+ // API, and the blob travels as base64 text because repos on Xet storage reject
+ // binary pushed through that route.
+ const TARGET = 'https://huggingface.co/datasets/acme/state/resolve/main/backups/freeapi.db.enc';
+
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ restoreEnv();
+ });
+
+ function seedDb(dir: string): string {
+ const dbPath = path.join(dir, 'freeapi.db');
+ const db = new Database(dbPath);
+ db.exec('CREATE TABLE items (name TEXT NOT NULL); INSERT INTO items (name) VALUES (\'survived\')');
+ db.close();
+ return dbPath;
+ }
+
+ it('uploads through the commit API as base64 and restores it', async () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'freeapi-hf-backup-'));
+ const dbPath = seedDb(dir);
+ process.env.ENCRYPTION_KEY = 'a'.repeat(64);
+ process.env.FREEAPI_DB_BACKUP_TARGET = TARGET;
+ process.env.FREEAPI_DB_BACKUP_TOKEN = 'hf_token';
+
+ let stored = '';
+ const fetchMock = vi.fn(async (url: string, init: RequestInit) => {
+ if (init.method === 'POST') {
+ stored = String(init.body);
+ return new Response('{}', { status: 200 });
+ }
+ return new Response(stored, { status: 200 });
+ });
+ vi.stubGlobal('fetch', fetchMock);
+
+ const db = new Database(dbPath);
+ expect((await backupDbNow(db, dbPath)).ok).toBe(true);
+ db.close();
+
+ const [uploadUrl, uploadInit] = fetchMock.mock.calls[0] as [string, RequestInit];
+ expect(uploadUrl).toBe('https://huggingface.co/api/datasets/acme/state/commit/main');
+ expect(uploadInit.method).toBe('POST');
+ expect((uploadInit.headers as Record<string, string>)['Content-Type']).toBe('application/x-ndjson');
+ expect((uploadInit.headers as Record<string, string>).Authorization).toBe('Bearer hf_token');
+
+ const [header, file] = String(uploadInit.body).split('\n').map(line => JSON.parse(line));
+ expect(header).toEqual({ key: 'header', value: { summary: 'chore: update backups/freeapi.db.enc' } });
+ expect(file.key).toBe('file');
+ expect(file.value.path).toBe('backups/freeapi.db.enc');
+ // Base64 text, not bytes — 'base64' here would make HF write the binary back.
+ expect(file.value.encoding).toBe('utf-8');
+ expect(file.value.content).toMatch(/^[A-Za-z0-9+/]+={0,2}$/);
+ expect(file.value.content).not.toContain('survived');
+
+ // What HF serves back from /resolve/ is that base64 text; the restore has to
+ // decode it before it is a database again.
+ stored = file.value.content;
+ fs.rmSync(dbPath);
+ expect((await restoreDbBackupIfNeeded(dbPath)).restored).toBe(true);
+
+ const restored = new Database(dbPath);
+ expect((restored.prepare('SELECT name FROM items').get() as { name: string }).name).toBe('survived');
+ restored.close();
+ });
+
+ it('still restores a blob that was stored raw', async () => {
+ // Anything a previous build managed to land is raw ciphertext. Sniffing the
+ // magic header keeps that path recoverable instead of base64-decoding it
+ // into garbage.
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'freeapi-hf-legacy-'));
+ const dbPath = seedDb(dir);
+ const blobPath = path.join(dir, 'backup.bin');
+ process.env.ENCRYPTION_KEY = 'a'.repeat(64);
+ process.env.FREEAPI_DB_BACKUP_PATH = blobPath;
+
+ const db = new Database(dbPath);
+ await backupDbNow(db, dbPath);
+ db.close();
+ const raw = fs.readFileSync(blobPath);
+
+ delete process.env.FREEAPI_DB_BACKUP_PATH;
+ process.env.FREEAPI_DB_BACKUP_TARGET = TARGET;
+ vi.stubGlobal('fetch', vi.fn(async () => new Response(raw, { status: 200 })));
+
+ fs.rmSync(dbPath);
+ expect((await restoreDbBackupIfNeeded(dbPath)).restored).toBe(true);
+ const restored = new Database(dbPath);
+ expect((restored.prepare('SELECT name FROM items').get() as { name: string }).name).toBe('survived');
+ restored.close();
+ });
+
+ it('refuses to upload without a token rather than 401ing on a schedule', async () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'freeapi-hf-token-'));
+ const dbPath = seedDb(dir);
+ process.env.ENCRYPTION_KEY = 'a'.repeat(64);
+ process.env.FREEAPI_DB_BACKUP_TARGET = TARGET;
+ delete process.env.FREEAPI_DB_BACKUP_TOKEN;
+
+ const fetchMock = vi.fn();
+ vi.stubGlobal('fetch', fetchMock);
+
+ const db = new Database(dbPath);
+ await expect(backupDbNow(db, dbPath)).rejects.toThrow(/FREEAPI_DB_BACKUP_TOKEN is required/);
+ db.close();
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it('surfaces a rejected commit', async () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'freeapi-hf-error-'));
+ const dbPath = seedDb(dir);
+ process.env.ENCRYPTION_KEY = 'a'.repeat(64);
+ process.env.FREEAPI_DB_BACKUP_TARGET = TARGET;
+ process.env.FREEAPI_DB_BACKUP_TOKEN = 'hf_token';
+ vi.stubGlobal('fetch', vi.fn(async () => new Response('repo not found', { status: 404 })));
+
+ const db = new Database(dbPath);
+ await expect(backupDbNow(db, dbPath)).rejects.toThrow(/HF commit HTTP 404 — repo not found/);
+ db.close();
+ });
+
+ it('leaves a plain HTTP target on the PUT path', async () => {
+ const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'freeapi-http-backup-'));
+ const dbPath = seedDb(dir);
+ process.env.ENCRYPTION_KEY = 'a'.repeat(64);
+ process.env.FREEAPI_DB_BACKUP_TARGET = 'https://storage.example.com/freeapi.db.enc';
+ process.env.FREEAPI_DB_BACKUP_TOKEN = 'plain_token';
+
+ const fetchMock = vi.fn(async () => new Response('', { status: 200 }));
+ vi.stubGlobal('fetch', fetchMock);
+
+ const db = new Database(dbPath);
+ expect((await backupDbNow(db, dbPath)).ok).toBe(true);
+ db.close();
+
+ const [url, init] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];
+ expect(url).toBe('https://storage.example.com/freeapi.db.enc');
+ expect(init.method).toBe('PUT');
+ expect((init.headers as Record<string, string>)['Content-Type']).toBe('application/octet-stream');
+ expect(Buffer.isBuffer(init.body)).toBe(true);
+ });
+});
diff --git a/server/src/__tests__/lib/error-classify-transport.test.ts b/server/src/__tests__/lib/error-classify-transport.test.ts
new file mode 100644
index 00000000..700d0e29
--- /dev/null
+++ b/server/src/__tests__/lib/error-classify-transport.test.ts
@@ -0,0 +1,147 @@
+import { describe, it, expect } from 'vitest';
+import {
+ isRetryableError,
+ isTransportError,
+ newClientAbortError,
+ newHedgeAbortError,
+} from '../../lib/error-classify.js';
+
+/** Build `new Error(message)` with an undici-style `code`, wrapped in `depth`
+ * generic wrappers whose own messages match NONE of the top-level substring
+ * rules — the exact shape that used to slip through as fatal. */
+function wrapped(depth: number, inner: Error): Error {
+ let cur: Error = inner;
+ for (let i = 0; i < depth; i++) {
+ cur = new Error('Provider request failed', { cause: cur });
+ }
+ return cur;
+}
+
+function coded(code: string, message = 'read ECONN'): Error {
+ const err = new Error(message);
+ (err as Error & { code?: string }).code = code;
+ return err;
+}
+
+describe('transport errors carried in err.cause', () => {
+ describe('codes anywhere in the chain are retryable', () => {
+ for (const code of ['ECONNRESET', 'ECONNREFUSED', 'EPIPE', 'ETIMEDOUT', 'EAI_AGAIN']) {
+ it(`${code} behind a generic wrapper`, () => {
+ const err = wrapped(1, coded(code, 'socket failure'));
+ expect(isTransportError(err)).toBe(true);
+ expect(isRetryableError(err)).toBe(true);
+ });
+ }
+
+ for (const code of ['UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']) {
+ it(`undici ${code} behind a generic wrapper`, () => {
+ const err = wrapped(1, coded(code, 'undici transport failure'));
+ expect(isRetryableError(err)).toBe(true);
+ });
+ }
+
+ it('matches a transport code on the top-level error itself (depth 0)', () => {
+ expect(isRetryableError(coded('ECONNRESET', 'boom'))).toBe(true);
+ });
+ });
+
+ describe('message hints anywhere in the chain are retryable', () => {
+ for (const message of [
+ 'socket hang up',
+ 'Premature close',
+ 'other side closed',
+ 'Client network socket disconnected before secure TLS connection was established',
+ ]) {
+ it(`"${message}" behind a generic wrapper`, () => {
+ const err = wrapped(1, new Error(message));
+ expect(isTransportError(err)).toBe(true);
+ expect(isRetryableError(err)).toBe(true);
+ });
+ }
+
+ it('undici\'s bare "terminated" body error is retryable', () => {
+ expect(isRetryableError(new Error('terminated'))).toBe(true);
+ expect(isRetryableError(wrapped(2, new Error('terminated')))).toBe(true);
+ });
+
+ // "terminated" is matched as a WHOLE message only: a terminated ACCOUNT is
+ // fatal and must not be retried around the entire chain.
+ it('does not treat "account has been terminated" as transport', () => {
+ const err: any = new Error('API error 403: your account has been terminated');
+ // Retryable via the 403 rule, but NOT because of the transport walk.
+ expect(isTransportError(err)).toBe(false);
+ });
+ });
+
+ describe('walk is bounded and cycle-safe', () => {
+ it('finds a cause nested at depth 5', () => {
+ expect(isTransportError(wrapped(5, coded('ECONNRESET', 'socket failure')))).toBe(true);
+ });
+
+ it('stops before an absurdly deep chain', () => {
+ expect(isTransportError(wrapped(40, coded('ECONNRESET', 'socket failure')))).toBe(false);
+ });
+
+ it('does not hang on a self-referential cause', () => {
+ const err: any = new Error('Provider request failed');
+ err.cause = err;
+ expect(isTransportError(err)).toBe(false);
+ });
+
+ it('does not hang on a two-link cause cycle', () => {
+ const a: any = new Error('Provider request failed');
+ const b: any = new Error('Also failed', { cause: a });
+ a.cause = b;
+ expect(isTransportError(a)).toBe(false);
+ });
+
+ it('tolerates non-object and null causes', () => {
+ expect(isTransportError(null)).toBe(false);
+ expect(isTransportError(undefined)).toBe(false);
+ expect(isTransportError(new Error('nope', { cause: 'ECONNRESET' }))).toBe(false);
+ expect(isTransportError(new Error('nope', { cause: undefined }))).toBe(false);
+ });
+ });
+
+ describe('fatal classes stay non-retryable', () => {
+ it('a 401 bad key is not made retryable by the cause walk', () => {
+ const err: any = new Error('API error 401: Invalid API key');
+ err.status = 401;
+ expect(isTransportError(err)).toBe(false);
+ expect(isRetryableError(err)).toBe(false);
+ });
+
+ // undici labels an aborted socket UND_ERR_ABORTED — the same family a real
+ // mid-flight socket death carries — so the marked abort errors must win.
+ it('a client abort wrapped in an undici UND_ERR_ABORTED stays non-retryable', () => {
+ const abort = newClientAbortError();
+ const err = new Error('Provider request failed', { cause: abort });
+ (err as any).code = 'UND_ERR_ABORTED';
+ expect(isTransportError(err)).toBe(false);
+ expect(isRetryableError(err)).toBe(false);
+ });
+
+ it('a bare client abort stays non-retryable', () => {
+ expect(isTransportError(newClientAbortError())).toBe(false);
+ expect(isRetryableError(newClientAbortError())).toBe(false);
+ });
+
+ it('the fallback time-budget hedge abort stays non-retryable', () => {
+ const hedge = newHedgeAbortError();
+ (hedge as any).code = 'UND_ERR_ABORTED';
+ expect(isTransportError(hedge)).toBe(false);
+ expect(isRetryableError(hedge)).toBe(false);
+ });
+
+ it('an ordinary validation 400 is untouched', () => {
+ const err: any = new Error('Bad Request: messages must be an array');
+ err.status = 400;
+ expect(isTransportError(err)).toBe(false);
+ expect(isRetryableError(err)).toBe(false);
+ });
+
+ it('a generic wrapper with no transport evidence stays non-retryable', () => {
+ expect(isTransportError(wrapped(3, new Error('model refused the prompt')))).toBe(false);
+ });
+ });
+});
diff --git a/server/src/__tests__/lib/gemini-schema-compat.test.ts b/server/src/__tests__/lib/gemini-schema-compat.test.ts
new file mode 100644
index 00000000..723112c1
--- /dev/null
+++ b/server/src/__tests__/lib/gemini-schema-compat.test.ts
@@ -0,0 +1,254 @@
+import { describe, it, expect } from 'vitest';
+import { sanitizeForGemini } from '../../lib/gemini-wire.js';
+
+describe('sanitizeForGemini type unions', () => {
+ it('collapses a nullable union into a type plus nullable at the top level', () => {
+ // Google's Schema proto rejects a repeated `type` with
+ // "Proto field is not repeating, cannot start list", so every schema built
+ // by a `.nullable()` helper 400s unless the union is collapsed.
+ expect(sanitizeForGemini({ type: ['number', 'null'] })).toEqual({
+ type: 'number',
+ nullable: true,
+ });
+ });
+
+ it('collapses nullable unions nested in properties, items, and anyOf branches', () => {
+ const input = {
+ type: 'object',
+ properties: {
+ age: { type: ['integer', 'null'], description: 'Age in years' },
+ address: {
+ type: ['object', 'null'],
+ properties: {
+ street: { type: ['string', 'null'] },
+ },
+ },
+ aliases: {
+ type: 'array',
+ items: { type: ['string', 'null'] },
+ },
+ contact: {
+ anyOf: [
+ { type: ['string', 'null'] },
+ { type: 'object', properties: { email: { type: ['string', 'null'] } } },
+ ],
+ },
+ },
+ required: ['age'],
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ age: { type: 'integer', description: 'Age in years', nullable: true },
+ address: {
+ type: 'object',
+ nullable: true,
+ properties: {
+ street: { type: 'string', nullable: true },
+ },
+ },
+ aliases: {
+ type: 'array',
+ items: { type: 'string', nullable: true },
+ },
+ contact: {
+ anyOf: [
+ { type: 'string', nullable: true },
+ { type: 'object', properties: { email: { type: 'string', nullable: true } } },
+ ],
+ },
+ },
+ required: ['age'],
+ });
+ });
+
+ it('keeps the first concrete member of a union that carries no null', () => {
+ expect(sanitizeForGemini({ type: ['string', 'number'] })).toEqual({ type: 'string' });
+ });
+
+ it('drops type entirely for a null-only union', () => {
+ expect(sanitizeForGemini({ type: ['null'], description: 'always empty' })).toEqual({
+ description: 'always empty',
+ nullable: true,
+ });
+ });
+
+ it('leaves a single-string type and an explicit nullable flag untouched', () => {
+ expect(sanitizeForGemini({ type: 'string', nullable: true })).toEqual({
+ type: 'string',
+ nullable: true,
+ });
+ });
+
+ it('does not treat a property literally named "type" as a type union', () => {
+ const input = {
+ type: 'object',
+ properties: {
+ type: { type: ['string', 'null'], enum: ['a', 'b'] },
+ },
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ type: { type: 'string', enum: ['a', 'b'], nullable: true },
+ },
+ });
+ });
+});
+
+describe('sanitizeForGemini $ref handling', () => {
+ it('inlines a $ref that points into $defs', () => {
+ const input = {
+ type: 'object',
+ $defs: {
+ Address: {
+ type: 'object',
+ properties: { city: { type: 'string' } },
+ required: ['city'],
+ },
+ },
+ properties: {
+ home: { $ref: '#/$defs/Address' },
+ },
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ home: {
+ type: 'object',
+ properties: { city: { type: 'string' } },
+ required: ['city'],
+ },
+ },
+ });
+ });
+
+ it('inlines a $ref into legacy definitions and sanitizes the target', () => {
+ const input = {
+ type: 'object',
+ definitions: {
+ Count: { type: ['integer', 'null'], exclusiveMinimum: 0, 'x-note': 'hi' },
+ },
+ properties: {
+ count: { $ref: '#/definitions/Count', description: 'How many' },
+ },
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ count: { type: 'integer', nullable: true, description: 'How many' },
+ },
+ });
+ });
+
+ it('reuses a definition across sibling properties', () => {
+ const input = {
+ type: 'object',
+ $defs: { Name: { type: 'string', minLength: 1 } },
+ properties: {
+ first: { $ref: '#/$defs/Name' },
+ last: { $ref: '#/$defs/Name' },
+ },
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ first: { type: 'string', minLength: 1 },
+ last: { type: 'string', minLength: 1 },
+ },
+ });
+ });
+
+ it('drops an orphan $ref and leaves a permissive schema behind', () => {
+ const input = {
+ type: 'object',
+ properties: {
+ missing: { $ref: '#/$defs/Nope' },
+ remote: { $ref: 'https://example.com/schema.json', description: 'kept' },
+ },
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ missing: {},
+ remote: { description: 'kept' },
+ },
+ });
+ });
+
+ it('stops expanding a self-referential definition', () => {
+ const input = {
+ type: 'object',
+ $defs: {
+ Node: {
+ type: 'object',
+ properties: {
+ value: { type: 'string' },
+ child: { $ref: '#/$defs/Node' },
+ },
+ },
+ },
+ properties: { root: { $ref: '#/$defs/Node' } },
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ root: {
+ type: 'object',
+ properties: {
+ value: { type: 'string' },
+ child: {},
+ },
+ },
+ },
+ });
+ });
+});
+
+describe('sanitizeForGemini vendor extensions and pass-through', () => {
+ it('strips x-* keys at every depth while keeping x-prefixed property names', () => {
+ const input = {
+ type: 'object',
+ 'x-tool-version': 3,
+ properties: {
+ 'x-request-id': {
+ type: ['string', 'null'],
+ 'X-Legacy-Hint': 'stripped case-insensitively',
+ },
+ nested: {
+ type: 'object',
+ properties: { mode: { type: 'string', 'x-provider': 'local' } },
+ },
+ },
+ required: ['x-request-id'],
+ };
+ expect(sanitizeForGemini(input)).toEqual({
+ type: 'object',
+ properties: {
+ 'x-request-id': { type: 'string', nullable: true },
+ nested: {
+ type: 'object',
+ properties: { mode: { type: 'string' } },
+ },
+ },
+ required: ['x-request-id'],
+ });
+ });
+
+ it('passes an already-valid schema through unchanged', () => {
+ const input = {
+ type: 'object',
+ description: 'Look up the weather',
+ properties: {
+ city: { type: 'string', description: 'City name', enum: ['Karachi', 'Lahore'] },
+ days: { type: 'integer', minimum: 1, maximum: 7 },
+ tags: { type: 'array', items: { type: 'string' }, minItems: 1 },
+ detail: { type: 'object', nullable: true, properties: { units: { type: 'string' } } },
+ },
+ required: ['city'],
+ };
+ // Byte-identical, not merely deep-equal: nothing may be reordered or
+ // re-synthesized on a schema Gemini already accepts.
+ expect(JSON.stringify(sanitizeForGemini(input))).toBe(JSON.stringify(input));
+ });
+});
diff --git a/server/src/__tests__/lib/proxy.test.ts b/server/src/__tests__/lib/proxy.test.ts
index a170fb94..b37ad6a2 100644
--- a/server/src/__tests__/lib/proxy.test.ts
+++ b/server/src/__tests__/lib/proxy.test.ts
@@ -11,6 +11,7 @@ import {
getNoProxyRules,
isProxyActive,
isSocksProxyUrl,
+ socksHostnameLookup,
PROXY_SCHEMES,
proxyFetch,
describeAbort,
@@ -219,6 +220,45 @@ describe('SOCKS socket timeout guard (#666)', () => {
});
});
+// socks-proxy-agent resolves the DESTINATION locally for the plain `socks5://`
+// and `socks4://` schemes and hands the proxy a bare IP. Rule-based proxy
+// clients (Clash and friends) route on the DOMAIN, so a pre-resolved IP loses
+// every routing rule the user wrote. socksFetch passes a `lookup` that echoes
+// the hostname, which is what makes the SOCKS path behave like socks5h
+// regardless of the scheme the user configured.
+describe('SOCKS destination hostname reaches the proxy unresolved', () => {
+ const lookupOf = (spy: ReturnType<typeof stubHttpsRequest>): any =>
+ (spy.mock.calls[0][0] as any).lookup;
+
+ it('returns the hostname it was handed, unchanged', () => {
+ const seen: unknown[] = [];
+ socksHostnameLookup('api.example.com', {}, (...args) => seen.push(args));
+ expect(seen).toEqual([[null, 'api.example.com', 4]]);
+ });
+
+ it('never performs a real DNS resolution', () => {
+ // A hostname that cannot resolve anywhere still comes straight back out,
+ // synchronously — proof the override short-circuits dns.lookup entirely.
+ let address: string | undefined;
+ socksHostnameLookup('this-host-does-not-exist.invalid', {}, (_e, addr) => { address = addr; });
+ expect(address).toBe('this-host-does-not-exist.invalid');
+ });
+
+ it('installs the override on every SOCKS scheme, including socks5', async () => {
+ for (const url of ['socks5://127.0.0.1:1080', 'socks5h://127.0.0.1:1080', 'socks4://127.0.0.1:1080']) {
+ applyProxyUrl(url);
+ const reqSpy = stubHttpsRequest();
+
+ await proxyFetch('https://api.example.com/v1', { method: 'POST' }, 'groq');
+
+ let resolved: string | undefined;
+ lookupOf(reqSpy)('api.example.com', {}, (_e: unknown, addr: string) => { resolved = addr; });
+ expect(resolved).toBe('api.example.com');
+ vi.restoreAllMocks();
+ }
+ });
+});
+
// #353: HTTPS_PROXY / HTTP_PROXY / ALL_PROXY / NO_PROXY are the de-facto
// standard for every other CLI on the box. Honouring them means a user who
// already exported them for curl/git gets a working app with no extra config —
diff --git a/server/src/__tests__/lib/unified-max-tokens.test.ts b/server/src/__tests__/lib/unified-max-tokens.test.ts
index d6f669a6..25289303 100644
--- a/server/src/__tests__/lib/unified-max-tokens.test.ts
+++ b/server/src/__tests__/lib/unified-max-tokens.test.ts
@@ -16,6 +16,8 @@ import {
unifiedMaxTokensCap,
resolveMaxTokens,
defaultMaxTokensFor,
+ maxTokensCapFor,
+ GITHUB_MAX_OUTPUT_TOKENS,
UNIFIED_MAX_TOKENS_SETTING,
UNIFIED_MAX_TOKENS_AUTO,
} from '../../lib/sampling-params.js';
@@ -95,3 +97,34 @@ describe('resolveMaxTokens under the cap', () => {
expect(resolveMaxTokens('groq', 16)).toBe(16);
});
});
+
+describe('per-platform max_tokens ceiling', () => {
+ it('is declared only for platforms whose API rejects large values', () => {
+ expect(maxTokensCapFor('github')).toBe(GITHUB_MAX_OUTPUT_TOKENS);
+ expect(maxTokensCapFor('groq')).toBeUndefined();
+ expect(maxTokensCapFor('nonsense-platform')).toBeUndefined();
+ });
+
+ it('clamps an oversized request even with the operator cap off', () => {
+ expect(unifiedMaxTokensCap()).toBeNull();
+ expect(resolveMaxTokens('github', 65536)).toBe(GITHUB_MAX_OUTPUT_TOKENS);
+ // Other platforms keep the historical pass-through behaviour.
+ expect(resolveMaxTokens('groq', 65536)).toBe(65536);
+ });
+
+ it('leaves a request at or below the platform ceiling alone', () => {
+ expect(resolveMaxTokens('github', 128)).toBe(128);
+ expect(resolveMaxTokens('github', GITHUB_MAX_OUTPUT_TOKENS)).toBe(GITHUB_MAX_OUTPUT_TOKENS);
+ });
+
+ it('sends nothing when the client sent nothing and the platform has no floor', () => {
+ expect(resolveMaxTokens('github', undefined)).toBeUndefined();
+ });
+
+ it('takes the tighter of the platform ceiling and the operator cap', () => {
+ settingStore.set(UNIFIED_MAX_TOKENS_SETTING, 'auto'); // 32768 — looser
+ expect(resolveMaxTokens('github', 65536)).toBe(GITHUB_MAX_OUTPUT_TOKENS);
+ settingStore.set(UNIFIED_MAX_TOKENS_SETTING, '128'); // tighter than the platform
+ expect(resolveMaxTokens('github', 65536)).toBe(128);
+ });
+});
diff --git a/server/src/__tests__/providers/zhipu.test.ts b/server/src/__tests__/providers/zhipu.test.ts
new file mode 100644
index 00000000..a0a56e2e
--- /dev/null
+++ b/server/src/__tests__/providers/zhipu.test.ts
@@ -0,0 +1,178 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { ZhipuProvider } from '../../providers/zhipu.js';
+
+const DOMESTIC = 'https://open.bigmodel.cn/api/paas/v4';
+const GLOBAL = 'https://api.z.ai/api/paas/v4';
+
+afterEach(() => {
+ vi.restoreAllMocks();
+});
+
+/** Minimal catalog/chat responses. status alone decides the validation verdict. */
+const catalogRes = (status: number, message?: string) => ({
+ ok: status < 400,
+ status,
+ statusText: status === 401 ? 'Unauthorized' : 'OK',
+ headers: new Headers(),
+ json: () => Promise.resolve(
+ status === 401 ? { error: { message: message ?? 'invalid api key' } } : { data: [] },
+ ),
+}) as unknown as Response;
+
+const chatRes = () => ({
+ ok: true,
+ status: 200,
+ headers: new Headers(),
+ json: () => Promise.resolve({
+ id: 'id', object: 'chat.completion', created: 1, model: 'glm-4.5-flash',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 },
+ }),
+}) as unknown as Response;
+
+/** Record every URL fetched and answer from a per-host table. */
+function mockHosts(table: Array<[string, () => Response]>) {
+ const urls: string[] = [];
+ vi.spyOn(global, 'fetch').mockImplementation((async (url: string | URL | Request) => {
+ const str = String(url);
+ urls.push(str);
+ const hit = table.find(([prefix]) => str.startsWith(prefix));
+ if (!hit) throw new Error(`unexpected host: ${str}`);
+ return hit[1]();
+ }) as typeof fetch);
+ return urls;
+}
+
+describe('ZhipuProvider console autodetect', () => {
+ it('keeps the domestic host when the domestic console accepts the key', async () => {
+ const provider = new ZhipuProvider();
+ const urls = mockHosts([[DOMESTIC, () => catalogRes(200)]]);
+
+ expect(await provider.validateKey('cn-key')).toBe(true);
+
+ // One probe, no speculative round-trip to api.z.ai for a key that works.
+ expect(urls).toEqual([`${DOMESTIC}/models`]);
+
+ await provider.chatCompletion('cn-key', [{ role: 'user', content: 'hi' }], 'glm-4.5-flash');
+ expect(urls[1]).toBe(`${DOMESTIC}/chat/completions`);
+ });
+
+ it('falls back to the global host on a domestic 401 and caches it for the key', async () => {
+ const provider = new ZhipuProvider();
+ const urls = mockHosts([
+ [DOMESTIC, () => catalogRes(401)],
+ [GLOBAL, () => catalogRes(200)],
+ ]);
+
+ expect(await provider.validateKey('zai-key')).toBe(true);
+
+ // Domestic first (it stays the default), global only as the fallback.
+ expect(urls).toEqual([`${DOMESTIC}/models`, `${GLOBAL}/models`]);
+
+ // The cached verdict has to reach ordinary traffic, not just validation.
+ await provider.chatCompletion('zai-key', [{ role: 'user', content: 'hi' }], 'glm-4.5-flash');
+ expect(urls[2]).toBe(`${GLOBAL}/chat/completions`);
+ });
+
+ it('streams a global key through the global host too', async () => {
+ const provider = new ZhipuProvider();
+ const urls = mockHosts([
+ [DOMESTIC, () => catalogRes(401)],
+ [GLOBAL, () => catalogRes(200)],
+ ]);
+ await provider.validateKey('zai-key');
+
+ const sse = [
+ 'data: {"id":"1","object":"chat.completion.chunk","created":1,"model":"glm-4.5-flash","choices":[{"index":0,"delta":{"content":"ok"},"finish_reason":null}]}\n\n',
+ 'data: {"id":"1","object":"chat.completion.chunk","created":1,"model":"glm-4.5-flash","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}\n\n',
+ 'data: [DONE]\n\n',
+ ].join('');
+ vi.spyOn(global, 'fetch').mockImplementation((async (url: string | URL | Request) => {
+ urls.push(String(url));
+ return {
+ ok: true, status: 200, headers: new Headers(),
+ body: new ReadableStream({
+ start(c) {
+ c.enqueue(new TextEncoder().encode(sse));
+ c.close();
+ },
+ }),
+ } as unknown as Response;
+ }) as typeof fetch);
+
+ for await (const _ of provider.streamChatCompletion('zai-key', [{ role: 'user', content: 'hi' }], 'glm-4.5-flash')) {
+ // drain
+ }
+ expect(urls.at(-1)).toBe(`${GLOBAL}/chat/completions`);
+ });
+
+ it('caches per key — a domestic key is unaffected by a global one', async () => {
+ const provider = new ZhipuProvider();
+ // Only 'zai-key' gets a cache entry; 'cn-key' is never validated here, so
+ // it must still take the pinned domestic default.
+ const urls = mockHosts([
+ [DOMESTIC, () => catalogRes(401)],
+ [GLOBAL, () => catalogRes(200)],
+ ]);
+ await provider.validateKey('zai-key');
+
+ vi.spyOn(global, 'fetch').mockImplementation((async (url: string | URL | Request) => {
+ urls.push(String(url));
+ return chatRes();
+ }) as typeof fetch);
+
+ await provider.chatCompletion('cn-key', [{ role: 'user', content: 'hi' }], 'glm-4.5-flash');
+ expect(urls.at(-1)).toBe(`${DOMESTIC}/chat/completions`);
+
+ await provider.chatCompletion('zai-key', [{ role: 'user', content: 'hi' }], 'glm-4.5-flash');
+ expect(urls.at(-1)).toBe(`${GLOBAL}/chat/completions`);
+ });
+
+ it('reports the domestic failure when both consoles reject the key', async () => {
+ const provider = new ZhipuProvider();
+ mockHosts([
+ [DOMESTIC, () => catalogRes(401, 'domestic says no')],
+ [GLOBAL, () => catalogRes(401, 'global says no')],
+ ]);
+
+ const result = await provider.validateKey('bad-key');
+ expect(result).not.toBe(true);
+ expect((result as { valid: false; error: string }).error).toContain('domestic says no');
+ });
+
+ it('reports the domestic rejection when the global host is unreachable', async () => {
+ const provider = new ZhipuProvider();
+ mockHosts([
+ [DOMESTIC, () => catalogRes(401, 'domestic says no')],
+ [GLOBAL, () => { throw new Error('ENETUNREACH'); }],
+ ]);
+
+ // An unreachable api.z.ai must not turn a plainly invalid key into a
+ // transport error — the domestic verdict still stands.
+ const result = await provider.validateKey('bad-key');
+ expect(result).not.toBe(true);
+ expect((result as { valid: false; error: string }).error).toContain('domestic says no');
+ });
+
+ it('re-checks from the domestic default when a key is validated again', async () => {
+ const provider = new ZhipuProvider();
+ const urls = mockHosts([
+ [DOMESTIC, () => catalogRes(401)],
+ [GLOBAL, () => catalogRes(200)],
+ ]);
+ await provider.validateKey('key');
+ urls.length = 0;
+
+ // The cached global verdict must not make the second validation skip the
+ // domestic probe — a key can be swapped in place for a domestic one.
+ await provider.validateKey('key');
+ expect(urls[0]).toBe(`${DOMESTIC}/models`);
+ });
+
+ it('registers under the zhipu platform with the Zhipu AI display name', () => {
+ const provider = new ZhipuProvider({ timeoutMs: 60_000 });
+ expect(provider.platform).toBe('zhipu');
+ expect(provider.name).toBe('Zhipu AI');
+ expect(provider.modelsUrl).toBe(`${DOMESTIC}/models`);
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-github-limits.test.ts b/server/src/__tests__/routes/proxy-github-limits.test.ts
new file mode 100644
index 00000000..5b2699e0
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-github-limits.test.ts
@@ -0,0 +1,131 @@
+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+import { GITHUB_MAX_INPUT_TOKENS } from '../../lib/content.js';
+import { GITHUB_MAX_OUTPUT_TOKENS } from '../../lib/sampling-params.js';
+import { mintDashboardToken } from '../helpers/auth.js';
+
+// GitHub Models is the strictest free platform in the catalog: it 400s on a
+// max_tokens above its own ceiling and 413s on a long history instead of
+// truncating it. Both are pre-dispatch rejections, so without these guards a
+// github hop is a guaranteed wasted attempt on every oversized request — and
+// the same oversized body rides the rest of the chain.
+
+let dashToken = '';
+
+async function request(app: Express, path: string, body: any, headers: Record<string, string>) {
+ const server = app.listen(0, '127.0.0.1');
+ if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', ...headers },
+ body: JSON.stringify(body),
+ });
+ const text = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(text); } catch { /* SSE body */ }
+ return { status: res.status, body: json };
+}
+
+describe('github platform limits', () => {
+ let app: Express;
+ const sent: any[] = [];
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ dashToken = mintDashboardToken();
+ });
+
+ beforeEach(async () => {
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM requests').run();
+ db.prepare('DELETE FROM rate_limit_cooldowns').run();
+ db.prepare('DELETE FROM rate_limit_usage').run();
+ sent.length = 0;
+ // Only a github key, so every route the chain can pick is a github one.
+ const { status } = await request(app, '/api/keys',
+ { platform: 'github', key: 'ghp_github_limits_test', label: 't' },
+ { Authorization: `Bearer ${dashToken}` });
+ expect(status).toBe(201);
+
+ const origFetch = global.fetch;
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (!urlStr.includes('models.github.ai')) return origFetch(url as any, init);
+ sent.push(JSON.parse(String((init as RequestInit).body)));
+ return new Response(JSON.stringify({
+ id: 'gh1', object: 'chat.completion', created: 1, model: 'gpt-4o',
+ choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 },
+ }), { status: 200, headers: { 'Content-Type': 'application/json' } });
+ });
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ });
+
+ it('clamps an aggressive max_tokens to the platform ceiling instead of 400ing', async () => {
+ const r = await request(app, '/v1/chat/completions', {
+ model: 'gpt-4o', max_tokens: 65536,
+ messages: [{ role: 'user', content: 'hello' }],
+ }, { Authorization: `Bearer ${getUnifiedApiKey()}` });
+
+ expect(r.status).toBe(200);
+ expect(sent).toHaveLength(1);
+ expect(sent[0].max_tokens).toBe(GITHUB_MAX_OUTPUT_TOKENS);
+ });
+
+ it('trims a long history to the platform input budget before dispatch', async () => {
+ // 1900 tokens per turn under the chars/4 estimate: four of them plus the
+ // question sit above the 7500-token input budget but still inside the
+ // model's 8000-token context window, so routing picks github and the
+ // guard — not the router — is what has to shrink the request. (max_tokens
+ // 1 keeps the routing output reserve out of the way.)
+ const turn = (c: string) => c.repeat(7600);
+ const messages = [
+ { role: 'system', content: 'be brief' },
+ { role: 'user', content: turn('a') },
+ { role: 'assistant', content: turn('b') },
+ { role: 'user', content: turn('c') },
+ { role: 'assistant', content: turn('d') },
+ { role: 'user', content: 'the actual question' },
+ ];
+
+ const r = await request(app, '/v1/chat/completions', {
+ model: 'gpt-4o', max_tokens: 1, messages,
+ }, { Authorization: `Bearer ${getUnifiedApiKey()}` });
+
+ expect(r.status).toBe(200);
+ expect(sent).toHaveLength(1);
+ const out = sent[0].messages as Array<{ role: string; content: string }>;
+ // System prompt and the newest turn survive; the oldest turns are gone.
+ expect(out[0].role).toBe('system');
+ expect(out[out.length - 1].content).toBe('the actual question');
+ expect(out.length).toBeLessThan(messages.length);
+ expect(out.some(m => m.content === turn('a'))).toBe(false);
+ const tokens = out.reduce((sum, m) => sum + Math.ceil(m.content.length / 4), 0);
+ expect(tokens).toBeLessThanOrEqual(GITHUB_MAX_INPUT_TOKENS);
+ });
+
+ it('leaves a history that already fits untouched', async () => {
+ const messages = [
+ { role: 'system', content: 'be brief' },
+ { role: 'user', content: 'first' },
+ { role: 'assistant', content: 'second' },
+ { role: 'user', content: 'third' },
+ ];
+ const r = await request(app, '/v1/chat/completions', {
+ model: 'gpt-4o', max_tokens: 1, messages,
+ }, { Authorization: `Bearer ${getUnifiedApiKey()}` });
+
+ expect(r.status).toBe(200);
+ expect(sent[0].messages.map((m: any) => m.content)).toEqual(messages.map(m => m.content));
+ });
+});
diff --git a/server/src/__tests__/routes/proxy-stream-integrity.test.ts b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
index 8254f95e..107dcb40 100644
--- a/server/src/__tests__/routes/proxy-stream-integrity.test.ts
+++ b/server/src/__tests__/routes/proxy-stream-integrity.test.ts
@@ -302,6 +302,79 @@ describe('proxy stream turn-integrity', () => {
expect(ledger.total).toBe(3278);
});
+ // Not every provider sends usage on a frame of its own: several attach it to
+ // the last choice-bearing frame instead. Reading it only off choice-less
+ // frames silently discarded those real counts and billed the chars/4
+ // estimate.
+ it('records usage bundled onto the final choice-bearing frame', async () => {
+ const usage = { prompt_tokens: 1111, completion_tokens: 22, total_tokens: 1133 };
+ mockUpstream([{
+ body: sse(
+ roleChunk,
+ textChunk('Hello'),
+ { ...finishChunk('stop'), usage },
+ '[DONE]',
+ ),
+ }]);
+
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true,
+ stream_options: { include_usage: true },
+ messages: [{ role: 'user', content: 'bundled usage on the finish frame' }],
+ });
+
+ expect(r.status).toBe(200);
+ const fs = frames(r.text);
+ // Exactly one usage frame, emitted last, and it carries no duplicated turn.
+ const withUsage = fs.filter(f => f.usage);
+ expect(withUsage).toHaveLength(1);
+ expect(withUsage[0].usage).toEqual(usage);
+ expect(fs[fs.length - 1]).toBe(withUsage[0]);
+ expect(withUsage[0].choices).toEqual([]);
+ // The turn itself is unchanged: one terminal finish_reason, text intact.
+ expect(fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('')).toBe('Hello');
+ expect(fs.map(f => f.choices?.[0]?.finish_reason).filter(Boolean)).toEqual(['stop']);
+
+ const row = getDb().prepare(
+ "SELECT input_tokens, output_tokens FROM requests WHERE status = 'success' ORDER BY id DESC LIMIT 1",
+ ).get() as { input_tokens: number; output_tokens: number };
+ expect(row).toEqual({ input_tokens: 1111, output_tokens: 22 });
+
+ const ledger = getDb().prepare(
+ "SELECT COALESCE(SUM(tokens), 0) AS total FROM rate_limit_usage WHERE kind = 'tokens'",
+ ).get() as { total: number };
+ expect(ledger.total).toBe(1133);
+ });
+
+ it('never emits usage twice when it rides a forwarded content frame', async () => {
+ const usage = { prompt_tokens: 7, completion_tokens: 8, total_tokens: 15 };
+ mockUpstream([{
+ body: sse(
+ roleChunk,
+ textChunk('Hello'),
+ { ...textChunk(' world'), usage },
+ finishChunk('stop'),
+ '[DONE]',
+ ),
+ }]);
+
+ const r = await request(app, '/v1/chat/completions', {
+ stream: true,
+ stream_options: { include_usage: true },
+ messages: [{ role: 'user', content: 'usage riding a content frame' }],
+ });
+
+ const fs = frames(r.text);
+ expect(fs.filter(f => f.usage)).toHaveLength(1);
+ expect(fs[fs.length - 1].usage).toEqual(usage);
+ // The content frame it arrived on is forwarded WITHOUT usage — the held
+ // copy is the single, last one the client sees (OpenAI ordering).
+ const contentFrame = fs.find(f => f.choices?.[0]?.delta?.content === ' world');
+ expect(contentFrame).toBeDefined();
+ expect(contentFrame).not.toHaveProperty('usage');
+ expect(fs.map(f => f.choices?.[0]?.delta?.content ?? '').join('')).toBe('Hello world');
+ });
+
it('rescues a non-streaming inline dialect answer into structured tool_calls', async () => {
const origFetch = global.fetch;
vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
diff --git a/server/src/__tests__/services/ratelimit-cooldown-error-kind.test.ts b/server/src/__tests__/services/ratelimit-cooldown-error-kind.test.ts
index 8a94dd05..002cf82a 100644
--- a/server/src/__tests__/services/ratelimit-cooldown-error-kind.test.ts
+++ b/server/src/__tests__/services/ratelimit-cooldown-error-kind.test.ts
@@ -81,11 +81,13 @@ describe('null-limits heuristic only fires on quota signals (#592)', () => {
});
it('real 429s still escalate — the Ollama-Cloud opaque-quota protection holds', () => {
- // 1st 429 → transient (no signal yet), then the ladder: 2m, 10m, 1h.
+ // 1st 429 → transient (no signal yet), then the ladder: 2m, 10m. This route
+ // publishes no RPD/TPD, so escalation stops at the unknown-limit 10m cap
+ // rather than climbing to the 1h/24h quarantine on an inferred verdict.
expect(cooldownDecisionForError(route, real429()).durationMs).toBe(TRANSIENT);
expect(cooldownDecisionForError(route, real429()).durationMs).toBe(2 * MINUTE);
expect(cooldownDecisionForError(route, real429()).durationMs).toBe(10 * MINUTE);
- expect(cooldownDecisionForError(route, real429()).durationMs).toBe(60 * MINUTE);
+ expect(cooldownDecisionForError(route, real429()).durationMs).toBe(10 * MINUTE);
});
});
diff --git a/server/src/__tests__/services/ratelimit-retention.test.ts b/server/src/__tests__/services/ratelimit-retention.test.ts
new file mode 100644
index 00000000..c7b53d86
--- /dev/null
+++ b/server/src/__tests__/services/ratelimit-retention.test.ts
@@ -0,0 +1,217 @@
+// Hot-path hygiene for the limiter's two record paths: the usage-table
+// retention sweep must not run on every insert, and the in-memory windows must
+// not accumulate rows nothing reads.
+
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import { initDb, getDb } from '../../db/index.js';
+import {
+ canMakeRequest,
+ canUseTokens,
+ recordRequest,
+ recordTokens,
+ setCooldown,
+ isOnCooldown,
+ cleanupExpiredCooldowns,
+} from '../../services/ratelimit.js';
+
+const MINUTE = 60 * 1000;
+const DAY = 24 * 60 * MINUTE;
+
+/** Insert a usage row at an explicit timestamp, bypassing recordRequest so the
+ * test controls how far in the past the row sits. */
+function seedUsage(platform: string, modelId: string, keyId: number, atMs: number) {
+ getDb().prepare(`
+ INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
+ VALUES (?, ?, ?, 'request', 0, ?)
+ `).run(platform, modelId, keyId, atMs);
+}
+
+function usageRowsAt(atMs: number): number {
+ const row = getDb()
+ .prepare('SELECT COUNT(*) AS n FROM rate_limit_usage WHERE created_at_ms = ?')
+ .get(atMs) as { n: number };
+ return row.n;
+}
+
+function cooldownRowCount(): number {
+ const row = getDb()
+ .prepare('SELECT COUNT(*) AS n FROM rate_limit_cooldowns')
+ .get() as { n: number };
+ return row.n;
+}
+
+describe('rate_limit_usage retention sweep is throttled', () => {
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it('prunes at most once a minute instead of on every insert', () => {
+ // The prune predicate (created_at_ms <= ?) cannot use the composite lookup
+ // index, so it is a full table scan; running it per insert put that scan on
+ // the request path. Rows outliving their DAY window by up to a minute are
+ // harmless — every counter query filters on created_at_ms itself.
+ vi.useFakeTimers();
+ const t0 = Date.UTC(2026, 3, 1, 12, 0, 0);
+ vi.setSystemTime(t0);
+ initDb(':memory:');
+
+ const keyId = Math.floor(Math.random() * 1_000_000);
+ const model = `prune-${keyId}`;
+
+ // First insert of the minute sweeps, establishing a known baseline.
+ const firstStale = t0 - 2 * DAY;
+ seedUsage('groq', model, keyId, firstStale);
+ recordRequest('groq', model, keyId);
+ expect(usageRowsAt(firstStale)).toBe(0);
+
+ // Second insert lands inside the same throttle window: the expired row it
+ // would otherwise have collected is still there.
+ const secondStale = t0 - 3 * DAY;
+ seedUsage('groq', model, keyId, secondStale);
+ vi.setSystemTime(t0 + 30 * 1000);
+ recordRequest('groq', model, keyId);
+ expect(usageRowsAt(secondStale)).toBe(1);
+
+ // Past the interval, the sweep runs again and clears the backlog.
+ vi.setSystemTime(t0 + 61 * 1000);
+ recordRequest('groq', model, keyId);
+ expect(usageRowsAt(secondStale)).toBe(0);
+ });
+
+ it('sweeps again after the clock steps backwards', () => {
+ // An NTP correction or a resumed host can move Date.now() back; a plain
+ // "now - last >= interval" test would wedge the sweep off until real time
+ // caught up with the pre-jump timestamp.
+ vi.useFakeTimers();
+ const t0 = Date.UTC(2026, 5, 1, 12, 0, 0);
+ vi.setSystemTime(t0);
+ initDb(':memory:');
+
+ const keyId = Math.floor(Math.random() * 1_000_000);
+ const model = `rewind-${keyId}`;
+ recordRequest('groq', model, keyId); // baseline sweep at t0
+
+ vi.setSystemTime(t0 - 10 * MINUTE);
+ const stale = t0 - 10 * MINUTE - 2 * DAY;
+ seedUsage('groq', model, keyId, stale);
+ recordRequest('groq', model, keyId);
+ expect(usageRowsAt(stale)).toBe(0);
+ });
+});
+
+describe('in-memory windows stay bounded', () => {
+ let keyId: number;
+ let model: string;
+
+ beforeEach(() => {
+ initDb(':memory:');
+ keyId = Math.floor(Math.random() * 1_000_000);
+ model = `mem-${keyId}`;
+ });
+
+ it('records nothing in memory while the DB is answering', () => {
+ const limits = { rpm: 5, rpd: null, tpm: null, tpd: null };
+ for (let i = 0; i < 5; i++) recordRequest('groq', model, keyId);
+ // Persisted counters see all five.
+ expect(canMakeRequest('groq', model, keyId, limits)).toBe(false);
+
+ // Drop the DB and the limiter falls back to its in-memory windows. Those
+ // are the degraded-mode store only: because nothing was mirrored into them
+ // while the DB was healthy, they start clean rather than holding five
+ // timestamps that no read path would ever have pruned.
+ getDb().close();
+ expect(canMakeRequest('groq', model, keyId, limits)).toBe(true);
+ });
+
+ it('records nothing in memory for tokens while the DB is answering', () => {
+ const limits = { tpm: 1000, tpd: null };
+ recordTokens('groq', model, keyId, 900);
+ expect(canUseTokens('groq', model, keyId, 200, limits)).toBe(false);
+
+ getDb().close();
+ expect(canUseTokens('groq', model, keyId, 200, limits)).toBe(true);
+ });
+
+ it('still counts requests in memory while the DB is unavailable', () => {
+ getDb().close();
+ const limits = { rpm: 3, rpd: null, tpm: null, tpd: null };
+ recordRequest('groq', model, keyId);
+ recordRequest('groq', model, keyId);
+ expect(canMakeRequest('groq', model, keyId, limits)).toBe(true);
+ recordRequest('groq', model, keyId);
+ expect(canMakeRequest('groq', model, keyId, limits)).toBe(false);
+ });
+
+ it('still counts tokens in memory while the DB is unavailable', () => {
+ getDb().close();
+ const limits = { tpm: 1000, tpd: null };
+ recordTokens('groq', model, keyId, 900);
+ expect(canUseTokens('groq', model, keyId, 200, limits)).toBe(false);
+ expect(canUseTokens('groq', model, keyId, 50, limits)).toBe(true);
+ });
+
+ it('drops timestamps that fell out of the window when the next one arrives', () => {
+ vi.useFakeTimers();
+ try {
+ const t0 = Date.UTC(2026, 7, 1, 9, 0, 0);
+ vi.setSystemTime(t0);
+ getDb().close();
+
+ const limits = { rpm: 2, rpd: null, tpm: null, tpd: null };
+ recordRequest('groq', model, keyId);
+ recordRequest('groq', model, keyId);
+ expect(canMakeRequest('groq', model, keyId, limits)).toBe(false);
+
+ // Two minutes on, the earlier pair is outside the rpm window, so the
+ // window holds one timestamp again rather than growing to three.
+ vi.setSystemTime(t0 + 2 * MINUTE);
+ recordRequest('groq', model, keyId);
+ expect(canMakeRequest('groq', model, keyId, limits)).toBe(true);
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+});
+
+describe('cleanupExpiredCooldowns', () => {
+ let keyId: number;
+
+ beforeEach(() => {
+ initDb(':memory:');
+ keyId = Math.floor(Math.random() * 1_000_000);
+ });
+
+ it('removes expired rows and reports how many went', () => {
+ // Cooldown expiry is otherwise collected only by isOnCooldown, and only for
+ // the exact route asked about — rows for a retired model or a deleted key
+ // are never asked about again, so they sit in the table forever.
+ setCooldown('groq', 'expired-a', keyId, -1000);
+ setCooldown('groq', 'expired-b', keyId, -5 * MINUTE);
+ expect(cooldownRowCount()).toBe(2);
+
+ expect(cleanupExpiredCooldowns()).toBe(2);
+ expect(cooldownRowCount()).toBe(0);
+ });
+
+ it('leaves an active bench alone', () => {
+ setCooldown('groq', 'still-benched', keyId, 10 * MINUTE);
+ setCooldown('groq', 'expired', keyId, -1000);
+
+ expect(cleanupExpiredCooldowns()).toBe(1);
+ expect(isOnCooldown('groq', 'still-benched', keyId)).toBe(true);
+ expect(isOnCooldown('groq', 'expired', keyId)).toBe(false);
+ });
+
+ it('clears the in-memory expiry too, not just the row', () => {
+ setCooldown('groq', 'mem-expired', keyId, -1000);
+ cleanupExpiredCooldowns();
+ // With the DB gone the in-memory map is authoritative; the sweep must have
+ // emptied it as well or the route stays benched for the whole process.
+ getDb().close();
+ expect(isOnCooldown('groq', 'mem-expired', keyId)).toBe(false);
+ });
+
+ it('is a no-op on an empty table', () => {
+ expect(cleanupExpiredCooldowns()).toBe(0);
+ });
+});
diff --git a/server/src/__tests__/services/ratelimit.test.ts b/server/src/__tests__/services/ratelimit.test.ts
index ff21bd6f..7ce2c9d7 100644
--- a/server/src/__tests__/services/ratelimit.test.ts
+++ b/server/src/__tests__/services/ratelimit.test.ts
@@ -145,14 +145,16 @@ describe('Rate Limiter', () => {
).toBe(90 * 1000);
});
- it('escalates null-limit 429s through the same ladder as documented RPD exhaustion', () => {
+ it('escalates null-limit 429s only as far as the unknown-limit 10m cap', () => {
// Documented RPD path (see "escalates only once the daily request limit
// is actually reached"): 1st call after counter ≥ limit → 2min (idx=0),
// 2nd → 10min (idx=1), 3rd → HOUR (idx=2), 4th+ → DAY (idx=3).
//
- // Null-limit path mirrors that sequence starting at the 2nd 429 (1st
- // is transient — no signal yet). Uses an independent counter so the
- // ladder index isn't inflated by the heuristic's own hits.
+ // Null-limit path enters that ladder at the 2nd 429 (1st is transient —
+ // no signal yet) but is capped at 10min: with no published RPD/TPD the
+ // "exhausted" verdict is inferred from repeated 429s alone, and RPM
+ // jitter is indistinguishable from a spent daily quota, so the guess is
+ // never allowed to reach the 1h/24h quarantine steps.
const id = Math.floor(Math.random() * 1_000_000);
const platform = 'ollama';
const model = `nolimit-esc-${id}`;
@@ -162,12 +164,26 @@ describe('Rate Limiter', () => {
expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(2 * 60 * 1000);
// 3rd 429 → 10min (idx=1)
expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(10 * 60 * 1000);
- // 4th 429 → HOUR (idx=2)
- expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(60 * 60 * 1000);
- // 5th 429 → DAY (idx=3, capped)
- expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(24 * 60 * 60 * 1000);
- // 6th+ stays at DAY
- expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(24 * 60 * 60 * 1000);
+ // 4th 429 — ladder says HOUR, cap holds it at 10min
+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(10 * 60 * 1000);
+ // 5th 429 — ladder says DAY, cap holds it at 10min
+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(10 * 60 * 1000);
+ // 6th+ stays at the cap, never the 24h quarantine
+ expect(getCooldownDurationForLimit(platform, model, id, { rpd: null, tpd: null })).toBe(10 * 60 * 1000);
+ });
+
+ it('keeps the full 24h ladder for a model with a real, exhausted RPD', () => {
+ // The cap is for GUESSED exhaustion only. A published rpd whose counter
+ // is spent is measured evidence, so the quarantine steps stay reachable.
+ const id = Math.floor(Math.random() * 1_000_000);
+ const platform = 'openrouter';
+ const model = `daily-esc-${id}`;
+ for (let i = 0; i < 3; i++) recordRequest(platform, model, id);
+ const limits = { rpd: 3, tpd: null };
+ expect(getCooldownDurationForLimit(platform, model, id, limits)).toBe(2 * 60 * 1000);
+ expect(getCooldownDurationForLimit(platform, model, id, limits)).toBe(10 * 60 * 1000);
+ expect(getCooldownDurationForLimit(platform, model, id, limits)).toBe(60 * 60 * 1000);
+ expect(getCooldownDurationForLimit(platform, model, id, limits)).toBe(24 * 60 * 60 * 1000);
});
it('does NOT trigger the heuristic when limits are known (even after 5+ hits)', () => {
diff --git a/server/src/__tests__/services/router-priority-penalty.test.ts b/server/src/__tests__/services/router-priority-penalty.test.ts
new file mode 100644
index 00000000..b81b04e6
--- /dev/null
+++ b/server/src/__tests__/services/router-priority-penalty.test.ts
@@ -0,0 +1,129 @@
+import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
+import {
+ routeRequest, refreshStatsCache, setRoutingStrategy,
+ recordRateLimitHit, recordSuccess,
+} from '../../services/router.js';
+import { getDb, initDb } from '../../db/index.js';
+
+vi.mock('../../services/ratelimit.js', async () => {
+ const actual = await vi.importActual('../../services/ratelimit.js');
+ return {
+ ...actual,
+ canMakeRequest: vi.fn(() => true),
+ canUseTokens: vi.fn(() => true),
+ isOnCooldown: vi.fn(() => false),
+ };
+});
+
+vi.mock('../../lib/crypto.js', async () => {
+ const actual = await vi.importActual('../../lib/crypto.js');
+ return { ...actual, decrypt: vi.fn(() => 'mocked-api-key') };
+});
+
+const ORIGINAL_DEV_MODE = process.env.DEV_MODE;
+const ORIGINAL_NODE_ENV = process.env.NODE_ENV;
+
+// Track every model we penalize so afterEach can drain it: the penalty map is
+// module-level state that outlives the in-memory db, and ids restart at 1 for
+// each fresh db — a leaked penalty would land on an unrelated model next test.
+const penalized: number[] = [];
+
+function penalize(modelDbId: number, hits: number) {
+ for (let i = 0; i < hits; i++) recordRateLimitHit(modelDbId);
+ penalized.push(modelDbId);
+}
+
+function addModel(opts: { platform: string; modelId: string; priority: number }): number {
+ const db = getDb();
+ db.prepare(`
+ INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, size_label, monthly_token_budget, enabled, supports_vision, supports_tools)
+ VALUES (?, ?, ?, 3, 1, 'Large', '~50M', 1, 0, 1)
+ `).run(opts.platform, opts.modelId, opts.modelId.toUpperCase());
+ const id = (db.prepare('SELECT id FROM models WHERE platform = ? AND model_id = ?')
+ .get(opts.platform, opts.modelId) as { id: number }).id;
+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(id, opts.priority);
+ db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES (?, 'k', 'enc', 'iv', 'tag', 'healthy', 1)
+ `).run(opts.platform);
+ return id;
+}
+
+describe('priority-mode routing penalty under spaced priorities', () => {
+ beforeEach(() => {
+ process.env.DEV_MODE = 'true';
+ process.env.NODE_ENV = 'test';
+ initDb(':memory:');
+ getDb().exec('DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests;');
+ vi.clearAllMocks();
+ setRoutingStrategy('priority');
+ });
+
+ afterEach(() => {
+ // MAX_PENALTY is 10 and recordSuccess decrements by 1, so 12 drains any entry.
+ for (const id of penalized) for (let i = 0; i < 12; i++) recordSuccess(id);
+ penalized.length = 0;
+ if (ORIGINAL_DEV_MODE === undefined) delete process.env.DEV_MODE; else process.env.DEV_MODE = ORIGINAL_DEV_MODE;
+ if (ORIGINAL_NODE_ENV === undefined) delete process.env.NODE_ENV; else process.env.NODE_ENV = ORIGINAL_NODE_ENV;
+ });
+
+ // The bug: the penalty is denominated in priority POSITIONS and capped at 10,
+ // so adding it to the raw priority could never reorder a chain the user (or
+ // the enabled-model filter) had spaced wider than that. A model 429-ing every
+ // request stayed pinned at the head of the chain forever.
+ it('demotes a penalized model even when priorities are spaced 10/20/30', () => {
+ const a = addModel({ platform: 'google', modelId: 'a', priority: 10 });
+ addModel({ platform: 'groq', modelId: 'b', priority: 20 });
+ addModel({ platform: 'cerebras', modelId: 'c', priority: 30 });
+ refreshStatsCache(getDb(), true);
+
+ expect(routeRequest(100).modelId).toBe('a');
+
+ // One 429 costs PENALTY_PER_429 = 3 positions: rank 1 + 3 = 4, behind b's 2.
+ penalize(a, 1);
+ expect(routeRequest(100).modelId).toBe('b');
+ });
+
+ it('demotes a penalized model when the enabled subset leaves holes in the sequence', () => {
+ // The realistic shape: the catalog was numbered 1..N densely, then most
+ // models were switched off, leaving the survivors far apart.
+ const a = addModel({ platform: 'google', modelId: 'a', priority: 3 });
+ addModel({ platform: 'groq', modelId: 'b', priority: 87 });
+ refreshStatsCache(getDb(), true);
+
+ expect(routeRequest(100).modelId).toBe('a');
+ penalize(a, 1);
+ expect(routeRequest(100).modelId).toBe('b');
+ });
+
+ it('leaves an unpenalized chain in exactly the order the user arranged', () => {
+ addModel({ platform: 'google', modelId: 'a', priority: 30 });
+ addModel({ platform: 'groq', modelId: 'b', priority: 10 });
+ addModel({ platform: 'cerebras', modelId: 'c', priority: 20 });
+ refreshStatsCache(getDb(), true);
+
+ expect(routeRequest(100).modelId).toBe('b');
+ });
+
+ it('keeps a dense chain behaving exactly as before', () => {
+ const a = addModel({ platform: 'google', modelId: 'a', priority: 1 });
+ addModel({ platform: 'groq', modelId: 'b', priority: 2 });
+ refreshStatsCache(getDb(), true);
+
+ expect(routeRequest(100).modelId).toBe('a');
+ penalize(a, 1);
+ expect(routeRequest(100).modelId).toBe('b');
+ });
+
+ it('does not demote past a model penalized even harder', () => {
+ const a = addModel({ platform: 'google', modelId: 'a', priority: 10 });
+ const b = addModel({ platform: 'groq', modelId: 'b', priority: 20 });
+ addModel({ platform: 'cerebras', modelId: 'c', priority: 30 });
+ refreshStatsCache(getDb(), true);
+
+ penalize(a, 1); // rank 1 + 3 = 4
+ penalize(b, 3); // rank 2 + 9 = 11
+ // c is unpenalized at rank 3 — the cheapest of the three.
+ expect(routeRequest(100).modelId).toBe('c');
+ });
+});
diff --git a/server/src/index.ts b/server/src/index.ts
index 50800a73..900a8d5b 100644
--- a/server/src/index.ts
+++ b/server/src/index.ts
@@ -17,6 +17,7 @@ import { generateSetupCode } from './lib/setup-code.js';
import { warnOnEnvDrift } from './lib/env-drift.js';
import { warnOnRoutingOverrideDrift } from './services/model-weight-overrides.js';
import { installLogRedaction } from './lib/log-redaction.js';
+import { cleanupExpiredCooldowns } from './services/ratelimit.js';
// Before any other statement runs, so no provider key can reach stdout — users
// paste server output into bug reports. Module scope, not inside main(), so it
@@ -44,6 +45,16 @@ async function main() {
// After initDb: the unknown-model half of this check reads the catalog.
warnOnRoutingOverrideDrift();
+ // Cooldowns persist across restarts on purpose, but their expiry is collected
+ // lazily (isOnCooldown, per model+key). Rows for routes nothing asks about
+ // again — retired models, deleted keys, a shutdown taken while everything was
+ // benched — would otherwise stay in the table forever and weigh down every
+ // cooldown rollup. One sweep at boot, while the DB is quiet.
+ const expiredCooldowns = cleanupExpiredCooldowns();
+ if (expiredCooldowns > 0) {
+ console.log(`[ratelimit] cleared ${expiredCooldowns} expired cooldown${expiredCooldowns === 1 ? '' : 's'}`);
+ }
+
// First-run hardening: when the dashboard is still unclaimed, mint a one-time
// setup code and log it. A loopback browser can finish setup without it; a
// remote caller must supply it (see routes/auth.ts). Regenerated each boot.
diff --git a/server/src/lib/content.ts b/server/src/lib/content.ts
index 696eaf47..a2c7eb69 100644
--- a/server/src/lib/content.ts
+++ b/server/src/lib/content.ts
@@ -71,6 +71,75 @@ export function stripImagesFromMessages(messages: ChatMessage[]): ChatMessage[]
});
}
+// GitHub Models refuses a long history outright (413) rather than truncating
+// it the way most providers do, so an oversized body turns every github hop in
+// the fallback chain into a wasted attempt. 7500 is the input budget we trim
+// to: comfortably inside the ~8000-token ceiling observed live, with room for
+// the tool schemas and the system prompt the gateway prepends.
+export const GITHUB_MAX_INPUT_TOKENS = 7500;
+
+// The proxy's own chars/4 input estimate, per message — same arithmetic the
+// routing/budget paths use, so the guard below trims against the number the
+// rest of the gateway reasons about.
+function estimateMessageTokens(message: ChatMessage): number {
+ return Math.ceil(contentToString(message.content).length / 4);
+}
+
+// Trim one message's text down to a token budget. Non-string content
+// (multimodal envelopes) is returned untouched — dropping or rewriting image
+// blocks here would corrupt the envelope, and text is the only thing this
+// guard is allowed to shorten.
+function truncateMessageText(message: ChatMessage, budget: number): ChatMessage {
+ if (typeof message.content !== 'string') return message;
+ const maxChars = Math.max(0, budget) * 4;
+ return message.content.length <= maxChars
+ ? message
+ : { ...message, content: message.content.slice(0, maxChars) };
+}
+
+/**
+ * Fit a message list inside GitHub Models' input ceiling before dispatch.
+ * Keeps the leading system prompt verbatim (its instructions are the whole
+ * point of prepending one) and then as many of the NEWEST messages as the
+ * remaining budget allows, dropping the oldest turns first — the same trade a
+ * client's own context window makes. A single message that blows the budget on
+ * its own is truncated rather than dropped, so the turn is never dispatched
+ * with nothing to answer.
+ *
+ * Returns the ORIGINAL array when the request already fits, which is both the
+ * common case and the cheap one: callers can hand every github-routed request
+ * through here without copying anything.
+ */
+export function truncateMessagesForGithub(
+ messages: ChatMessage[],
+ budget: number = GITHUB_MAX_INPUT_TOKENS,
+): ChatMessage[] {
+ if (messages.length === 0) return messages;
+ const total = messages.reduce((sum, m) => sum + estimateMessageTokens(m), 0);
+ if (total <= budget) return messages;
+
+ const systemIdx = messages.findIndex(m => m.role === 'system');
+ const systemMessage = systemIdx >= 0 ? messages[systemIdx] : null;
+ const rest = messages.filter((_, i) => i !== systemIdx);
+
+ const systemTokens = systemMessage ? estimateMessageTokens(systemMessage) : 0;
+ let used = systemTokens;
+ const kept: ChatMessage[] = [];
+ // Newest → oldest: the most recent context is the context the answer needs.
+ for (let i = rest.length - 1; i >= 0; i--) {
+ const tokens = estimateMessageTokens(rest[i]);
+ if (used + tokens > budget) break;
+ kept.unshift(rest[i]);
+ used += tokens;
+ }
+ // Nothing fit alongside the system prompt: keep the newest message in
+ // truncated form instead of sending an empty conversation.
+ if (kept.length === 0 && rest.length > 0) {
+ kept.push(truncateMessageText(rest[rest.length - 1], budget - systemTokens));
+ }
+ return systemMessage ? [systemMessage, ...kept] : kept;
+}
+
// Harden the OUTBOUND envelope so strict OpenAI clients don't choke on the
// shape variations free-tier providers emit. Complements normalizeOutboundContent
// (which fixes array content) by fixing the *frame* fields, not the content:
diff --git a/server/src/lib/crypto.ts b/server/src/lib/crypto.ts
index 3bef4136..db4c6738 100644
--- a/server/src/lib/crypto.ts
+++ b/server/src/lib/crypto.ts
@@ -195,7 +195,21 @@ export function decrypt(encrypted: string, iv: string, authTag: string): string
return decrypted;
}
+// A masked key is display-only — it feeds the `maskedKey` field every key,
+// media, embedding and client-profile route hands to the dashboard — so it must
+// never be enough to reconstruct the secret. The old short-key branch was
+// `'****' + key.slice(-4)`, and slice(-4) keeps the last four characters no
+// matter how short the input is: a 4-character key was echoed back IN FULL, and
+// a 5-character key gave away 4 of its 5 characters. Short keys are not
+// hypothetical — proxy credentials, self-hosted Ollama tokens and local dev
+// keys all land here.
+//
+// So reveal nothing at all below 5 characters, and at most the last TWO up to
+// 8: enough tail to tell two rows apart in the UI, not enough to be a
+// meaningful share of the secret. Long keys keep the familiar prefix+suffix
+// form, where 8 revealed characters out of 20+ is a fixed, small fraction.
export function maskKey(key: string): string {
- if (key.length <= 8) return '****' + key.slice(-4);
+ if (key.length < 5) return '****';
+ if (key.length <= 8) return '****' + key.slice(-2);
return key.slice(0, 4) + '...' + key.slice(-4);
}
diff --git a/server/src/lib/db-backup.ts b/server/src/lib/db-backup.ts
index f988d4a8..6e1c2ac0 100644
--- a/server/src/lib/db-backup.ts
+++ b/server/src/lib/db-backup.ts
@@ -77,6 +77,89 @@ function decryptBackup(payload: Buffer): Buffer {
return Buffer.concat([decipher.update(ciphertext), decipher.final()]);
}
+/**
+ * Split a Hugging Face `/resolve/` download URL into the pieces the commit API
+ * needs, or null for anything else.
+ *
+ * A HF repo serves downloads at `/{type}/{ns}/{repo}/resolve/{rev}/{path}`, but
+ * that route is read-only: a PUT to it is answered 404/405 and the blob never
+ * lands. Writes go through the commit API instead. Both halves are derived from
+ * the one URL the operator configures, so FREEAPI_DB_BACKUP_TARGET stays the
+ * plain download URL they can paste into a browser.
+ *
+ * Models are the type with no prefix segment (`/{ns}/{repo}/resolve/...`);
+ * datasets and spaces name themselves. The API path pluralizes the type.
+ */
+export function parseHuggingFaceTarget(target: string): { commitUrl: string; filePath: string } | null {
+ let url: URL;
+ try {
+ url = new URL(target);
+ } catch {
+ return null;
+ }
+ if (url.hostname !== 'huggingface.co') return null;
+
+ const segments = url.pathname.split('/').filter(Boolean);
+ const prefixed = segments[0] === 'datasets' || segments[0] === 'spaces';
+ const type = prefixed ? segments[0] : 'models';
+ const rest = prefixed ? segments.slice(1) : segments;
+ // ns / repo / 'resolve' / rev / path…
+ if (rest.length < 5 || rest[2] !== 'resolve') return null;
+ const [namespace, repo, , revision] = rest;
+ const filePath = rest.slice(4).map(decodeURIComponent).join('/');
+ if (!namespace || !repo || !revision || !filePath) return null;
+
+ return {
+ commitUrl: `https://huggingface.co/api/${type}/${namespace}/${repo}/commit/${revision}`,
+ filePath,
+ };
+}
+
+/**
+ * Upload one file through the HF commit API: newline-delimited JSON, a header
+ * line naming the commit followed by a line per file.
+ *
+ * The blob is sent as base64 TEXT rather than as bytes — `encoding: 'utf-8'`,
+ * NOT 'base64', because that flag would tell HF to decode the content back to
+ * binary before writing it, which is exactly what we are avoiding: repos on Xet
+ * storage reject binary blobs pushed through this route. Base64 costs a third
+ * more bytes and buys a file the route will actually accept. readTarget()
+ * reverses it.
+ */
+async function uploadToHuggingFace(
+ commitUrl: string,
+ filePath: string,
+ payload: Buffer,
+ token: string,
+): Promise<void> {
+ const lines = [
+ { key: 'header', value: { summary: `chore: update ${filePath}` } },
+ { key: 'file', value: { path: filePath, content: payload.toString('base64'), encoding: 'utf-8' } },
+ ];
+ const res = await fetch(commitUrl, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/x-ndjson',
+ Authorization: `Bearer ${token}`,
+ },
+ body: lines.map(line => JSON.stringify(line)).join('\n'),
+ signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+ });
+ if (!res.ok) {
+ const detail = await res.text().catch(() => '');
+ throw new Error(`backup upload failed: HF commit HTTP ${res.status}${detail ? ` — ${detail.slice(0, 200)}` : ''}`);
+ }
+}
+
+/** Undo the base64 wrapper uploadToHuggingFace applies. Sniffed rather than
+ * assumed: a blob left behind by an older build is raw and starts with MAGIC,
+ * and a restore that silently base64-decoded it would turn a recoverable
+ * database into a corrupt one. */
+function decodeHuggingFacePayload(raw: Buffer): Buffer {
+ if (raw.subarray(0, MAGIC.length).equals(MAGIC)) return raw;
+ return Buffer.from(raw.toString('utf8'), 'base64');
+}
+
async function readTarget(target: string): Promise<Buffer | null> {
if (isHttpTarget(target)) {
const headers: Record<string, string> = {};
@@ -85,7 +168,8 @@ async function readTarget(target: string): Promise<Buffer | null> {
const res = await fetch(target, { method: 'GET', headers, signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
if (res.status === 404 || res.status === 204) return null;
if (!res.ok) throw new Error(`backup restore failed: HTTP ${res.status}`);
- return Buffer.from(await res.arrayBuffer());
+ const raw = Buffer.from(await res.arrayBuffer());
+ return parseHuggingFaceTarget(target) ? decodeHuggingFacePayload(raw) : raw;
}
if (!fs.existsSync(target)) return null;
@@ -94,8 +178,17 @@ async function readTarget(target: string): Promise<Buffer | null> {
async function writeTarget(target: string, payload: Buffer): Promise<void> {
if (isHttpTarget(target)) {
- const headers: Record<string, string> = { 'Content-Type': 'application/octet-stream' };
const token = process.env.FREEAPI_DB_BACKUP_TOKEN?.trim();
+ const huggingFace = parseHuggingFaceTarget(target);
+ if (huggingFace) {
+ // Anonymous writes do not exist on HF; without a token every upload would
+ // 401 on a schedule and only ever be noticed as an empty restore.
+ if (!token) throw new Error('FREEAPI_DB_BACKUP_TOKEN is required to upload a backup to Hugging Face');
+ await uploadToHuggingFace(huggingFace.commitUrl, huggingFace.filePath, payload, token);
+ return;
+ }
+
+ const headers: Record<string, string> = { 'Content-Type': 'application/octet-stream' };
if (token) headers.Authorization = `Bearer ${token}`;
const res = await fetch(target, {
method: 'PUT',
diff --git a/server/src/lib/error-classify.ts b/server/src/lib/error-classify.ts
index d1a9cdd6..81049125 100644
--- a/server/src/lib/error-classify.ts
+++ b/server/src/lib/error-classify.ts
@@ -102,7 +102,108 @@ export function isRetryableError(err: any): boolean {
// byte reached the client, and a different model usually gets the same
// call right — the thrower marks the model skipped for this request, since
// a sibling key would misbehave identically.
- || msg.includes('invalid tool arguments');
+ || msg.includes('invalid tool arguments')
+ // A transport failure undici buried in `err.cause` rather than the
+ // top-level message (see isTransportError). Purely additive: every rule
+ // above is unchanged, and this is a second, structured signal for the
+ // errors whose real cause never reaches the text we match on.
+ || isTransportError(err);
+}
+
+// ── Transport failures hidden in the cause chain (undici) ────────────────────
+// Every rule in isRetryableError above reads the TOP-LEVEL message, and undici
+// does not always put the real failure there. The INITIAL-CONNECT case is
+// covered by luck: undici words a connection that never opened "fetch failed",
+// and the allowlist matches that string. A socket that dies MID-request does
+// not get the same treatment — it surfaces as a generic wrapper (a bare
+// `TypeError: terminated`, or an adapter's own re-throw) whose `.cause` carries
+// the actual ECONNRESET / EPIPE / "socket hang up" / "premature close" /
+// "other side closed" error, sometimes nested a link or two deeper still.
+// Those fell through every rule and classified FATAL: the client got a 502
+// while the healthy paid routes queued behind the dead one were never tried.
+//
+// Everything matched here is a transient network fault that says nothing about
+// the request or the model, so the next candidate in the chain can serve it.
+// The walk is bounded and cycle-safe: `cause` is attacker-adjacent data (it can
+// come from a provider's own error object) and a self-referential chain on this
+// hot path would hang the request, not just slow it.
+const TRANSPORT_CAUSE_MAX_DEPTH = 5;
+
+const TRANSPORT_ERROR_CODES = new Set([
+ // Node socket-level codes. A dead/refused/unreachable host is transient from
+ // the chain's point of view either way: the next candidate is a DIFFERENT
+ // host, so even a permanently bad one here is worth failing over rather than
+ // 502-ing the caller.
+ 'ECONNRESET', 'ECONNREFUSED', 'ECONNABORTED', 'EPIPE', 'ETIMEDOUT',
+ 'EAI_AGAIN', 'ENOTFOUND', 'EHOSTUNREACH', 'ENETUNREACH', 'EADDRNOTAVAIL',
+]);
+
+// undici stamps its own transport failures with a `UND_ERR_*` code
+// (UND_ERR_SOCKET, UND_ERR_CONNECT_TIMEOUT, UND_ERR_HEADERS_TIMEOUT,
+// UND_ERR_BODY_TIMEOUT, …). Matched by PREFIX rather than enumerated: the list
+// grows with undici releases, and every member of it is a transport condition.
+const UNDICI_ERROR_CODE_PREFIX = 'UND_ERR_';
+
+const TRANSPORT_MESSAGE_HINTS = [
+ 'socket hang up',
+ 'premature close',
+ 'other side closed',
+ // A socket dropped before the TLS handshake finished — undici's full wording
+ // is "Client network socket disconnected before secure TLS connection was
+ // established". The peer closed the TCP connection mid-handshake; transient.
+ 'client network socket disconnected',
+ // The same codes as above, for the links that carry them in text only (an
+ // error stringified across a boundary keeps the code in its message but
+ // loses the `code` property).
+ 'econnreset', 'econnrefused', 'epipe', 'etimedout', 'eai_again',
+];
+
+/** Walk an error's `cause` chain, yielding each link's `code` and `message`.
+ * Includes the error itself as the first link. Bounded to
+ * TRANSPORT_CAUSE_MAX_DEPTH hops and cycle-safe. */
+function errorChainLinks(err: unknown): Array<{ code: string; message: string }> {
+ const links: Array<{ code: string; message: string }> = [];
+ const seen = new Set<unknown>();
+ let cur: any = err;
+ for (let depth = 0; cur != null && depth <= TRANSPORT_CAUSE_MAX_DEPTH; depth++) {
+ if (typeof cur !== 'object' && typeof cur !== 'function') break;
+ if (seen.has(cur)) break;
+ seen.add(cur);
+ links.push({
+ code: typeof cur.code === 'string' ? cur.code : '',
+ message: typeof cur.message === 'string' ? cur.message : '',
+ });
+ cur = cur.cause;
+ }
+ return links;
+}
+
+/** True when the error — or anything in its bounded cause chain — is a
+ * transient network transport failure: a reset, dropped, refused or timed-out
+ * socket, a failed TLS handshake, or an undici transport code. */
+export function isTransportError(err: any): boolean {
+ if (err == null) return false;
+ // A client hang-up and the fallback time-budget hedge BOTH reach undici as an
+ // aborted socket, and undici labels that abort UND_ERR_ABORTED — the very
+ // same code a genuine mid-flight socket death carries. Neither is provider
+ // health, so classifying either retryable would resurrect exactly what the
+ // two marked abort errors exist to prevent (no cooldown, no health penalty,
+ // no failure stats for a request the GATEWAY canceled). The structured
+ // markers are authoritative and win over any transport evidence below.
+ if (isClientAbortError(err) || isHedgeAbortError(err)) return false;
+ for (const { code, message } of errorChainLinks(err)) {
+ if (code && (TRANSPORT_ERROR_CODES.has(code) || code.startsWith(UNDICI_ERROR_CODE_PREFIX))) return true;
+ const msg = message.toLowerCase();
+ if (!msg) continue;
+ if (TRANSPORT_MESSAGE_HINTS.some(hint => msg.includes(hint))) return true;
+ // undici's wording for a response body whose socket died mid-read is the
+ // bare word "terminated". Matched as the WHOLE message and never as a
+ // substring: "your account has been terminated" is a fatal billing/auth
+ // condition, and retrying that around the entire chain would burn every
+ // candidate on a request that can never succeed.
+ if (msg.trim() === 'terminated') return true;
+ }
+ return false;
}
// A genuine provider QUOTA signal: a structured 429 or rate-limit/quota wording.
diff --git a/server/src/lib/gemini-wire.ts b/server/src/lib/gemini-wire.ts
index d9784820..eb42e236 100644
--- a/server/src/lib/gemini-wire.ts
+++ b/server/src/lib/gemini-wire.ts
@@ -107,22 +107,88 @@ const UNSUPPORTED_SCHEMA_KEYS = new Set([
const VENDOR_EXTENSION_SCHEMA_KEY = /^x-/i;
+// Google's Schema proto types `type` as a single enum, but OpenAI-style
+// clients emit JSON Schema unions (`.nullable()` builders produce
+// `"type": ["number", "null"]`), which 400s with "Proto field is not
+// repeating, cannot start list". Collapse the union to its first concrete
+// member plus Gemini's own `nullable` flag — the inverse of what
+// normalizeGeminiSchema does on the inbound direction.
+interface CollapsedType {
+ type?: string;
+ nullable: boolean;
+}
+
+function collapseTypeUnion(value: unknown[]): CollapsedType {
+ const names = value.filter((entry): entry is string => typeof entry === 'string');
+ const nullable = names.some(name => name.toLowerCase() === 'null');
+ // A union of two concrete types has no Gemini equivalent; keeping the first
+ // is lossy but still describes the argument, where dropping `type` entirely
+ // would let the model emit anything.
+ const concrete = names.find(name => name.toLowerCase() !== 'null');
+ return { type: concrete, nullable };
+}
+
+// Best-effort JSON pointer resolution against the schema root, so a `$ref`
+// into `$defs`/`definitions` keeps its structure instead of collapsing to a
+// permissive `{}`. Anything else (remote refs, missing targets) still falls
+// back to dropping the keyword, like the other unsupported keywords do.
+function resolvePointer(root: unknown, ref: string): unknown {
+ if (!ref.startsWith('#/')) return undefined;
+ let node: unknown = root;
+ for (const rawSegment of ref.slice(2).split('/')) {
+ if (!node || typeof node !== 'object' || Array.isArray(node)) return undefined;
+ const segment = decodeURIComponent(rawSegment).replace(/~1/g, '/').replace(/~0/g, '~');
+ node = (node as Record<string, unknown>)[segment];
+ }
+ return node;
+}
+
+interface SanitizeContext {
+ root: unknown;
+ // Refs expanded on the current path, so a self-referential definition drops
+ // out instead of recursing forever.
+ expanding: ReadonlySet<string>;
+}
+
export function sanitizeForGemini(schema: unknown): unknown {
- return sanitizeSchema(schema, false);
+ return sanitizeSchema(schema, false, { root: schema, expanding: new Set() });
}
-function sanitizeSchema(schema: unknown, insidePropertiesMap: boolean): unknown {
- if (Array.isArray(schema)) return schema.map(value => sanitizeSchema(value, false));
+function sanitizeSchema(schema: unknown, insidePropertiesMap: boolean, ctx: SanitizeContext): unknown {
+ if (Array.isArray(schema)) return schema.map(value => sanitizeSchema(value, false, ctx));
if (!schema || typeof schema !== 'object') return schema;
const out: Record<string, unknown> = {};
+ let nullable = false;
+ let inlined: Record<string, unknown> | undefined;
for (const [key, value] of Object.entries(schema as Record<string, unknown>)) {
if (insidePropertiesMap) {
- out[key] = sanitizeSchema(value, false);
- } else if (!UNSUPPORTED_SCHEMA_KEYS.has(key) && !VENDOR_EXTENSION_SCHEMA_KEY.test(key)) {
- out[key] = sanitizeSchema(value, key === 'properties');
+ out[key] = sanitizeSchema(value, false, ctx);
+ continue;
+ }
+ if (key === '$ref' && typeof value === 'string' && !ctx.expanding.has(value)) {
+ const target = resolvePointer(ctx.root, value);
+ if (target && typeof target === 'object' && !Array.isArray(target)) {
+ inlined = sanitizeSchema(target, false, {
+ root: ctx.root,
+ expanding: new Set([...ctx.expanding, value]),
+ }) as Record<string, unknown>;
+ }
+ continue;
+ }
+ if (key === 'type' && Array.isArray(value)) {
+ const collapsed = collapseTypeUnion(value);
+ if (collapsed.type !== undefined) out.type = collapsed.type;
+ nullable = nullable || collapsed.nullable;
+ continue;
+ }
+ if (!UNSUPPORTED_SCHEMA_KEYS.has(key) && !VENDOR_EXTENSION_SCHEMA_KEY.test(key)) {
+ out[key] = sanitizeSchema(value, key === 'properties', ctx);
}
}
- return out;
+ if (nullable) out.nullable = true;
+ // Keywords written alongside a `$ref` (description, overrides) win over the
+ // definition they point at.
+ return inlined ? { ...inlined, ...out } : out;
}
function serializeResponse(value: unknown): string {
diff --git a/server/src/lib/proxy.ts b/server/src/lib/proxy.ts
index 8599f105..ed98e235 100644
--- a/server/src/lib/proxy.ts
+++ b/server/src/lib/proxy.ts
@@ -361,6 +361,31 @@ function enrichAbort(
return enriched;
}
+/**
+ * DNS `lookup` override for the SOCKS fallback path: hand back the hostname it
+ * was asked to resolve, unchanged.
+ *
+ * socks-proxy-agent resolves the DESTINATION locally for the `socks5://` and
+ * `socks4://` schemes (`shouldLookup`) and sends the proxy a bare IP; only
+ * `socks5h://`/`socks4a://` pass the name through. That local resolution is
+ * what breaks rule-based proxy clients (Clash and friends), which match routing
+ * rules on the domain and have nothing to match once the name is gone — and on
+ * a DNS-poisoned network it resolves to the poisoned address as well.
+ *
+ * `http.request` forwards this to the agent as `opts.lookup`, so echoing the
+ * hostname makes every SOCKS scheme reach the proxy with the domain intact,
+ * i.e. behave like its `h`/`a` variant. The agent only forwards the "address"
+ * as the SOCKS destination host — it never inspects the address family, so the
+ * `4` is a placeholder the callback signature requires.
+ */
+export function socksHostnameLookup(
+ hostname: string,
+ _options: unknown,
+ callback: (err: null, address: string, family: number) => void,
+): void {
+ callback(null, hostname, 4);
+}
+
function socksFetch(
urlStr: string,
init: RequestInit | undefined,
@@ -427,6 +452,11 @@ function socksFetch(
servername: isTls ? url.hostname : undefined,
rejectUnauthorized: true,
timeout: socketTimeoutMs,
+ // Keep the destination hostname unresolved so the SOCKS proxy does the
+ // DNS. `agent` here is always a SocksProxyAgent (every socksFetch caller
+ // is behind an `isSocks` branch), and the agent is the only consumer of
+ // this hook — the connection to the proxy itself still resolves normally.
+ lookup: socksHostnameLookup,
}, (res) => {
if (signal?.aborted) {
res.destroy();
diff --git a/server/src/lib/sampling-params.ts b/server/src/lib/sampling-params.ts
index f436f664..1e1ca80b 100644
--- a/server/src/lib/sampling-params.ts
+++ b/server/src/lib/sampling-params.ts
@@ -218,8 +218,22 @@ export interface PlatformParamPolicy {
// effect once its adapter routes max_tokens through that helper (they all
// do).
defaultMaxTokens?: number;
+ // Output-token CEILING this platform's API enforces itself, applied by
+ // resolveMaxTokens() whatever the client asked for. The mirror image of
+ // defaultMaxTokens: a floor rescues a client that sent nothing, a cap
+ // rescues one that sent too much. Without it an aggressive max_tokens
+ // (Open WebUI's 65536 default) is a guaranteed 400 on every hop that lands
+ // on such a platform, and the fallback chain cannot repair it because the
+ // same value rides every candidate. Effective max_tokens is min(requested,
+ // this cap, the operator's unified cap).
+ maxTokensCap?: number;
}
+/** GitHub Models' own output-token ceiling: asking for more 400s ("max_tokens
+ * is too large"), so the request never reaches the model. Wired into the
+ * github policy below as maxTokensCap. */
+export const GITHUB_MAX_OUTPUT_TOKENS = 400;
+
// Keyed by Platform (not string) so a typo'd platform id fails tsc instead of
// silently no-op'ing the policy; the string-typed accessors below cast at the
// boundary since routes carry platform ids as plain strings.
@@ -237,8 +251,14 @@ export const PLATFORM_PARAM_POLICIES: Partial<Record<Platform, PlatformParamPoli
// GitHub Models sits on Azure OpenAI, which 400s "Unrecognized request
// argument" for knobs outside the OpenAI set. Its reasoning_effort enum is
// the older low/medium/high one, so 'none'/'minimal' are clamped rather
- // than sent.
- github: { drop: ['top_k', 'min_p', 'repetition_penalty'], reasoningEfforts: ['low', 'medium', 'high'] },
+ // than sent. Its free tier also refuses any max_tokens above
+ // GITHUB_MAX_OUTPUT_TOKENS, so the cap is clamped here instead of being
+ // spent as a wasted fallback hop.
+ github: {
+ drop: ['top_k', 'min_p', 'repetition_penalty'],
+ reasoningEfforts: ['low', 'medium', 'high'],
+ maxTokensCap: GITHUB_MAX_OUTPUT_TOKENS,
+ },
// Gemini's generationConfig has no equivalents for these; the adapter
// translates the rest natively (topK, seed, penalties, responseSchema, and
// reasoning_effort → thinkingConfig — see toGeminiExtendedConfig).
@@ -314,13 +334,20 @@ export function defaultMaxTokensFor(platform: string): number | undefined {
return PLATFORM_PARAM_POLICIES[platform as Platform]?.defaultMaxTokens;
}
+/** This platform's own output-token ceiling, or undefined when it accepts
+ * whatever max_tokens the client asks for. */
+export function maxTokensCapFor(platform: string): number | undefined {
+ return PLATFORM_PARAM_POLICIES[platform as Platform]?.maxTokensCap;
+}
+
/**
* The max_tokens to put on the wire for one request: whatever the client asked
* for, or the platform's floor when the client asked for nothing (#553), then
- * lowered to the unified output cap when the operator configured one. With the
- * cap off (the default) nothing is clamped — a client-set value passes through
- * untouched in both directions, and the gateway's own guardrails (token budget,
- * routing reserve) have already had their say by the time an adapter calls this.
+ * lowered to the tightest ceiling that applies — the platform's own
+ * maxTokensCap, the operator's unified cap, or both. With neither in play
+ * nothing is clamped — a client-set value passes through untouched in both
+ * directions, and the gateway's own guardrails (token budget, routing reserve)
+ * have already had their say by the time an adapter calls this.
*
* EVERY adapter must send max_tokens through here, or the cap is not unified:
* openai-compat (and its subclasses), cloudflare, cohere, google and aihorde
@@ -329,8 +356,10 @@ export function defaultMaxTokensFor(platform: string): number | undefined {
export function resolveMaxTokens(platform: string, requested: number | undefined): number | undefined {
const resolved = requested ?? defaultMaxTokensFor(platform);
if (resolved == null) return resolved;
- const cap = unifiedMaxTokensCap();
- return cap == null ? resolved : Math.min(resolved, cap);
+ // The tighter ceiling wins: a platform's hard reject applies even with the
+ // operator cap off, and an operator cap below it applies everywhere.
+ const caps = [unifiedMaxTokensCap(), maxTokensCapFor(platform)].filter((c): c is number => c != null);
+ return caps.length === 0 ? resolved : Math.min(resolved, ...caps);
}
// ── Unified output-token cap ─────────────────────────────────────────────────
diff --git a/server/src/providers/index.ts b/server/src/providers/index.ts
index 07a46339..f39848fa 100644
--- a/server/src/providers/index.ts
+++ b/server/src/providers/index.ts
@@ -7,6 +7,7 @@ import { CloudflareProvider } from './cloudflare.js';
import { AIHordeProvider } from './aihorde.js';
import { ModelScopeProvider } from './modelscope.js';
import { PollinationsProvider } from './pollinations.js';
+import { ZhipuProvider } from './zhipu.js';
const providers = new Map<Platform, BaseProvider>();
@@ -116,19 +117,18 @@ register(new CohereProvider());
// Cloudflare Workers AI - OpenAI-compatible endpoint (key = "account_id:token")
register(new CloudflareProvider());
-// Zhipu (Z.ai / bigmodel.cn) - OpenAI-compatible
+// Zhipu (Z.ai / bigmodel.cn) - OpenAI-compatible. ZhipuProvider is stock
+// openai-compat chat routing plus console autodetect: the domestic
+// open.bigmodel.cn host stays the default, and a key it rejects is re-probed
+// against the global api.z.ai host during validation instead of being written
+// off as invalid (the two consoles don't share a key namespace).
//
// glm-4.7-flash is a hidden-reasoning model: it burns through a long
// reasoning_content before the first answer byte (live-probed 41s TTFB on a
// one-word completion, 2026-07-11), and Zhipu buffers that phase even when
// streaming — so the default 15s timeout aborted every attempt. 60s covers
// the observed worst case with headroom.
-register(new OpenAICompatProvider({
- platform: 'zhipu',
- name: 'Zhipu AI',
- baseUrl: 'https://open.bigmodel.cn/api/paas/v4',
- timeoutMs: 60_000,
-}));
+register(new ZhipuProvider({ timeoutMs: 60_000 }));
// Hugging Face Inference Providers router — re-added in V13. The V4 removal
// reason ("tool-call format issues") was the legacy serverless route that
diff --git a/server/src/providers/zhipu.ts b/server/src/providers/zhipu.ts
new file mode 100644
index 00000000..dd8b0f6f
--- /dev/null
+++ b/server/src/providers/zhipu.ts
@@ -0,0 +1,109 @@
+import { OpenAICompatProvider } from './openai-compat.js';
+import type { KeyValidationResult, ProviderFetchOptions } from './base.js';
+import type { QuotaObservationContext } from '../services/provider-quota.js';
+
+/** Domestic console (bigmodel.cn) — the historical registration default. */
+const DOMESTIC_BASE_URL = 'https://open.bigmodel.cn/api/paas/v4';
+/** Global console (z.ai). Same OpenAI-compatible surface, different key namespace. */
+const GLOBAL_BASE_URL = 'https://api.z.ai/api/paas/v4';
+
+/**
+ * Zhipu AI / Z.ai. One platform, two consoles that do NOT share a key
+ * namespace: a key minted on z.ai is a 401 at open.bigmodel.cn and vice versa.
+ * Pinning the platform to the domestic host therefore made every global-console
+ * key look like a bad key, with a 401 that says nothing about which host it
+ * came from.
+ *
+ * So the domestic host stays the default — an existing bigmodel.cn deployment
+ * behaves exactly as before, one request, same URL — and only a key the
+ * domestic host actually REJECTS is re-probed against the global host during
+ * validation. The verdict is remembered per key so the key's chat, streaming
+ * and catalog traffic follows it.
+ */
+export class ZhipuProvider extends OpenAICompatProvider {
+ /**
+ * Keys proven to belong to the global console. Deliberately in-memory: it is
+ * a cache of something validateKey can always re-derive, and health checks
+ * re-run validateKey, so a restart costs one extra probe per global key and
+ * nothing else. Everything absent from this set uses the domestic host.
+ */
+ private readonly globalKeys = new Set<string>();
+
+ constructor(opts: { timeoutMs?: number } = {}) {
+ super({
+ platform: 'zhipu',
+ name: 'Zhipu AI',
+ baseUrl: DOMESTIC_BASE_URL,
+ timeoutMs: opts.timeoutMs,
+ });
+ }
+
+ /**
+ * Recover the bearer from a request this provider built. OpenAICompatProvider
+ * composes every URL from its own private base URL, so this is the only place
+ * where the outgoing URL and the key that should choose it are both in scope.
+ */
+ private static apiKeyOf(init: RequestInit): string | undefined {
+ const raw = init.headers;
+ const auth = raw instanceof Headers
+ ? raw.get('authorization')
+ : (raw as Record<string, string> | undefined)?.['Authorization']
+ ?? (raw as Record<string, string> | undefined)?.['authorization'];
+ return auth?.startsWith('Bearer ') ? auth.slice('Bearer '.length) : undefined;
+ }
+
+ /**
+ * Swap the pinned domestic host for the global one when this key validated
+ * there. A URL that already names a host explicitly — the validateKey probes
+ * below — is left alone, which keeps validation free of its own cache.
+ */
+ private resolveUrl(url: string, init: RequestInit): string {
+ if (!url.startsWith(DOMESTIC_BASE_URL)) return url;
+ const apiKey = ZhipuProvider.apiKeyOf(init);
+ if (!apiKey || !this.globalKeys.has(apiKey)) return url;
+ return GLOBAL_BASE_URL + url.slice(DOMESTIC_BASE_URL.length);
+ }
+
+ protected override fetchWithTimeout(
+ url: string,
+ init: RequestInit,
+ timeoutMs?: number,
+ fetchOpts?: ProviderFetchOptions,
+ ): Promise<Response> {
+ return super.fetchWithTimeout(this.resolveUrl(url, init), init, timeoutMs, fetchOpts);
+ }
+
+ override async validateKey(apiKey: string, quotaContext?: QuotaObservationContext): Promise<KeyValidationResult> {
+ // Drop any earlier verdict first: a re-check must start from the domestic
+ // default (a key can be replaced in place), and it keeps resolveUrl from
+ // rewriting the domestic probe below.
+ this.globalKeys.delete(apiKey);
+
+ const domesticRes = await this.fetchCatalogEndpoint(`${DOMESTIC_BASE_URL}/models`, apiKey, quotaContext);
+ if (domesticRes.status !== 401 && domesticRes.status !== 403) {
+ return this.validationResult(domesticRes);
+ }
+
+ // Only reached when the domestic host said "not my key" — which is exactly
+ // what it says about a valid global-console key. Ask the global host before
+ // calling the key invalid.
+ let globalRes: Response;
+ try {
+ globalRes = await this.fetchCatalogEndpoint(`${GLOBAL_BASE_URL}/models`, apiKey, quotaContext);
+ } catch {
+ // api.z.ai unreachable (no route, blocked, timeout). Nothing new learned,
+ // so report the domestic rejection rather than turning a plainly invalid
+ // key into a transport error that health.ts would log as 'error'.
+ return this.validationResult(domesticRes);
+ }
+
+ if (globalRes.status !== 401 && globalRes.status !== 403) {
+ this.globalKeys.add(apiKey);
+ return this.validationResult(globalRes);
+ }
+
+ // Rejected by both consoles. Report the domestic failure: that is the host
+ // the key is used against by default, so its message is the relevant one.
+ return this.validationResult(domesticRes);
+ }
+}
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 1c52ad85..22ea63a3 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -10,7 +10,7 @@ import { runImageGeneration, runSpeech, runTranscription, MediaError, MAX_TRANSC
import multer from 'multer';
import { getDb } from '../db/index.js';
import { resolveAuth, prependSystemPrompt, type ResolvedAuth } from '../lib/system-prompt.js';
-import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse } from '../lib/content.js';
+import { contentToString, messageHasImage, normalizeOutboundContent, sanitizeResponse, truncateMessagesForGithub } from '../lib/content.js';
import { normalizeMessageImages } from '../lib/image-normalize.js';
import { repairToolArguments, toolSchemaMap } from '../lib/tool-args.js';
import { invalidToolArgumentsError, invalidToolCallReasons, isToolArgumentValidationEnabled } from '../lib/tool-validate.js';
@@ -1002,6 +1002,12 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
requestedModel: attempt === 0 ? requestedModelLabel : undefined,
});
+ // Same GitHub input ceiling as /chat/completions below: trim the
+ // dispatched copy so a long legacy prompt doesn't 413 the github hop.
+ const dispatchMessages = route.platform === 'github'
+ ? truncateMessagesForGithub(messages)
+ : messages;
+
if (stream) {
let totalOutputTokens = 0;
let headerSent = false;
@@ -1032,7 +1038,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
try {
const gen = route.provider.streamChatCompletion(
route.apiKey,
- messages,
+ dispatchMessages,
route.modelId,
{ temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
@@ -1132,7 +1138,7 @@ proxyRouter.post('/completions', async (req: Request, res: Response) => {
const result = await route.provider.chatCompletion(
route.apiKey,
- messages,
+ dispatchMessages,
route.modelId,
{ temperature, max_tokens, top_p, stop, signal: AbortSignal.any([clientAbort.signal, hedgeAbort.signal]) },
quotaContextForRoute(route, 'chat/completions'),
@@ -1846,6 +1852,16 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
outboundMessages = restoreSessionReasoning(outboundMessages, rememberedReasoning, route.platform);
}
+ // GitHub Models 413s a history above its input ceiling instead of
+ // truncating it, so a long conversation burns the github hop of every
+ // chain it appears in. Trim the outbound copy to what the platform will
+ // accept — scoped to this attempt, so the next candidate still sees the
+ // client's full history. A no-op returning the same array when the
+ // request already fits.
+ if (route.platform === 'github') {
+ outboundMessages = truncateMessagesForGithub(outboundMessages);
+ }
+
if (stream) {
// — Stream turn-integrity (#231 audit) —
// The old loop forwarded upstream chunks verbatim and called any
@@ -1955,13 +1971,20 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
if (anyChunk.id) lastMeta = { id: anyChunk.id, model: anyChunk.model, created: anyChunk.created };
+ // Usage arrives either on its own frame (OpenAI's
+ // stream_options.include_usage shape) or bundled onto the last
+ // choice-bearing frame — several providers do the latter, and
+ // reading it only off choice-less frames threw their real token
+ // counts away and left accounting on the chars/4 estimate. Capture
+ // it wherever it lands, reduced to a usage-only frame: the frame's
+ // deltas are re-emitted through our own framing below, so holding
+ // the original verbatim would duplicate content (or a finish_reason
+ // the client already saw) when it is written back after our finish
+ // chunk to preserve OpenAI ordering.
+ if (anyChunk.usage) usageChunk = { ...anyChunk, choices: [], usage: anyChunk.usage };
+
const choice = anyChunk.choices?.[0];
- if (!choice) {
- // Usage-only frame (stream_options.include_usage) — held and
- // re-emitted after our finish chunk to preserve OpenAI ordering.
- if (anyChunk.usage) usageChunk = anyChunk;
- continue;
- }
+ if (!choice) continue;
if (choice.finish_reason) upstreamFinish = choice.finish_reason;
@@ -2004,7 +2027,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// #764: thinking-only chunks still consumed tokens — count them.
if (reasoning.length > 0) totalOutputTokens += Math.ceil(reasoning.length / 4);
if (choice.delta && Object.keys(choice.delta).some(k => k !== 'content' && k !== 'tool_calls' && choice.delta[k] != null)) {
- const cleaned = { ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] };
+ // `usage: undefined` (dropped by JSON.stringify): it was held
+ // above and is re-emitted once, after our finish chunk.
+ const cleaned = { ...anyChunk, usage: undefined, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] };
if (headerSent) writeChunk(cleaned); else preamble.push(cleaned);
}
continue;
@@ -2016,7 +2041,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
totalOutputTokens += Math.ceil((text.length + reasoning.length) / 4);
if (mode === 'passthrough') {
- writeChunk({ ...anyChunk, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] });
+ // Same rule as the preamble path: usage rides the held frame,
+ // not this one, so the client sees it exactly once and last.
+ writeChunk({ ...anyChunk, usage: undefined, choices: [{ ...choice, delta: { ...choice.delta, tool_calls: undefined }, finish_reason: null }] });
continue;
}
diff --git a/server/src/services/ratelimit.ts b/server/src/services/ratelimit.ts
index 90369d84..dbb28eb8 100644
--- a/server/src/services/ratelimit.ts
+++ b/server/src/services/ratelimit.ts
@@ -192,6 +192,18 @@ function provisionalProviderTokens(platform: string, keyId: number, now: number)
return total;
}
+// Retention sweep for rate_limit_usage. The only index on the table is
+// idx_rate_limit_usage_lookup(platform, model_id, key_id, kind, created_at_ms),
+// which a bare `created_at_ms <= ?` predicate cannot use — the prune is a full
+// table scan. Running it inside every insert put that scan on the hot request
+// path, so it is throttled: rows are kept for a DAY and every counter query is
+// bounded by its own `created_at_ms > ?` filter, so letting expired rows linger
+// for up to a minute costs nothing but a few stale rows.
+const USAGE_PRUNE_INTERVAL_MS = MINUTE;
+let lastUsagePruneMs = 0;
+
+/** True when the row actually reached the DB. Callers use this to decide
+ * whether the in-memory windows have to carry the count instead. */
function recordUsage(
platform: string,
modelId: string,
@@ -199,14 +211,21 @@ function recordUsage(
kind: UsageKind,
tokens: number,
now: number,
-) {
- withDb(db => {
+): boolean {
+ return withDb(db => {
db.prepare(`
INSERT INTO rate_limit_usage (platform, model_id, key_id, kind, tokens, created_at_ms)
VALUES (?, ?, ?, ?, ?, ?)
`).run(platform, modelId, keyId, kind, tokens, now);
- db.prepare('DELETE FROM rate_limit_usage WHERE created_at_ms <= ?').run(now - DAY);
- });
+ // `now < lastUsagePruneMs` covers a clock step backwards (NTP correction,
+ // a suspended host resuming) — without it a jump forward would wedge the
+ // prune off until real time caught up again.
+ if (now - lastUsagePruneMs >= USAGE_PRUNE_INTERVAL_MS || now < lastUsagePruneMs) {
+ lastUsagePruneMs = now;
+ db.prepare('DELETE FROM rate_limit_usage WHERE created_at_ms <= ?').run(now - DAY);
+ }
+ return true;
+ }) ?? false;
}
function countPersistedRequests(
@@ -572,16 +591,38 @@ export function canUseProviderTokens(
return used + providerBilledTokens(platform, modelId, estimatedTokens) <= cap;
}
+// ── Degraded-mode memory windows ─────────────────────────────────────────────
+// The in-memory windows are a FALLBACK for the persisted counters: requestCount
+// / tokenCount / providerDailyRequestCount all prefer the DB and only consult
+// memory when the DB refuses to answer. So on a healthy server nothing ever
+// reads these arrays — and pruning happens only on the read path, which means
+// nothing ever trimmed them either: every recorded request leaked one timestamp
+// (and one {ts, tokens} object) for the lifetime of the process.
+//
+// Record in memory only when the persisted write actually failed, and prune the
+// window as we push so a long DB outage cannot grow it past its own width
+// either. Degraded-mode behavior is unchanged: the same timestamps land in the
+// same windows whenever the DB is the one that could not record them.
+function pushMemoryRequest(key: string, windowMs: number, now: number) {
+ const w = getWindow(key);
+ w.timestamps = pruneTimestamps(w.timestamps, windowMs, now);
+ w.timestamps.push(now);
+}
+
+function pushMemoryTokens(key: string, windowMs: number, now: number, tokens: number) {
+ const w = getWindow(key);
+ w.tokenTimestamps = w.tokenTimestamps.filter(t => t.ts > now - windowMs);
+ w.tokenTimestamps.push({ ts: now, tokens });
+}
+
export function recordRequest(platform: string, modelId: string, keyId: number) {
const now = Date.now();
- const rpmKey = `${platform}:${modelId}:${keyId}:rpm`;
- getWindow(rpmKey).timestamps.push(now);
-
- const rpdKey = `${platform}:${modelId}:${keyId}:rpd`;
- getWindow(rpdKey).timestamps.push(now);
+ if (!recordUsage(platform, modelId, keyId, 'request', 0, now)) {
+ pushMemoryRequest(`${platform}:${modelId}:${keyId}:rpm`, MINUTE, now);
+ pushMemoryRequest(`${platform}:${modelId}:${keyId}:rpd`, DAY, now);
+ }
- recordUsage(platform, modelId, keyId, 'request', 0, now);
clearNullLimitHits(platform, modelId, keyId);
clearCooldownHits(platform, modelId, keyId);
}
@@ -594,13 +635,10 @@ export function recordTokens(
) {
const now = Date.now();
- const tpmKey = `${platform}:${modelId}:${keyId}:tpm`;
- getWindow(tpmKey).tokenTimestamps.push({ ts: now, tokens });
-
- const tpdKey = `${platform}:${modelId}:${keyId}:tpd`;
- getWindow(tpdKey).tokenTimestamps.push({ ts: now, tokens });
-
- recordUsage(platform, modelId, keyId, 'tokens', tokens, now);
+ if (!recordUsage(platform, modelId, keyId, 'tokens', tokens, now)) {
+ pushMemoryTokens(`${platform}:${modelId}:${keyId}:tpm`, MINUTE, now, tokens);
+ pushMemoryTokens(`${platform}:${modelId}:${keyId}:tpd`, DAY, now, tokens);
+ }
}
// Cooldown: when a provider returns 429, block that model+key for a period
@@ -659,6 +697,17 @@ function clearCooldownHits(platform: string, modelId: string, keyId: number): vo
// Short cooldown for a transient (per-minute) 429 — recovers within ~one window.
const TRANSIENT_COOLDOWN_MS = 90 * 1000;
+// Ceiling for the null-limits escalation path. A provider that publishes no
+// RPD/TPD gives us no counter to check, so "daily-exhausted" there is inferred
+// from repeated 429s alone — and per-minute jitter under load looks exactly the
+// same as a spent daily quota. Letting that guess climb the full ladder means a
+// short burst of RPM 429s can quarantine a model+key for 24h with nothing able
+// to contradict it (a benched route serves no request, so no success arrives to
+// clear the counter). Cap the guess at the ladder's 10-minute step; a genuinely
+// dead route just re-benches every 10 minutes instead of hammering the 90s loop.
+// Real RPD/TPD exhaustion is measured, not guessed, and keeps the full ladder.
+const UNKNOWN_LIMIT_MAX_COOLDOWN_MS = 10 * MINUTE;
+
// Long cooldown for a 402 Payment Required (provider/key out of credits). Unlike
// a 429, this won't clear on the next minute/day window — it needs a top-up or
// billing reset. Bench the model+key for a full day so the router fails over to
@@ -678,8 +727,9 @@ export const MODEL_FORBIDDEN_COOLDOWN_MS = DAY;
// not yet seeded — common for ollama, cloudflare, nvidia, huggingface, mistral,
// kilo, llm7, pollinations), we cannot check a counter against a cap. Fall back
// to a hit-count heuristic: after 2+ 429s within this rolling window, treat as
-// "effectively daily-exhausted" and enter the standard escalation ladder at
-// the same step the documented-RPD path would. Without this, these providers
+// "effectively daily-exhausted" and escalate — but only as far as
+// UNKNOWN_LIMIT_MAX_COOLDOWN_MS, since the verdict is a guess. Without this,
+// these providers
// stay stuck at TRANSIENT_COOLDOWN_MS forever even when every request is a
// 429 (observed in production: ollama 130× 429s in 1h with all 90s cooldowns
// expired before the next request). Cheaper than waiting for the operator to
@@ -841,9 +891,16 @@ export function getCooldownDecisionForLimit(
heuristicallyExhausted =
recentHitCount(platform, modelId, keyId, now) >= NULL_LIMIT_HIT_THRESHOLD;
}
- const base = (rpdExhausted || tpdExhausted || heuristicallyExhausted)
- ? getNextCooldownDuration(platform, modelId, keyId)
- : TRANSIENT_COOLDOWN_MS;
+ const dailyExhausted = rpdExhausted || tpdExhausted;
+ let base = TRANSIENT_COOLDOWN_MS;
+ if (dailyExhausted || heuristicallyExhausted) {
+ // The ladder step is taken either way, so a route whose real limits get
+ // seeded (or learned) later resumes escalating where it left off rather
+ // than restarting at 2 minutes.
+ base = getNextCooldownDuration(platform, modelId, keyId);
+ // Guessed exhaustion (no published RPD/TPD) never benches past the cap.
+ if (!dailyExhausted) base = Math.min(base, UNKNOWN_LIMIT_MAX_COOLDOWN_MS);
+ }
// Honor an upstream Retry-After as a floor: never bench shorter than our own
// heuristic, but extend (capped at a day) when the provider explicitly asks
// to wait longer than we otherwise would.
@@ -901,6 +958,29 @@ function clearPersistedCooldown(platform: string, modelId: string, keyId: number
});
}
+/**
+ * Delete every already-expired cooldown, on disk and in memory, and return how
+ * many rows went. Expiry is otherwise purely lazy: isOnCooldown drops a row only
+ * when something asks about that exact (platform, model, key), so a row whose
+ * model was retired, whose key was deleted, or that simply outlived a process
+ * that had every route benched is never collected — the table only grows, and
+ * every cooldown query (status rollups, the probe scan, penalty inspector) pays
+ * for the dead rows. Called once at boot (index.ts) so a restart starts clean.
+ */
+export function cleanupExpiredCooldowns(now = Date.now()): number {
+ const cleared = withDb(db => {
+ const result = db.prepare('DELETE FROM rate_limit_cooldowns WHERE expires_at_ms <= ?').run(now);
+ return Number(result.changes ?? 0);
+ }) ?? 0;
+
+ // Only the expired entries: this is a sweep, not clearCooldownsForKey — an
+ // active bench must survive it.
+ for (const [key, expiry] of [...cooldowns]) {
+ if (expiry <= now) cooldowns.delete(key);
+ }
+ return cleared;
+}
+
export function setCooldown(
platform: string,
modelId: string,
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 66137eff..879863ce 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -871,10 +871,38 @@ function orderChain(chain: ChainRow[], strategy: RoutingStrategy, sampled = true
const tier = (e: ChainRow) => e.match_tier ?? 0;
const weights = weightsFor(strategy);
if (!weights) {
- // Legacy priority mode: base priority + 429 penalty, ascending.
+ // Legacy priority mode: manual chain order + the 429/failure penalty,
+ // ascending.
+ //
+ // The penalty is denominated in PRIORITY POSITIONS — PENALTY_PER_429 = 3
+ // positions per rate limit, PENALTY_PER_FAIL = 1 per upstream failure,
+ // capped at MAX_PENALTY = 10 — so adding it to the RAW priority only ever
+ // reorders a chain whose neighbours sit within 10 of each other. Nothing
+ // guarantees that, and several ordinary paths guarantee the opposite:
+ // - PUT /api/fallback validates `priority` as a bare z.number(), so any
+ // spacing the caller likes (10 / 20 / 30) is persisted verbatim;
+ // - the seed and sort-preset paths number the WHOLE catalog 1..N, while
+ // this chain is only the ENABLED subset (`JOIN models m ON
+ // m.enabled = 1`) — switching models off punches arbitrarily large
+ // holes in the surviving sequence;
+ // - resolveModelGroupCandidates hydrates scattered group members with
+ // whatever COALESCE(fc.priority, 0) they happen to carry.
+ // On any of those the penalty was silently INERT: a model 429-ing every
+ // single request stayed pinned at the head of the chain forever, and the
+ // routing panel's "effective priority" claimed a demotion that never
+ // happened.
+ //
+ // So rank first, then penalize. Sort by the manual priority, re-number the
+ // survivors densely 1..N, and add the penalty to THAT rank. Dense ranking
+ // is monotonic in priority, so an unpenalized chain comes out in exactly
+ // the order the user arranged (tier still dominates as the outer sort key,
+ // and the raw priority remains the tiebreaker); the difference is that one
+ // penalty position now means what it says — one position.
return chain
- .map(e => ({ e, eff: e.priority + getPenalty(e.model_db_id) }))
- .sort((a, b) => tier(a.e) - tier(b.e) || a.eff - b.eff || a.e.priority - b.e.priority)
+ .map((e, i) => ({ e, i }))
+ .sort((a, b) => a.e.priority - b.e.priority || a.i - b.i)
+ .map(({ e, i }, rank) => ({ e, i, eff: rank + 1 + getPenalty(e.model_db_id) }))
+ .sort((a, b) => tier(a.e) - tier(b.e) || a.eff - b.eff || a.e.priority - b.e.priority || a.i - b.i)
.map(x => x.e);
}