7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
fix(keys): cascade custom models out of the fallback chain on key deletion (#197)
Tashfeen9b882fbe2ff2
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.tsindex a536e4e9..e86bb958 100644--- a/server/src/__tests__/providers/openai-compat.test.ts+++ b/server/src/__tests__/providers/openai-compat.test.ts@@ -105,6 +105,21 @@ describe('OpenAICompatProvider', () => {).rejects.toThrow(/Too many requests/);});+ it('explains a non-JSON 200 body instead of surfacing the raw parse error (#189)', async () => {+ // e.g. a custom base URL pointing at Ollama's native NDJSON /api endpoint:+ // real fetch's res.json() rejects with "Unexpected non-whitespace character+ // after JSON at position …", which is useless to the user.+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({+ ok: true,+ status: 200,+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 583 (line 27 column 2)')),+ } as any);++ await expect(+ provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model')+ ).rejects.toThrow(/not OpenAI-compatible/);+ });+it('should validate key using models endpoint', async () => {vi.spyOn(global, 'fetch').mockResolvedValueOnce({ ok: true, status: 200 } as any);expect(await provider.validateKey('valid')).toBe(true);diff --git a/server/src/__tests__/routes/custom-provider.test.ts b/server/src/__tests__/routes/custom-provider.test.tsindex e3f59c35..8f31345f 100644--- a/server/src/__tests__/routes/custom-provider.test.ts+++ b/server/src/__tests__/routes/custom-provider.test.ts@@ -1,7 +1,8 @@import { describe, it, expect, beforeAll } from 'vitest';+import http from 'node:http';import type { Express } from 'express';import { createApp } from '../../app.js';-import { initDb, getDb } from '../../db/index.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';import { routeRequest } from '../../services/router.js';import { resolveProvider, getProvider } from '../../providers/index.js';import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';@@ -35,6 +36,18 @@ async function get(app: Express, path: string) {return { status: res.status, body: data };}+async function del(app: Express, path: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'DELETE',+ headers: isGatedApiPath(path) ? { Authorization: `Bearer ${dashToken}` } : {},+ });+ const data = await res.json().catch(() => null);+ server.close();+ return { status: res.status, body: data };+}+describe('resolveProvider (#117)', () => {it('builds a custom provider bound to the supplied base URL', () => {const p = resolveProvider('custom', 'http://127.0.0.1:8080/v1');@@ -111,4 +124,87 @@ describe('POST /api/keys/custom (#117)', () => {expect((route.provider as any).baseUrl).toBe('http://127.0.0.1:11434/v1');expect(['qwen3:4b', 'llama3:8b']).toContain(route.modelId);});++ it('deleting the custom key cascades its models out of the fallback chain (#189)', async () => {+ const db = getDb();+ const key = db.prepare("SELECT id FROM api_keys WHERE platform = 'custom'").get() as { id: number };+ const customModelIds = (db.prepare("SELECT id FROM models WHERE platform = 'custom'").all() as { id: number }[]).map(r => r.id);+ expect(customModelIds.length).toBe(2); // qwen3:4b + llama3:8b from earlier tests+ const builtinModels = (db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform != 'custom'").get() as { n: number }).n;++ const { status } = await del(app, `/api/keys/${key.id}`);+ expect(status).toBe(200);++ // Custom models and their fallback entries are gone — not orphaned.+ expect((db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform = 'custom'").get() as { n: number }).n).toBe(0);+ const placeholders = customModelIds.map(() => '?').join(',');+ expect((db.prepare(`SELECT COUNT(*) AS n FROM fallback_config WHERE model_db_id IN (${placeholders})`).get(...customModelIds) as { n: number }).n).toBe(0);+ // Built-in catalog rows are untouched.+ expect((db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform != 'custom'").get() as { n: number }).n).toBe(builtinModels);+ });++ it('deleting a built-in platform key does NOT cascade its catalog models', async () => {+ const db = getDb();+ const r = db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'test', 'x', 'x', 'x', 'unknown', 1)+ `).run();+ const groqModels = (db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform = 'groq'").get() as { n: number }).n;+ expect(groqModels).toBeGreaterThan(0);++ const { status } = await del(app, `/api/keys/${r.lastInsertRowid}`);+ expect(status).toBe(200);+ expect((db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform = 'groq'").get() as { n: number }).n).toBe(groqModels);+ });++ it('re-adding a custom provider after deletion starts a fresh chain entry', async () => {+ const { status, body } = await post(app, '/api/keys/custom', {+ baseUrl: 'http://127.0.0.1:8080/v1',+ model: 'mistral:7b',+ });+ expect(status).toBe(201);+ const db = getDb();+ expect((db.prepare("SELECT COUNT(*) AS n FROM api_keys WHERE platform = 'custom'").get() as { n: number }).n).toBe(1);+ const fc = db.prepare('SELECT * FROM fallback_config WHERE model_db_id = ?').get(body.modelDbId);+ expect(fc).toBeDefined();+ });++ it('surfaces a clear error when the custom endpoint speaks NDJSON, not OpenAI (#189)', async () => {+ // Real upstream that answers like Ollama's native /api/chat: HTTP 200,+ // newline-delimited JSON documents — res.json() in the provider would die+ // with "Unexpected non-whitespace character after JSON at position …".+ const upstream = http.createServer((_req, res) => {+ res.writeHead(200, { 'Content-Type': 'application/x-ndjson' });+ res.end(+ JSON.stringify({ model: 'qwen3:4b', message: { role: 'assistant', content: 'hi' } }, null, 2) ++ '\n' ++ JSON.stringify({ done: true }) ++ '\n',+ );+ });+ await new Promise<void>(resolve => upstream.listen(0, resolve));+ const upstreamPort = (upstream.address() as any).port;++ // Point the custom provider at the NDJSON upstream and pin its model.+ const reg = await post(app, '/api/keys/custom', {+ baseUrl: `http://127.0.0.1:${upstreamPort}/v1`,+ model: 'ndjson-model',+ });+ expect(reg.status).toBe(201);++ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getUnifiedApiKey()}` },+ body: JSON.stringify({ model: 'ndjson-model', messages: [{ role: 'user', content: 'hi' }] }),+ });+ const body = await res.json().catch(() => null);+ server.close();++ upstream.close();+ expect(res.status).toBe(502);+ expect(JSON.stringify(body)).toMatch(/not OpenAI-compatible/);+ expect(JSON.stringify(body)).not.toMatch(/Unexpected non-whitespace/);+ });});diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.tsindex 6db880f6..717ea78f 100644--- a/server/src/providers/openai-compat.ts+++ b/server/src/providers/openai-compat.ts@@ -77,7 +77,19 @@ export class OpenAICompatProvider extends BaseProvider {throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);}- const data = await res.json() as ChatCompletionResponse;+ let data: ChatCompletionResponse;+ try {+ data = await res.json() as ChatCompletionResponse;+ } catch {+ // A 200 whose body isn't a single JSON document — typically a base URL+ // pointing at a non-OpenAI-compatible API (e.g. Ollama's native NDJSON+ // /api endpoints instead of /v1, #189). Surface what's wrong instead of+ // the raw JSON.parse position error.+ throw new Error(+ `${this.name} returned 200 with a non-JSON body — the endpoint is not OpenAI-compatible. ` ++ `Check the base URL (for Ollama use http://host:11434/v1, for llama.cpp/vLLM/LM Studio the /v1 path).`,+ );+ }normalizeChoices(data);data._routed_via = { platform: this.platform, model: modelId };return data;diff --git a/server/src/routes/keys.ts b/server/src/routes/keys.tsindex fbf66334..35c91c0f 100644--- a/server/src/routes/keys.ts+++ b/server/src/routes/keys.ts@@ -206,13 +206,29 @@ keysRouter.delete('/:id', (req: Request, res: Response) => {}const db = getDb();- const result = db.prepare('DELETE FROM api_keys WHERE id = ?').run(id);-- if (result.changes === 0) {+ const row = db.prepare('SELECT platform FROM api_keys WHERE id = ?').get(id) as { platform: string } | undefined;+ if (!row) {res.status(404).json({ error: { message: 'Key not found' } });return;}+ const remove = db.transaction(() => {+ db.prepare('DELETE FROM api_keys WHERE id = ?').run(id);+ // Custom models exist only because POST /custom registered them alongside+ // this endpoint key (#117) — they can't route without it. Built-in+ // platforms keep their seeded catalog rows, but once the last custom key+ // is gone, orphaned custom models would linger in the fallback chain+ // forever (#189), so cascade them away.+ if (row.platform === 'custom') {+ const remaining = db.prepare("SELECT COUNT(*) AS n FROM api_keys WHERE platform = 'custom'").get() as { n: number };+ if (remaining.n === 0) {+ db.prepare("DELETE FROM fallback_config WHERE model_db_id IN (SELECT id FROM models WHERE platform = 'custom')").run();+ db.prepare("DELETE FROM models WHERE platform = 'custom'").run();+ }+ }+ });+ remove();+res.json({ success: true });});