Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

fix(keys): cascade custom models out of the fallback chain on key deletion (#197)

Tashfeen9b882fbe2ff2
diff --git a/server/src/__tests__/providers/openai-compat.test.ts b/server/src/__tests__/providers/openai-compat.test.ts
index a536e4e9..e86bb958 100644
--- a/server/src/__tests__/providers/openai-compat.test.ts
+++ b/server/src/__tests__/providers/openai-compat.test.ts
@@ -105,6 +105,21 @@ describe('OpenAICompatProvider', () => {
).rejects.toThrow(/Too many requests/);
});
+ it('explains a non-JSON 200 body instead of surfacing the raw parse error (#189)', async () => {
+ // e.g. a custom base URL pointing at Ollama's native NDJSON /api endpoint:
+ // real fetch's res.json() rejects with "Unexpected non-whitespace character
+ // after JSON at position …", which is useless to the user.
+ vi.spyOn(global, 'fetch').mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ json: () => Promise.reject(new SyntaxError('Unexpected non-whitespace character after JSON at position 583 (line 27 column 2)')),
+ } as any);
+
+ await expect(
+ provider.chatCompletion('key', [{ role: 'user', content: 'hi' }], 'model')
+ ).rejects.toThrow(/not OpenAI-compatible/);
+ });
+
it('should validate key using models endpoint', async () => {
vi.spyOn(global, 'fetch').mockResolvedValueOnce({ ok: true, status: 200 } as any);
expect(await provider.validateKey('valid')).toBe(true);
diff --git a/server/src/__tests__/routes/custom-provider.test.ts b/server/src/__tests__/routes/custom-provider.test.ts
index e3f59c35..8f31345f 100644
--- a/server/src/__tests__/routes/custom-provider.test.ts
+++ b/server/src/__tests__/routes/custom-provider.test.ts
@@ -1,7 +1,8 @@
import { describe, it, expect, beforeAll } from 'vitest';
+import http from 'node:http';
import type { Express } from 'express';
import { createApp } from '../../app.js';
-import { initDb, getDb } from '../../db/index.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
import { routeRequest } from '../../services/router.js';
import { resolveProvider, getProvider } from '../../providers/index.js';
import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';
@@ -35,6 +36,18 @@ async function get(app: Express, path: string) {
return { status: res.status, body: data };
}
+async function del(app: Express, path: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'DELETE',
+ headers: isGatedApiPath(path) ? { Authorization: `Bearer ${dashToken}` } : {},
+ });
+ const data = await res.json().catch(() => null);
+ server.close();
+ return { status: res.status, body: data };
+}
+
describe('resolveProvider (#117)', () => {
it('builds a custom provider bound to the supplied base URL', () => {
const p = resolveProvider('custom', 'http://127.0.0.1:8080/v1');
@@ -111,4 +124,87 @@ describe('POST /api/keys/custom (#117)', () => {
expect((route.provider as any).baseUrl).toBe('http://127.0.0.1:11434/v1');
expect(['qwen3:4b', 'llama3:8b']).toContain(route.modelId);
});
+
+ it('deleting the custom key cascades its models out of the fallback chain (#189)', async () => {
+ const db = getDb();
+ const key = db.prepare("SELECT id FROM api_keys WHERE platform = 'custom'").get() as { id: number };
+ const customModelIds = (db.prepare("SELECT id FROM models WHERE platform = 'custom'").all() as { id: number }[]).map(r => r.id);
+ expect(customModelIds.length).toBe(2); // qwen3:4b + llama3:8b from earlier tests
+ const builtinModels = (db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform != 'custom'").get() as { n: number }).n;
+
+ const { status } = await del(app, `/api/keys/${key.id}`);
+ expect(status).toBe(200);
+
+ // Custom models and their fallback entries are gone — not orphaned.
+ expect((db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform = 'custom'").get() as { n: number }).n).toBe(0);
+ const placeholders = customModelIds.map(() => '?').join(',');
+ expect((db.prepare(`SELECT COUNT(*) AS n FROM fallback_config WHERE model_db_id IN (${placeholders})`).get(...customModelIds) as { n: number }).n).toBe(0);
+ // Built-in catalog rows are untouched.
+ expect((db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform != 'custom'").get() as { n: number }).n).toBe(builtinModels);
+ });
+
+ it('deleting a built-in platform key does NOT cascade its catalog models', async () => {
+ const db = getDb();
+ const r = db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)
+ VALUES ('groq', 'test', 'x', 'x', 'x', 'unknown', 1)
+ `).run();
+ const groqModels = (db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform = 'groq'").get() as { n: number }).n;
+ expect(groqModels).toBeGreaterThan(0);
+
+ const { status } = await del(app, `/api/keys/${r.lastInsertRowid}`);
+ expect(status).toBe(200);
+ expect((db.prepare("SELECT COUNT(*) AS n FROM models WHERE platform = 'groq'").get() as { n: number }).n).toBe(groqModels);
+ });
+
+ it('re-adding a custom provider after deletion starts a fresh chain entry', async () => {
+ const { status, body } = await post(app, '/api/keys/custom', {
+ baseUrl: 'http://127.0.0.1:8080/v1',
+ model: 'mistral:7b',
+ });
+ expect(status).toBe(201);
+ const db = getDb();
+ expect((db.prepare("SELECT COUNT(*) AS n FROM api_keys WHERE platform = 'custom'").get() as { n: number }).n).toBe(1);
+ const fc = db.prepare('SELECT * FROM fallback_config WHERE model_db_id = ?').get(body.modelDbId);
+ expect(fc).toBeDefined();
+ });
+
+ it('surfaces a clear error when the custom endpoint speaks NDJSON, not OpenAI (#189)', async () => {
+ // Real upstream that answers like Ollama's native /api/chat: HTTP 200,
+ // newline-delimited JSON documents — res.json() in the provider would die
+ // with "Unexpected non-whitespace character after JSON at position …".
+ const upstream = http.createServer((_req, res) => {
+ res.writeHead(200, { 'Content-Type': 'application/x-ndjson' });
+ res.end(
+ JSON.stringify({ model: 'qwen3:4b', message: { role: 'assistant', content: 'hi' } }, null, 2) +
+ '\n' +
+ JSON.stringify({ done: true }) +
+ '\n',
+ );
+ });
+ await new Promise<void>(resolve => upstream.listen(0, resolve));
+ const upstreamPort = (upstream.address() as any).port;
+
+ // Point the custom provider at the NDJSON upstream and pin its model.
+ const reg = await post(app, '/api/keys/custom', {
+ baseUrl: `http://127.0.0.1:${upstreamPort}/v1`,
+ model: 'ndjson-model',
+ });
+ expect(reg.status).toBe(201);
+
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}/v1/chat/completions`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${getUnifiedApiKey()}` },
+ body: JSON.stringify({ model: 'ndjson-model', messages: [{ role: 'user', content: 'hi' }] }),
+ });
+ const body = await res.json().catch(() => null);
+ server.close();
+
+ upstream.close();
+ expect(res.status).toBe(502);
+ expect(JSON.stringify(body)).toMatch(/not OpenAI-compatible/);
+ expect(JSON.stringify(body)).not.toMatch(/Unexpected non-whitespace/);
+ });
});
diff --git a/server/src/providers/openai-compat.ts b/server/src/providers/openai-compat.ts
index 6db880f6..717ea78f 100644
--- a/server/src/providers/openai-compat.ts
+++ b/server/src/providers/openai-compat.ts
@@ -77,7 +77,19 @@ export class OpenAICompatProvider extends BaseProvider {
throw new Error(`${this.name} API error ${res.status}: ${(err as any).error?.message ?? res.statusText}`);
}
- const data = await res.json() as ChatCompletionResponse;
+ let data: ChatCompletionResponse;
+ try {
+ data = await res.json() as ChatCompletionResponse;
+ } catch {
+ // A 200 whose body isn't a single JSON document — typically a base URL
+ // pointing at a non-OpenAI-compatible API (e.g. Ollama's native NDJSON
+ // /api endpoints instead of /v1, #189). Surface what's wrong instead of
+ // the raw JSON.parse position error.
+ throw new Error(
+ `${this.name} returned 200 with a non-JSON body — the endpoint is not OpenAI-compatible. ` +
+ `Check the base URL (for Ollama use http://host:11434/v1, for llama.cpp/vLLM/LM Studio the /v1 path).`,
+ );
+ }
normalizeChoices(data);
data._routed_via = { platform: this.platform, model: modelId };
return data;
diff --git a/server/src/routes/keys.ts b/server/src/routes/keys.ts
index fbf66334..35c91c0f 100644
--- a/server/src/routes/keys.ts
+++ b/server/src/routes/keys.ts
@@ -206,13 +206,29 @@ keysRouter.delete('/:id', (req: Request, res: Response) => {
}
const db = getDb();
- const result = db.prepare('DELETE FROM api_keys WHERE id = ?').run(id);
-
- if (result.changes === 0) {
+ const row = db.prepare('SELECT platform FROM api_keys WHERE id = ?').get(id) as { platform: string } | undefined;
+ if (!row) {
res.status(404).json({ error: { message: 'Key not found' } });
return;
}
+ const remove = db.transaction(() => {
+ db.prepare('DELETE FROM api_keys WHERE id = ?').run(id);
+ // Custom models exist only because POST /custom registered them alongside
+ // this endpoint key (#117) — they can't route without it. Built-in
+ // platforms keep their seeded catalog rows, but once the last custom key
+ // is gone, orphaned custom models would linger in the fallback chain
+ // forever (#189), so cascade them away.
+ if (row.platform === 'custom') {
+ const remaining = db.prepare("SELECT COUNT(*) AS n FROM api_keys WHERE platform = 'custom'").get() as { n: number };
+ if (remaining.n === 0) {
+ db.prepare("DELETE FROM fallback_config WHERE model_db_id IN (SELECT id FROM models WHERE platform = 'custom')").run();
+ db.prepare("DELETE FROM models WHERE platform = 'custom'").run();
+ }
+ }
+ });
+ remove();
+
res.json({ success: true });
});