Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(providers): add generic custom OpenAI-compatible provider (#145)

Tashfeena89cc64c1e9f
diff --git a/client/src/pages/KeysPage.tsx b/client/src/pages/KeysPage.tsx
index 25d854fe..3dc7854c 100644
--- a/client/src/pages/KeysPage.tsx
+++ b/client/src/pages/KeysPage.tsx
@@ -29,6 +29,13 @@ const PLATFORMS: { value: Platform; label: string }[] = [
{ value: 'huggingface', label: 'HuggingFace Router' },
]
+// 'custom' is configured through its own form (base URL + model), not the
+// generic key dropdown — but it still appears in the grouped provider list.
+const CUSTOM_GROUP: { value: Platform; label: string } = {
+ value: 'custom',
+ label: 'Custom (OpenAI-compatible)',
+}
+
const statusDot: Record<string, string> = {
healthy: 'bg-emerald-500',
rate_limited: 'bg-amber-500',
@@ -136,6 +143,89 @@ function UnifiedKeySection() {
)
}
+function CustomProviderSection() {
+ const queryClient = useQueryClient()
+ const [baseUrl, setBaseUrl] = useState('')
+ const [model, setModel] = useState('')
+ const [displayName, setDisplayName] = useState('')
+ const [apiKey, setApiKey] = useState('')
+
+ const addCustom = useMutation({
+ mutationFn: (body: { baseUrl: string; model: string; displayName?: string; apiKey?: string }) =>
+ apiFetch('/api/keys/custom', { method: 'POST', body: JSON.stringify(body) }),
+ onSuccess: () => {
+ queryClient.invalidateQueries({ queryKey: ['keys'] })
+ queryClient.invalidateQueries({ queryKey: ['health'] })
+ queryClient.invalidateQueries({ queryKey: ['fallback'] })
+ queryClient.invalidateQueries({ queryKey: ['models'] })
+ setModel('')
+ setDisplayName('')
+ },
+ })
+
+ const submit = (e: React.FormEvent) => {
+ e.preventDefault()
+ if (!baseUrl || !model) return
+ addCustom.mutate({ baseUrl, model, displayName: displayName || undefined, apiKey: apiKey || undefined })
+ }
+
+ return (
+ <section>
+ <h2 className="text-sm font-medium mb-1">Add a custom OpenAI-compatible model</h2>
+ <p className="text-xs text-muted-foreground mb-3">
+ Point at any OpenAI-compatible endpoint — llama.cpp, LM Studio, vLLM, a local Ollama, or a remote
+ gateway. Add each model you want routed; they all share the one endpoint. The API key is optional
+ (most local servers don't need one).
+ </p>
+ <form onSubmit={submit} className="flex flex-wrap items-end gap-3 rounded-lg border p-4 bg-card">
+ <div className="space-y-1.5 flex-1 min-w-[240px]">
+ <Label className="text-xs">Base URL</Label>
+ <Input
+ value={baseUrl}
+ onChange={e => setBaseUrl(e.target.value)}
+ placeholder="http://127.0.0.1:11434/v1"
+ className="font-mono text-xs"
+ />
+ </div>
+ <div className="space-y-1.5">
+ <Label className="text-xs">Model</Label>
+ <Input
+ value={model}
+ onChange={e => setModel(e.target.value)}
+ placeholder="qwen3:4b"
+ className="w-[180px] font-mono text-xs"
+ />
+ </div>
+ <div className="space-y-1.5">
+ <Label className="text-xs">Display name</Label>
+ <Input
+ value={displayName}
+ onChange={e => setDisplayName(e.target.value)}
+ placeholder="optional"
+ className="w-[150px]"
+ />
+ </div>
+ <div className="space-y-1.5">
+ <Label className="text-xs">API key</Label>
+ <Input
+ type="password"
+ value={apiKey}
+ onChange={e => setApiKey(e.target.value)}
+ placeholder="optional"
+ className="w-[150px] font-mono text-xs"
+ />
+ </div>
+ <Button type="submit" size="sm" disabled={!baseUrl || !model || addCustom.isPending}>
+ {addCustom.isPending ? 'Adding…' : 'Add model'}
+ </Button>
+ </form>
+ {addCustom.isError && (
+ <p className="text-destructive text-xs mt-2">{(addCustom.error as Error).message}</p>
+ )}
+ </section>
+ )
+}
+
export default function KeysPage() {
const queryClient = useQueryClient()
const [platform, setPlatform] = useState<Platform | ''>('')
@@ -256,7 +346,7 @@ export default function KeysPage() {
const healthKeyMap = new Map<number, { status: string; lastCheckedAt: string | null }>()
for (const k of healthData?.keys ?? []) healthKeyMap.set(k.id, k)
- const grouped = PLATFORMS.map(p => ({
+ const grouped = [...PLATFORMS, CUSTOM_GROUP].map(p => ({
...p,
keys: keys.filter(k => k.platform === p.value),
})).filter(p => p.keys.length > 0)
@@ -333,6 +423,8 @@ export default function KeysPage() {
)}
</section>
+ <CustomProviderSection />
+
<section>
<h2 className="text-sm font-medium mb-3">Configured providers</h2>
{isLoading ? (
diff --git a/server/src/__tests__/routes/custom-provider.test.ts b/server/src/__tests__/routes/custom-provider.test.ts
new file mode 100644
index 00000000..bae7163f
--- /dev/null
+++ b/server/src/__tests__/routes/custom-provider.test.ts
@@ -0,0 +1,105 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb } from '../../db/index.js';
+import { routeRequest } from '../../services/router.js';
+import { resolveProvider, getProvider } from '../../providers/index.js';
+
+async function post(app: Express, path: string, body: any) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify(body),
+ });
+ const data = await res.json().catch(() => null);
+ server.close();
+ return { status: res.status, body: data };
+}
+
+async function get(app: Express, path: string) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`);
+ const data = await res.json().catch(() => null);
+ server.close();
+ return { status: res.status, body: data };
+}
+
+describe('resolveProvider (#117)', () => {
+ it('builds a custom provider bound to the supplied base URL', () => {
+ const p = resolveProvider('custom', 'http://127.0.0.1:8080/v1');
+ expect(p).toBeDefined();
+ expect(p!.platform).toBe('custom');
+ expect((p as any).baseUrl).toBe('http://127.0.0.1:8080/v1');
+ });
+
+ it('returns undefined for a custom provider with no base URL', () => {
+ expect(resolveProvider('custom', null)).toBeUndefined();
+ expect(resolveProvider('custom', ' ')).toBeUndefined();
+ });
+
+ it('returns the registered singleton for built-in platforms', () => {
+ expect(resolveProvider('groq')).toBe(getProvider('groq'));
+ });
+});
+
+describe('POST /api/keys/custom (#117)', () => {
+ let app: Express;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ });
+
+ it('rejects an invalid base URL', async () => {
+ const { status } = await post(app, '/api/keys/custom', { baseUrl: 'not-a-url', model: 'm' });
+ expect(status).toBe(400);
+ });
+
+ it('registers a custom endpoint, model, and fallback entry', async () => {
+ const { status, body } = await post(app, '/api/keys/custom', {
+ baseUrl: 'http://127.0.0.1:11434/v1/',
+ model: 'qwen3:4b',
+ displayName: 'Local Qwen3 4B',
+ });
+ expect(status).toBe(201);
+ expect(body.platform).toBe('custom');
+ expect(body.baseUrl).toBe('http://127.0.0.1:11434/v1'); // trailing slash trimmed
+ expect(body.model).toBe('qwen3:4b');
+
+ const db = getDb();
+ const key = db.prepare("SELECT * FROM api_keys WHERE platform = 'custom'").get() as any;
+ expect(key.base_url).toBe('http://127.0.0.1:11434/v1');
+ const model = db.prepare("SELECT * FROM models WHERE platform = 'custom' AND model_id = 'qwen3:4b'").get() as any;
+ expect(model).toBeDefined();
+ const fc = db.prepare('SELECT * FROM fallback_config WHERE model_db_id = ?').get(model.id);
+ expect(fc).toBeDefined();
+ });
+
+ it('reuses the single custom key when a second model is added', async () => {
+ await post(app, '/api/keys/custom', { baseUrl: 'http://127.0.0.1:11434/v1', model: 'llama3:8b' });
+ const db = getDb();
+ const keys = db.prepare("SELECT * FROM api_keys WHERE platform = 'custom'").all();
+ expect(keys.length).toBe(1); // not a second key
+ const models = db.prepare("SELECT * FROM models WHERE platform = 'custom'").all();
+ expect(models.length).toBe(2);
+ });
+
+ it('surfaces baseUrl in the keys listing', async () => {
+ const { body } = await get(app, '/api/keys');
+ const custom = body.find((k: any) => k.platform === 'custom');
+ expect(custom.baseUrl).toBe('http://127.0.0.1:11434/v1');
+ });
+
+ it('routes a request to the custom model through its base URL', () => {
+ // The seeded built-in models have no keys, so the only routable model is
+ // the custom one we registered above.
+ const route = routeRequest(1000);
+ expect(route.platform).toBe('custom');
+ expect((route.provider as any).baseUrl).toBe('http://127.0.0.1:11434/v1');
+ expect(['qwen3:4b', 'llama3:8b']).toContain(route.modelId);
+ });
+});
diff --git a/server/src/db/index.ts b/server/src/db/index.ts
index dd1ae01d..33a109bd 100644
--- a/server/src/db/index.ts
+++ b/server/src/db/index.ts
@@ -143,6 +143,7 @@ function createTables(db: Database.Database) {
`);
ensureRequestKeyIdColumn(db);
+ ensureApiKeysBaseUrlColumn(db);
}
function ensureRequestKeyIdColumn(db: Database.Database) {
@@ -153,6 +154,15 @@ function ensureRequestKeyIdColumn(db: Database.Database) {
db.prepare('CREATE INDEX IF NOT EXISTS idx_requests_key_id ON requests(key_id)').run();
}
+// `base_url` is the upstream endpoint for the user-configured 'custom' provider
+// (#117). NULL for every built-in platform — they use their hardcoded base URL.
+function ensureApiKeysBaseUrlColumn(db: Database.Database) {
+ const columns = db.prepare('PRAGMA table_info(api_keys)').all() as { name: string }[];
+ if (!columns.some(col => col.name === 'base_url')) {
+ db.prepare('ALTER TABLE api_keys ADD COLUMN base_url TEXT').run();
+ }
+}
+
function seedModels(db: Database.Database) {
const count = db.prepare('SELECT COUNT(*) as cnt FROM models').get() as { cnt: number };
if (count.cnt > 0) return;
diff --git a/server/src/providers/index.ts b/server/src/providers/index.ts
index 0e3177e5..a149c9d9 100644
--- a/server/src/providers/index.ts
+++ b/server/src/providers/index.ts
@@ -148,10 +148,43 @@ register(new OpenAICompatProvider({
// $0.0, please pay with fiat or send tao". The "free" tier requires a
// non-zero balance, which conflicts with the project's no-card criterion.
+// Placeholder so getProvider('custom')/hasProvider('custom')/getAllProviders()
+// behave — but the real instance is built per-key by resolveProvider(), since
+// a custom provider's base URL is user-supplied and lives on the api_keys row.
+register(new OpenAICompatProvider({
+ platform: 'custom',
+ name: 'Custom (OpenAI-compatible)',
+ baseUrl: '',
+}));
+
+// Locally-hosted inference (llama.cpp / vLLM / Ollama on CPU) can be slow, so
+// custom providers get the same extended timeout as Ollama Cloud.
+const CUSTOM_PROVIDER_TIMEOUT_MS = 120000;
+
export function getProvider(platform: Platform): BaseProvider | undefined {
return providers.get(platform);
}
+/**
+ * Resolve the provider for a route. Built-in platforms return their registered
+ * singleton; the 'custom' platform builds a fresh OpenAICompatProvider bound to
+ * the caller-supplied base URL (stored per api_keys row). Returns undefined for
+ * a custom provider with no base URL configured.
+ */
+export function resolveProvider(platform: Platform, baseUrl?: string | null): BaseProvider | undefined {
+ if (platform === 'custom') {
+ const trimmed = baseUrl?.trim();
+ if (!trimmed) return undefined;
+ return new OpenAICompatProvider({
+ platform: 'custom',
+ name: 'Custom (OpenAI-compatible)',
+ baseUrl: trimmed,
+ timeoutMs: CUSTOM_PROVIDER_TIMEOUT_MS,
+ });
+ }
+ return providers.get(platform);
+}
+
export function getAllProviders(): BaseProvider[] {
return Array.from(providers.values());
}
diff --git a/server/src/routes/keys.ts b/server/src/routes/keys.ts
index f7b1b9d2..caff9631 100644
--- a/server/src/routes/keys.ts
+++ b/server/src/routes/keys.ts
@@ -12,7 +12,7 @@ export const keysRouter = Router();
const PLATFORMS = [
'google', 'groq', 'cerebras', 'sambanova', 'nvidia', 'mistral',
'openrouter', 'github', 'cohere', 'cloudflare', 'zhipu', 'ollama',
- 'kilo', 'pollinations', 'llm7', 'huggingface',
+ 'kilo', 'pollinations', 'llm7', 'huggingface', 'custom',
] as const;
const addKeySchema = z.object({
@@ -46,6 +46,7 @@ keysRouter.get('/', (_req: Request, res: Response) => {
platform: row.platform,
label: row.label,
maskedKey,
+ baseUrl: row.base_url ?? null,
status: row.status,
enabled: row.enabled === 1,
createdAt: row.created_at,
@@ -83,6 +84,86 @@ keysRouter.post('/', (req: Request, res: Response) => {
});
});
+// ── Custom OpenAI-compatible provider (#117) ──────────────────────────────
+// A single user-configured endpoint (llama.cpp / LM Studio / vLLM / Ollama /
+// any OpenAI-compatible base_url). The endpoint lives on one 'custom' api_keys
+// row; each call registers another model that routes through it.
+const customProviderSchema = z.object({
+ baseUrl: z.string().url('baseUrl must be a valid URL'),
+ model: z.string().min(1, 'model is required'),
+ displayName: z.string().optional(),
+ apiKey: z.string().optional(),
+ label: z.string().optional(),
+});
+
+keysRouter.post('/custom', (req: Request, res: Response) => {
+ const parsed = customProviderSchema.safeParse(req.body);
+ if (!parsed.success) {
+ res.status(400).json({ error: { message: parsed.error.errors.map(e => e.message).join(', ') } });
+ return;
+ }
+
+ const baseUrl = parsed.data.baseUrl.trim().replace(/\/+$/, '');
+ const modelId = parsed.data.model.trim();
+ const displayName = (parsed.data.displayName ?? modelId).trim();
+ // Local servers often need no key; keep a sentinel so there's always a bearer.
+ const rawKey = parsed.data.apiKey?.trim() || 'no-key';
+ const label = parsed.data.label ?? 'Custom';
+
+ const db = getDb();
+ const upsert = db.transaction(() => {
+ // One shared 'custom' key holds the endpoint URL. Reuse it across models;
+ // update its base_url/key when re-submitted.
+ const existing = db.prepare("SELECT id FROM api_keys WHERE platform = 'custom' LIMIT 1").get() as { id: number } | undefined;
+ let keyId: number;
+ if (existing) {
+ const { encrypted, iv, authTag } = encrypt(rawKey);
+ db.prepare("UPDATE api_keys SET base_url = ?, encrypted_key = ?, iv = ?, auth_tag = ?, status = 'unknown', enabled = 1 WHERE id = ?")
+ .run(baseUrl, encrypted, iv, authTag, existing.id);
+ keyId = existing.id;
+ } else {
+ const { encrypted, iv, authTag } = encrypt(rawKey);
+ const r = db.prepare(`
+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, base_url)
+ VALUES ('custom', ?, ?, ?, ?, 'unknown', 1, ?)
+ `).run(label, encrypted, iv, authTag, baseUrl);
+ keyId = Number(r.lastInsertRowid);
+ }
+
+ // Register the model (idempotent on platform+model_id). Custom models carry
+ // no rate limits and sort last in the intelligence preset (size_label tier).
+ db.prepare(`
+ INSERT OR IGNORE INTO models
+ (platform, model_id, display_name, intelligence_rank, speed_rank, size_label,
+ rpm_limit, rpd_limit, tpm_limit, tpd_limit, monthly_token_budget, context_window, enabled)
+ VALUES ('custom', ?, ?, 50, 50, 'Custom', NULL, NULL, NULL, NULL, '', NULL, 1)
+ `).run(modelId, displayName);
+
+ const modelRow = db.prepare("SELECT id FROM models WHERE platform = 'custom' AND model_id = ?").get(modelId) as { id: number };
+
+ // Append to the fallback chain if not already present.
+ const inChain = db.prepare('SELECT 1 FROM fallback_config WHERE model_db_id = ?').get(modelRow.id);
+ if (!inChain) {
+ const max = db.prepare('SELECT COALESCE(MAX(priority), 0) AS m FROM fallback_config').get() as { m: number };
+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(modelRow.id, max.m + 1);
+ }
+
+ return { keyId, modelDbId: modelRow.id };
+ });
+
+ const { keyId, modelDbId } = upsert();
+ res.status(201).json({
+ success: true,
+ keyId,
+ modelDbId,
+ platform: 'custom',
+ baseUrl,
+ model: modelId,
+ displayName,
+ maskedKey: maskKey(rawKey),
+ });
+});
+
// Delete a key
keysRouter.delete('/:id', (req: Request, res: Response) => {
const id = parseInt(req.params.id as string, 10);
diff --git a/server/src/services/health.ts b/server/src/services/health.ts
index 5677e641..00caecca 100644
--- a/server/src/services/health.ts
+++ b/server/src/services/health.ts
@@ -1,5 +1,5 @@
import { getDb } from '../db/index.js';
-import { getProvider } from '../providers/index.js';
+import { resolveProvider } from '../providers/index.js';
import { decrypt } from '../lib/crypto.js';
import type { Platform, KeyStatus } from '@freellmapi/shared/types.js';
@@ -14,7 +14,7 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {
const row = db.prepare('SELECT * FROM api_keys WHERE id = ?').get(keyId) as any;
if (!row) return 'error';
- const provider = getProvider(row.platform as Platform);
+ const provider = resolveProvider(row.platform as Platform, row.base_url);
if (!provider) return 'error';
try {
diff --git a/server/src/services/router.ts b/server/src/services/router.ts
index 26168fa1..5629f621 100644
--- a/server/src/services/router.ts
+++ b/server/src/services/router.ts
@@ -1,5 +1,5 @@
import { getDb } from '../db/index.js';
-import { getProvider } from '../providers/index.js';
+import { getProvider, resolveProvider } from '../providers/index.js';
import { decrypt } from '../lib/crypto.js';
import { canMakeRequest, canUseTokens, isOnCooldown } from './ratelimit.js';
import type { BaseProvider } from '../providers/base.js';
@@ -23,6 +23,7 @@ interface KeyRow {
auth_tag: string;
status: string;
enabled: number;
+ base_url: string | null;
}
interface FallbackRow {
@@ -208,10 +209,18 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, pre
continue;
}
+ // For the 'custom' platform the real provider is built from this key's
+ // base_url (the registered instance is just a placeholder). A custom key
+ // with no base_url can't be routed — skip it.
+ const resolvedProvider = model.platform === 'custom'
+ ? resolveProvider('custom', key.base_url)
+ : provider;
+ if (!resolvedProvider) continue;
+
// We found a working key for this model!
roundRobinIndex.set(rrKey, idx);
return {
- provider,
+ provider: resolvedProvider,
modelId: model.model_id,
modelDbId: model.id,
apiKey: decryptedKey,
diff --git a/shared/types.ts b/shared/types.ts
index fe663d95..9982b510 100644
--- a/shared/types.ts
+++ b/shared/types.ts
@@ -21,7 +21,10 @@ export type Platform =
| 'kilo'
| 'pollinations'
| 'llm7'
- | 'huggingface';
+ | 'huggingface'
+ // User-configured OpenAI-compatible endpoint (llama.cpp, LM Studio, vLLM,
+ // Ollama, any base_url). The endpoint URL lives on the api_keys row; see #117.
+ | 'custom';
export interface Model {
id: number;