7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(providers): add generic custom OpenAI-compatible provider (#145)
Tashfeena89cc64c1e9f
diff --git a/client/src/pages/KeysPage.tsx b/client/src/pages/KeysPage.tsxindex 25d854fe..3dc7854c 100644--- a/client/src/pages/KeysPage.tsx+++ b/client/src/pages/KeysPage.tsx@@ -29,6 +29,13 @@ const PLATFORMS: { value: Platform; label: string }[] = [{ value: 'huggingface', label: 'HuggingFace Router' },]+// 'custom' is configured through its own form (base URL + model), not the+// generic key dropdown — but it still appears in the grouped provider list.+const CUSTOM_GROUP: { value: Platform; label: string } = {+ value: 'custom',+ label: 'Custom (OpenAI-compatible)',+}+const statusDot: Record<string, string> = {healthy: 'bg-emerald-500',rate_limited: 'bg-amber-500',@@ -136,6 +143,89 @@ function UnifiedKeySection() {)}+function CustomProviderSection() {+ const queryClient = useQueryClient()+ const [baseUrl, setBaseUrl] = useState('')+ const [model, setModel] = useState('')+ const [displayName, setDisplayName] = useState('')+ const [apiKey, setApiKey] = useState('')++ const addCustom = useMutation({+ mutationFn: (body: { baseUrl: string; model: string; displayName?: string; apiKey?: string }) =>+ apiFetch('/api/keys/custom', { method: 'POST', body: JSON.stringify(body) }),+ onSuccess: () => {+ queryClient.invalidateQueries({ queryKey: ['keys'] })+ queryClient.invalidateQueries({ queryKey: ['health'] })+ queryClient.invalidateQueries({ queryKey: ['fallback'] })+ queryClient.invalidateQueries({ queryKey: ['models'] })+ setModel('')+ setDisplayName('')+ },+ })++ const submit = (e: React.FormEvent) => {+ e.preventDefault()+ if (!baseUrl || !model) return+ addCustom.mutate({ baseUrl, model, displayName: displayName || undefined, apiKey: apiKey || undefined })+ }++ return (+ <section>+ <h2 className="text-sm font-medium mb-1">Add a custom OpenAI-compatible model</h2>+ <p className="text-xs text-muted-foreground mb-3">+ Point at any OpenAI-compatible endpoint — llama.cpp, LM Studio, vLLM, a local Ollama, or a remote+ gateway. Add each model you want routed; they all share the one endpoint. The API key is optional+ (most local servers don't need one).+ </p>+ <form onSubmit={submit} className="flex flex-wrap items-end gap-3 rounded-lg border p-4 bg-card">+ <div className="space-y-1.5 flex-1 min-w-[240px]">+ <Label className="text-xs">Base URL</Label>+ <Input+ value={baseUrl}+ onChange={e => setBaseUrl(e.target.value)}+ placeholder="http://127.0.0.1:11434/v1"+ className="font-mono text-xs"+ />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">Model</Label>+ <Input+ value={model}+ onChange={e => setModel(e.target.value)}+ placeholder="qwen3:4b"+ className="w-[180px] font-mono text-xs"+ />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">Display name</Label>+ <Input+ value={displayName}+ onChange={e => setDisplayName(e.target.value)}+ placeholder="optional"+ className="w-[150px]"+ />+ </div>+ <div className="space-y-1.5">+ <Label className="text-xs">API key</Label>+ <Input+ type="password"+ value={apiKey}+ onChange={e => setApiKey(e.target.value)}+ placeholder="optional"+ className="w-[150px] font-mono text-xs"+ />+ </div>+ <Button type="submit" size="sm" disabled={!baseUrl || !model || addCustom.isPending}>+ {addCustom.isPending ? 'Adding…' : 'Add model'}+ </Button>+ </form>+ {addCustom.isError && (+ <p className="text-destructive text-xs mt-2">{(addCustom.error as Error).message}</p>+ )}+ </section>+ )+}+export default function KeysPage() {const queryClient = useQueryClient()const [platform, setPlatform] = useState<Platform | ''>('')@@ -256,7 +346,7 @@ export default function KeysPage() {const healthKeyMap = new Map<number, { status: string; lastCheckedAt: string | null }>()for (const k of healthData?.keys ?? []) healthKeyMap.set(k.id, k)- const grouped = PLATFORMS.map(p => ({+ const grouped = [...PLATFORMS, CUSTOM_GROUP].map(p => ({...p,keys: keys.filter(k => k.platform === p.value),})).filter(p => p.keys.length > 0)@@ -333,6 +423,8 @@ export default function KeysPage() {)}</section>+ <CustomProviderSection />+<section><h2 className="text-sm font-medium mb-3">Configured providers</h2>{isLoading ? (diff --git a/server/src/__tests__/routes/custom-provider.test.ts b/server/src/__tests__/routes/custom-provider.test.tsnew file mode 100644index 00000000..bae7163f--- /dev/null+++ b/server/src/__tests__/routes/custom-provider.test.ts@@ -0,0 +1,105 @@+import { describe, it, expect, beforeAll } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb } from '../../db/index.js';+import { routeRequest } from '../../services/router.js';+import { resolveProvider, getProvider } from '../../providers/index.js';++async function post(app: Express, path: string, body: any) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`, {+ method: 'POST',+ headers: { 'Content-Type': 'application/json' },+ body: JSON.stringify(body),+ });+ const data = await res.json().catch(() => null);+ server.close();+ return { status: res.status, body: data };+}++async function get(app: Express, path: string) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const res = await fetch(`http://127.0.0.1:${addr.port}${path}`);+ const data = await res.json().catch(() => null);+ server.close();+ return { status: res.status, body: data };+}++describe('resolveProvider (#117)', () => {+ it('builds a custom provider bound to the supplied base URL', () => {+ const p = resolveProvider('custom', 'http://127.0.0.1:8080/v1');+ expect(p).toBeDefined();+ expect(p!.platform).toBe('custom');+ expect((p as any).baseUrl).toBe('http://127.0.0.1:8080/v1');+ });++ it('returns undefined for a custom provider with no base URL', () => {+ expect(resolveProvider('custom', null)).toBeUndefined();+ expect(resolveProvider('custom', ' ')).toBeUndefined();+ });++ it('returns the registered singleton for built-in platforms', () => {+ expect(resolveProvider('groq')).toBe(getProvider('groq'));+ });+});++describe('POST /api/keys/custom (#117)', () => {+ let app: Express;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ });++ it('rejects an invalid base URL', async () => {+ const { status } = await post(app, '/api/keys/custom', { baseUrl: 'not-a-url', model: 'm' });+ expect(status).toBe(400);+ });++ it('registers a custom endpoint, model, and fallback entry', async () => {+ const { status, body } = await post(app, '/api/keys/custom', {+ baseUrl: 'http://127.0.0.1:11434/v1/',+ model: 'qwen3:4b',+ displayName: 'Local Qwen3 4B',+ });+ expect(status).toBe(201);+ expect(body.platform).toBe('custom');+ expect(body.baseUrl).toBe('http://127.0.0.1:11434/v1'); // trailing slash trimmed+ expect(body.model).toBe('qwen3:4b');++ const db = getDb();+ const key = db.prepare("SELECT * FROM api_keys WHERE platform = 'custom'").get() as any;+ expect(key.base_url).toBe('http://127.0.0.1:11434/v1');+ const model = db.prepare("SELECT * FROM models WHERE platform = 'custom' AND model_id = 'qwen3:4b'").get() as any;+ expect(model).toBeDefined();+ const fc = db.prepare('SELECT * FROM fallback_config WHERE model_db_id = ?').get(model.id);+ expect(fc).toBeDefined();+ });++ it('reuses the single custom key when a second model is added', async () => {+ await post(app, '/api/keys/custom', { baseUrl: 'http://127.0.0.1:11434/v1', model: 'llama3:8b' });+ const db = getDb();+ const keys = db.prepare("SELECT * FROM api_keys WHERE platform = 'custom'").all();+ expect(keys.length).toBe(1); // not a second key+ const models = db.prepare("SELECT * FROM models WHERE platform = 'custom'").all();+ expect(models.length).toBe(2);+ });++ it('surfaces baseUrl in the keys listing', async () => {+ const { body } = await get(app, '/api/keys');+ const custom = body.find((k: any) => k.platform === 'custom');+ expect(custom.baseUrl).toBe('http://127.0.0.1:11434/v1');+ });++ it('routes a request to the custom model through its base URL', () => {+ // The seeded built-in models have no keys, so the only routable model is+ // the custom one we registered above.+ const route = routeRequest(1000);+ expect(route.platform).toBe('custom');+ expect((route.provider as any).baseUrl).toBe('http://127.0.0.1:11434/v1');+ expect(['qwen3:4b', 'llama3:8b']).toContain(route.modelId);+ });+});diff --git a/server/src/db/index.ts b/server/src/db/index.tsindex dd1ae01d..33a109bd 100644--- a/server/src/db/index.ts+++ b/server/src/db/index.ts@@ -143,6 +143,7 @@ function createTables(db: Database.Database) {`);ensureRequestKeyIdColumn(db);+ ensureApiKeysBaseUrlColumn(db);}function ensureRequestKeyIdColumn(db: Database.Database) {@@ -153,6 +154,15 @@ function ensureRequestKeyIdColumn(db: Database.Database) {db.prepare('CREATE INDEX IF NOT EXISTS idx_requests_key_id ON requests(key_id)').run();}+// `base_url` is the upstream endpoint for the user-configured 'custom' provider+// (#117). NULL for every built-in platform — they use their hardcoded base URL.+function ensureApiKeysBaseUrlColumn(db: Database.Database) {+ const columns = db.prepare('PRAGMA table_info(api_keys)').all() as { name: string }[];+ if (!columns.some(col => col.name === 'base_url')) {+ db.prepare('ALTER TABLE api_keys ADD COLUMN base_url TEXT').run();+ }+}+function seedModels(db: Database.Database) {const count = db.prepare('SELECT COUNT(*) as cnt FROM models').get() as { cnt: number };if (count.cnt > 0) return;diff --git a/server/src/providers/index.ts b/server/src/providers/index.tsindex 0e3177e5..a149c9d9 100644--- a/server/src/providers/index.ts+++ b/server/src/providers/index.ts@@ -148,10 +148,43 @@ register(new OpenAICompatProvider({// $0.0, please pay with fiat or send tao". The "free" tier requires a// non-zero balance, which conflicts with the project's no-card criterion.+// Placeholder so getProvider('custom')/hasProvider('custom')/getAllProviders()+// behave — but the real instance is built per-key by resolveProvider(), since+// a custom provider's base URL is user-supplied and lives on the api_keys row.+register(new OpenAICompatProvider({+ platform: 'custom',+ name: 'Custom (OpenAI-compatible)',+ baseUrl: '',+}));++// Locally-hosted inference (llama.cpp / vLLM / Ollama on CPU) can be slow, so+// custom providers get the same extended timeout as Ollama Cloud.+const CUSTOM_PROVIDER_TIMEOUT_MS = 120000;+export function getProvider(platform: Platform): BaseProvider | undefined {return providers.get(platform);}+/**+ * Resolve the provider for a route. Built-in platforms return their registered+ * singleton; the 'custom' platform builds a fresh OpenAICompatProvider bound to+ * the caller-supplied base URL (stored per api_keys row). Returns undefined for+ * a custom provider with no base URL configured.+ */+export function resolveProvider(platform: Platform, baseUrl?: string | null): BaseProvider | undefined {+ if (platform === 'custom') {+ const trimmed = baseUrl?.trim();+ if (!trimmed) return undefined;+ return new OpenAICompatProvider({+ platform: 'custom',+ name: 'Custom (OpenAI-compatible)',+ baseUrl: trimmed,+ timeoutMs: CUSTOM_PROVIDER_TIMEOUT_MS,+ });+ }+ return providers.get(platform);+}+export function getAllProviders(): BaseProvider[] {return Array.from(providers.values());}diff --git a/server/src/routes/keys.ts b/server/src/routes/keys.tsindex f7b1b9d2..caff9631 100644--- a/server/src/routes/keys.ts+++ b/server/src/routes/keys.ts@@ -12,7 +12,7 @@ export const keysRouter = Router();const PLATFORMS = ['google', 'groq', 'cerebras', 'sambanova', 'nvidia', 'mistral','openrouter', 'github', 'cohere', 'cloudflare', 'zhipu', 'ollama',- 'kilo', 'pollinations', 'llm7', 'huggingface',+ 'kilo', 'pollinations', 'llm7', 'huggingface', 'custom',] as const;const addKeySchema = z.object({@@ -46,6 +46,7 @@ keysRouter.get('/', (_req: Request, res: Response) => {platform: row.platform,label: row.label,maskedKey,+ baseUrl: row.base_url ?? null,status: row.status,enabled: row.enabled === 1,createdAt: row.created_at,@@ -83,6 +84,86 @@ keysRouter.post('/', (req: Request, res: Response) => {});});+// ── Custom OpenAI-compatible provider (#117) ──────────────────────────────+// A single user-configured endpoint (llama.cpp / LM Studio / vLLM / Ollama /+// any OpenAI-compatible base_url). The endpoint lives on one 'custom' api_keys+// row; each call registers another model that routes through it.+const customProviderSchema = z.object({+ baseUrl: z.string().url('baseUrl must be a valid URL'),+ model: z.string().min(1, 'model is required'),+ displayName: z.string().optional(),+ apiKey: z.string().optional(),+ label: z.string().optional(),+});++keysRouter.post('/custom', (req: Request, res: Response) => {+ const parsed = customProviderSchema.safeParse(req.body);+ if (!parsed.success) {+ res.status(400).json({ error: { message: parsed.error.errors.map(e => e.message).join(', ') } });+ return;+ }++ const baseUrl = parsed.data.baseUrl.trim().replace(/\/+$/, '');+ const modelId = parsed.data.model.trim();+ const displayName = (parsed.data.displayName ?? modelId).trim();+ // Local servers often need no key; keep a sentinel so there's always a bearer.+ const rawKey = parsed.data.apiKey?.trim() || 'no-key';+ const label = parsed.data.label ?? 'Custom';++ const db = getDb();+ const upsert = db.transaction(() => {+ // One shared 'custom' key holds the endpoint URL. Reuse it across models;+ // update its base_url/key when re-submitted.+ const existing = db.prepare("SELECT id FROM api_keys WHERE platform = 'custom' LIMIT 1").get() as { id: number } | undefined;+ let keyId: number;+ if (existing) {+ const { encrypted, iv, authTag } = encrypt(rawKey);+ db.prepare("UPDATE api_keys SET base_url = ?, encrypted_key = ?, iv = ?, auth_tag = ?, status = 'unknown', enabled = 1 WHERE id = ?")+ .run(baseUrl, encrypted, iv, authTag, existing.id);+ keyId = existing.id;+ } else {+ const { encrypted, iv, authTag } = encrypt(rawKey);+ const r = db.prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled, base_url)+ VALUES ('custom', ?, ?, ?, ?, 'unknown', 1, ?)+ `).run(label, encrypted, iv, authTag, baseUrl);+ keyId = Number(r.lastInsertRowid);+ }++ // Register the model (idempotent on platform+model_id). Custom models carry+ // no rate limits and sort last in the intelligence preset (size_label tier).+ db.prepare(`+ INSERT OR IGNORE INTO models+ (platform, model_id, display_name, intelligence_rank, speed_rank, size_label,+ rpm_limit, rpd_limit, tpm_limit, tpd_limit, monthly_token_budget, context_window, enabled)+ VALUES ('custom', ?, ?, 50, 50, 'Custom', NULL, NULL, NULL, NULL, '', NULL, 1)+ `).run(modelId, displayName);++ const modelRow = db.prepare("SELECT id FROM models WHERE platform = 'custom' AND model_id = ?").get(modelId) as { id: number };++ // Append to the fallback chain if not already present.+ const inChain = db.prepare('SELECT 1 FROM fallback_config WHERE model_db_id = ?').get(modelRow.id);+ if (!inChain) {+ const max = db.prepare('SELECT COALESCE(MAX(priority), 0) AS m FROM fallback_config').get() as { m: number };+ db.prepare('INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, ?, 1)').run(modelRow.id, max.m + 1);+ }++ return { keyId, modelDbId: modelRow.id };+ });++ const { keyId, modelDbId } = upsert();+ res.status(201).json({+ success: true,+ keyId,+ modelDbId,+ platform: 'custom',+ baseUrl,+ model: modelId,+ displayName,+ maskedKey: maskKey(rawKey),+ });+});+// Delete a keykeysRouter.delete('/:id', (req: Request, res: Response) => {const id = parseInt(req.params.id as string, 10);diff --git a/server/src/services/health.ts b/server/src/services/health.tsindex 5677e641..00caecca 100644--- a/server/src/services/health.ts+++ b/server/src/services/health.ts@@ -1,5 +1,5 @@import { getDb } from '../db/index.js';-import { getProvider } from '../providers/index.js';+import { resolveProvider } from '../providers/index.js';import { decrypt } from '../lib/crypto.js';import type { Platform, KeyStatus } from '@freellmapi/shared/types.js';@@ -14,7 +14,7 @@ export async function checkKeyHealth(keyId: number): Promise<KeyStatus> {const row = db.prepare('SELECT * FROM api_keys WHERE id = ?').get(keyId) as any;if (!row) return 'error';- const provider = getProvider(row.platform as Platform);+ const provider = resolveProvider(row.platform as Platform, row.base_url);if (!provider) return 'error';try {diff --git a/server/src/services/router.ts b/server/src/services/router.tsindex 26168fa1..5629f621 100644--- a/server/src/services/router.ts+++ b/server/src/services/router.ts@@ -1,5 +1,5 @@import { getDb } from '../db/index.js';-import { getProvider } from '../providers/index.js';+import { getProvider, resolveProvider } from '../providers/index.js';import { decrypt } from '../lib/crypto.js';import { canMakeRequest, canUseTokens, isOnCooldown } from './ratelimit.js';import type { BaseProvider } from '../providers/base.js';@@ -23,6 +23,7 @@ interface KeyRow {auth_tag: string;status: string;enabled: number;+ base_url: string | null;}interface FallbackRow {@@ -208,10 +209,18 @@ export function routeRequest(estimatedTokens = 1000, skipKeys?: Set<string>, precontinue;}+ // For the 'custom' platform the real provider is built from this key's+ // base_url (the registered instance is just a placeholder). A custom key+ // with no base_url can't be routed — skip it.+ const resolvedProvider = model.platform === 'custom'+ ? resolveProvider('custom', key.base_url)+ : provider;+ if (!resolvedProvider) continue;+// We found a working key for this model!roundRobinIndex.set(rrKey, idx);return {- provider,+ provider: resolvedProvider,modelId: model.model_id,modelDbId: model.id,apiKey: decryptedKey,diff --git a/shared/types.ts b/shared/types.tsindex fe663d95..9982b510 100644--- a/shared/types.ts+++ b/shared/types.ts@@ -21,7 +21,10 @@ export type Platform =| 'kilo'| 'pollinations'| 'llm7'- | 'huggingface';+ | 'huggingface'+ // User-configured OpenAI-compatible endpoint (llama.cpp, LM Studio, vLLM,+ // Ollama, any base_url). The endpoint URL lives on the api_keys row; see #117.+ | 'custom';export interface Model {id: number;