7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
List named fallback chains as auto:<name> models (#986)
suantea (asuan_suan)cc1e985e04eb
diff --git a/server/src/__tests__/routes/models-management.test.ts b/server/src/__tests__/routes/models-management.test.tsindex 7ba0521a..97dab4a7 100644--- a/server/src/__tests__/routes/models-management.test.ts+++ b/server/src/__tests__/routes/models-management.test.ts@@ -1,7 +1,8 @@-import { describe, it, expect, beforeAll } from 'vitest';+import { describe, it, expect, afterEach, beforeAll, vi } from 'vitest';import type { Express } from 'express';import { createApp } from '../../app.js';-import { initDb, getDb, getSetting, setSetting } from '../../db/index.js';+import { initDb, getDb, getSetting, setSetting, getUnifiedApiKey } from '../../db/index.js';+import { encrypt } from '../../lib/crypto.js';import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';import { applyAllModelOverrides } from '../../services/model-state.js';@@ -37,11 +38,17 @@ async function request(app: Express, method: string, path: string, body?: any) {const addr = server.address() as any;const url = `http://127.0.0.1:${addr.port}${path}`;+ // /v1/* inference endpoints authenticate with the unified key (or a client+ // profile key), NOT the dashboard token; /api/* admin endpoints take the+ // dashboard token. isGatedApiPath covers the admin side, and /v1 paths are+ // simply excluded from it — so add the unified key explicitly for those.+ const isV1 = path.startsWith('/v1/');const res = await fetch(url, {method,headers: {...(body ? { 'Content-Type': 'application/json' } : {}),...(isGatedApiPath(path) ? { Authorization: `Bearer ${dashToken}` } : {}),+ ...(isV1 ? { Authorization: `Bearer ${getUnifiedApiKey()}` } : {}),},body: body ? JSON.stringify(body) : undefined,});@@ -61,6 +68,10 @@ describe('Model management API', () => {dashToken = mintDashboardToken();});+ afterEach(() => {+ vi.restoreAllMocks();+ });+it('updates catalog model metadata and records durable overrides', async () => {const target = getDb().prepare(`SELECT id FROM models@@ -319,4 +330,144 @@ describe('Model management API', () => {WHERE kind = 'chat' AND platform = ? AND model_id = ?`).get(target.platform, target.model_id)).toBeDefined();});++ // The named-chain listing and routing tests below need a live groq key: a+ // chain is only "available" when one of its members can serve a request now.+ function seedGroqKey(): void {+ const secret = encrypt('gsk_named_chain_test');+ getDb().prepare(`+ INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled)+ VALUES ('groq', 'named-chain', ?, ?, ?, 'healthy', 1)+ `).run(secret.encrypted, secret.iv, secret.authTag);+ }++ // A model that exists ONLY inside the custom profile, so "the served model is+ // this one" proves the request went through the named chain rather than the+ // active/default chain.+ function seedChainOnlyModel(modelId: string, contextWindow: number): number {+ const inserted = getDb().prepare(`+ INSERT INTO models (+ platform, model_id, display_name, intelligence_rank, speed_rank, size_label,+ context_window, enabled, supports_tools+ ) VALUES ('groq', ?, ?, 100, 100, 'Small', ?, 1, 1)+ `).run(modelId, `Named chain ${modelId}`, contextWindow);+ return Number(inserted.lastInsertRowid);+ }++ function newProfile(name: string, sortOrder: number): number {+ const inserted = getDb().prepare(`+ INSERT INTO profiles (name, emoji, color, type, sort_order) VALUES (?, '', '#6366f1', 'custom', ?)+ `).run(name, sortOrder);+ return Number(inserted.lastInsertRowid);+ }++ function addToProfile(profileId: number, modelDbId: number, enabled = 1): void {+ getDb().prepare('INSERT INTO profile_models (profile_id, model_db_id, priority, enabled) VALUES (?, ?, 1, ?)')+ .run(profileId, modelDbId, enabled);+ }++ it('exposes custom profiles as auto:<name> chains in /v1/models (#960/#895)', async () => {+ seedGroqKey();++ // A chain whose single member is enabled and backed by an enabled key.+ const servableId = seedChainOnlyModel('named-chain-servable', 111000);+ addToProfile(newProfile('my-group', 5), servableId);++ // A chain whose single member is switched off in the chain: nothing in it+ // can serve a request, so the entry must say so rather than claim ready.+ const prunedId = seedChainOnlyModel('named-chain-pruned', 222000);+ addToProfile(newProfile('off-group', 6), prunedId, 0);++ const listed = await request(app, 'GET', '/v1/models');+ expect(listed.status).toBe(200);++ const chain = listed.body.data.find((m: any) => m.id === 'auto:my-group');+ expect(chain).toBeDefined();+ expect(chain.name).toContain('my-group');+ expect(chain.object).toBe('model');+ expect(chain.owned_by).toBe('freellmapi');+ // The real point of the entry: is this chain usable right now, and how big+ // a context can it take? Both are asserted, not merely present.+ expect(chain.available).toBe(true);+ expect(chain.unavailable_reason).toBeNull();+ expect(chain.context_window).toBe(111000);+ expect(chain.context_length).toBe(111000);++ const off = listed.body.data.find((m: any) => m.id === 'auto:off-group');+ expect(off).toBeDefined();+ expect(off.available).toBe(false);+ expect(off.unavailable_reason).toBe('no_models');++ // The virtual auto: id must not collide with real catalog ids.+ expect(listed.body.data.filter((m: any) => m.id === 'auto:my-group')).toHaveLength(1);+ });++ it('lists the Claude family ids on the plain OpenAI /v1/models too (#880)', async () => {+ // routes/anthropic.ts only answers GET /v1/models when the caller sends an+ // `anthropic-version` header. Claude Desktop's gateway picker does not, so+ // the OpenAI-shaped listing has to carry the same Claude-shaped ids or the+ // picker reports "found 0 models".+ const listed = await request(app, 'GET', '/v1/models');+ expect(listed.status).toBe(200);++ for (const id of ['claude-opus-4-5', 'claude-sonnet-4-5', 'claude-haiku-4-5']) {+ const entry = listed.body.data.find((m: any) => m.id === id);+ expect(entry, `${id} must be listed`).toBeDefined();+ expect(entry.object).toBe('model');+ expect(entry.owned_by).toBe('freellmapi');+ expect(entry.available).toBe(true);+ // The display name says where the request really goes — none of these is+ // hosted Claude.+ expect(entry.name).toMatch(/slot/);+ }++ // Still one row per id.+ const ids = listed.body.data.map((m: any) => m.id);+ expect(new Set(ids).size).toBe(ids.length);+ });++ it('routes a request to model auto:<name> through that named chain (#960/#895)', async () => {+ const chainOnlyId = seedChainOnlyModel('named-chain-routed', 128000);+ addToProfile(newProfile('route-group', 7), chainOnlyId);++ const origFetch = global.fetch;+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const u = typeof url === 'string' ? url : url.toString();+ if (u.includes('api.groq.com')) {+ return {+ ok: true,+ status: 200,+ headers: new Headers(),+ json: () => Promise.resolve({+ id: 'chatcmpl-chain', object: 'chat.completion', created: 1, model: 'named-chain-routed',+ choices: [{ index: 0, message: { role: 'assistant', content: 'routed via route-group' }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 2, completion_tokens: 1, total_tokens: 3 },+ }),+ } as any;+ }+ return origFetch(url, init);+ });++ const routed = await request(app, 'POST', '/v1/chat/completions', {+ model: 'auto:route-group',+ messages: [{ role: 'user', content: 'hi' }],+ });+ expect(routed.status).toBe(200);+ expect(routed.body.choices[0].message.content).toBe('routed via route-group');++ // The served model is the one that exists ONLY in this profile, so the+ // request really went down the named chain and not the active one.+ const served = getDb().prepare('SELECT model_id FROM requests ORDER BY id DESC LIMIT 1')+ .get() as { model_id: string } | undefined;+ expect(served?.model_id).toBe('named-chain-routed');++ // An unknown chain name must fail loudly, not silently fall back to the+ // active chain.+ const missing = await request(app, 'POST', '/v1/chat/completions', {+ model: 'auto:no-such-group',+ messages: [{ role: 'user', content: 'hi' }],+ });+ expect(missing.status).toBe(400);+ expect(JSON.stringify(missing.body)).toContain('no-such-group');+ });});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 7ee36a53..29af8596 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -31,6 +31,7 @@ import type { Platform } from '@freellmapi/shared/types.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdForDispatch } from '../services/model-groups.js';import { buildModelListing } from '../services/model-listing.js';+import { claudeFamilyDiscoveryEntries } from '../services/anthropic-map.js';import { compressRequest, formatCompressionHeader } from '../services/compression/pipeline.js';export const proxyRouter = Router();@@ -310,6 +311,59 @@ proxyRouter.get('/models', (req: Request, res: Response) => {const onlyAvailable = q === '1' || q === 'true' || q === 'yes';const listed = onlyAvailable ? allListed.filter(m => m.available === 1) : allListed;+ // Named fallback chains (#960/#895): every user-defined profile is exposed+ // as an `auto:<name>` model so a client can pick a specific fallback chain+ // per request (auto:my-group) instead of only the active one. Available iff+ // at least one model in that profile's chain can serve a request right now.+ const profileRows = getDb().prepare(`+ SELECT p.id, p.name,+ EXISTS (+ SELECT 1+ FROM profile_models pm+ JOIN models m ON m.id = pm.model_db_id AND m.enabled = 1+ WHERE pm.profile_id = p.id AND pm.enabled = 1+ AND EXISTS (+ SELECT 1 FROM api_keys k+ WHERE k.platform = m.platform AND k.enabled = 1+ AND (m.key_id IS NULL OR k.id = m.key_id)+ )+ ) AS usable,+ (SELECT MAX(m2.context_window)+ FROM profile_models pm2+ JOIN models m2 ON m2.id = pm2.model_db_id AND m2.enabled = 1+ WHERE pm2.profile_id = p.id AND pm2.enabled = 1) AS max_ctx+ FROM profiles p+ WHERE p.type = 'custom'+ ORDER BY p.sort_order, p.id+ `).all() as { id: number; name: string; usable: number; max_ctx: number | null }[];++ // Claude-family discovery entries (#880). The Anthropic-shaped GET /v1/models+ // in routes/anthropic.ts already lists one id per Claude family so clients+ // that only accept Claude-looking ids can discover anything at all — but that+ // handler only answers when the caller sends an `anthropic-version` header.+ // Claude Desktop's gateway picker fetches this path WITHOUT that header, so+ // it fell through to the OpenAI-shaped listing below and still saw zero+ // Claude-shaped ids. Emit the same entries here, from the same builder, so+ // both shapes agree on what the gateway will serve. Listed only when+ // something can actually serve them, and never when the id would collide+ // with a real catalog row.+ const listedIds = new Set(listed.map(m => m.id));+ const claudeFamilyEntries = allListed.some(m => m.available === 1)+ ? claudeFamilyDiscoveryEntries()+ .filter(a => !listedIds.has(a.id))+ .map(a => ({+ id: a.id,+ object: 'model' as const,+ created: 0,+ owned_by: 'freellmapi',+ name: a.displayName,+ context_window: autoContextWindow,+ context_length: autoContextWindow,+ available: true,+ unavailable_reason: null,+ }))+ : [];+res.json({object: 'list',data: [@@ -339,6 +393,18 @@ proxyRouter.get('/models', (req: Request, res: Response) => {available: autoContextWindow != null,unavailable_reason: autoContextWindow != null ? null : 'no_models',},+ ...claudeFamilyEntries,+ ...profileRows.map(p => ({+ id: `auto:${p.name.toLowerCase()}`,+ object: 'model',+ created: 0,+ owned_by: 'freellmapi',+ name: `Auto: ${p.name} (named fallback chain)`,+ context_window: p.max_ctx,+ context_length: p.max_ctx,+ available: p.usable === 1,+ unavailable_reason: p.usable === 1 ? null : 'no_models',+ })),...listed.map(m => ({id: m.id,object: 'model',