7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(models): serve GET /v1/models/:id (OpenAI models.retrieve) from the shared catalog builder (#1632)
tashdroidd1eeab29759b
diff --git a/server/src/__tests__/routes/models-retrieve.test.ts b/server/src/__tests__/routes/models-retrieve.test.tsnew file mode 100644index 00000000..09f96ea5--- /dev/null+++ b/server/src/__tests__/routes/models-retrieve.test.ts@@ -0,0 +1,130 @@+import { describe, it, expect, beforeAll, beforeEach, afterEach, vi } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';++// GET /v1/models/:id — OpenAI `models.retrieve`. Competitors all expose it+// (OpenAI itself, OpenRouter's GET /api/v1/models/{author}/{slug}, Together),+// and OpenAI-SDK-based gateways call it before routing. Served from the same+// builder as the list so the two endpoints can never disagree.+//+// OpenAI-compatible /models endpoint (used by Hermes for metadata)+// shows API models which is linked by the user++let dashToken = '';++async function request(app: Express, method: string, path: string, body?: any, headers: Record<string, string> = {}) {+ const server = app.listen(0, '127.0.0.1');+ if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));+ const addr = server.address() as any;+ const url = `http://127.0.0.1:${addr.port}${path}`;++ const res = await fetch(url, {+ method,+ headers: { ...(body ? { 'Content-Type': 'application/json' } : {}), ...(isGatedApiPath(path) && !('Authorization' in headers) ? { Authorization: `Bearer ${dashToken}` } : {}), ...headers },+ body: body ? JSON.stringify(body) : undefined,+ });++ const data = await res.text();+ server.close();++ let json: any = null;+ try { json = JSON.parse(data); } catch {}++ return { status: res.status, body: json, raw: data };+}++function authHeaders() {+ return { Authorization: `Bearer ${getUnifiedApiKey()}` };+}++describe('GET /v1/models/:id (models.retrieve)', () => {+ let app: Express;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ });++ beforeEach(async () => {+ initDb(':memory:');+ app = createApp();+ dashToken = mintDashboardToken();+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare('DELETE FROM profile_models').run();+ db.prepare('DELETE FROM fallback_config').run();+ db.prepare('DELETE FROM models').run();+ const addKey = await request(app, 'POST', '/api/keys', {+ platform: 'groq',+ key: 'gsk_retrieve_test',+ label: 'retrieve',+ });+ expect(addKey.status).toBe(201);+ const addModel = db.prepare(`+ INSERT INTO models (+ platform, model_id, display_name, intelligence_rank, speed_rank, size_label,+ rpm_limit, rpd_limit, tpm_limit, tpd_limit, monthly_token_budget,+ context_window, enabled, supports_vision, supports_tools+ )+ VALUES ('groq', 'groq/retrieve-test-model', 'Retrieve Test', 1, 1, 'Small',+ NULL, NULL, NULL, NULL, '~1M', 4096, 1, 0, 1)+ `).run();+ expect(addModel.changes).toBe(1);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ });++ it('returns the entry a client just listed, field for field', async () => {+ const listed = await request(app, 'GET', '/v1/models', undefined, authHeaders());+ expect(listed.status).toBe(200);+ // The retrieve must cover every id the list hands out, so walk the list:+ // each catalog id must be retrievable with the identical shape.+ // The unified listing publishes the canonical slug for the seeded row.+ const row = listed.body.data.find((m: any) => m.id === 'retrieve-test');+ expect(row).toBeTruthy();++ const got = await request(app, 'GET', `/v1/models/${encodeURIComponent(row.id)}`, undefined, authHeaders());+ expect(got.status).toBe(200);+ expect(got.body.object).toBe('model');+ expect(got.body.id).toBe(row.id);+ expect(got.body.owned_by).toBe(row.owned_by);+ expect(got.body.context_window).toBe(4096);+ expect(got.body.available).toBe(true);+ expect(got.body.execution_status).toBe(row.execution_status);+ });++ it('resolves the virtual router entries too', async () => {+ const auto = await request(app, 'GET', '/v1/models/auto', undefined, authHeaders());+ expect(auto.status).toBe(200);+ expect(auto.body).toMatchObject({ id: 'auto', object: 'model', owned_by: 'freellmapi' });++ const fusion = await request(app, 'GET', '/v1/models/fusion', undefined, authHeaders());+ expect(fusion.status).toBe(200);+ expect(fusion.body.id).toBe('fusion');+ });++ it('answers an unknown id with OpenAI\'s 404 shape', async () => {+ const res = await request(app, 'GET', '/v1/models/not-a-real-model', undefined, authHeaders());+ expect(res.status).toBe(404);+ expect(res.body.error).toMatchObject({+ type: 'invalid_request_error',+ code: 'model_not_found',+ param: 'id',+ });+ expect(res.body.error.message).toContain('not-a-real-model');+ });++ it('requires inference auth', async () => {+ const { status } = await request(app, 'GET', '/v1/models/auto');+ expect(status).toBe(401);+ });++ it('still answers the list endpoint (the /:id route must not shadow it)', async () => {+ const res = await request(app, 'GET', '/v1/models', undefined, authHeaders());+ expect(res.status).toBe(200);+ expect(res.body.object).toBe('list');+ });+});diff --git a/server/src/docs/openapi.ts b/server/src/docs/openapi.tsindex 5588863f..c647ee40 100644--- a/server/src/docs/openapi.ts+++ b/server/src/docs/openapi.ts@@ -624,6 +624,40 @@ export const openapiSpec = {},},},+ '/v1/models/{id}': {+ get: {+ tags: ['Models'],+ operationId: 'retrieveModel',+ summary: 'Retrieve a single model (OpenAI models.retrieve)',+ description:+ 'Returns one catalog entry by id in the OpenAI model shape, so ' ++ 'OpenAI-SDK clients (``client.models.retrieve(id)``) can resolve a ' ++ 'model before routing. Includes the virtual `auto` and `fusion` ids ' ++ 'and any catalog id with a slash (URL-encode the slash). Served from ' ++ 'the same builder as the list, so the two never disagree.',+ parameters: [+ {+ name: 'id',+ in: 'path',+ required: true,+ description: 'Model id (e.g. `auto`, or `vendor/model` — slash URL-encoded).',+ schema: { type: 'string' },+ },+ ],+ responses: {+ '200': {+ description: 'The model.',+ content: {+ 'application/json': {+ schema: { $ref: '#/components/schemas/Model' },+ },+ },+ },+ '401': { $ref: '#/components/responses/Unauthorized' },+ '404': { description: 'No model with this id (OpenAI error shape: `model_not_found`).' },+ },+ },+ },'/v1/usage': {get: {tags: ['Models'],diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 16084e1b..daf06144 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -326,17 +326,66 @@ proxyRouter.get('/models', (req: Request, res: Response) => {// "which models can this instance actually serve now". `available` is computed as// "enabled AND an enabled key can serve it"; dedup prefers an available// instance of a model id over a disabled/keyless one.- // Shared catalog listing (one source of truth for the OpenAI and Anthropic- // /v1/models endpoints — see services/model-listing.ts). `autoContextWindow`- // is the honest ceiling for the virtual "auto" model: the largest context- // window among models that can serve a request right now. Advertising null- // makes OpenAI-compatible clients (opencode, Continue) fall back to their own- // conservative default and truncate long inputs before they reach us (#282).- const { models: allListed, autoContextWindow } = buildModelListing();-const q = String(req.query.available ?? req.query.connected ?? req.query.ready ?? '').toLowerCase();const onlyAvailable = q === '1' || q === 'true' || q === 'yes';- const listed = onlyAvailable ? allListed.filter(m => m.available === 1) : allListed;+ const esValues: Record<string, NormalizedModel['executionStatus']> = {+ ready: 'ready', needskey: 'needsKey', exhausted: 'exhausted',+ };+ const es = esValues[String(req.query.execution_status ?? '').toLowerCase()];++ res.json({+ object: 'list',+ data: buildCatalogEntries({ onlyAvailable, executionStatus: es }),+ });+});++// GET /v1/models/:id — OpenAI's `models.retrieve` (single-model lookup).+// Without it, `client.models.retrieve(id)` and every gateway that validates a+// model id before routing (LiteLLM proxy config checks, OpenAI Agents SDK+// model caps, eval harnesses) answers 404 even though chat completions for+// the same id work fine. Served from the SAME builder as the list, so what a+// client lists is byte-for-byte what it retrieves; unknown ids get OpenAI's+// error shape (type: invalid_request_error). The id may itself contain+// slashes (openrouter-style ids): the wildcard capture rejoins them, and a+// URL-encoded %2F arrives decoded through Express's plain :id capture.+const MODEL_RETRIEVE_MAX_ID_LEN = 300;+proxyRouter.get(['/models/:id', '/models/*rest'], (req: Request, res: Response) => {+ if (!requireInferenceAuth(req, res)) return;+ // Express 5 hands a wildcard capture as a string array; a plain :id capture+ // as a string. Rejoin the segments so "vendor/ns/model" round-trips.+ const capture = (req.params as Record<string, unknown>).id+ ?? (req.params as Record<string, unknown>).rest;+ const id = Array.isArray(capture) ? capture.join('/') : String(capture ?? '');+ if (!id || id.length > MODEL_RETRIEVE_MAX_ID_LEN) {+ res.status(404).json({+ error: { message: `Model not found: ${id.slice(0, 80)}`, type: 'invalid_request_error', code: 'model_not_found', param: 'id' },+ });+ return;+ }+ const entry = buildCatalogEntries().find(m => m.id === id);+ if (!entry) {+ res.status(404).json({+ error: { message: `Model not found: ${id}`, type: 'invalid_request_error', code: 'model_not_found', param: 'id' },+ });+ return;+ }+ res.json({ ...entry, object: 'model' });+});++// The whole OpenAI-shaped /v1/models body, as one array of entries: the two+// router entries (`auto`, `fusion`), the Claude-family discovery aliases (#880),+// one entry per named fallback chain (`auto:<name>`), then the catalog rows.+// Shared by GET /v1/models and GET /v1/models/:id so a model a client can LIST+// is a model it can RETRIEVE with the identical shape — the retrieve endpoint+// is OpenAI's `models.retrieve`, which LiteLLM-style gateways and eval+// harnesses (lm-evaluation-harness, OpenAI SDK `.models.retrieve()`) call+// before routing; a 404 there breaks those clients even when chat works.+// The `?available=` / `?execution_status=` filters narrow only the catalog+// rows — `auto`, `fusion` and the named chains are router entries, not models+// with keys of their own.+function buildCatalogEntries(filters: { onlyAvailable?: boolean; executionStatus?: NormalizedModel['executionStatus'] } = {}): Array<Record<string, unknown>> {+ const { models: allListed, autoContextWindow } = buildModelListing();+ const listed = filters.onlyAvailable ? allListed.filter(m => m.available === 1) : allListed;// Named fallback chains (#960/#895): every user-defined profile is exposed// as an `auto:<name>` model so a client can pick a specific fallback chain@@ -398,78 +447,78 @@ proxyRouter.get('/models', (req: Request, res: Response) => {// the value itself is camelCase; an unrecognised value filters nothing, the// same way an unrecognised `?available=` does. Like `?available=`, this// narrows only the catalog rows: `auto`, `fusion` and the named chains are- // router entries, not models with keys of their own.- const esValues: Record<string, NormalizedModel['executionStatus']> = {- ready: 'ready', needskey: 'needsKey', exhausted: 'exhausted',- };- const es = esValues[String(req.query.execution_status ?? '').toLowerCase()];- const esFiltered = es ? listed.filter(m => m.executionStatus === es) : listed;+ // Machine-readable execution filter: `?execution_status=ready` narrows to+ // models a request can actually serve RIGHT NOW (a key that is not scoped+ // away, cooling down or out of window). `ready` implies available;+ // `needsKey`/`exhausted` select the rest. Matched case-insensitively because+ // the value itself is camelCase; an unrecognised value filters nothing, the+ // same way an unrecognised `?available=` does.+ const esFiltered = filters.executionStatus+ ? listed.filter(m => m.executionStatus === filters.executionStatus)+ : listed;- res.json({- object: 'list',- data: [- {- id: AUTO_MODEL_ID,- object: 'model',- created: 0,- owned_by: 'freellmapi',- name: 'Auto (router picks the best available model)',- context_window: autoContextWindow,- // `context_length` is OpenRouter's field name and the one most- // OpenAI-compatible clients read; emit both so whichever a client- // looks for is populated. Additive — clients ignore unknown fields.- context_length: autoContextWindow,- available: true,- unavailable_reason: null,- },- {- id: FUSION_MODEL_ID,- object: 'model',- created: 0,- owned_by: 'freellmapi',- name: 'Fusion (panel of models answer in parallel, a judge synthesizes one answer)',- context_window: autoContextWindow,- context_length: autoContextWindow,- // Available whenever auto is — fusion needs at least one routable model.- available: autoContextWindow != null,- unavailable_reason: autoContextWindow != null ? null : 'no_models',- },- ...claudeFamilyEntries,- ...profileRows.map(p => ({- id: `auto:${p.name.toLowerCase()}`,- object: 'model',- created: 0,- owned_by: 'freellmapi',- name: `Auto: ${p.name} (named fallback chain)`,- context_window: p.max_ctx,- context_length: p.max_ctx,- available: p.usable === 1,- unavailable_reason: p.usable === 1 ? null : 'no_models',- })),- ...esFiltered.map(m => ({- id: m.id,- object: 'model',- created: 0,- owned_by: m.ownedBy,- name: m.name,- context_window: m.contextWindow,- context_length: m.contextWindow,- // Non-standard but additive: OpenAI clients ignore unknown fields.- available: m.available === 1,- unavailable_reason: m.available === 1 ? null : (m.enabled === 1 ? 'no_key' : 'disabled'),- // Machine-readable dynamic status: 'ready' | 'needsKey' | 'exhausted'- // (see services/model-listing.ts). Agents can filter with- // ?execution_status=ready and route around exhausted models.- execution_status: m.executionStatus,- // OpenRouter's field name; agents use it to pick knobs per model. For- // a unify group this is the intersection over member platforms — a- // param is only advertised when every platform the router might pick- // honors it.- supported_parameters: supportedParametersForPlatforms(m.platforms, { tools: m.supportsTools }),- })),- ],- });-});+ return [+ {+ id: AUTO_MODEL_ID,+ object: 'model',+ created: 0,+ owned_by: 'freellmapi',+ name: 'Auto (router picks the best available model)',+ context_window: autoContextWindow,+ // `context_length` is OpenRouter's field name and the one most+ // OpenAI-compatible clients read; emit both so whichever a client+ // looks for is populated. Additive — clients ignore unknown fields.+ context_length: autoContextWindow,+ available: true,+ unavailable_reason: null,+ },+ {+ id: FUSION_MODEL_ID,+ object: 'model',+ created: 0,+ owned_by: 'freellmapi',+ name: 'Fusion (panel of models answer in parallel, a judge synthesizes one answer)',+ context_window: autoContextWindow,+ context_length: autoContextWindow,+ // Available whenever auto is — fusion needs at least one routable model.+ available: autoContextWindow != null,+ unavailable_reason: autoContextWindow != null ? null : 'no_models',+ },+ ...claudeFamilyEntries,+ ...profileRows.map(p => ({+ id: `auto:${p.name.toLowerCase()}`,+ object: 'model',+ created: 0,+ owned_by: 'freellmapi',+ name: `Auto: ${p.name} (named fallback chain)`,+ context_window: p.max_ctx,+ context_length: p.max_ctx,+ available: p.usable === 1,+ unavailable_reason: p.usable === 1 ? null : 'no_models',+ })),+ ...esFiltered.map(m => ({+ id: m.id,+ object: 'model',+ created: 0,+ owned_by: m.ownedBy,+ name: m.name,+ context_window: m.contextWindow,+ context_length: m.contextWindow,+ // Non-standard but additive: OpenAI clients ignore unknown fields.+ available: m.available === 1,+ unavailable_reason: m.available === 1 ? null : (m.enabled === 1 ? 'no_key' : 'disabled'),+ // Machine-readable dynamic status: 'ready' | 'needsKey' | 'exhausted'+ // (see services/model-listing.ts). Agents can filter with+ // ?execution_status=ready and route around exhausted models.+ execution_status: m.executionStatus,+ // OpenRouter's field name; agents use it to pick knobs per model. For+ // a unify group this is the intersection over member platforms — a+ // param is only advertised when every platform the router might pick+ // honors it.+ supported_parameters: supportedParametersForPlatforms(m.platforms, { tools: m.supportsTools }),+ })),+ ];+}// GET /v1/usage — self-service consumption for inference clients (#1568).