Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(server): MCP server at /mcp - gateway introspection for coding agents (#517)

Tashfeen2094ea0eda83
diff --git a/README.md b/README.md
index d139b521..b381bc22 100644
--- a/README.md
+++ b/README.md
@@ -123,6 +123,7 @@ The full, always-current list lives at **[freellmapi.co/models](https://freellma
- **Admin dashboard** — React + Vite UI to manage keys, reorder the fallback chain, inspect analytics, and run prompts in a playground. Dark mode included.
- **Analytics** — Per-request logging with latency (p50 / p95 and time-to-first-token for streams), token counts, success rate, estimated cost savings, and per-provider / per-model / per-key breakdowns over 24h to 90d windows.
- **Interactive API docs** — `GET /v1/docs` serves a dependency-free OpenAPI viewer covering every proxy endpoint; the spec itself lives at `GET /v1/openapi.json`.
+- **MCP server** — `POST /mcp` speaks the Model Context Protocol (Streamable HTTP), so MCP-capable agents can ask the router which free models are usable right now (with per-model `supported_parameters`), check provider/key health and cooldowns, read usage and cache stats, and switch the routing strategy mid-session. See [Coding agents](#coding-agents).
- **Encrypted DB backups** — optional periodic encrypted snapshots of the SQLite database to a local path or HTTP target, restored automatically on a fresh boot (`FREEAPI_DB_BACKUP_*` env vars).
- **Context handoff on model switch** — Optional. When a session falls over to a different model, injects one compact system message so the new model knows it is continuing an existing task. Disabled by default; enable with `FREELLMAPI_CONTEXT_HANDOFF=on_model_switch`. See [Context Handoff](#context-handoff).
- **Runs anywhere Node 20+ runs** — Windows, macOS, Linux servers, or a small ARM SBC (Raspberry Pi included). ~40 MB RSS at idle behind PM2 / systemd / whatever supervisor you prefer.
@@ -417,6 +418,18 @@ a model (`auto` lets the router pick).
| **opencode** | OpenAI-compatible provider with the same base URL and key |
| **Cursor** | paste the unified key under a custom OpenAI base URL — but note Cursor verifies and calls the API **from its servers**, so your router must be reachable from the internet (a tunnel or a host with a public address), not just `localhost` |
+On top of inference, the router is an **MCP server**: agents can introspect it mid-session
+(usable models and the params each one honors, provider health, usage and cache stats,
+routing strategy). For Claude Code:
+
+```bash
+claude mcp add --transport http freellmapi http://localhost:3001/mcp \
+ --header "Authorization: Bearer freellmapi-your-unified-key"
+```
+
+Any MCP client that speaks Streamable HTTP works the same way: point it at `/mcp` with the
+unified key as a Bearer token.
+
FreeLLMAPI is local-first and single-user by design. Your provider keys stay in
your SQLite database, encrypted at rest, and requests go from your machine to the
upstream providers you enabled.
diff --git a/server/src/__tests__/routes/mcp.test.ts b/server/src/__tests__/routes/mcp.test.ts
new file mode 100644
index 00000000..7d427b0c
--- /dev/null
+++ b/server/src/__tests__/routes/mcp.test.ts
@@ -0,0 +1,139 @@
+import { describe, it, expect, beforeAll } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getUnifiedApiKey } from '../../db/index.js';
+
+let app: Express;
+
+async function rpc(message: unknown, opts: { auth?: boolean } = {}) {
+ const server = app.listen(0);
+ const addr = server.address() as { port: number };
+ const res = await fetch(`http://127.0.0.1:${addr.port}/mcp`, {
+ method: 'POST',
+ headers: {
+ 'Content-Type': 'application/json',
+ ...(opts.auth === false ? {} : { Authorization: `Bearer ${getUnifiedApiKey()}` }),
+ },
+ body: JSON.stringify(message),
+ });
+ const text = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(text); } catch { /* 202 empty */ }
+ return { status: res.status, body: json };
+}
+
+function toolResultJson(body: any): any {
+ expect(body.result?.content?.[0]?.type).toBe('text');
+ return JSON.parse(body.result.content[0].text);
+}
+
+describe('MCP server (/mcp, stateless Streamable HTTP)', () => {
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ initDb(':memory:');
+ app = createApp();
+ });
+
+ it('rejects requests without the unified key', async () => {
+ const { status, body } = await rpc({ jsonrpc: '2.0', id: 1, method: 'tools/list' }, { auth: false });
+ expect(status).toBe(401);
+ expect(body.error.code).toBe(-32001);
+ });
+
+ it('initialize negotiates a supported protocol version and advertises tools', async () => {
+ const { status, body } = await rpc({
+ jsonrpc: '2.0', id: 1, method: 'initialize',
+ params: { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 't', version: '0' } },
+ });
+ expect(status).toBe(200);
+ expect(body.result.protocolVersion).toBe('2025-03-26');
+ expect(body.result.capabilities.tools).toBeDefined();
+ expect(body.result.serverInfo.name).toBe('freellmapi');
+ });
+
+ it('falls back to the default protocol version for an unknown one', async () => {
+ const { body } = await rpc({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '1999-01-01' } });
+ expect(body.result.protocolVersion).toBe('2025-06-18');
+ });
+
+ it('accepts notifications with a 202 and no body', async () => {
+ const { status, body } = await rpc({ jsonrpc: '2.0', method: 'notifications/initialized' });
+ expect(status).toBe(202);
+ expect(body).toBeNull();
+ });
+
+ it('lists the six gateway tools with schemas', async () => {
+ const { body } = await rpc({ jsonrpc: '2.0', id: 2, method: 'tools/list' });
+ const names = body.result.tools.map((t: any) => t.name).sort();
+ expect(names).toEqual(['cache_stats', 'list_models', 'provider_health', 'routing_info', 'set_routing_strategy', 'usage_summary']);
+ for (const tool of body.result.tools) {
+ expect(tool.description.length).toBeGreaterThan(20);
+ expect(tool.inputSchema.type).toBe('object');
+ }
+ });
+
+ it('list_models returns catalog entries with supported_parameters', async () => {
+ const { body } = await rpc({
+ jsonrpc: '2.0', id: 3, method: 'tools/call',
+ params: { name: 'list_models', arguments: { available_only: false } },
+ });
+ const data = toolResultJson(body);
+ expect(data.count).toBeGreaterThan(0);
+ expect(data.auto.id).toBe('auto');
+ const model = data.models[0];
+ expect(model.id).toBeTruthy();
+ expect(Array.isArray(model.supported_parameters)).toBe(true);
+ expect(model.supported_parameters).toContain('temperature');
+ });
+
+ it('set_routing_strategy switches and routing_info reflects it', async () => {
+ const set = await rpc({
+ jsonrpc: '2.0', id: 4, method: 'tools/call',
+ params: { name: 'set_routing_strategy', arguments: { strategy: 'fastest' } },
+ });
+ expect(toolResultJson(set.body)).toEqual({ strategy: 'fastest' });
+
+ const info = await rpc({ jsonrpc: '2.0', id: 5, method: 'tools/call', params: { name: 'routing_info' } });
+ expect(toolResultJson(info.body).strategy).toBe('fastest');
+ });
+
+ it('an invalid strategy is a tool-level error, not a protocol error', async () => {
+ const { body } = await rpc({
+ jsonrpc: '2.0', id: 6, method: 'tools/call',
+ params: { name: 'set_routing_strategy', arguments: { strategy: 'chaotic' } },
+ });
+ expect(body.result.isError).toBe(true);
+ expect(body.result.content[0].text).toContain('strategy must be one of');
+ });
+
+ it('usage_summary and provider_health and cache_stats answer on an empty install', async () => {
+ for (const name of ['usage_summary', 'provider_health', 'cache_stats']) {
+ const { body } = await rpc({ jsonrpc: '2.0', id: 7, method: 'tools/call', params: { name } });
+ expect(body.result.content[0].type).toBe('text');
+ expect(body.result.isError).toBeUndefined();
+ }
+ });
+
+ it('unknown tool and unknown method return JSON-RPC errors', async () => {
+ const tool = await rpc({ jsonrpc: '2.0', id: 8, method: 'tools/call', params: { name: 'rm_rf' } });
+ expect(tool.body.error.code).toBe(-32602);
+ const method = await rpc({ jsonrpc: '2.0', id: 9, method: 'resources/list' });
+ expect(method.body.error.code).toBe(-32601);
+ });
+
+ it('rejects batches and non-request bodies', async () => {
+ const batch = await rpc([{ jsonrpc: '2.0', id: 1, method: 'ping' }]);
+ expect(batch.status).toBe(400);
+ const junk = await rpc({ hello: 'world' });
+ expect(junk.status).toBe(400);
+ });
+
+ it('GET /mcp is 405 (stateless: no server-initiated stream)', async () => {
+ const server = app.listen(0);
+ const addr = server.address() as { port: number };
+ const res = await fetch(`http://127.0.0.1:${addr.port}/mcp`);
+ server.close();
+ expect(res.status).toBe(405);
+ });
+});
diff --git a/server/src/app.ts b/server/src/app.ts
index 815d955d..3956007b 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -19,6 +19,7 @@ import { premiumRouter } from './routes/premium.js';
import { cacheRouter } from './routes/cache.js';
import { authRouter } from './routes/auth.js';
import { docsRouter } from './routes/docs.js';
+import { mcpRouter } from './routes/mcp.js';
import { requireAuth } from './middleware/requireAuth.js';
import { createProxyRateLimiter } from './middleware/rateLimit.js';
import { errorHandler } from './middleware/errorHandler.js';
@@ -101,6 +102,11 @@ export function createApp(config?: Config) {
// OpenAI Responses API shim (Codex CLI requires wire_api="responses"; see #96)
app.use('/v1', responsesRouter);
+ // MCP server (Model Context Protocol over stateless Streamable HTTP):
+ // gateway introspection tools for MCP-speaking agents. Unified-key auth,
+ // like /v1 — NOT behind the dashboard session gate.
+ app.use('/mcp', mcpRouter);
+
// Health check
app.get('/api/ping', (_req, res) => {
res.json({ status: 'ok', timestamp: new Date().toISOString() });
diff --git a/server/src/routes/mcp.ts b/server/src/routes/mcp.ts
new file mode 100644
index 00000000..41592763
--- /dev/null
+++ b/server/src/routes/mcp.ts
@@ -0,0 +1,308 @@
+import { Router } from 'express';
+import type { Request, Response } from 'express';
+import { getDb, getUnifiedApiKey } from '../db/index.js';
+import { extractApiToken, timingSafeStringEqual } from './proxy.js';
+import { buildModelListing } from '../services/model-listing.js';
+import { supportedParametersForPlatforms } from '../lib/sampling-params.js';
+import { getRoutingScores, getRoutingStrategy, setRoutingStrategy } from '../services/router.js';
+import type { RoutingStrategy } from '../services/scoring.js';
+import { getCacheStats } from '../services/cache.js';
+
+// ─────────────────────────────────────────────────────────────────────────
+// MCP server for the gateway (POST /mcp) — Model Context Protocol over
+// Streamable HTTP, stateless mode.
+//
+// Lets MCP-speaking agents (Claude Code, Cursor, Cline…) ask the router
+// questions mid-session: which free models are usable right now and with
+// which parameters, how healthy the provider pool is, what the routing
+// strategy is, and how much quota the cache has saved — plus one control
+// knob (switching the routing strategy). Inference itself stays on the
+// OpenAI/Anthropic surfaces; these tools are the gateway's introspection.
+//
+// Hand-rolled JSON-RPC instead of the MCP SDK for the same reason the
+// OpenAPI viewer is dependency-free (#482): the desktop bundle and the
+// Node-20 CI matrix punish heavy/new dependencies, and a tools-only
+// stateless MCP server is ~five methods of plain JSON-RPC. No sessions, no
+// server-initiated streams (GET → 405), single JSON responses.
+//
+// Auth mirrors /v1: the unified API key as a Bearer token (or x-api-key).
+// ─────────────────────────────────────────────────────────────────────────
+
+export const mcpRouter = Router();
+
+const SUPPORTED_PROTOCOL_VERSIONS = new Set(['2025-06-18', '2025-03-26', '2024-11-05']);
+const DEFAULT_PROTOCOL_VERSION = '2025-06-18';
+
+interface JsonRpcRequest {
+ jsonrpc?: string;
+ id?: number | string | null;
+ method?: string;
+ params?: Record<string, unknown>;
+}
+
+function rpcResult(id: number | string | null, result: unknown) {
+ return { jsonrpc: '2.0', id, result };
+}
+
+function rpcError(id: number | string | null, code: number, message: string) {
+ return { jsonrpc: '2.0', id, error: { code, message } };
+}
+
+// One text block carrying pretty-printed JSON — the standard shape for
+// machine-readable MCP tool output.
+function toolJson(data: unknown) {
+ return { content: [{ type: 'text', text: JSON.stringify(data, null, 2) }] };
+}
+
+function toolError(message: string) {
+ return { content: [{ type: 'text', text: message }], isError: true };
+}
+
+// ── Tool implementations ─────────────────────────────────────────────────
+
+function listModels(args: Record<string, unknown>): unknown {
+ const availableOnly = args.available_only !== false; // default true: agents want what they can use
+ const { models, autoContextWindow } = buildModelListing();
+ const rows = (availableOnly ? models.filter(m => m.available === 1) : models).map(m => ({
+ id: m.id,
+ name: m.name,
+ context_window: m.contextWindow,
+ available: m.available === 1,
+ platforms: m.platforms,
+ supports_tools: m.supportsTools,
+ supported_parameters: supportedParametersForPlatforms(m.platforms, { tools: m.supportsTools }),
+ }));
+ return {
+ auto: { id: 'auto', description: 'router picks the best available model', context_window: autoContextWindow },
+ count: rows.length,
+ models: rows,
+ };
+}
+
+function providerHealth(): unknown {
+ const db = getDb();
+ const now = Date.now();
+ const keys = db.prepare(`
+ SELECT platform, status, COUNT(*) AS n
+ FROM api_keys WHERE enabled = 1
+ GROUP BY platform, status
+ `).all() as Array<{ platform: string; status: string; n: number }>;
+ const cooldowns = db.prepare(`
+ SELECT platform, COUNT(*) AS n
+ FROM rate_limit_cooldowns WHERE expires_at_ms > ?
+ GROUP BY platform
+ `).all(now) as Array<{ platform: string; n: number }>;
+ const availableModels = db.prepare(`
+ SELECT m.platform, COUNT(*) AS n
+ FROM models m
+ WHERE m.enabled = 1 AND EXISTS (
+ SELECT 1 FROM api_keys k
+ WHERE k.platform = m.platform AND k.enabled = 1
+ AND (m.key_id IS NULL OR k.id = m.key_id)
+ )
+ GROUP BY m.platform
+ `).all() as Array<{ platform: string; n: number }>;
+
+ const byPlatform = new Map<string, { keys: Record<string, number>; active_cooldowns: number; available_models: number }>();
+ const entry = (p: string) => {
+ let e = byPlatform.get(p);
+ if (!e) { e = { keys: {}, active_cooldowns: 0, available_models: 0 }; byPlatform.set(p, e); }
+ return e;
+ };
+ for (const k of keys) entry(k.platform).keys[k.status] = k.n;
+ for (const c of cooldowns) entry(c.platform).active_cooldowns = c.n;
+ for (const m of availableModels) entry(m.platform).available_models = m.n;
+ return Object.fromEntries([...byPlatform.entries()].sort(([a], [b]) => a.localeCompare(b)));
+}
+
+const USAGE_RANGES: Record<string, number> = { '24h': 24, '7d': 24 * 7, '30d': 24 * 30 };
+
+function usageSummary(args: Record<string, unknown>): unknown {
+ const range = typeof args.range === 'string' && USAGE_RANGES[args.range] ? args.range : '24h';
+ const db = getDb();
+ const since = new Date(Date.now() - USAGE_RANGES[range] * 3600_000).toISOString();
+ const totals = db.prepare(`
+ SELECT COALESCE(SUM(total_requests), 0) AS requests,
+ COALESCE(SUM(success_count), 0) AS successes,
+ COALESCE(SUM(input_tokens), 0) AS input_tokens,
+ COALESCE(SUM(output_tokens), 0) AS output_tokens
+ FROM request_hourly WHERE hour >= ?
+ `).get(since.slice(0, 13) + ':00:00') as { requests: number; successes: number; input_tokens: number; output_tokens: number };
+ const topModels = db.prepare(`
+ SELECT platform, model_id, COUNT(*) AS requests,
+ SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successes
+ FROM requests WHERE created_at >= ?
+ GROUP BY platform, model_id ORDER BY requests DESC LIMIT 5
+ `).all(since) as Array<{ platform: string; model_id: string; requests: number; successes: number }>;
+ return {
+ range,
+ requests: totals.requests,
+ success_rate: totals.requests > 0 ? Math.round((totals.successes / totals.requests) * 1000) / 10 : null,
+ input_tokens: totals.input_tokens,
+ output_tokens: totals.output_tokens,
+ top_models: topModels,
+ };
+}
+
+function routingInfo(): unknown {
+ const scores = getRoutingScores();
+ return {
+ strategy: scores.strategy,
+ top_models: scores.scores
+ .filter(s => s.enabled)
+ .slice(0, 10)
+ .map(s => ({ model: s.modelId, platform: s.platform, score: Math.round(s.score * 1000) / 1000 })),
+ };
+}
+
+const ROUTING_STRATEGIES = ['priority', 'balanced', 'smartest', 'fastest', 'reliable', 'custom'] as const;
+
+function setStrategy(args: Record<string, unknown>): unknown {
+ const strategy = args.strategy;
+ if (typeof strategy !== 'string' || !(ROUTING_STRATEGIES as readonly string[]).includes(strategy)) {
+ throw new Error(`strategy must be one of: ${ROUTING_STRATEGIES.join(', ')}`);
+ }
+ setRoutingStrategy(strategy as RoutingStrategy);
+ return { strategy: getRoutingStrategy() };
+}
+
+// ── Tool registry ────────────────────────────────────────────────────────
+
+interface McpTool {
+ description: string;
+ inputSchema: Record<string, unknown>;
+ run: (args: Record<string, unknown>) => unknown;
+}
+
+const TOOLS: Record<string, McpTool> = {
+ list_models: {
+ description: 'List the models this FreeLLMAPI router can serve, with context windows, tool support, and the parameters each model honors (supported_parameters). Defaults to only models that are usable right now.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ available_only: { type: 'boolean', description: 'false to include models that are configured but not currently usable (no key, disabled)', default: true },
+ },
+ },
+ run: listModels,
+ },
+ provider_health: {
+ description: 'Per-provider key statuses (healthy/rate_limited/invalid/error/unknown), active cooldowns, and how many models each provider can serve right now.',
+ inputSchema: { type: 'object', properties: {} },
+ run: () => providerHealth(),
+ },
+ usage_summary: {
+ description: 'Request/token totals, success rate, and the top models by traffic for a recent window.',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ range: { type: 'string', enum: ['24h', '7d', '30d'], default: '24h' },
+ },
+ },
+ run: usageSummary,
+ },
+ routing_info: {
+ description: 'The active routing strategy and the current top-scored models in the fallback chain.',
+ inputSchema: { type: 'object', properties: {} },
+ run: () => routingInfo(),
+ },
+ set_routing_strategy: {
+ description: 'Switch the routing strategy (priority = manual chain order; balanced / smartest / fastest / reliable are scored presets; custom uses the saved weight vector).',
+ inputSchema: {
+ type: 'object',
+ properties: {
+ strategy: { type: 'string', enum: [...ROUTING_STRATEGIES] },
+ },
+ required: ['strategy'],
+ },
+ run: setStrategy,
+ },
+ cache_stats: {
+ description: 'Response-cache statistics: entries, total hits, and the prompt/completion tokens the cache has saved.',
+ inputSchema: { type: 'object', properties: {} },
+ run: () => getCacheStats(),
+ },
+};
+
+// ── JSON-RPC dispatch ────────────────────────────────────────────────────
+
+function handleRpc(msg: JsonRpcRequest): unknown | undefined {
+ const id = msg.id ?? null;
+ switch (msg.method) {
+ case 'initialize': {
+ const requested = (msg.params?.protocolVersion as string) ?? '';
+ const version = SUPPORTED_PROTOCOL_VERSIONS.has(requested) ? requested : DEFAULT_PROTOCOL_VERSION;
+ return rpcResult(id, {
+ protocolVersion: version,
+ capabilities: { tools: {} },
+ serverInfo: { name: 'freellmapi', version: '1.0.0' },
+ instructions: 'FreeLLMAPI gateway introspection: list usable free models (with per-model supported_parameters), check provider/key health, read usage and cache stats, and switch the routing strategy. Inference goes through the OpenAI-compatible /v1 endpoints, not MCP.',
+ });
+ }
+ case 'ping':
+ return rpcResult(id, {});
+ case 'tools/list':
+ return rpcResult(id, {
+ tools: Object.entries(TOOLS).map(([name, t]) => ({
+ name,
+ description: t.description,
+ inputSchema: t.inputSchema,
+ })),
+ });
+ case 'tools/call': {
+ const name = msg.params?.name as string;
+ const tool = TOOLS[name];
+ if (!tool) return rpcError(id, -32602, `Unknown tool: ${name}`);
+ try {
+ const args = (msg.params?.arguments as Record<string, unknown>) ?? {};
+ return rpcResult(id, toolJson(tool.run(args)));
+ } catch (err: any) {
+ // Tool-level failures are results with isError, not protocol errors.
+ return rpcResult(id, toolError(err?.message ?? 'tool failed'));
+ }
+ }
+ default:
+ if (msg.method?.startsWith('notifications/')) return undefined; // ack silently
+ return rpcError(id, -32601, `Method not found: ${msg.method}`);
+ }
+}
+
+function authenticate(req: Request, res: Response): boolean {
+ const token = extractApiToken(req);
+ const unifiedKey = getUnifiedApiKey();
+ if (!token || !timingSafeStringEqual(token, unifiedKey)) {
+ res.status(401).json(rpcError(null, -32001, 'Invalid API key. Authenticate with the unified key as a Bearer token.'));
+ return false;
+ }
+ return true;
+}
+
+mcpRouter.post('/', (req: Request, res: Response) => {
+ if (!authenticate(req, res)) return;
+
+ const body = req.body;
+ // The 2025-06-18 revision removed JSON-RPC batching; a single message per
+ // POST is the interoperable shape.
+ if (Array.isArray(body)) {
+ res.status(400).json(rpcError(null, -32600, 'Batch requests are not supported'));
+ return;
+ }
+ if (!body || typeof body !== 'object' || typeof body.method !== 'string') {
+ res.status(400).json(rpcError(null, -32600, 'Expected a JSON-RPC request object'));
+ return;
+ }
+
+ const response = handleRpc(body as JsonRpcRequest);
+ if (response === undefined) {
+ res.status(202).end(); // notification — accepted, nothing to say
+ return;
+ }
+ res.json(response);
+});
+
+// Stateless server: no server-initiated stream, no sessions to delete.
+mcpRouter.get('/', (_req: Request, res: Response) => {
+ res.status(405).json(rpcError(null, -32000, 'This MCP server is stateless: POST JSON-RPC messages to /mcp.'));
+});
+mcpRouter.delete('/', (_req: Request, res: Response) => {
+ res.status(405).json(rpcError(null, -32000, 'This MCP server is stateless: there is no session to delete.'));
+});