7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(server): MCP server at /mcp - gateway introspection for coding agents (#517)
Tashfeen2094ea0eda83
diff --git a/README.md b/README.mdindex d139b521..b381bc22 100644--- a/README.md+++ b/README.md@@ -123,6 +123,7 @@ The full, always-current list lives at **[freellmapi.co/models](https://freellma- **Admin dashboard** — React + Vite UI to manage keys, reorder the fallback chain, inspect analytics, and run prompts in a playground. Dark mode included.- **Analytics** — Per-request logging with latency (p50 / p95 and time-to-first-token for streams), token counts, success rate, estimated cost savings, and per-provider / per-model / per-key breakdowns over 24h to 90d windows.- **Interactive API docs** — `GET /v1/docs` serves a dependency-free OpenAPI viewer covering every proxy endpoint; the spec itself lives at `GET /v1/openapi.json`.+- **MCP server** — `POST /mcp` speaks the Model Context Protocol (Streamable HTTP), so MCP-capable agents can ask the router which free models are usable right now (with per-model `supported_parameters`), check provider/key health and cooldowns, read usage and cache stats, and switch the routing strategy mid-session. See [Coding agents](#coding-agents).- **Encrypted DB backups** — optional periodic encrypted snapshots of the SQLite database to a local path or HTTP target, restored automatically on a fresh boot (`FREEAPI_DB_BACKUP_*` env vars).- **Context handoff on model switch** — Optional. When a session falls over to a different model, injects one compact system message so the new model knows it is continuing an existing task. Disabled by default; enable with `FREELLMAPI_CONTEXT_HANDOFF=on_model_switch`. See [Context Handoff](#context-handoff).- **Runs anywhere Node 20+ runs** — Windows, macOS, Linux servers, or a small ARM SBC (Raspberry Pi included). ~40 MB RSS at idle behind PM2 / systemd / whatever supervisor you prefer.@@ -417,6 +418,18 @@ a model (`auto` lets the router pick).| **opencode** | OpenAI-compatible provider with the same base URL and key || **Cursor** | paste the unified key under a custom OpenAI base URL — but note Cursor verifies and calls the API **from its servers**, so your router must be reachable from the internet (a tunnel or a host with a public address), not just `localhost` |+On top of inference, the router is an **MCP server**: agents can introspect it mid-session+(usable models and the params each one honors, provider health, usage and cache stats,+routing strategy). For Claude Code:++```bash+claude mcp add --transport http freellmapi http://localhost:3001/mcp \+ --header "Authorization: Bearer freellmapi-your-unified-key"+```++Any MCP client that speaks Streamable HTTP works the same way: point it at `/mcp` with the+unified key as a Bearer token.+FreeLLMAPI is local-first and single-user by design. Your provider keys stay inyour SQLite database, encrypted at rest, and requests go from your machine to theupstream providers you enabled.diff --git a/server/src/__tests__/routes/mcp.test.ts b/server/src/__tests__/routes/mcp.test.tsnew file mode 100644index 00000000..7d427b0c--- /dev/null+++ b/server/src/__tests__/routes/mcp.test.ts@@ -0,0 +1,139 @@+import { describe, it, expect, beforeAll } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getUnifiedApiKey } from '../../db/index.js';++let app: Express;++async function rpc(message: unknown, opts: { auth?: boolean } = {}) {+ const server = app.listen(0);+ const addr = server.address() as { port: number };+ const res = await fetch(`http://127.0.0.1:${addr.port}/mcp`, {+ method: 'POST',+ headers: {+ 'Content-Type': 'application/json',+ ...(opts.auth === false ? {} : { Authorization: `Bearer ${getUnifiedApiKey()}` }),+ },+ body: JSON.stringify(message),+ });+ const text = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(text); } catch { /* 202 empty */ }+ return { status: res.status, body: json };+}++function toolResultJson(body: any): any {+ expect(body.result?.content?.[0]?.type).toBe('text');+ return JSON.parse(body.result.content[0].text);+}++describe('MCP server (/mcp, stateless Streamable HTTP)', () => {+ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ initDb(':memory:');+ app = createApp();+ });++ it('rejects requests without the unified key', async () => {+ const { status, body } = await rpc({ jsonrpc: '2.0', id: 1, method: 'tools/list' }, { auth: false });+ expect(status).toBe(401);+ expect(body.error.code).toBe(-32001);+ });++ it('initialize negotiates a supported protocol version and advertises tools', async () => {+ const { status, body } = await rpc({+ jsonrpc: '2.0', id: 1, method: 'initialize',+ params: { protocolVersion: '2025-03-26', capabilities: {}, clientInfo: { name: 't', version: '0' } },+ });+ expect(status).toBe(200);+ expect(body.result.protocolVersion).toBe('2025-03-26');+ expect(body.result.capabilities.tools).toBeDefined();+ expect(body.result.serverInfo.name).toBe('freellmapi');+ });++ it('falls back to the default protocol version for an unknown one', async () => {+ const { body } = await rpc({ jsonrpc: '2.0', id: 1, method: 'initialize', params: { protocolVersion: '1999-01-01' } });+ expect(body.result.protocolVersion).toBe('2025-06-18');+ });++ it('accepts notifications with a 202 and no body', async () => {+ const { status, body } = await rpc({ jsonrpc: '2.0', method: 'notifications/initialized' });+ expect(status).toBe(202);+ expect(body).toBeNull();+ });++ it('lists the six gateway tools with schemas', async () => {+ const { body } = await rpc({ jsonrpc: '2.0', id: 2, method: 'tools/list' });+ const names = body.result.tools.map((t: any) => t.name).sort();+ expect(names).toEqual(['cache_stats', 'list_models', 'provider_health', 'routing_info', 'set_routing_strategy', 'usage_summary']);+ for (const tool of body.result.tools) {+ expect(tool.description.length).toBeGreaterThan(20);+ expect(tool.inputSchema.type).toBe('object');+ }+ });++ it('list_models returns catalog entries with supported_parameters', async () => {+ const { body } = await rpc({+ jsonrpc: '2.0', id: 3, method: 'tools/call',+ params: { name: 'list_models', arguments: { available_only: false } },+ });+ const data = toolResultJson(body);+ expect(data.count).toBeGreaterThan(0);+ expect(data.auto.id).toBe('auto');+ const model = data.models[0];+ expect(model.id).toBeTruthy();+ expect(Array.isArray(model.supported_parameters)).toBe(true);+ expect(model.supported_parameters).toContain('temperature');+ });++ it('set_routing_strategy switches and routing_info reflects it', async () => {+ const set = await rpc({+ jsonrpc: '2.0', id: 4, method: 'tools/call',+ params: { name: 'set_routing_strategy', arguments: { strategy: 'fastest' } },+ });+ expect(toolResultJson(set.body)).toEqual({ strategy: 'fastest' });++ const info = await rpc({ jsonrpc: '2.0', id: 5, method: 'tools/call', params: { name: 'routing_info' } });+ expect(toolResultJson(info.body).strategy).toBe('fastest');+ });++ it('an invalid strategy is a tool-level error, not a protocol error', async () => {+ const { body } = await rpc({+ jsonrpc: '2.0', id: 6, method: 'tools/call',+ params: { name: 'set_routing_strategy', arguments: { strategy: 'chaotic' } },+ });+ expect(body.result.isError).toBe(true);+ expect(body.result.content[0].text).toContain('strategy must be one of');+ });++ it('usage_summary and provider_health and cache_stats answer on an empty install', async () => {+ for (const name of ['usage_summary', 'provider_health', 'cache_stats']) {+ const { body } = await rpc({ jsonrpc: '2.0', id: 7, method: 'tools/call', params: { name } });+ expect(body.result.content[0].type).toBe('text');+ expect(body.result.isError).toBeUndefined();+ }+ });++ it('unknown tool and unknown method return JSON-RPC errors', async () => {+ const tool = await rpc({ jsonrpc: '2.0', id: 8, method: 'tools/call', params: { name: 'rm_rf' } });+ expect(tool.body.error.code).toBe(-32602);+ const method = await rpc({ jsonrpc: '2.0', id: 9, method: 'resources/list' });+ expect(method.body.error.code).toBe(-32601);+ });++ it('rejects batches and non-request bodies', async () => {+ const batch = await rpc([{ jsonrpc: '2.0', id: 1, method: 'ping' }]);+ expect(batch.status).toBe(400);+ const junk = await rpc({ hello: 'world' });+ expect(junk.status).toBe(400);+ });++ it('GET /mcp is 405 (stateless: no server-initiated stream)', async () => {+ const server = app.listen(0);+ const addr = server.address() as { port: number };+ const res = await fetch(`http://127.0.0.1:${addr.port}/mcp`);+ server.close();+ expect(res.status).toBe(405);+ });+});diff --git a/server/src/app.ts b/server/src/app.tsindex 815d955d..3956007b 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -19,6 +19,7 @@ import { premiumRouter } from './routes/premium.js';import { cacheRouter } from './routes/cache.js';import { authRouter } from './routes/auth.js';import { docsRouter } from './routes/docs.js';+import { mcpRouter } from './routes/mcp.js';import { requireAuth } from './middleware/requireAuth.js';import { createProxyRateLimiter } from './middleware/rateLimit.js';import { errorHandler } from './middleware/errorHandler.js';@@ -101,6 +102,11 @@ export function createApp(config?: Config) {// OpenAI Responses API shim (Codex CLI requires wire_api="responses"; see #96)app.use('/v1', responsesRouter);+ // MCP server (Model Context Protocol over stateless Streamable HTTP):+ // gateway introspection tools for MCP-speaking agents. Unified-key auth,+ // like /v1 — NOT behind the dashboard session gate.+ app.use('/mcp', mcpRouter);+// Health checkapp.get('/api/ping', (_req, res) => {res.json({ status: 'ok', timestamp: new Date().toISOString() });diff --git a/server/src/routes/mcp.ts b/server/src/routes/mcp.tsnew file mode 100644index 00000000..41592763--- /dev/null+++ b/server/src/routes/mcp.ts@@ -0,0 +1,308 @@+import { Router } from 'express';+import type { Request, Response } from 'express';+import { getDb, getUnifiedApiKey } from '../db/index.js';+import { extractApiToken, timingSafeStringEqual } from './proxy.js';+import { buildModelListing } from '../services/model-listing.js';+import { supportedParametersForPlatforms } from '../lib/sampling-params.js';+import { getRoutingScores, getRoutingStrategy, setRoutingStrategy } from '../services/router.js';+import type { RoutingStrategy } from '../services/scoring.js';+import { getCacheStats } from '../services/cache.js';++// ─────────────────────────────────────────────────────────────────────────+// MCP server for the gateway (POST /mcp) — Model Context Protocol over+// Streamable HTTP, stateless mode.+//+// Lets MCP-speaking agents (Claude Code, Cursor, Cline…) ask the router+// questions mid-session: which free models are usable right now and with+// which parameters, how healthy the provider pool is, what the routing+// strategy is, and how much quota the cache has saved — plus one control+// knob (switching the routing strategy). Inference itself stays on the+// OpenAI/Anthropic surfaces; these tools are the gateway's introspection.+//+// Hand-rolled JSON-RPC instead of the MCP SDK for the same reason the+// OpenAPI viewer is dependency-free (#482): the desktop bundle and the+// Node-20 CI matrix punish heavy/new dependencies, and a tools-only+// stateless MCP server is ~five methods of plain JSON-RPC. No sessions, no+// server-initiated streams (GET → 405), single JSON responses.+//+// Auth mirrors /v1: the unified API key as a Bearer token (or x-api-key).+// ─────────────────────────────────────────────────────────────────────────++export const mcpRouter = Router();++const SUPPORTED_PROTOCOL_VERSIONS = new Set(['2025-06-18', '2025-03-26', '2024-11-05']);+const DEFAULT_PROTOCOL_VERSION = '2025-06-18';++interface JsonRpcRequest {+ jsonrpc?: string;+ id?: number | string | null;+ method?: string;+ params?: Record<string, unknown>;+}++function rpcResult(id: number | string | null, result: unknown) {+ return { jsonrpc: '2.0', id, result };+}++function rpcError(id: number | string | null, code: number, message: string) {+ return { jsonrpc: '2.0', id, error: { code, message } };+}++// One text block carrying pretty-printed JSON — the standard shape for+// machine-readable MCP tool output.+function toolJson(data: unknown) {+ return { content: [{ type: 'text', text: JSON.stringify(data, null, 2) }] };+}++function toolError(message: string) {+ return { content: [{ type: 'text', text: message }], isError: true };+}++// ── Tool implementations ─────────────────────────────────────────────────++function listModels(args: Record<string, unknown>): unknown {+ const availableOnly = args.available_only !== false; // default true: agents want what they can use+ const { models, autoContextWindow } = buildModelListing();+ const rows = (availableOnly ? models.filter(m => m.available === 1) : models).map(m => ({+ id: m.id,+ name: m.name,+ context_window: m.contextWindow,+ available: m.available === 1,+ platforms: m.platforms,+ supports_tools: m.supportsTools,+ supported_parameters: supportedParametersForPlatforms(m.platforms, { tools: m.supportsTools }),+ }));+ return {+ auto: { id: 'auto', description: 'router picks the best available model', context_window: autoContextWindow },+ count: rows.length,+ models: rows,+ };+}++function providerHealth(): unknown {+ const db = getDb();+ const now = Date.now();+ const keys = db.prepare(`+ SELECT platform, status, COUNT(*) AS n+ FROM api_keys WHERE enabled = 1+ GROUP BY platform, status+ `).all() as Array<{ platform: string; status: string; n: number }>;+ const cooldowns = db.prepare(`+ SELECT platform, COUNT(*) AS n+ FROM rate_limit_cooldowns WHERE expires_at_ms > ?+ GROUP BY platform+ `).all(now) as Array<{ platform: string; n: number }>;+ const availableModels = db.prepare(`+ SELECT m.platform, COUNT(*) AS n+ FROM models m+ WHERE m.enabled = 1 AND EXISTS (+ SELECT 1 FROM api_keys k+ WHERE k.platform = m.platform AND k.enabled = 1+ AND (m.key_id IS NULL OR k.id = m.key_id)+ )+ GROUP BY m.platform+ `).all() as Array<{ platform: string; n: number }>;++ const byPlatform = new Map<string, { keys: Record<string, number>; active_cooldowns: number; available_models: number }>();+ const entry = (p: string) => {+ let e = byPlatform.get(p);+ if (!e) { e = { keys: {}, active_cooldowns: 0, available_models: 0 }; byPlatform.set(p, e); }+ return e;+ };+ for (const k of keys) entry(k.platform).keys[k.status] = k.n;+ for (const c of cooldowns) entry(c.platform).active_cooldowns = c.n;+ for (const m of availableModels) entry(m.platform).available_models = m.n;+ return Object.fromEntries([...byPlatform.entries()].sort(([a], [b]) => a.localeCompare(b)));+}++const USAGE_RANGES: Record<string, number> = { '24h': 24, '7d': 24 * 7, '30d': 24 * 30 };++function usageSummary(args: Record<string, unknown>): unknown {+ const range = typeof args.range === 'string' && USAGE_RANGES[args.range] ? args.range : '24h';+ const db = getDb();+ const since = new Date(Date.now() - USAGE_RANGES[range] * 3600_000).toISOString();+ const totals = db.prepare(`+ SELECT COALESCE(SUM(total_requests), 0) AS requests,+ COALESCE(SUM(success_count), 0) AS successes,+ COALESCE(SUM(input_tokens), 0) AS input_tokens,+ COALESCE(SUM(output_tokens), 0) AS output_tokens+ FROM request_hourly WHERE hour >= ?+ `).get(since.slice(0, 13) + ':00:00') as { requests: number; successes: number; input_tokens: number; output_tokens: number };+ const topModels = db.prepare(`+ SELECT platform, model_id, COUNT(*) AS requests,+ SUM(CASE WHEN status = 'success' THEN 1 ELSE 0 END) AS successes+ FROM requests WHERE created_at >= ?+ GROUP BY platform, model_id ORDER BY requests DESC LIMIT 5+ `).all(since) as Array<{ platform: string; model_id: string; requests: number; successes: number }>;+ return {+ range,+ requests: totals.requests,+ success_rate: totals.requests > 0 ? Math.round((totals.successes / totals.requests) * 1000) / 10 : null,+ input_tokens: totals.input_tokens,+ output_tokens: totals.output_tokens,+ top_models: topModels,+ };+}++function routingInfo(): unknown {+ const scores = getRoutingScores();+ return {+ strategy: scores.strategy,+ top_models: scores.scores+ .filter(s => s.enabled)+ .slice(0, 10)+ .map(s => ({ model: s.modelId, platform: s.platform, score: Math.round(s.score * 1000) / 1000 })),+ };+}++const ROUTING_STRATEGIES = ['priority', 'balanced', 'smartest', 'fastest', 'reliable', 'custom'] as const;++function setStrategy(args: Record<string, unknown>): unknown {+ const strategy = args.strategy;+ if (typeof strategy !== 'string' || !(ROUTING_STRATEGIES as readonly string[]).includes(strategy)) {+ throw new Error(`strategy must be one of: ${ROUTING_STRATEGIES.join(', ')}`);+ }+ setRoutingStrategy(strategy as RoutingStrategy);+ return { strategy: getRoutingStrategy() };+}++// ── Tool registry ────────────────────────────────────────────────────────++interface McpTool {+ description: string;+ inputSchema: Record<string, unknown>;+ run: (args: Record<string, unknown>) => unknown;+}++const TOOLS: Record<string, McpTool> = {+ list_models: {+ description: 'List the models this FreeLLMAPI router can serve, with context windows, tool support, and the parameters each model honors (supported_parameters). Defaults to only models that are usable right now.',+ inputSchema: {+ type: 'object',+ properties: {+ available_only: { type: 'boolean', description: 'false to include models that are configured but not currently usable (no key, disabled)', default: true },+ },+ },+ run: listModels,+ },+ provider_health: {+ description: 'Per-provider key statuses (healthy/rate_limited/invalid/error/unknown), active cooldowns, and how many models each provider can serve right now.',+ inputSchema: { type: 'object', properties: {} },+ run: () => providerHealth(),+ },+ usage_summary: {+ description: 'Request/token totals, success rate, and the top models by traffic for a recent window.',+ inputSchema: {+ type: 'object',+ properties: {+ range: { type: 'string', enum: ['24h', '7d', '30d'], default: '24h' },+ },+ },+ run: usageSummary,+ },+ routing_info: {+ description: 'The active routing strategy and the current top-scored models in the fallback chain.',+ inputSchema: { type: 'object', properties: {} },+ run: () => routingInfo(),+ },+ set_routing_strategy: {+ description: 'Switch the routing strategy (priority = manual chain order; balanced / smartest / fastest / reliable are scored presets; custom uses the saved weight vector).',+ inputSchema: {+ type: 'object',+ properties: {+ strategy: { type: 'string', enum: [...ROUTING_STRATEGIES] },+ },+ required: ['strategy'],+ },+ run: setStrategy,+ },+ cache_stats: {+ description: 'Response-cache statistics: entries, total hits, and the prompt/completion tokens the cache has saved.',+ inputSchema: { type: 'object', properties: {} },+ run: () => getCacheStats(),+ },+};++// ── JSON-RPC dispatch ────────────────────────────────────────────────────++function handleRpc(msg: JsonRpcRequest): unknown | undefined {+ const id = msg.id ?? null;+ switch (msg.method) {+ case 'initialize': {+ const requested = (msg.params?.protocolVersion as string) ?? '';+ const version = SUPPORTED_PROTOCOL_VERSIONS.has(requested) ? requested : DEFAULT_PROTOCOL_VERSION;+ return rpcResult(id, {+ protocolVersion: version,+ capabilities: { tools: {} },+ serverInfo: { name: 'freellmapi', version: '1.0.0' },+ instructions: 'FreeLLMAPI gateway introspection: list usable free models (with per-model supported_parameters), check provider/key health, read usage and cache stats, and switch the routing strategy. Inference goes through the OpenAI-compatible /v1 endpoints, not MCP.',+ });+ }+ case 'ping':+ return rpcResult(id, {});+ case 'tools/list':+ return rpcResult(id, {+ tools: Object.entries(TOOLS).map(([name, t]) => ({+ name,+ description: t.description,+ inputSchema: t.inputSchema,+ })),+ });+ case 'tools/call': {+ const name = msg.params?.name as string;+ const tool = TOOLS[name];+ if (!tool) return rpcError(id, -32602, `Unknown tool: ${name}`);+ try {+ const args = (msg.params?.arguments as Record<string, unknown>) ?? {};+ return rpcResult(id, toolJson(tool.run(args)));+ } catch (err: any) {+ // Tool-level failures are results with isError, not protocol errors.+ return rpcResult(id, toolError(err?.message ?? 'tool failed'));+ }+ }+ default:+ if (msg.method?.startsWith('notifications/')) return undefined; // ack silently+ return rpcError(id, -32601, `Method not found: ${msg.method}`);+ }+}++function authenticate(req: Request, res: Response): boolean {+ const token = extractApiToken(req);+ const unifiedKey = getUnifiedApiKey();+ if (!token || !timingSafeStringEqual(token, unifiedKey)) {+ res.status(401).json(rpcError(null, -32001, 'Invalid API key. Authenticate with the unified key as a Bearer token.'));+ return false;+ }+ return true;+}++mcpRouter.post('/', (req: Request, res: Response) => {+ if (!authenticate(req, res)) return;++ const body = req.body;+ // The 2025-06-18 revision removed JSON-RPC batching; a single message per+ // POST is the interoperable shape.+ if (Array.isArray(body)) {+ res.status(400).json(rpcError(null, -32600, 'Batch requests are not supported'));+ return;+ }+ if (!body || typeof body !== 'object' || typeof body.method !== 'string') {+ res.status(400).json(rpcError(null, -32600, 'Expected a JSON-RPC request object'));+ return;+ }++ const response = handleRpc(body as JsonRpcRequest);+ if (response === undefined) {+ res.status(202).end(); // notification — accepted, nothing to say+ return;+ }+ res.json(response);+});++// Stateless server: no server-initiated stream, no sessions to delete.+mcpRouter.get('/', (_req: Request, res: Response) => {+ res.status(405).json(rpcError(null, -32000, 'This MCP server is stateless: POST JSON-RPC messages to /mcp.'));+});+mcpRouter.delete('/', (_req: Request, res: Response) => {+ res.status(405).json(rpcError(null, -32000, 'This MCP server is stateless: there is no session to delete.'));+});