7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat: opt-in response cache for /v1/chat/completions (#481)
Tashfeend5a89d53c835
diff --git a/.env.example b/.env.exampleindex c48e05e9..db84f424 100644--- a/.env.example+++ b/.env.example@@ -38,6 +38,23 @@ PORT=3001# set to 0 to disable the cap. Example — cap OpenRouter at 50 requests/day:# PROVIDER_DAILY_REQUEST_CAP_OPENROUTER=50+# Opt-in response cache. When on, a successful non-streaming /v1/chat/completions+# answer is stored in a bounded in-memory LRU keyed by a canonical hash of the+# request, so an IDENTICAL later request is served from memory without spending+# any provider quota. Exact match only (no fuzzy matching), so a near-miss never+# returns a different prompt's answer. Off by default; set to true to enable.+# Can also be toggled at runtime via the dashboard (PUT /api/cache/config), and+# per-request with the `X-FreeLLM-Cache: on|off` header. A restart flushes it.+# RESPONSE_CACHE=false+# How long a cached answer stays fresh, in seconds (default: 3600 = 1 hour).+# RESPONSE_CACHE_TTL_SECONDS=3600+# Only cache requests at/below this temperature; higher temperatures want fresh+# variety, so they are never cached. Default 1.0 caches everything when enabled;+# lower it (e.g. 0.2) to cache only near-deterministic calls.+# RESPONSE_CACHE_MAX_TEMPERATURE=1.0+# Hard cap on stored entries; the least-recently-used are evicted past this.+# RESPONSE_CACHE_MAX_ENTRIES=5000+# Custom-provider URL policy. Cloud metadata and link-local addresses# (169.254.169.254, metadata.google.internal, fe80::/10, ...) are ALWAYS# blocked as custom provider base URLs. By default localhost and private LANdiff --git a/README.md b/README.mdindex ec78cbe8..88833b62 100644--- a/README.md+++ b/README.md@@ -850,6 +850,7 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full migration CLI and workflow<a href="https://github.com/iisyw"><img src="https://images.weserv.nl/?url=github.com/iisyw.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@iisyw" /></a><a href="https://github.com/xsfhacg"><img src="https://images.weserv.nl/?url=github.com/xsfhacg.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@xsfhacg" /></a><a href="https://github.com/noobix"><img src="https://images.weserv.nl/?url=github.com/noobix.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@noobix" /></a>+<a href="https://github.com/nandukmelath"><img src="https://images.weserv.nl/?url=github.com/nandukmelath.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@nandukmelath" /></a>## Terms of Service reviewdiff --git a/server/src/__tests__/routes/proxy-cache.test.ts b/server/src/__tests__/routes/proxy-cache.test.tsnew file mode 100644index 00000000..72766cfa--- /dev/null+++ b/server/src/__tests__/routes/proxy-cache.test.ts@@ -0,0 +1,240 @@+import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';+import type { Express } from 'express';+import { createApp } from '../../app.js';+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';+import { clearCache } from '../../services/cache.js';+import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';++let dashToken = '';++async function request(app: Express, method: string, path: string, body?: any, headers: Record<string, string> = {}) {+ const server = app.listen(0);+ const addr = server.address() as any;+ const url = `http://127.0.0.1:${addr.port}${path}`;+ const res = await fetch(url, {+ method,+ headers: {+ ...(body ? { 'Content-Type': 'application/json' } : {}),+ ...(isGatedApiPath(path) && !('Authorization' in headers) ? { Authorization: `Bearer ${dashToken}` } : {}),+ ...headers,+ },+ body: body ? JSON.stringify(body) : undefined,+ });+ const raw = await res.text();+ server.close();+ let json: any = null;+ try { json = JSON.parse(raw); } catch { /* SSE or empty body */ }+ return { status: res.status, body: json, headers: res.headers, raw };+}++function authHeaders() {+ return { Authorization: `Bearer ${getUnifiedApiKey()}` };+}++// Mock every groq chat-completion call, counting how many actually reach the+// provider (separately for streaming vs non-streaming). The whole point of the+// cache is that a repeat non-streaming request does NOT reach the provider.+function mockGroq(content: string) {+ const origFetch = global.fetch;+ const counter = { calls: 0, streamCalls: 0 };+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {+ const urlStr = typeof url === 'string' ? url : url.toString();+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {+ const reqBody = JSON.parse(String((init as RequestInit).body));+ if (reqBody.stream) {+ counter.streamCalls++;+ const sse = [+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: { role: 'assistant', content: null }, finish_reason: null }] },+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: { content }, finish_reason: null }] },+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 } },+ ].map(f => `data: ${JSON.stringify(f)}\n\n`).join('') + 'data: [DONE]\n\n';+ return new Response(sse, { status: 200, headers: { 'Content-Type': 'text/event-stream' } }) as any;+ }+ counter.calls++;+ return {+ ok: true,+ json: () => Promise.resolve({+ id: 'chatcmpl-cache-it',+ object: 'chat.completion',+ created: 123,+ model: 'openai/gpt-oss-120b',+ choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 },+ }),+ } as any;+ }+ return origFetch(url, init);+ });+ return counter;+}++describe('Response cache (proxy integration)', () => {+ let app: Express;++ beforeAll(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ process.env.RESPONSE_CACHE = 'on';+ initDb(':memory:');+ app = createApp();+ dashToken = mintDashboardToken();+ });++ afterAll(() => {+ delete process.env.RESPONSE_CACHE;+ });++ beforeEach(async () => {+ const db = getDb();+ db.prepare('DELETE FROM api_keys').run();+ db.prepare('DELETE FROM requests').run();+ db.prepare('DELETE FROM rate_limit_usage').run();+ clearCache();+ const addKey = await request(app, 'POST', '/api/keys', { platform: 'groq', key: 'gsk_cache_test', label: 'cache' });+ expect(addKey.status).toBe(201);+ });++ afterEach(() => {+ vi.restoreAllMocks();+ clearCache();+ });++ const chat = (extra: Record<string, unknown> = {}, headers: Record<string, string> = {}) =>+ request(app, 'POST', '/v1/chat/completions', {+ model: 'auto',+ messages: [{ role: 'user', content: 'what is 2+2?' }],+ ...extra,+ }, { ...authHeaders(), ...headers });++ it('serves an identical second request from cache without calling the provider', async () => {+ const counter = mockGroq('the answer is four');++ const first = await chat();+ expect(first.status).toBe(200);+ expect(first.body.choices[0].message.content).toBe('the answer is four');+ expect(first.headers.get('x-freellm-cache')).toBe('MISS');+ expect(counter.calls).toBe(1);++ const second = await chat();+ expect(second.status).toBe(200);+ expect(second.body.choices[0].message.content).toBe('the answer is four');+ expect(second.headers.get('x-freellm-cache')).toBe('HIT');+ expect(second.headers.get('x-routed-via')).toBe('cache');+ // The provider was NOT hit again; the whole point.+ expect(counter.calls).toBe(1);+ });++ it('does not record rate-limit usage for a cache hit (quota preserved)', async () => {+ mockGroq('four');+ await chat();+ const usageAfterMiss = (getDb().prepare("SELECT COUNT(*) AS c FROM rate_limit_usage WHERE kind='request'").get() as { c: number }).c;+ await chat(); // hit+ const usageAfterHit = (getDb().prepare("SELECT COUNT(*) AS c FROM rate_limit_usage WHERE kind='request'").get() as { c: number }).c;+ expect(usageAfterHit).toBe(usageAfterMiss); // no new provider request recorded+ });++ it('treats a different prompt as a miss and calls the provider again', async () => {+ const counter = mockGroq('depends on the prompt');+ await chat({ messages: [{ role: 'user', content: 'first question' }] });+ await chat({ messages: [{ role: 'user', content: 'second question' }] });+ expect(counter.calls).toBe(2);+ });++ it('requests differing only in response_format, seed, or stop never collide', async () => {+ const counter = mockGroq('knob-sensitive');+ await chat(); // plain → MISS, populates+ // Same prompt with response_format json_object must be a MISS, not a+ // replay of the cached plain-text answer.+ const jsonReq = await chat({ response_format: { type: 'json_object' } });+ expect(jsonReq.headers.get('x-freellm-cache')).toBe('MISS');+ expect(counter.calls).toBe(2);+ // Same prompt, different seed / stop → also fresh generations.+ await chat({ seed: 42 });+ await chat({ stop: 'END' });+ expect(counter.calls).toBe(4);+ // But an exact repeat (with a knob present) is still a HIT.+ const repeat = await chat({ seed: 42 });+ expect(repeat.headers.get('x-freellm-cache')).toBe('HIT');+ expect(counter.calls).toBe(4);+ });++ it('X-FreeLLM-Cache: off bypasses the cache on both read and write', async () => {+ const counter = mockGroq('bypassed');+ await chat({}, { 'X-FreeLLM-Cache': 'off' });+ await chat({}, { 'X-FreeLLM-Cache': 'off' });+ // Neither call read nor populated the cache → provider hit twice.+ expect(counter.calls).toBe(2);+ });++ it('X-FreeLLM-Cache: on forces caching even when the cache is globally off', async () => {+ process.env.RESPONSE_CACHE = '';+ try {+ const counter = mockGroq('forced-on');+ const first = await chat({}, { 'X-FreeLLM-Cache': 'on' });+ expect(first.headers.get('x-freellm-cache')).toBe('MISS');+ const second = await chat({}, { 'X-FreeLLM-Cache': 'on' });+ expect(second.headers.get('x-freellm-cache')).toBe('HIT');+ expect(counter.calls).toBe(1);+ } finally {+ process.env.RESPONSE_CACHE = 'on';+ }+ });++ it('does not cache a high-temperature request', async () => {+ process.env.RESPONSE_CACHE_MAX_TEMPERATURE = '0.2';+ try {+ const counter = mockGroq('varies');+ await chat({ temperature: 0.9 });+ await chat({ temperature: 0.9 });+ expect(counter.calls).toBe(2); // above the cap → never cached+ } finally {+ delete process.env.RESPONSE_CACHE_MAX_TEMPERATURE;+ }+ });++ it('a streaming request bypasses the cache: never served from it, never populates it', async () => {+ const counter = mockGroq('cached four');++ // A non-streaming request populates the cache.+ const first = await chat();+ expect(first.headers.get('x-freellm-cache')).toBe('MISS');+ expect(counter.calls).toBe(1);++ // An identical STREAMING request must not be served from the cache, so it+ // reaches the provider's streaming endpoint despite the cached entry.+ const streamed = await chat({ stream: true });+ expect(streamed.status).toBe(200);+ expect(counter.streamCalls).toBe(1);++ // And it did not overwrite/populate a cache entry: a following identical+ // non-streaming request is still a HIT served from the original entry, with+ // no new non-streaming provider call.+ const third = await chat();+ expect(third.headers.get('x-freellm-cache')).toBe('HIT');+ expect(third.body.choices[0].message.content).toBe('cached four');+ expect(counter.calls).toBe(1);+ });++ it('exposes cache stats and savings on the admin endpoint', async () => {+ mockGroq('four');+ await chat(); // MISS, populates+ await chat(); // HIT (1 hit, saves 5 prompt + 3 completion tokens)+ const stats = await request(app, 'GET', '/api/cache/stats');+ expect(stats.status).toBe(200);+ expect(stats.body.enabled).toBe(true);+ expect(stats.body.entries).toBe(1);+ expect(stats.body.totalHits).toBe(1);+ expect(stats.body.savedTokens).toBe(8); // 5 + 3+ });++ it('DELETE /api/cache flushes stored entries', async () => {+ mockGroq('four');+ await chat();+ let stats = await request(app, 'GET', '/api/cache/stats');+ expect(stats.body.entries).toBe(1);+ const cleared = await request(app, 'DELETE', '/api/cache');+ expect(cleared.status).toBe(200);+ expect(cleared.body.cleared).toBe(1);+ stats = await request(app, 'GET', '/api/cache/stats');+ expect(stats.body.entries).toBe(0);+ });+});diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.tsnew file mode 100644index 00000000..df867dff--- /dev/null+++ b/server/src/__tests__/services/cache.test.ts@@ -0,0 +1,342 @@+import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { initDb, setSetting } from '../../db/index.js';+import {+ computeCacheKey,+ getCachedResponse,+ storeCachedResponse,+ getCacheStats,+ clearCache,+ isCacheableTemperature,+ parseCacheDirective,+ cacheActive,+ isCacheEnabled,+ CACHE_ENABLED_SETTING,+} from '../../services/cache.js';+import type { ChatMessage } from '@freellmapi/shared/types.js';++const CACHE_ENV = [+ 'RESPONSE_CACHE',+ 'RESPONSE_CACHE_TTL_SECONDS',+ 'RESPONSE_CACHE_MAX_TEMPERATURE',+ 'RESPONSE_CACHE_MAX_ENTRIES',+] as const;++function msg(role: ChatMessage['role'], content: string): ChatMessage {+ return { role, content } as ChatMessage;+}++const sampleBody = (text: string) => ({+ id: 'chatcmpl-test',+ object: 'chat.completion',+ choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }],+ usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },+});++const store = (key: string, text: string, now?: number) =>+ storeCachedResponse(key, {+ body: sampleBody(text),+ platform: 'groq',+ modelId: 'llama-3.3-70b',+ keyId: 7,+ promptTokens: 10,+ completionTokens: 5,+ }, now);++describe('response cache', () => {+ const saved: Record<string, string | undefined> = {};++ beforeEach(() => {+ process.env.ENCRYPTION_KEY = '0'.repeat(64);+ for (const k of CACHE_ENV) saved[k] = process.env[k];+ // Fresh in-memory DB per test so the settings-backed toggle is isolated.+ initDb(':memory:');+ // The cache is module-level in-memory state: flush it between tests.+ clearCache();+ });++ afterEach(() => {+ clearCache();+ for (const k of CACHE_ENV) {+ if (saved[k] === undefined) delete process.env[k];+ else process.env[k] = saved[k]!;+ }+ });++ describe('computeCacheKey', () => {+ it('is stable regardless of object key order', () => {+ const messages = [msg('user', 'hello')];+ const a = computeCacheKey({ model: 'auto', messages, temperature: 0.2, top_p: 1 });+ const b = computeCacheKey({ messages, top_p: 1, temperature: 0.2, model: 'auto' });+ expect(a).toBe(b);+ });++ it('treats omitted model and "auto" as the same bucket', () => {+ const messages = [msg('user', 'hello')];+ expect(computeCacheKey({ model: undefined, messages }))+ .toBe(computeCacheKey({ model: 'auto', messages }));+ });++ it('changes when the prompt changes', () => {+ expect(computeCacheKey({ model: 'auto', messages: [msg('user', 'a')] }))+ .not.toBe(computeCacheKey({ model: 'auto', messages: [msg('user', 'b')] }));+ });++ it('changes when sampling params change', () => {+ const messages = [msg('user', 'hello')];+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0 }))+ .not.toBe(computeCacheKey({ model: 'auto', messages, temperature: 0.9 }));+ });++ it('distinguishes a pinned model from auto', () => {+ const messages = [msg('user', 'hello')];+ expect(computeCacheKey({ model: 'gpt-oss-120b', messages }))+ .not.toBe(computeCacheKey({ model: 'auto', messages }));+ });++ // Tools policy: a tool-bearing request is cacheable, but its tool set is+ // part of the key, so a different tool set is a MISS and never serves+ // another request's cached answer.+ it('changes when the tool set changes (no cross-tool contamination)', () => {+ const messages = [msg('user', 'call a tool')];+ const toolsA = [{ type: 'function', function: { name: 'Read' } }];+ const toolsB = [{ type: 'function', function: { name: 'Write' } }];+ expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))+ .not.toBe(computeCacheKey({ model: 'auto', messages, tools: toolsB }));+ // Same tools + same prompt collapse to the same key.+ expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))+ .toBe(computeCacheKey({ model: 'auto', messages, tools: toolsA }));+ // A tools request differs from the tool-free version of the same prompt.+ expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))+ .not.toBe(computeCacheKey({ model: 'auto', messages }));+ });++ // Collision guard for the remaining sampling/format knobs: requests that+ // differ ONLY in one of these must never share a key, or one could be+ // served the other's cached answer (worst case: a response_format+ // json_object request replayed a cached plain-text reply).+ it('changes when response_format changes (json_object vs none)', () => {+ const messages = [msg('user', 'give me data')];+ expect(computeCacheKey({ model: 'auto', messages, response_format: { type: 'json_object' } }))+ .not.toBe(computeCacheKey({ model: 'auto', messages }));+ expect(computeCacheKey({ model: 'auto', messages, response_format: { type: 'json_object' } }))+ .not.toBe(computeCacheKey({ model: 'auto', messages, response_format: { type: 'text' } }));+ });++ it('changes when stop, seed, or n change', () => {+ const messages = [msg('user', 'hello')];+ const base = computeCacheKey({ model: 'auto', messages });+ expect(computeCacheKey({ model: 'auto', messages, stop: ['\n'] })).not.toBe(base);+ expect(computeCacheKey({ model: 'auto', messages, stop: 'END' }))+ .not.toBe(computeCacheKey({ model: 'auto', messages, stop: 'STOP' }));+ expect(computeCacheKey({ model: 'auto', messages, seed: 42 })).not.toBe(base);+ expect(computeCacheKey({ model: 'auto', messages, seed: 42 }))+ .not.toBe(computeCacheKey({ model: 'auto', messages, seed: 43 }));+ expect(computeCacheKey({ model: 'auto', messages, n: 2 })).not.toBe(base);+ });++ it('changes when penalties, logit_bias, or logprobs knobs change', () => {+ const messages = [msg('user', 'hello')];+ const base = computeCacheKey({ model: 'auto', messages });+ expect(computeCacheKey({ model: 'auto', messages, presence_penalty: 0.5 })).not.toBe(base);+ expect(computeCacheKey({ model: 'auto', messages, frequency_penalty: 0.5 })).not.toBe(base);+ expect(computeCacheKey({ model: 'auto', messages, logit_bias: { '50256': -100 } })).not.toBe(base);+ expect(computeCacheKey({ model: 'auto', messages, logprobs: true })).not.toBe(base);+ expect(computeCacheKey({ model: 'auto', messages, logprobs: true, top_logprobs: 5 }))+ .not.toBe(computeCacheKey({ model: 'auto', messages, logprobs: true }));+ });++ it('identical requests with the new knobs present collapse to the same key', () => {+ const messages = [msg('user', 'give me data')];+ const full = () => computeCacheKey({+ model: 'auto', messages, temperature: 0.1,+ response_format: { type: 'json_object' },+ stop: ['END', 'STOP'], n: 1, seed: 42,+ presence_penalty: 0.25, frequency_penalty: 0.5,+ logit_bias: { '50256': -100 }, logprobs: true, top_logprobs: 3,+ });+ expect(full()).toBe(full());+ // Key-order independence holds for the new fields too.+ expect(computeCacheKey({ seed: 42, stop: 'END', model: 'auto', messages } as any))+ .toBe(computeCacheKey({ model: 'auto', messages, stop: 'END', seed: 42 }));+ // And absent knobs still hash like before (undefined dropped, not null-ed).+ expect(computeCacheKey({ model: 'auto', messages, seed: undefined }))+ .toBe(computeCacheKey({ model: 'auto', messages }));+ });+ });++ describe('store / get round-trip', () => {+ it('returns null on a miss', () => {+ expect(getCachedResponse('does-not-exist')).toBeNull();+ });++ it('returns the stored body on a hit', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });+ store(key, 'cached answer');+ const hit = getCachedResponse(key);+ expect(hit).not.toBeNull();+ expect((hit!.body as any).choices[0].message.content).toBe('cached answer');+ expect(hit!.platform).toBe('groq');+ expect(hit!.modelId).toBe('llama-3.3-70b');+ expect(hit!.keyId).toBe(7);+ expect(hit!.promptTokens).toBe(10);+ expect(hit!.completionTokens).toBe(5);+ });++ it('overwrites and refreshes an existing entry', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });+ store(key, 'first');+ store(key, 'second');+ expect((getCachedResponse(key)!.body as any).choices[0].message.content).toBe('second');+ expect(getCacheStats().entries).toBe(1);+ });++ it('skips an unserializable body rather than throwing', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });+ const circular: any = {};+ circular.self = circular;+ storeCachedResponse(key, {+ body: circular, platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,+ });+ expect(getCacheStats().entries).toBe(0);+ expect(getCachedResponse(key)).toBeNull();+ });+ });++ describe('TTL expiry', () => {+ it('treats an entry older than the TTL as a miss and deletes it', () => {+ process.env.RESPONSE_CACHE_TTL_SECONDS = '1';+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });+ const t0 = 1_000_000;+ store(key, 'stale', t0);+ // 2s later, past the 1s TTL → miss.+ expect(getCachedResponse(key, t0 + 2000)).toBeNull();+ // The expired entry was purged.+ expect(getCacheStats().entries).toBe(0);+ });++ it('serves an entry still within the TTL', () => {+ process.env.RESPONSE_CACHE_TTL_SECONDS = '60';+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });+ const t0 = 1_000_000;+ store(key, 'fresh', t0);+ expect(getCachedResponse(key, t0 + 30_000)).not.toBeNull();+ });+ });++ describe('hit accounting', () => {+ it('counts hits and tallies saved tokens', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });+ store(key, 'answer');+ getCachedResponse(key);+ getCachedResponse(key);+ getCachedResponse(key);+ const stats = getCacheStats();+ expect(stats.entries).toBe(1);+ expect(stats.totalHits).toBe(3);+ expect(stats.savedPromptTokens).toBe(30); // 3 hits × 10+ expect(stats.savedCompletionTokens).toBe(15); // 3 hits × 5+ });+ });++ describe('entry cap eviction (LRU)', () => {+ it('evicts the oldest entries past RESPONSE_CACHE_MAX_ENTRIES', () => {+ process.env.RESPONSE_CACHE_MAX_ENTRIES = '3';+ // Realistic, increasing timestamps so reads stay inside the default TTL;+ // the point under test is eviction by count, not TTL expiry.+ const base = 1_700_000_000_000;+ const keys = ['a', 'b', 'c', 'd', 'e'].map(t =>+ computeCacheKey({ model: 'auto', messages: [msg('user', t)] }));+ keys.forEach((k, i) => store(k, `v${i}`, base + i));++ expect(getCacheStats().entries).toBe(3);+ // Two oldest gone, three newest survive.+ expect(getCachedResponse(keys[0]!, base + 10)).toBeNull();+ expect(getCachedResponse(keys[1]!, base + 10)).toBeNull();+ expect(getCachedResponse(keys[4]!, base + 10)).not.toBeNull();+ });++ it('spares a recently-read entry from eviction (true LRU, not FIFO)', () => {+ process.env.RESPONSE_CACHE_MAX_ENTRIES = '3';+ const base = 1_700_000_000_000;+ const keys = ['a', 'b', 'c'].map(t =>+ computeCacheKey({ model: 'auto', messages: [msg('user', t)] }));+ keys.forEach((k, i) => store(k, `v${i}`, base + i)); // a,b,c+ // Touch 'a' so it becomes most-recently-used; 'b' is now the coldest.+ expect(getCachedResponse(keys[0]!, base + 5)).not.toBeNull();+ // Insert a 4th → evicts the LRU, which is now 'b', NOT 'a'.+ const d = computeCacheKey({ model: 'auto', messages: [msg('user', 'd')] });+ store(d, 'v3', base + 6);+ expect(getCachedResponse(keys[0]!, base + 7)).not.toBeNull(); // 'a' survived+ expect(getCachedResponse(keys[1]!, base + 7)).toBeNull(); // 'b' evicted+ });+ });++ describe('clearCache', () => {+ it('removes every entry', () => {+ store(computeCacheKey({ model: 'auto', messages: [msg('user', 'x')] }), 'x');+ store(computeCacheKey({ model: 'auto', messages: [msg('user', 'y')] }), 'y');+ expect(clearCache()).toBe(2);+ expect(getCacheStats().entries).toBe(0);+ });+ });++ describe('temperature guard', () => {+ it('caches omitted temperature', () => {+ expect(isCacheableTemperature(undefined)).toBe(true);+ });+ it('respects RESPONSE_CACHE_MAX_TEMPERATURE', () => {+ process.env.RESPONSE_CACHE_MAX_TEMPERATURE = '0.5';+ expect(isCacheableTemperature(0.2)).toBe(true);+ expect(isCacheableTemperature(0.5)).toBe(true);+ expect(isCacheableTemperature(0.9)).toBe(false);+ });+ });++ describe('directive parsing', () => {+ it('parses off/on aliases', () => {+ expect(parseCacheDirective('off')).toBe('off');+ expect(parseCacheDirective('bypass')).toBe('off');+ expect(parseCacheDirective('on')).toBe('on');+ expect(parseCacheDirective('force')).toBe('on');+ expect(parseCacheDirective(undefined)).toBe('default');+ });++ it('treats Cache-Control: no-store as off', () => {+ expect(parseCacheDirective(undefined, 'no-store')).toBe('off');+ });++ it('takes the first value of a repeated header', () => {+ expect(parseCacheDirective(['off', 'on'])).toBe('off');+ });+ });++ describe('cacheActive (env switch)', () => {+ it('off directive always wins; on directive forces; default follows env', () => {+ delete process.env.RESPONSE_CACHE;+ expect(isCacheEnabled()).toBe(false);+ expect(cacheActive('off')).toBe(false);+ expect(cacheActive('on')).toBe(true);+ expect(cacheActive('default')).toBe(false);+ process.env.RESPONSE_CACHE = 'on';+ expect(cacheActive('default')).toBe(true);+ expect(cacheActive('off')).toBe(false);+ });+ });++ describe('settings-backed toggle (no restart)', () => {+ it('the stored setting enables the cache even with the env var unset', () => {+ delete process.env.RESPONSE_CACHE;+ expect(isCacheEnabled()).toBe(false);+ setSetting(CACHE_ENABLED_SETTING, '1');+ expect(isCacheEnabled()).toBe(true);+ });++ it('the stored setting wins over the env var', () => {+ process.env.RESPONSE_CACHE = 'on';+ setSetting(CACHE_ENABLED_SETTING, '0');+ expect(isCacheEnabled()).toBe(false); // explicit off beats env on+ setSetting(CACHE_ENABLED_SETTING, ''); // cleared → fall back to env+ expect(isCacheEnabled()).toBe(true);+ });+ });+});diff --git a/server/src/app.ts b/server/src/app.tsindex d54fd12a..1f64e1e5 100644--- a/server/src/app.ts+++ b/server/src/app.ts@@ -16,6 +16,7 @@ import { analyticsRouter } from './routes/analytics.js';import { healthRouter } from './routes/health.js';import { settingsRouter } from './routes/settings.js';import { premiumRouter } from './routes/premium.js';+import { cacheRouter } from './routes/cache.js';import { authRouter } from './routes/auth.js';import { requireAuth } from './middleware/requireAuth.js';import { createProxyRateLimiter } from './middleware/rateLimit.js';@@ -77,6 +78,7 @@ export function createApp(config?: Config) {app.use('/api/health', requireAuth, healthRouter);app.use('/api/settings', requireAuth, settingsRouter);app.use('/api/premium', requireAuth, premiumRouter);+ app.use('/api/cache', requireAuth, cacheRouter);// OpenAI-compatible proxy. Per-IP rate limiting (#35 item #6) runs first so// it throttles unauthenticated brute-force / flood attempts before anydiff --git a/server/src/routes/cache.ts b/server/src/routes/cache.tsnew file mode 100644index 00000000..c5f74e6e--- /dev/null+++ b/server/src/routes/cache.ts@@ -0,0 +1,50 @@+import { Router } from 'express';+import type { Request, Response } from 'express';+import { setSetting } from '../db/index.js';+import {+ getCacheStats,+ clearCache,+ isCacheEnabled,+ cacheTtlMs,+ cacheMaxEntries,+ cacheMaxTemperature,+ CACHE_ENABLED_SETTING,+} from '../services/cache.js';++export const cacheRouter = Router();++// Cache status + savings for the dashboard. "saved" tokens are provider tokens+// that hits avoided spending, i.e. the free-tier quota the cache gave back.+cacheRouter.get('/stats', (_req: Request, res: Response) => {+ const stats = getCacheStats();+ res.json({+ enabled: isCacheEnabled(),+ ttlSeconds: Math.round(cacheTtlMs() / 1000),+ maxEntries: cacheMaxEntries(),+ maxTemperature: cacheMaxTemperature(),+ ...stats,+ savedTokens: stats.savedPromptTokens + stats.savedCompletionTokens,+ });+});++// Toggle the cache at runtime (no restart). Persisted in settings, which wins+// over the RESPONSE_CACHE env var. Clears the value to fall back to the env+// default when `enabled` is null.+cacheRouter.put('/config', (req: Request, res: Response) => {+ const { enabled } = req.body ?? {};+ if (enabled === null) {+ setSetting(CACHE_ENABLED_SETTING, '');+ } else if (typeof enabled === 'boolean') {+ setSetting(CACHE_ENABLED_SETTING, enabled ? '1' : '0');+ } else {+ res.status(400).json({ error: { message: '`enabled` must be a boolean or null', type: 'invalid_request_error' } });+ return;+ }+ res.json({ enabled: isCacheEnabled() });+});++// Flush the cache (e.g. after changing keys/models, or to force fresh answers).+cacheRouter.delete('/', (_req: Request, res: Response) => {+ const removed = clearCache();+ res.json({ cleared: removed });+});diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 8ec837b2..6c4dc739 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -16,6 +16,7 @@ import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandofimport { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL_ID } from '../services/fusion.js';import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isProviderBadRequestError } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';+import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';import type { Platform } from '@freellmapi/shared/types.js';import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from '../services/model-groups.js';@@ -1239,6 +1240,49 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {return;}+ // ── Response cache (services/cache.ts) ──+ // Opt-in exact-match cache. An identical earlier request is replayed from an+ // in-memory LRU without spending any provider quota. Computed here, after+ // message + sampling-param normalization but before any routing/session work,+ // so a hit short-circuits the whole pipeline. Only NON-streaming requests at a+ // cacheable temperature are eligible (v1 scope: streaming always bypasses); a+ // per-request `X-FreeLLM-Cache` header can force or bypass. Off unless enabled+ // via the RESPONSE_CACHE env var or the response_cache_enabled setting.+ const cacheDirective = parseCacheDirective(req.headers['x-freellm-cache'], req.headers['cache-control']);+ const cacheKey = (!stream && cacheActive(cacheDirective) && isCacheableTemperature(temperature))+ ? computeCacheKey({+ model: requestedModel, messages, temperature, top_p, max_tokens, tools, tool_choice,+ // Normalized stop (providerSafeStop), i.e. what is actually forwarded.+ stop,+ // The knobs below are NOT in chatCompletionSchema, so zod strips them+ // from parsed.data; read them from the raw body. They still change what+ // answer the client is asking for, so requests differing only in one of+ // them must never collide on a cached entry. Explicit null is coerced+ // to undefined (dropped from the key) to match how the proxy treats+ // null-valued optional knobs as absent.+ response_format: req.body?.response_format ?? undefined,+ n: req.body?.n ?? undefined,+ seed: req.body?.seed ?? undefined,+ presence_penalty: req.body?.presence_penalty ?? undefined,+ frequency_penalty: req.body?.frequency_penalty ?? undefined,+ logit_bias: req.body?.logit_bias ?? undefined,+ logprobs: req.body?.logprobs ?? undefined,+ top_logprobs: req.body?.top_logprobs ?? undefined,+ })+ : null;+ if (cacheKey) {+ const hit = getCachedResponse(cacheKey);+ if (hit) {+ // A hit consumes NO provider quota, so recordRequest/recordTokens are+ // deliberately skipped and the reply is not re-logged as provider usage.+ // The savings are reported separately by GET /api/cache/stats.+ res.setHeader('X-Routed-Via', 'cache');+ res.setHeader('X-FreeLLM-Cache', 'HIT');+ res.json(hit.body);+ return;+ }+ }+// Optional client-managed session affinity (see getSessionKey). Express// lower-cases header names; a repeated header arrives as an array — take// the first value.@@ -1741,7 +1785,24 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}}// Normalize array-shaped message.content to a string on the way out (#166).- res.json(sanitizeResponse(normalizeOutboundContent(result)));+ const outboundBody = sanitizeResponse(normalizeOutboundContent(result));+ res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');+ res.json(outboundBody);++ // Cache the freshly-generated answer so an identical later request is+ // served from memory without spending another free-tier slot. A+ // truncated turn (finish_reason 'length') is NOT cached: replaying a+ // cut-off answer forever would be worse than regenerating.+ if (cacheKey && result.choices?.[0]?.finish_reason !== 'length') {+ storeCachedResponse(cacheKey, {+ body: outboundBody,+ platform: route.platform,+ modelId: route.modelId,+ keyId: route.keyId,+ promptTokens: result.usage?.prompt_tokens ?? 0,+ completionTokens: result.usage?.completion_tokens ?? 0,+ });+ }traceRouteEvent('Proxy', {event: 'ok',diff --git a/server/src/services/cache.ts b/server/src/services/cache.tsnew file mode 100644index 00000000..78e175b5--- /dev/null+++ b/server/src/services/cache.ts@@ -0,0 +1,342 @@+// Opt-in exact-match response cache.+//+// A free-tier-stacking proxy lives or dies by how far it stretches scarce+// quota. Re-asking a model the *same* prompt burns a free-tier slot for an+// answer we already have, which is pure waste. This cache stores successful+// completions keyed by a canonical hash of the request and serves an identical+// later request straight from memory, without touching any provider: zero quota+// cost, near-zero latency, and one fewer 429 on the way to the daily reset.+//+// Design choices that keep it SAFE:+// - Exact match only. No embeddings / fuzzy matching, so a near-miss can never+// return a different prompt's answer. The key is a SHA-256 over the+// canonicalized request, so a single token of difference is a miss.+// - The key is the REQUEST, not the route. Any model's good answer to an+// identical prompt is a valid hit, which is what maximizes the hit rate for+// auto-routed traffic. platform/model_id/key_id are stored for attribution+// and savings accounting only.+// - Opt-in. Off unless enabled via the RESPONSE_CACHE env var or the+// response_cache_enabled setting, so existing installs see no behavior+// change. A per-request header overrides either way.+// - Temperature-gated. High-temperature requests are asking for variety, so+// replaying one frozen answer would defeat that. Cached only when the+// temperature is omitted or at/below RESPONSE_CACHE_MAX_TEMPERATURE.+// - In-memory and bounded. Entries live in a size-capped LRU map (oldest use+// evicted first), so the cache can never grow without bound and a restart+// simply flushes it. No schema, no migration, no persisted response blobs.+// - Fail-safe reads. The settings lookup that decides the master switch is+// wrapped, so a not-yet-initialized DB disables the cache rather than+// throwing in the proxy hot path (mirrors services/ratelimit.ts).++import crypto from 'crypto';+import { getSetting } from '../db/index.js';+import type { ChatMessage } from '@freellmapi/shared/types.js';++// ── Config (read on each call so tests and the dashboard can toggle live) ──++function envFlag(name: string, fallback: boolean): boolean {+ const raw = process.env[name];+ if (raw === undefined || raw.trim() === '') return fallback;+ return /^(1|true|on|yes)$/i.test(raw.trim());+}++function envNum(name: string, fallback: number): number {+ const raw = process.env[name];+ if (raw === undefined || raw.trim() === '') return fallback;+ const n = Number(raw);+ return Number.isFinite(n) && n >= 0 ? n : fallback;+}++// DB-absent-safe settings read: a not-yet-initialized DB (or any read error)+// must not throw on the proxy hot path, so it degrades to "no setting stored".+function readSetting(key: string): string | undefined {+ try {+ return getSetting(key);+ } catch {+ return undefined;+ }+}++// Setting key that lets the dashboard toggle the cache at runtime, no restart.+export const CACHE_ENABLED_SETTING = 'response_cache_enabled';++/**+ * Master switch. Default off so adopting the cache is an explicit choice. The+ * settings-table value wins when present (dashboard toggle, no restart), then+ * the RESPONSE_CACHE env var, then off.+ */+export function isCacheEnabled(): boolean {+ const stored = readSetting(CACHE_ENABLED_SETTING);+ if (stored !== undefined && stored.trim() !== '') {+ return /^(1|true|on|yes)$/i.test(stored.trim());+ }+ return envFlag('RESPONSE_CACHE', false);+}++/** Entry lifetime. Default 1h: long enough to absorb retries and agent re-runs,+ * short enough that a refreshed catalog or key changes answers soon after. */+export function cacheTtlMs(): number {+ return envNum('RESPONSE_CACHE_TTL_SECONDS', 3600) * 1000;+}++/** Above this temperature a request wants variety, so it is never cached.+ * Default 1.0 caches everything when enabled (max quota savings); lower it to+ * restrict caching to (near-)deterministic calls. */+export function cacheMaxTemperature(): number {+ return envNum('RESPONSE_CACHE_MAX_TEMPERATURE', 1.0);+}++/** Hard cap on stored entries; least-recently-used are evicted past this.+ * Bounds memory use. */+export function cacheMaxEntries(): number {+ return Math.floor(envNum('RESPONSE_CACHE_MAX_ENTRIES', 5000));+}++// A request is cacheable only when its temperature is omitted (caller accepts+// the provider default and is fine with a stable answer) or at/below the cap.+export function isCacheableTemperature(temperature?: number | null): boolean {+ if (temperature === undefined || temperature === null) return true;+ return temperature <= cacheMaxTemperature();+}++// ── Per-request directive ──+// `X-FreeLLM-Cache: off|on` (and the standard `Cache-Control: no-store`) let a+// caller override the global switch for one request, e.g. force a fresh+// generation, or opt a single call into caching on an otherwise cache-off+// install.+export type CacheDirective = 'default' | 'off' | 'on';++export function parseCacheDirective(+ header: string | string[] | undefined,+ cacheControl?: string | string[] | undefined,+): CacheDirective {+ const cc = (Array.isArray(cacheControl) ? cacheControl[0] : cacheControl)?.toLowerCase() ?? '';+ if (cc.includes('no-store') || cc.includes('no-cache')) return 'off';+ const raw = (Array.isArray(header) ? header[0] : header)?.trim().toLowerCase();+ if (!raw) return 'default';+ if (/^(off|no|0|false|bypass|skip)$/.test(raw)) return 'off';+ if (/^(on|yes|1|true|force)$/.test(raw)) return 'on';+ return 'default';+}++/** Resolve the global switch + per-request directive into a single yes/no. */+export function cacheActive(directive: CacheDirective): boolean {+ if (directive === 'off') return false;+ if (directive === 'on') return true;+ return isCacheEnabled();+}++// ── Canonical key ──++// Deterministic JSON: object keys sorted recursively and undefined dropped, so+// two requests that differ only in key order or omitted-vs-undefined fields+// hash identically. (JSON.stringify alone preserves insertion order, which+// varies between clients and would scatter otherwise-identical requests.)+function stableStringify(value: unknown): string {+ if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;+ const obj = value as Record<string, unknown>;+ const parts = Object.keys(obj)+ .sort()+ .filter(k => obj[k] !== undefined)+ .map(k => `${JSON.stringify(k)}:${stableStringify(obj[k])}`);+ return `{${parts.join(',')}}`;+}++export interface CacheKeyInput {+ model: string | undefined; // the client's `model` field ('auto'/pinned/omitted)+ messages: ChatMessage[];+ temperature?: number;+ top_p?: number;+ max_tokens?: number;+ tools?: unknown;+ tool_choice?: unknown;+ // Every remaining sampling/format knob a client can send. All are part of the+ // key so two requests that differ ONLY in one of these can never be served+ // each other's cached answer (worst case otherwise: a response_format+ // json_object request gets a cached plain-text reply). Wrong-answer+ // collisions are worse than missed hits, so these are keyed even when the+ // proxy does not currently forward them upstream. Loosely typed on purpose:+ // several arrive un-validated straight from the request body.+ stop?: unknown;+ response_format?: unknown;+ n?: unknown;+ seed?: unknown;+ presence_penalty?: unknown;+ frequency_penalty?: unknown;+ logit_bias?: unknown;+ logprobs?: unknown;+ top_logprobs?: unknown;+}++function normModel(model: string | undefined): string {+ // Omitted and the explicit "auto" sentinel mean the same thing (let the router+ // decide), so they must share a cache bucket.+ return !model || model === 'auto' ? 'auto' : model;+}++export function computeCacheKey(input: CacheKeyInput): string {+ const canonical = stableStringify({+ v: 2, // bump to invalidate every entry if the cached shape ever changes+ model: normModel(input.model),+ messages: input.messages,+ temperature: input.temperature,+ top_p: input.top_p,+ max_tokens: input.max_tokens,+ // tools/tool_choice are part of the key so a request with a different tool+ // set never collides with (or is served) another's cached answer.+ tools: input.tools,+ tool_choice: input.tool_choice,+ // Remaining knobs (see CacheKeyInput). Absent/undefined fields are dropped+ // by stableStringify, so requests without them keep hashing identically.+ stop: input.stop,+ response_format: input.response_format,+ n: input.n,+ seed: input.seed,+ presence_penalty: input.presence_penalty,+ frequency_penalty: input.frequency_penalty,+ logit_bias: input.logit_bias,+ logprobs: input.logprobs,+ top_logprobs: input.top_logprobs,+ });+ return crypto.createHash('sha256').update(canonical).digest('hex');+}++// ── Store ──++export interface CachedResponse {+ body: unknown; // the full OpenAI-shaped completion JSON, replayed verbatim+ platform: string;+ modelId: string;+ keyId: number | null;+ promptTokens: number;+ completionTokens: number;+}++export interface StoreInput {+ body: unknown;+ platform: string;+ modelId: string;+ keyId: number | null;+ promptTokens: number;+ completionTokens: number;+}++interface CacheEntry {+ body: unknown;+ platform: string;+ modelId: string;+ keyId: number | null;+ promptTokens: number;+ completionTokens: number;+ hitCount: number;+ createdAtMs: number;+ lastHitAtMs: number | null;+}++// Insertion-ordered map used as an LRU: the first key is the least-recently+// used, the last is the most-recently used. A read or write re-inserts its key+// at the end (delete + set), so eviction from the front drops the coldest entry.+const store = new Map<string, CacheEntry>();++/**+ * Look up a cached completion. Returns null on a miss or when the entry has aged+ * past the TTL (expired entries are deleted lazily on read). A hit bumps the+ * entry's hit_count and moves it to most-recently-used.+ */+export function getCachedResponse(cacheKey: string, now = Date.now()): CachedResponse | null {+ const entry = store.get(cacheKey);+ if (!entry) return null;++ if (now - entry.createdAtMs > cacheTtlMs()) {+ store.delete(cacheKey);+ return null;+ }++ entry.hitCount += 1;+ entry.lastHitAtMs = now;+ // Move to most-recently-used.+ store.delete(cacheKey);+ store.set(cacheKey, entry);++ return {+ body: entry.body,+ platform: entry.platform,+ modelId: entry.modelId,+ keyId: entry.keyId,+ promptTokens: entry.promptTokens,+ completionTokens: entry.completionTokens,+ };+}++/**+ * Store a successful completion. Overwrites any existing entry for the key (a+ * re-generation refreshes the cached answer, its TTL, and its hit count).+ * Enforces the entry cap by evicting the least-recently-used entries. Best-+ * effort: an unserializable body is skipped so caching can never break a+ * request that already succeeded.+ */+export function storeCachedResponse(cacheKey: string, input: StoreInput, now = Date.now()): void {+ // Reject bodies that can't be JSON-serialized; a hit must be replayable.+ try {+ JSON.stringify(input.body);+ } catch {+ return;+ }++ // Delete-then-set so an overwrite also refreshes recency order.+ store.delete(cacheKey);+ store.set(cacheKey, {+ body: input.body,+ platform: input.platform,+ modelId: input.modelId,+ keyId: input.keyId,+ promptTokens: input.promptTokens,+ completionTokens: input.completionTokens,+ hitCount: 0,+ createdAtMs: now,+ lastHitAtMs: null,+ });++ // Evict least-recently-used beyond the cap. The count only drifts by one per+ // insert, so at most one entry is removed per call in steady state.+ const cap = cacheMaxEntries();+ while (store.size > cap) {+ const oldest = store.keys().next().value as string | undefined;+ if (oldest === undefined) break;+ store.delete(oldest);+ }+}++// ── Stats / admin ──++export interface CacheStats {+ entries: number;+ totalHits: number;+ savedPromptTokens: number;+ savedCompletionTokens: number;+}++/**+ * Aggregate cache stats for the dashboard. "saved" tokens are the provider+ * tokens that hits avoided spending: hit_count x the entry's token counts,+ * summed, i.e. the free-tier quota the cache gave back.+ */+export function getCacheStats(): CacheStats {+ let totalHits = 0;+ let savedPromptTokens = 0;+ let savedCompletionTokens = 0;+ for (const entry of store.values()) {+ totalHits += entry.hitCount;+ savedPromptTokens += entry.hitCount * entry.promptTokens;+ savedCompletionTokens += entry.hitCount * entry.completionTokens;+ }+ return { entries: store.size, totalHits, savedPromptTokens, savedCompletionTokens };+}++/** Drop every cached entry. Returns the number removed. */+export function clearCache(): number {+ const removed = store.size;+ store.clear();+ return removed;+}