Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat: opt-in response cache for /v1/chat/completions (#481)

Tashfeend5a89d53c835
diff --git a/.env.example b/.env.example
index c48e05e9..db84f424 100644
--- a/.env.example
+++ b/.env.example
@@ -38,6 +38,23 @@ PORT=3001
# set to 0 to disable the cap. Example — cap OpenRouter at 50 requests/day:
# PROVIDER_DAILY_REQUEST_CAP_OPENROUTER=50
+# Opt-in response cache. When on, a successful non-streaming /v1/chat/completions
+# answer is stored in a bounded in-memory LRU keyed by a canonical hash of the
+# request, so an IDENTICAL later request is served from memory without spending
+# any provider quota. Exact match only (no fuzzy matching), so a near-miss never
+# returns a different prompt's answer. Off by default; set to true to enable.
+# Can also be toggled at runtime via the dashboard (PUT /api/cache/config), and
+# per-request with the `X-FreeLLM-Cache: on|off` header. A restart flushes it.
+# RESPONSE_CACHE=false
+# How long a cached answer stays fresh, in seconds (default: 3600 = 1 hour).
+# RESPONSE_CACHE_TTL_SECONDS=3600
+# Only cache requests at/below this temperature; higher temperatures want fresh
+# variety, so they are never cached. Default 1.0 caches everything when enabled;
+# lower it (e.g. 0.2) to cache only near-deterministic calls.
+# RESPONSE_CACHE_MAX_TEMPERATURE=1.0
+# Hard cap on stored entries; the least-recently-used are evicted past this.
+# RESPONSE_CACHE_MAX_ENTRIES=5000
+
# Custom-provider URL policy. Cloud metadata and link-local addresses
# (169.254.169.254, metadata.google.internal, fe80::/10, ...) are ALWAYS
# blocked as custom provider base URLs. By default localhost and private LAN
diff --git a/README.md b/README.md
index ec78cbe8..88833b62 100644
--- a/README.md
+++ b/README.md
@@ -850,6 +850,7 @@ See [CONTRIBUTING.md](./CONTRIBUTING.md) for the full migration CLI and workflow
<a href="https://github.com/iisyw"><img src="https://images.weserv.nl/?url=github.com/iisyw.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@iisyw" /></a>
<a href="https://github.com/xsfhacg"><img src="https://images.weserv.nl/?url=github.com/xsfhacg.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@xsfhacg" /></a>
<a href="https://github.com/noobix"><img src="https://images.weserv.nl/?url=github.com/noobix.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@noobix" /></a>
+<a href="https://github.com/nandukmelath"><img src="https://images.weserv.nl/?url=github.com/nandukmelath.png&w=60&h=60&fit=cover&mask=circle" width="60" alt="@nandukmelath" /></a>
## Terms of Service review
diff --git a/server/src/__tests__/routes/proxy-cache.test.ts b/server/src/__tests__/routes/proxy-cache.test.ts
new file mode 100644
index 00000000..72766cfa
--- /dev/null
+++ b/server/src/__tests__/routes/proxy-cache.test.ts
@@ -0,0 +1,240 @@
+import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
+import type { Express } from 'express';
+import { createApp } from '../../app.js';
+import { initDb, getDb, getUnifiedApiKey } from '../../db/index.js';
+import { clearCache } from '../../services/cache.js';
+import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';
+
+let dashToken = '';
+
+async function request(app: Express, method: string, path: string, body?: any, headers: Record<string, string> = {}) {
+ const server = app.listen(0);
+ const addr = server.address() as any;
+ const url = `http://127.0.0.1:${addr.port}${path}`;
+ const res = await fetch(url, {
+ method,
+ headers: {
+ ...(body ? { 'Content-Type': 'application/json' } : {}),
+ ...(isGatedApiPath(path) && !('Authorization' in headers) ? { Authorization: `Bearer ${dashToken}` } : {}),
+ ...headers,
+ },
+ body: body ? JSON.stringify(body) : undefined,
+ });
+ const raw = await res.text();
+ server.close();
+ let json: any = null;
+ try { json = JSON.parse(raw); } catch { /* SSE or empty body */ }
+ return { status: res.status, body: json, headers: res.headers, raw };
+}
+
+function authHeaders() {
+ return { Authorization: `Bearer ${getUnifiedApiKey()}` };
+}
+
+// Mock every groq chat-completion call, counting how many actually reach the
+// provider (separately for streaming vs non-streaming). The whole point of the
+// cache is that a repeat non-streaming request does NOT reach the provider.
+function mockGroq(content: string) {
+ const origFetch = global.fetch;
+ const counter = { calls: 0, streamCalls: 0 };
+ vi.spyOn(global, 'fetch').mockImplementation(async (url, init) => {
+ const urlStr = typeof url === 'string' ? url : url.toString();
+ if (urlStr.includes('api.groq.com/openai/v1/chat/completions')) {
+ const reqBody = JSON.parse(String((init as RequestInit).body));
+ if (reqBody.stream) {
+ counter.streamCalls++;
+ const sse = [
+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: { role: 'assistant', content: null }, finish_reason: null }] },
+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: { content }, finish_reason: null }] },
+ { id: 'c1', object: 'chat.completion.chunk', created: 1, model: 'openai/gpt-oss-120b', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 } },
+ ].map(f => `data: ${JSON.stringify(f)}\n\n`).join('') + 'data: [DONE]\n\n';
+ return new Response(sse, { status: 200, headers: { 'Content-Type': 'text/event-stream' } }) as any;
+ }
+ counter.calls++;
+ return {
+ ok: true,
+ json: () => Promise.resolve({
+ id: 'chatcmpl-cache-it',
+ object: 'chat.completion',
+ created: 123,
+ model: 'openai/gpt-oss-120b',
+ choices: [{ index: 0, message: { role: 'assistant', content }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 5, completion_tokens: 3, total_tokens: 8 },
+ }),
+ } as any;
+ }
+ return origFetch(url, init);
+ });
+ return counter;
+}
+
+describe('Response cache (proxy integration)', () => {
+ let app: Express;
+
+ beforeAll(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ process.env.RESPONSE_CACHE = 'on';
+ initDb(':memory:');
+ app = createApp();
+ dashToken = mintDashboardToken();
+ });
+
+ afterAll(() => {
+ delete process.env.RESPONSE_CACHE;
+ });
+
+ beforeEach(async () => {
+ const db = getDb();
+ db.prepare('DELETE FROM api_keys').run();
+ db.prepare('DELETE FROM requests').run();
+ db.prepare('DELETE FROM rate_limit_usage').run();
+ clearCache();
+ const addKey = await request(app, 'POST', '/api/keys', { platform: 'groq', key: 'gsk_cache_test', label: 'cache' });
+ expect(addKey.status).toBe(201);
+ });
+
+ afterEach(() => {
+ vi.restoreAllMocks();
+ clearCache();
+ });
+
+ const chat = (extra: Record<string, unknown> = {}, headers: Record<string, string> = {}) =>
+ request(app, 'POST', '/v1/chat/completions', {
+ model: 'auto',
+ messages: [{ role: 'user', content: 'what is 2+2?' }],
+ ...extra,
+ }, { ...authHeaders(), ...headers });
+
+ it('serves an identical second request from cache without calling the provider', async () => {
+ const counter = mockGroq('the answer is four');
+
+ const first = await chat();
+ expect(first.status).toBe(200);
+ expect(first.body.choices[0].message.content).toBe('the answer is four');
+ expect(first.headers.get('x-freellm-cache')).toBe('MISS');
+ expect(counter.calls).toBe(1);
+
+ const second = await chat();
+ expect(second.status).toBe(200);
+ expect(second.body.choices[0].message.content).toBe('the answer is four');
+ expect(second.headers.get('x-freellm-cache')).toBe('HIT');
+ expect(second.headers.get('x-routed-via')).toBe('cache');
+ // The provider was NOT hit again; the whole point.
+ expect(counter.calls).toBe(1);
+ });
+
+ it('does not record rate-limit usage for a cache hit (quota preserved)', async () => {
+ mockGroq('four');
+ await chat();
+ const usageAfterMiss = (getDb().prepare("SELECT COUNT(*) AS c FROM rate_limit_usage WHERE kind='request'").get() as { c: number }).c;
+ await chat(); // hit
+ const usageAfterHit = (getDb().prepare("SELECT COUNT(*) AS c FROM rate_limit_usage WHERE kind='request'").get() as { c: number }).c;
+ expect(usageAfterHit).toBe(usageAfterMiss); // no new provider request recorded
+ });
+
+ it('treats a different prompt as a miss and calls the provider again', async () => {
+ const counter = mockGroq('depends on the prompt');
+ await chat({ messages: [{ role: 'user', content: 'first question' }] });
+ await chat({ messages: [{ role: 'user', content: 'second question' }] });
+ expect(counter.calls).toBe(2);
+ });
+
+ it('requests differing only in response_format, seed, or stop never collide', async () => {
+ const counter = mockGroq('knob-sensitive');
+ await chat(); // plain → MISS, populates
+ // Same prompt with response_format json_object must be a MISS, not a
+ // replay of the cached plain-text answer.
+ const jsonReq = await chat({ response_format: { type: 'json_object' } });
+ expect(jsonReq.headers.get('x-freellm-cache')).toBe('MISS');
+ expect(counter.calls).toBe(2);
+ // Same prompt, different seed / stop → also fresh generations.
+ await chat({ seed: 42 });
+ await chat({ stop: 'END' });
+ expect(counter.calls).toBe(4);
+ // But an exact repeat (with a knob present) is still a HIT.
+ const repeat = await chat({ seed: 42 });
+ expect(repeat.headers.get('x-freellm-cache')).toBe('HIT');
+ expect(counter.calls).toBe(4);
+ });
+
+ it('X-FreeLLM-Cache: off bypasses the cache on both read and write', async () => {
+ const counter = mockGroq('bypassed');
+ await chat({}, { 'X-FreeLLM-Cache': 'off' });
+ await chat({}, { 'X-FreeLLM-Cache': 'off' });
+ // Neither call read nor populated the cache → provider hit twice.
+ expect(counter.calls).toBe(2);
+ });
+
+ it('X-FreeLLM-Cache: on forces caching even when the cache is globally off', async () => {
+ process.env.RESPONSE_CACHE = '';
+ try {
+ const counter = mockGroq('forced-on');
+ const first = await chat({}, { 'X-FreeLLM-Cache': 'on' });
+ expect(first.headers.get('x-freellm-cache')).toBe('MISS');
+ const second = await chat({}, { 'X-FreeLLM-Cache': 'on' });
+ expect(second.headers.get('x-freellm-cache')).toBe('HIT');
+ expect(counter.calls).toBe(1);
+ } finally {
+ process.env.RESPONSE_CACHE = 'on';
+ }
+ });
+
+ it('does not cache a high-temperature request', async () => {
+ process.env.RESPONSE_CACHE_MAX_TEMPERATURE = '0.2';
+ try {
+ const counter = mockGroq('varies');
+ await chat({ temperature: 0.9 });
+ await chat({ temperature: 0.9 });
+ expect(counter.calls).toBe(2); // above the cap → never cached
+ } finally {
+ delete process.env.RESPONSE_CACHE_MAX_TEMPERATURE;
+ }
+ });
+
+ it('a streaming request bypasses the cache: never served from it, never populates it', async () => {
+ const counter = mockGroq('cached four');
+
+ // A non-streaming request populates the cache.
+ const first = await chat();
+ expect(first.headers.get('x-freellm-cache')).toBe('MISS');
+ expect(counter.calls).toBe(1);
+
+ // An identical STREAMING request must not be served from the cache, so it
+ // reaches the provider's streaming endpoint despite the cached entry.
+ const streamed = await chat({ stream: true });
+ expect(streamed.status).toBe(200);
+ expect(counter.streamCalls).toBe(1);
+
+ // And it did not overwrite/populate a cache entry: a following identical
+ // non-streaming request is still a HIT served from the original entry, with
+ // no new non-streaming provider call.
+ const third = await chat();
+ expect(third.headers.get('x-freellm-cache')).toBe('HIT');
+ expect(third.body.choices[0].message.content).toBe('cached four');
+ expect(counter.calls).toBe(1);
+ });
+
+ it('exposes cache stats and savings on the admin endpoint', async () => {
+ mockGroq('four');
+ await chat(); // MISS, populates
+ await chat(); // HIT (1 hit, saves 5 prompt + 3 completion tokens)
+ const stats = await request(app, 'GET', '/api/cache/stats');
+ expect(stats.status).toBe(200);
+ expect(stats.body.enabled).toBe(true);
+ expect(stats.body.entries).toBe(1);
+ expect(stats.body.totalHits).toBe(1);
+ expect(stats.body.savedTokens).toBe(8); // 5 + 3
+ });
+
+ it('DELETE /api/cache flushes stored entries', async () => {
+ mockGroq('four');
+ await chat();
+ let stats = await request(app, 'GET', '/api/cache/stats');
+ expect(stats.body.entries).toBe(1);
+ const cleared = await request(app, 'DELETE', '/api/cache');
+ expect(cleared.status).toBe(200);
+ expect(cleared.body.cleared).toBe(1);
+ stats = await request(app, 'GET', '/api/cache/stats');
+ expect(stats.body.entries).toBe(0);
+ });
+});
diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.ts
new file mode 100644
index 00000000..df867dff
--- /dev/null
+++ b/server/src/__tests__/services/cache.test.ts
@@ -0,0 +1,342 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { initDb, setSetting } from '../../db/index.js';
+import {
+ computeCacheKey,
+ getCachedResponse,
+ storeCachedResponse,
+ getCacheStats,
+ clearCache,
+ isCacheableTemperature,
+ parseCacheDirective,
+ cacheActive,
+ isCacheEnabled,
+ CACHE_ENABLED_SETTING,
+} from '../../services/cache.js';
+import type { ChatMessage } from '@freellmapi/shared/types.js';
+
+const CACHE_ENV = [
+ 'RESPONSE_CACHE',
+ 'RESPONSE_CACHE_TTL_SECONDS',
+ 'RESPONSE_CACHE_MAX_TEMPERATURE',
+ 'RESPONSE_CACHE_MAX_ENTRIES',
+] as const;
+
+function msg(role: ChatMessage['role'], content: string): ChatMessage {
+ return { role, content } as ChatMessage;
+}
+
+const sampleBody = (text: string) => ({
+ id: 'chatcmpl-test',
+ object: 'chat.completion',
+ choices: [{ index: 0, message: { role: 'assistant', content: text }, finish_reason: 'stop' }],
+ usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 },
+});
+
+const store = (key: string, text: string, now?: number) =>
+ storeCachedResponse(key, {
+ body: sampleBody(text),
+ platform: 'groq',
+ modelId: 'llama-3.3-70b',
+ keyId: 7,
+ promptTokens: 10,
+ completionTokens: 5,
+ }, now);
+
+describe('response cache', () => {
+ const saved: Record<string, string | undefined> = {};
+
+ beforeEach(() => {
+ process.env.ENCRYPTION_KEY = '0'.repeat(64);
+ for (const k of CACHE_ENV) saved[k] = process.env[k];
+ // Fresh in-memory DB per test so the settings-backed toggle is isolated.
+ initDb(':memory:');
+ // The cache is module-level in-memory state: flush it between tests.
+ clearCache();
+ });
+
+ afterEach(() => {
+ clearCache();
+ for (const k of CACHE_ENV) {
+ if (saved[k] === undefined) delete process.env[k];
+ else process.env[k] = saved[k]!;
+ }
+ });
+
+ describe('computeCacheKey', () => {
+ it('is stable regardless of object key order', () => {
+ const messages = [msg('user', 'hello')];
+ const a = computeCacheKey({ model: 'auto', messages, temperature: 0.2, top_p: 1 });
+ const b = computeCacheKey({ messages, top_p: 1, temperature: 0.2, model: 'auto' });
+ expect(a).toBe(b);
+ });
+
+ it('treats omitted model and "auto" as the same bucket', () => {
+ const messages = [msg('user', 'hello')];
+ expect(computeCacheKey({ model: undefined, messages }))
+ .toBe(computeCacheKey({ model: 'auto', messages }));
+ });
+
+ it('changes when the prompt changes', () => {
+ expect(computeCacheKey({ model: 'auto', messages: [msg('user', 'a')] }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages: [msg('user', 'b')] }));
+ });
+
+ it('changes when sampling params change', () => {
+ const messages = [msg('user', 'hello')];
+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0 }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages, temperature: 0.9 }));
+ });
+
+ it('distinguishes a pinned model from auto', () => {
+ const messages = [msg('user', 'hello')];
+ expect(computeCacheKey({ model: 'gpt-oss-120b', messages }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages }));
+ });
+
+ // Tools policy: a tool-bearing request is cacheable, but its tool set is
+ // part of the key, so a different tool set is a MISS and never serves
+ // another request's cached answer.
+ it('changes when the tool set changes (no cross-tool contamination)', () => {
+ const messages = [msg('user', 'call a tool')];
+ const toolsA = [{ type: 'function', function: { name: 'Read' } }];
+ const toolsB = [{ type: 'function', function: { name: 'Write' } }];
+ expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages, tools: toolsB }));
+ // Same tools + same prompt collapse to the same key.
+ expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))
+ .toBe(computeCacheKey({ model: 'auto', messages, tools: toolsA }));
+ // A tools request differs from the tool-free version of the same prompt.
+ expect(computeCacheKey({ model: 'auto', messages, tools: toolsA }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages }));
+ });
+
+ // Collision guard for the remaining sampling/format knobs: requests that
+ // differ ONLY in one of these must never share a key, or one could be
+ // served the other's cached answer (worst case: a response_format
+ // json_object request replayed a cached plain-text reply).
+ it('changes when response_format changes (json_object vs none)', () => {
+ const messages = [msg('user', 'give me data')];
+ expect(computeCacheKey({ model: 'auto', messages, response_format: { type: 'json_object' } }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages }));
+ expect(computeCacheKey({ model: 'auto', messages, response_format: { type: 'json_object' } }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages, response_format: { type: 'text' } }));
+ });
+
+ it('changes when stop, seed, or n change', () => {
+ const messages = [msg('user', 'hello')];
+ const base = computeCacheKey({ model: 'auto', messages });
+ expect(computeCacheKey({ model: 'auto', messages, stop: ['\n'] })).not.toBe(base);
+ expect(computeCacheKey({ model: 'auto', messages, stop: 'END' }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages, stop: 'STOP' }));
+ expect(computeCacheKey({ model: 'auto', messages, seed: 42 })).not.toBe(base);
+ expect(computeCacheKey({ model: 'auto', messages, seed: 42 }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages, seed: 43 }));
+ expect(computeCacheKey({ model: 'auto', messages, n: 2 })).not.toBe(base);
+ });
+
+ it('changes when penalties, logit_bias, or logprobs knobs change', () => {
+ const messages = [msg('user', 'hello')];
+ const base = computeCacheKey({ model: 'auto', messages });
+ expect(computeCacheKey({ model: 'auto', messages, presence_penalty: 0.5 })).not.toBe(base);
+ expect(computeCacheKey({ model: 'auto', messages, frequency_penalty: 0.5 })).not.toBe(base);
+ expect(computeCacheKey({ model: 'auto', messages, logit_bias: { '50256': -100 } })).not.toBe(base);
+ expect(computeCacheKey({ model: 'auto', messages, logprobs: true })).not.toBe(base);
+ expect(computeCacheKey({ model: 'auto', messages, logprobs: true, top_logprobs: 5 }))
+ .not.toBe(computeCacheKey({ model: 'auto', messages, logprobs: true }));
+ });
+
+ it('identical requests with the new knobs present collapse to the same key', () => {
+ const messages = [msg('user', 'give me data')];
+ const full = () => computeCacheKey({
+ model: 'auto', messages, temperature: 0.1,
+ response_format: { type: 'json_object' },
+ stop: ['END', 'STOP'], n: 1, seed: 42,
+ presence_penalty: 0.25, frequency_penalty: 0.5,
+ logit_bias: { '50256': -100 }, logprobs: true, top_logprobs: 3,
+ });
+ expect(full()).toBe(full());
+ // Key-order independence holds for the new fields too.
+ expect(computeCacheKey({ seed: 42, stop: 'END', model: 'auto', messages } as any))
+ .toBe(computeCacheKey({ model: 'auto', messages, stop: 'END', seed: 42 }));
+ // And absent knobs still hash like before (undefined dropped, not null-ed).
+ expect(computeCacheKey({ model: 'auto', messages, seed: undefined }))
+ .toBe(computeCacheKey({ model: 'auto', messages }));
+ });
+ });
+
+ describe('store / get round-trip', () => {
+ it('returns null on a miss', () => {
+ expect(getCachedResponse('does-not-exist')).toBeNull();
+ });
+
+ it('returns the stored body on a hit', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
+ store(key, 'cached answer');
+ const hit = getCachedResponse(key);
+ expect(hit).not.toBeNull();
+ expect((hit!.body as any).choices[0].message.content).toBe('cached answer');
+ expect(hit!.platform).toBe('groq');
+ expect(hit!.modelId).toBe('llama-3.3-70b');
+ expect(hit!.keyId).toBe(7);
+ expect(hit!.promptTokens).toBe(10);
+ expect(hit!.completionTokens).toBe(5);
+ });
+
+ it('overwrites and refreshes an existing entry', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
+ store(key, 'first');
+ store(key, 'second');
+ expect((getCachedResponse(key)!.body as any).choices[0].message.content).toBe('second');
+ expect(getCacheStats().entries).toBe(1);
+ });
+
+ it('skips an unserializable body rather than throwing', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
+ const circular: any = {};
+ circular.self = circular;
+ storeCachedResponse(key, {
+ body: circular, platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
+ });
+ expect(getCacheStats().entries).toBe(0);
+ expect(getCachedResponse(key)).toBeNull();
+ });
+ });
+
+ describe('TTL expiry', () => {
+ it('treats an entry older than the TTL as a miss and deletes it', () => {
+ process.env.RESPONSE_CACHE_TTL_SECONDS = '1';
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
+ const t0 = 1_000_000;
+ store(key, 'stale', t0);
+ // 2s later, past the 1s TTL → miss.
+ expect(getCachedResponse(key, t0 + 2000)).toBeNull();
+ // The expired entry was purged.
+ expect(getCacheStats().entries).toBe(0);
+ });
+
+ it('serves an entry still within the TTL', () => {
+ process.env.RESPONSE_CACHE_TTL_SECONDS = '60';
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
+ const t0 = 1_000_000;
+ store(key, 'fresh', t0);
+ expect(getCachedResponse(key, t0 + 30_000)).not.toBeNull();
+ });
+ });
+
+ describe('hit accounting', () => {
+ it('counts hits and tallies saved tokens', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'hi')] });
+ store(key, 'answer');
+ getCachedResponse(key);
+ getCachedResponse(key);
+ getCachedResponse(key);
+ const stats = getCacheStats();
+ expect(stats.entries).toBe(1);
+ expect(stats.totalHits).toBe(3);
+ expect(stats.savedPromptTokens).toBe(30); // 3 hits × 10
+ expect(stats.savedCompletionTokens).toBe(15); // 3 hits × 5
+ });
+ });
+
+ describe('entry cap eviction (LRU)', () => {
+ it('evicts the oldest entries past RESPONSE_CACHE_MAX_ENTRIES', () => {
+ process.env.RESPONSE_CACHE_MAX_ENTRIES = '3';
+ // Realistic, increasing timestamps so reads stay inside the default TTL;
+ // the point under test is eviction by count, not TTL expiry.
+ const base = 1_700_000_000_000;
+ const keys = ['a', 'b', 'c', 'd', 'e'].map(t =>
+ computeCacheKey({ model: 'auto', messages: [msg('user', t)] }));
+ keys.forEach((k, i) => store(k, `v${i}`, base + i));
+
+ expect(getCacheStats().entries).toBe(3);
+ // Two oldest gone, three newest survive.
+ expect(getCachedResponse(keys[0]!, base + 10)).toBeNull();
+ expect(getCachedResponse(keys[1]!, base + 10)).toBeNull();
+ expect(getCachedResponse(keys[4]!, base + 10)).not.toBeNull();
+ });
+
+ it('spares a recently-read entry from eviction (true LRU, not FIFO)', () => {
+ process.env.RESPONSE_CACHE_MAX_ENTRIES = '3';
+ const base = 1_700_000_000_000;
+ const keys = ['a', 'b', 'c'].map(t =>
+ computeCacheKey({ model: 'auto', messages: [msg('user', t)] }));
+ keys.forEach((k, i) => store(k, `v${i}`, base + i)); // a,b,c
+ // Touch 'a' so it becomes most-recently-used; 'b' is now the coldest.
+ expect(getCachedResponse(keys[0]!, base + 5)).not.toBeNull();
+ // Insert a 4th → evicts the LRU, which is now 'b', NOT 'a'.
+ const d = computeCacheKey({ model: 'auto', messages: [msg('user', 'd')] });
+ store(d, 'v3', base + 6);
+ expect(getCachedResponse(keys[0]!, base + 7)).not.toBeNull(); // 'a' survived
+ expect(getCachedResponse(keys[1]!, base + 7)).toBeNull(); // 'b' evicted
+ });
+ });
+
+ describe('clearCache', () => {
+ it('removes every entry', () => {
+ store(computeCacheKey({ model: 'auto', messages: [msg('user', 'x')] }), 'x');
+ store(computeCacheKey({ model: 'auto', messages: [msg('user', 'y')] }), 'y');
+ expect(clearCache()).toBe(2);
+ expect(getCacheStats().entries).toBe(0);
+ });
+ });
+
+ describe('temperature guard', () => {
+ it('caches omitted temperature', () => {
+ expect(isCacheableTemperature(undefined)).toBe(true);
+ });
+ it('respects RESPONSE_CACHE_MAX_TEMPERATURE', () => {
+ process.env.RESPONSE_CACHE_MAX_TEMPERATURE = '0.5';
+ expect(isCacheableTemperature(0.2)).toBe(true);
+ expect(isCacheableTemperature(0.5)).toBe(true);
+ expect(isCacheableTemperature(0.9)).toBe(false);
+ });
+ });
+
+ describe('directive parsing', () => {
+ it('parses off/on aliases', () => {
+ expect(parseCacheDirective('off')).toBe('off');
+ expect(parseCacheDirective('bypass')).toBe('off');
+ expect(parseCacheDirective('on')).toBe('on');
+ expect(parseCacheDirective('force')).toBe('on');
+ expect(parseCacheDirective(undefined)).toBe('default');
+ });
+
+ it('treats Cache-Control: no-store as off', () => {
+ expect(parseCacheDirective(undefined, 'no-store')).toBe('off');
+ });
+
+ it('takes the first value of a repeated header', () => {
+ expect(parseCacheDirective(['off', 'on'])).toBe('off');
+ });
+ });
+
+ describe('cacheActive (env switch)', () => {
+ it('off directive always wins; on directive forces; default follows env', () => {
+ delete process.env.RESPONSE_CACHE;
+ expect(isCacheEnabled()).toBe(false);
+ expect(cacheActive('off')).toBe(false);
+ expect(cacheActive('on')).toBe(true);
+ expect(cacheActive('default')).toBe(false);
+ process.env.RESPONSE_CACHE = 'on';
+ expect(cacheActive('default')).toBe(true);
+ expect(cacheActive('off')).toBe(false);
+ });
+ });
+
+ describe('settings-backed toggle (no restart)', () => {
+ it('the stored setting enables the cache even with the env var unset', () => {
+ delete process.env.RESPONSE_CACHE;
+ expect(isCacheEnabled()).toBe(false);
+ setSetting(CACHE_ENABLED_SETTING, '1');
+ expect(isCacheEnabled()).toBe(true);
+ });
+
+ it('the stored setting wins over the env var', () => {
+ process.env.RESPONSE_CACHE = 'on';
+ setSetting(CACHE_ENABLED_SETTING, '0');
+ expect(isCacheEnabled()).toBe(false); // explicit off beats env on
+ setSetting(CACHE_ENABLED_SETTING, ''); // cleared → fall back to env
+ expect(isCacheEnabled()).toBe(true);
+ });
+ });
+});
diff --git a/server/src/app.ts b/server/src/app.ts
index d54fd12a..1f64e1e5 100644
--- a/server/src/app.ts
+++ b/server/src/app.ts
@@ -16,6 +16,7 @@ import { analyticsRouter } from './routes/analytics.js';
import { healthRouter } from './routes/health.js';
import { settingsRouter } from './routes/settings.js';
import { premiumRouter } from './routes/premium.js';
+import { cacheRouter } from './routes/cache.js';
import { authRouter } from './routes/auth.js';
import { requireAuth } from './middleware/requireAuth.js';
import { createProxyRateLimiter } from './middleware/rateLimit.js';
@@ -77,6 +78,7 @@ export function createApp(config?: Config) {
app.use('/api/health', requireAuth, healthRouter);
app.use('/api/settings', requireAuth, settingsRouter);
app.use('/api/premium', requireAuth, premiumRouter);
+ app.use('/api/cache', requireAuth, cacheRouter);
// OpenAI-compatible proxy. Per-IP rate limiting (#35 item #6) runs first so
// it throttles unauthenticated brute-force / flood attempts before any
diff --git a/server/src/routes/cache.ts b/server/src/routes/cache.ts
new file mode 100644
index 00000000..c5f74e6e
--- /dev/null
+++ b/server/src/routes/cache.ts
@@ -0,0 +1,50 @@
+import { Router } from 'express';
+import type { Request, Response } from 'express';
+import { setSetting } from '../db/index.js';
+import {
+ getCacheStats,
+ clearCache,
+ isCacheEnabled,
+ cacheTtlMs,
+ cacheMaxEntries,
+ cacheMaxTemperature,
+ CACHE_ENABLED_SETTING,
+} from '../services/cache.js';
+
+export const cacheRouter = Router();
+
+// Cache status + savings for the dashboard. "saved" tokens are provider tokens
+// that hits avoided spending, i.e. the free-tier quota the cache gave back.
+cacheRouter.get('/stats', (_req: Request, res: Response) => {
+ const stats = getCacheStats();
+ res.json({
+ enabled: isCacheEnabled(),
+ ttlSeconds: Math.round(cacheTtlMs() / 1000),
+ maxEntries: cacheMaxEntries(),
+ maxTemperature: cacheMaxTemperature(),
+ ...stats,
+ savedTokens: stats.savedPromptTokens + stats.savedCompletionTokens,
+ });
+});
+
+// Toggle the cache at runtime (no restart). Persisted in settings, which wins
+// over the RESPONSE_CACHE env var. Clears the value to fall back to the env
+// default when `enabled` is null.
+cacheRouter.put('/config', (req: Request, res: Response) => {
+ const { enabled } = req.body ?? {};
+ if (enabled === null) {
+ setSetting(CACHE_ENABLED_SETTING, '');
+ } else if (typeof enabled === 'boolean') {
+ setSetting(CACHE_ENABLED_SETTING, enabled ? '1' : '0');
+ } else {
+ res.status(400).json({ error: { message: '`enabled` must be a boolean or null', type: 'invalid_request_error' } });
+ return;
+ }
+ res.json({ enabled: isCacheEnabled() });
+});
+
+// Flush the cache (e.g. after changing keys/models, or to force fresh answers).
+cacheRouter.delete('/', (_req: Request, res: Response) => {
+ const removed = clearCache();
+ res.json({ cleared: removed });
+});
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 8ec837b2..6c4dc739 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -16,6 +16,7 @@ import { getContextHandoffMode, recordIncomingMessages, maybeInjectContextHandof
import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL_ID } from '../services/fusion.js';
import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isProviderBadRequestError } from '../lib/error-classify.js';
import { logRequest } from '../lib/request-log.js';
+import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';
import type { Platform } from '@freellmapi/shared/types.js';
import { inferQuotaPoolKey, type QuotaObservationContext } from '../services/provider-quota.js';
import { isUnifyEnabled, getModelGroups, resolveRequestedIdToMembers } from '../services/model-groups.js';
@@ -1239,6 +1240,49 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
return;
}
+ // ── Response cache (services/cache.ts) ──
+ // Opt-in exact-match cache. An identical earlier request is replayed from an
+ // in-memory LRU without spending any provider quota. Computed here, after
+ // message + sampling-param normalization but before any routing/session work,
+ // so a hit short-circuits the whole pipeline. Only NON-streaming requests at a
+ // cacheable temperature are eligible (v1 scope: streaming always bypasses); a
+ // per-request `X-FreeLLM-Cache` header can force or bypass. Off unless enabled
+ // via the RESPONSE_CACHE env var or the response_cache_enabled setting.
+ const cacheDirective = parseCacheDirective(req.headers['x-freellm-cache'], req.headers['cache-control']);
+ const cacheKey = (!stream && cacheActive(cacheDirective) && isCacheableTemperature(temperature))
+ ? computeCacheKey({
+ model: requestedModel, messages, temperature, top_p, max_tokens, tools, tool_choice,
+ // Normalized stop (providerSafeStop), i.e. what is actually forwarded.
+ stop,
+ // The knobs below are NOT in chatCompletionSchema, so zod strips them
+ // from parsed.data; read them from the raw body. They still change what
+ // answer the client is asking for, so requests differing only in one of
+ // them must never collide on a cached entry. Explicit null is coerced
+ // to undefined (dropped from the key) to match how the proxy treats
+ // null-valued optional knobs as absent.
+ response_format: req.body?.response_format ?? undefined,
+ n: req.body?.n ?? undefined,
+ seed: req.body?.seed ?? undefined,
+ presence_penalty: req.body?.presence_penalty ?? undefined,
+ frequency_penalty: req.body?.frequency_penalty ?? undefined,
+ logit_bias: req.body?.logit_bias ?? undefined,
+ logprobs: req.body?.logprobs ?? undefined,
+ top_logprobs: req.body?.top_logprobs ?? undefined,
+ })
+ : null;
+ if (cacheKey) {
+ const hit = getCachedResponse(cacheKey);
+ if (hit) {
+ // A hit consumes NO provider quota, so recordRequest/recordTokens are
+ // deliberately skipped and the reply is not re-logged as provider usage.
+ // The savings are reported separately by GET /api/cache/stats.
+ res.setHeader('X-Routed-Via', 'cache');
+ res.setHeader('X-FreeLLM-Cache', 'HIT');
+ res.json(hit.body);
+ return;
+ }
+ }
+
// Optional client-managed session affinity (see getSessionKey). Express
// lower-cases header names; a repeated header arrives as an array — take
// the first value.
@@ -1741,7 +1785,24 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
}
}
// Normalize array-shaped message.content to a string on the way out (#166).
- res.json(sanitizeResponse(normalizeOutboundContent(result)));
+ const outboundBody = sanitizeResponse(normalizeOutboundContent(result));
+ res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');
+ res.json(outboundBody);
+
+ // Cache the freshly-generated answer so an identical later request is
+ // served from memory without spending another free-tier slot. A
+ // truncated turn (finish_reason 'length') is NOT cached: replaying a
+ // cut-off answer forever would be worse than regenerating.
+ if (cacheKey && result.choices?.[0]?.finish_reason !== 'length') {
+ storeCachedResponse(cacheKey, {
+ body: outboundBody,
+ platform: route.platform,
+ modelId: route.modelId,
+ keyId: route.keyId,
+ promptTokens: result.usage?.prompt_tokens ?? 0,
+ completionTokens: result.usage?.completion_tokens ?? 0,
+ });
+ }
traceRouteEvent('Proxy', {
event: 'ok',
diff --git a/server/src/services/cache.ts b/server/src/services/cache.ts
new file mode 100644
index 00000000..78e175b5
--- /dev/null
+++ b/server/src/services/cache.ts
@@ -0,0 +1,342 @@
+// Opt-in exact-match response cache.
+//
+// A free-tier-stacking proxy lives or dies by how far it stretches scarce
+// quota. Re-asking a model the *same* prompt burns a free-tier slot for an
+// answer we already have, which is pure waste. This cache stores successful
+// completions keyed by a canonical hash of the request and serves an identical
+// later request straight from memory, without touching any provider: zero quota
+// cost, near-zero latency, and one fewer 429 on the way to the daily reset.
+//
+// Design choices that keep it SAFE:
+// - Exact match only. No embeddings / fuzzy matching, so a near-miss can never
+// return a different prompt's answer. The key is a SHA-256 over the
+// canonicalized request, so a single token of difference is a miss.
+// - The key is the REQUEST, not the route. Any model's good answer to an
+// identical prompt is a valid hit, which is what maximizes the hit rate for
+// auto-routed traffic. platform/model_id/key_id are stored for attribution
+// and savings accounting only.
+// - Opt-in. Off unless enabled via the RESPONSE_CACHE env var or the
+// response_cache_enabled setting, so existing installs see no behavior
+// change. A per-request header overrides either way.
+// - Temperature-gated. High-temperature requests are asking for variety, so
+// replaying one frozen answer would defeat that. Cached only when the
+// temperature is omitted or at/below RESPONSE_CACHE_MAX_TEMPERATURE.
+// - In-memory and bounded. Entries live in a size-capped LRU map (oldest use
+// evicted first), so the cache can never grow without bound and a restart
+// simply flushes it. No schema, no migration, no persisted response blobs.
+// - Fail-safe reads. The settings lookup that decides the master switch is
+// wrapped, so a not-yet-initialized DB disables the cache rather than
+// throwing in the proxy hot path (mirrors services/ratelimit.ts).
+
+import crypto from 'crypto';
+import { getSetting } from '../db/index.js';
+import type { ChatMessage } from '@freellmapi/shared/types.js';
+
+// ── Config (read on each call so tests and the dashboard can toggle live) ──
+
+function envFlag(name: string, fallback: boolean): boolean {
+ const raw = process.env[name];
+ if (raw === undefined || raw.trim() === '') return fallback;
+ return /^(1|true|on|yes)$/i.test(raw.trim());
+}
+
+function envNum(name: string, fallback: number): number {
+ const raw = process.env[name];
+ if (raw === undefined || raw.trim() === '') return fallback;
+ const n = Number(raw);
+ return Number.isFinite(n) && n >= 0 ? n : fallback;
+}
+
+// DB-absent-safe settings read: a not-yet-initialized DB (or any read error)
+// must not throw on the proxy hot path, so it degrades to "no setting stored".
+function readSetting(key: string): string | undefined {
+ try {
+ return getSetting(key);
+ } catch {
+ return undefined;
+ }
+}
+
+// Setting key that lets the dashboard toggle the cache at runtime, no restart.
+export const CACHE_ENABLED_SETTING = 'response_cache_enabled';
+
+/**
+ * Master switch. Default off so adopting the cache is an explicit choice. The
+ * settings-table value wins when present (dashboard toggle, no restart), then
+ * the RESPONSE_CACHE env var, then off.
+ */
+export function isCacheEnabled(): boolean {
+ const stored = readSetting(CACHE_ENABLED_SETTING);
+ if (stored !== undefined && stored.trim() !== '') {
+ return /^(1|true|on|yes)$/i.test(stored.trim());
+ }
+ return envFlag('RESPONSE_CACHE', false);
+}
+
+/** Entry lifetime. Default 1h: long enough to absorb retries and agent re-runs,
+ * short enough that a refreshed catalog or key changes answers soon after. */
+export function cacheTtlMs(): number {
+ return envNum('RESPONSE_CACHE_TTL_SECONDS', 3600) * 1000;
+}
+
+/** Above this temperature a request wants variety, so it is never cached.
+ * Default 1.0 caches everything when enabled (max quota savings); lower it to
+ * restrict caching to (near-)deterministic calls. */
+export function cacheMaxTemperature(): number {
+ return envNum('RESPONSE_CACHE_MAX_TEMPERATURE', 1.0);
+}
+
+/** Hard cap on stored entries; least-recently-used are evicted past this.
+ * Bounds memory use. */
+export function cacheMaxEntries(): number {
+ return Math.floor(envNum('RESPONSE_CACHE_MAX_ENTRIES', 5000));
+}
+
+// A request is cacheable only when its temperature is omitted (caller accepts
+// the provider default and is fine with a stable answer) or at/below the cap.
+export function isCacheableTemperature(temperature?: number | null): boolean {
+ if (temperature === undefined || temperature === null) return true;
+ return temperature <= cacheMaxTemperature();
+}
+
+// ── Per-request directive ──
+// `X-FreeLLM-Cache: off|on` (and the standard `Cache-Control: no-store`) let a
+// caller override the global switch for one request, e.g. force a fresh
+// generation, or opt a single call into caching on an otherwise cache-off
+// install.
+export type CacheDirective = 'default' | 'off' | 'on';
+
+export function parseCacheDirective(
+ header: string | string[] | undefined,
+ cacheControl?: string | string[] | undefined,
+): CacheDirective {
+ const cc = (Array.isArray(cacheControl) ? cacheControl[0] : cacheControl)?.toLowerCase() ?? '';
+ if (cc.includes('no-store') || cc.includes('no-cache')) return 'off';
+ const raw = (Array.isArray(header) ? header[0] : header)?.trim().toLowerCase();
+ if (!raw) return 'default';
+ if (/^(off|no|0|false|bypass|skip)$/.test(raw)) return 'off';
+ if (/^(on|yes|1|true|force)$/.test(raw)) return 'on';
+ return 'default';
+}
+
+/** Resolve the global switch + per-request directive into a single yes/no. */
+export function cacheActive(directive: CacheDirective): boolean {
+ if (directive === 'off') return false;
+ if (directive === 'on') return true;
+ return isCacheEnabled();
+}
+
+// ── Canonical key ──
+
+// Deterministic JSON: object keys sorted recursively and undefined dropped, so
+// two requests that differ only in key order or omitted-vs-undefined fields
+// hash identically. (JSON.stringify alone preserves insertion order, which
+// varies between clients and would scatter otherwise-identical requests.)
+function stableStringify(value: unknown): string {
+ if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null';
+ if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`;
+ const obj = value as Record<string, unknown>;
+ const parts = Object.keys(obj)
+ .sort()
+ .filter(k => obj[k] !== undefined)
+ .map(k => `${JSON.stringify(k)}:${stableStringify(obj[k])}`);
+ return `{${parts.join(',')}}`;
+}
+
+export interface CacheKeyInput {
+ model: string | undefined; // the client's `model` field ('auto'/pinned/omitted)
+ messages: ChatMessage[];
+ temperature?: number;
+ top_p?: number;
+ max_tokens?: number;
+ tools?: unknown;
+ tool_choice?: unknown;
+ // Every remaining sampling/format knob a client can send. All are part of the
+ // key so two requests that differ ONLY in one of these can never be served
+ // each other's cached answer (worst case otherwise: a response_format
+ // json_object request gets a cached plain-text reply). Wrong-answer
+ // collisions are worse than missed hits, so these are keyed even when the
+ // proxy does not currently forward them upstream. Loosely typed on purpose:
+ // several arrive un-validated straight from the request body.
+ stop?: unknown;
+ response_format?: unknown;
+ n?: unknown;
+ seed?: unknown;
+ presence_penalty?: unknown;
+ frequency_penalty?: unknown;
+ logit_bias?: unknown;
+ logprobs?: unknown;
+ top_logprobs?: unknown;
+}
+
+function normModel(model: string | undefined): string {
+ // Omitted and the explicit "auto" sentinel mean the same thing (let the router
+ // decide), so they must share a cache bucket.
+ return !model || model === 'auto' ? 'auto' : model;
+}
+
+export function computeCacheKey(input: CacheKeyInput): string {
+ const canonical = stableStringify({
+ v: 2, // bump to invalidate every entry if the cached shape ever changes
+ model: normModel(input.model),
+ messages: input.messages,
+ temperature: input.temperature,
+ top_p: input.top_p,
+ max_tokens: input.max_tokens,
+ // tools/tool_choice are part of the key so a request with a different tool
+ // set never collides with (or is served) another's cached answer.
+ tools: input.tools,
+ tool_choice: input.tool_choice,
+ // Remaining knobs (see CacheKeyInput). Absent/undefined fields are dropped
+ // by stableStringify, so requests without them keep hashing identically.
+ stop: input.stop,
+ response_format: input.response_format,
+ n: input.n,
+ seed: input.seed,
+ presence_penalty: input.presence_penalty,
+ frequency_penalty: input.frequency_penalty,
+ logit_bias: input.logit_bias,
+ logprobs: input.logprobs,
+ top_logprobs: input.top_logprobs,
+ });
+ return crypto.createHash('sha256').update(canonical).digest('hex');
+}
+
+// ── Store ──
+
+export interface CachedResponse {
+ body: unknown; // the full OpenAI-shaped completion JSON, replayed verbatim
+ platform: string;
+ modelId: string;
+ keyId: number | null;
+ promptTokens: number;
+ completionTokens: number;
+}
+
+export interface StoreInput {
+ body: unknown;
+ platform: string;
+ modelId: string;
+ keyId: number | null;
+ promptTokens: number;
+ completionTokens: number;
+}
+
+interface CacheEntry {
+ body: unknown;
+ platform: string;
+ modelId: string;
+ keyId: number | null;
+ promptTokens: number;
+ completionTokens: number;
+ hitCount: number;
+ createdAtMs: number;
+ lastHitAtMs: number | null;
+}
+
+// Insertion-ordered map used as an LRU: the first key is the least-recently
+// used, the last is the most-recently used. A read or write re-inserts its key
+// at the end (delete + set), so eviction from the front drops the coldest entry.
+const store = new Map<string, CacheEntry>();
+
+/**
+ * Look up a cached completion. Returns null on a miss or when the entry has aged
+ * past the TTL (expired entries are deleted lazily on read). A hit bumps the
+ * entry's hit_count and moves it to most-recently-used.
+ */
+export function getCachedResponse(cacheKey: string, now = Date.now()): CachedResponse | null {
+ const entry = store.get(cacheKey);
+ if (!entry) return null;
+
+ if (now - entry.createdAtMs > cacheTtlMs()) {
+ store.delete(cacheKey);
+ return null;
+ }
+
+ entry.hitCount += 1;
+ entry.lastHitAtMs = now;
+ // Move to most-recently-used.
+ store.delete(cacheKey);
+ store.set(cacheKey, entry);
+
+ return {
+ body: entry.body,
+ platform: entry.platform,
+ modelId: entry.modelId,
+ keyId: entry.keyId,
+ promptTokens: entry.promptTokens,
+ completionTokens: entry.completionTokens,
+ };
+}
+
+/**
+ * Store a successful completion. Overwrites any existing entry for the key (a
+ * re-generation refreshes the cached answer, its TTL, and its hit count).
+ * Enforces the entry cap by evicting the least-recently-used entries. Best-
+ * effort: an unserializable body is skipped so caching can never break a
+ * request that already succeeded.
+ */
+export function storeCachedResponse(cacheKey: string, input: StoreInput, now = Date.now()): void {
+ // Reject bodies that can't be JSON-serialized; a hit must be replayable.
+ try {
+ JSON.stringify(input.body);
+ } catch {
+ return;
+ }
+
+ // Delete-then-set so an overwrite also refreshes recency order.
+ store.delete(cacheKey);
+ store.set(cacheKey, {
+ body: input.body,
+ platform: input.platform,
+ modelId: input.modelId,
+ keyId: input.keyId,
+ promptTokens: input.promptTokens,
+ completionTokens: input.completionTokens,
+ hitCount: 0,
+ createdAtMs: now,
+ lastHitAtMs: null,
+ });
+
+ // Evict least-recently-used beyond the cap. The count only drifts by one per
+ // insert, so at most one entry is removed per call in steady state.
+ const cap = cacheMaxEntries();
+ while (store.size > cap) {
+ const oldest = store.keys().next().value as string | undefined;
+ if (oldest === undefined) break;
+ store.delete(oldest);
+ }
+}
+
+// ── Stats / admin ──
+
+export interface CacheStats {
+ entries: number;
+ totalHits: number;
+ savedPromptTokens: number;
+ savedCompletionTokens: number;
+}
+
+/**
+ * Aggregate cache stats for the dashboard. "saved" tokens are the provider
+ * tokens that hits avoided spending: hit_count x the entry's token counts,
+ * summed, i.e. the free-tier quota the cache gave back.
+ */
+export function getCacheStats(): CacheStats {
+ let totalHits = 0;
+ let savedPromptTokens = 0;
+ let savedCompletionTokens = 0;
+ for (const entry of store.values()) {
+ totalHits += entry.hitCount;
+ savedPromptTokens += entry.hitCount * entry.promptTokens;
+ savedCompletionTokens += entry.hitCount * entry.completionTokens;
+ }
+ return { entries: store.size, totalHits, savedPromptTokens, savedCompletionTokens };
+}
+
+/** Drop every cached entry. Returns the number removed. */
+export function clearCache(): number {
+ const removed = store.size;
+ store.clear();
+ return removed;
+}