7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
feat(cache): let stream:true requests hit the response cache (#1099)
suantea (asuan_suan)dd3d1e9666de
diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.tsindex ae566198..89dde4d8 100644--- a/server/src/__tests__/services/cache.test.ts+++ b/server/src/__tests__/services/cache.test.ts@@ -4,6 +4,8 @@ import {computeCacheKey,getCachedResponse,storeCachedResponse,+ getCachedStreamResponse,+ storeCachedStreamResponse,getCacheStats,clearCache,loadCacheFromDb,@@ -358,6 +360,70 @@ describe('response cache', () => {});});+ describe('streaming cache', () => {+ const frames = (text: string) => [+ `data: ${JSON.stringify({ choices: [{ delta: { content: text }, finish_reason: null }] })}\n\n`,+ 'data: [DONE]\n\n',+ ];++ it('returns null on a streaming miss', () => {+ expect(getCachedStreamResponse('does-not-exist')).toBeNull();+ });++ it('replays the exact SSE frames on a streaming hit', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'stream me')] });+ storeCachedStreamResponse(key, {+ frames: frames('streamed answer'),+ platform: 'groq',+ modelId: 'llama-3.3-70b',+ keyId: 7,+ promptTokens: 10,+ completionTokens: 5,+ });+ const hit = getCachedStreamResponse(key);+ expect(hit).not.toBeNull();+ expect(hit!.frames).toEqual(frames('streamed answer'));+ expect(hit!.platform).toBe('groq');+ expect(hit!.modelId).toBe('llama-3.3-70b');+ });++ it('keeps JSON and streaming entries separate for the same key', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'both')] });+ store(key, 'json answer');+ storeCachedStreamResponse(key, {+ frames: frames('stream answer'),+ platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,+ });+ // The JSON store still serves its entry; the stream store its own.+ expect((getCachedResponse(key)!.body as any).choices[0].message.content).toBe('json answer');+ expect(getCachedStreamResponse(key)!.frames).toEqual(frames('stream answer'));+ });++ it('rejects an empty frame list (nothing to replay)', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'empty')] });+ storeCachedStreamResponse(key, {+ frames: [],+ platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,+ });+ expect(getCachedStreamResponse(key)).toBeNull();+ });++ it('counts streaming hits in the aggregated stats and clearCache clears both', () => {+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'stats')] });+ storeCachedStreamResponse(key, {+ frames: frames('a'), platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 10, completionTokens: 5,+ });+ getCachedStreamResponse(key);+ getCachedStreamResponse(key);+ const stats = getCacheStats();+ expect(stats.entries).toBe(1);+ expect(stats.totalHits).toBe(2);+ expect(stats.savedPromptTokens).toBe(20); // 2 hits × 10+ expect(clearCache()).toBe(1);+ expect(getCacheStats().entries).toBe(0);+ });+ });+describe('settings-backed toggle (no restart)', () => {it('the stored setting enables the cache even with the env var unset', () => {delete process.env.RESPONSE_CACHE;diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.tsindex 8633ca0d..00a46dc8 100644--- a/server/src/routes/proxy.ts+++ b/server/src/routes/proxy.ts@@ -21,7 +21,7 @@ import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODELimport { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';import { logRequest } from '../lib/request-log.js';import { observeServedModel } from '../lib/served-model.js';-import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';+import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse, getCachedStreamResponse, storeCachedStreamResponse } from '../services/cache.js';import { normalizeIdempotencyKey, hashIdempotencyKey, computeIdempotencyFingerprint, lookupIdempotencyReplay, storeIdempotencyResult } from '../services/idempotency.js';import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';@@ -1774,7 +1774,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// per-request `X-FreeLLM-Cache` header can force or bypass. Off unless enabled// via the RESPONSE_CACHE env var or the response_cache_enabled setting.const cacheDirective = parseCacheDirective(req.headers['x-freellm-cache'], req.headers['cache-control']);- const cacheKey = (!stream && cacheActive(cacheDirective) && isCacheableTemperature(temperature))+ // Streaming requests participate in the cache too: same canonical key (the+ // request content is identical), but hits are looked up in the streaming+ // store and replayed as SSE rather than JSON (see below).+ const cacheKey = (cacheActive(cacheDirective) && isCacheableTemperature(temperature))? computeCacheKey({model: requestedModel, messages, temperature, top_p, max_tokens, tools, tool_choice,// Normalized stop (providerSafeStop), i.e. what is actually forwarded.@@ -1800,15 +1803,33 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {}): null;if (cacheKey) {- const hit = getCachedResponse(cacheKey);- if (hit) {- // A hit consumes NO provider quota, so recordRequest/recordTokens are- // deliberately skipped and the reply is not re-logged as provider usage.- // The savings are reported separately by GET /api/cache/stats.- res.setHeader('X-Routed-Via', 'cache');- res.setHeader('X-FreeLLM-Cache', 'HIT');- res.json(hit.body);- return;+ if (stream) {+ // Streaming hit: replay the captured SSE frame sequence verbatim —+ // same zero-quota rationale as a JSON hit, with first-byte semantics+ // preserved (the frames include the leading content chunks, so the+ // first token arrives immediately).+ const streamHit = getCachedStreamResponse(cacheKey);+ if (streamHit) {+ res.setHeader('Content-Type', 'text/event-stream');+ res.setHeader('Cache-Control', 'no-cache');+ res.setHeader('Connection', 'keep-alive');+ res.setHeader('X-Routed-Via', 'cache');+ res.setHeader('X-FreeLLM-Cache', 'HIT');+ for (const frame of streamHit.frames) res.write(frame);+ res.end();+ return;+ }+ } else {+ const hit = getCachedResponse(cacheKey);+ if (hit) {+ // A hit consumes NO provider quota, so recordRequest/recordTokens are+ // deliberately skipped and the reply is not re-logged as provider usage.+ // The savings are reported separately by GET /api/cache/stats.+ res.setHeader('X-Routed-Via', 'cache');+ res.setHeader('X-FreeLLM-Cache', 'HIT');+ res.json(hit.body);+ return;+ }}}@@ -2114,6 +2135,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {// Only evidence when a provider serves a different model than routed// (#534); compared/persisted on success via observeServedModel.let upstreamModel: string | null = null;+ // Every `data: ...` frame the client sees, captured for a possible+ // streaming cache store on success (exact SSE replay on a later hit).+ const streamFrames: string[] = [];const flushHeaders = () => {if (headerSent) return;@@ -2124,12 +2148,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {res.setHeader('Cache-Control', 'no-cache');res.setHeader('Connection', 'keep-alive');res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));+ res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');setFallbackHeaders(res, attempt, attemptLog);headerSent = true;// Committed: the answer is on its way, so the retry budget must no// longer cancel this attempt (it could not fail over now anyway).ctx.disarmHedge();- for (const p of preamble) res.write(`data: ${JSON.stringify(p)}\n\n`);+ for (const p of preamble) {+ const frame = `data: ${JSON.stringify(p)}\n\n`;+ streamFrames.push(frame);+ res.write(frame);+ }preamble.length = 0;};const mkChunk = (delta: Record<string, unknown>, finish: string | null) => ({@@ -2139,7 +2168,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {model: lastMeta.model ?? route.modelId,choices: [{ index: 0, delta, finish_reason: finish }],});- const writeChunk = (c: unknown) => res.write(`data: ${JSON.stringify(c)}\n\n`);+ const writeChunk = (c: unknown) => {+ const frame = `data: ${JSON.stringify(c)}\n\n`;+ streamFrames.push(frame);+ res.write(frame);+ };try {const gen = route.provider.streamChatCompletion(@@ -2444,7 +2477,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {},});}- res.write('data: [DONE]\n\n');+ const doneFrame = 'data: [DONE]\n\n';+ streamFrames.push(doneFrame);+ res.write(doneFrame);res.end();const upstreamUsage = (usageChunk as { usage?: TokenUsage } | null)?.usage;@@ -2452,6 +2487,23 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {const outputTokens = upstreamUsage?.completion_tokens ?? totalOutputTokens;const totalTokens = upstreamUsage?.total_tokens ?? (inputTokens + outputTokens);recordUpstreamSuccess(route, totalTokens);++ // Cache the freshly-generated SSE sequence so an identical later+ // stream request is replayed without spending another free-tier+ // slot. A truncated turn (finish 'length') is NOT cached, matching+ // the JSON cache policy — replaying a cut-off answer would be worse+ // than regenerating.+ if (cacheKey && finish !== 'length') {+ storeCachedStreamResponse(cacheKey, {+ frames: streamFrames,+ platform: route.platform,+ modelId: route.modelId,+ keyId: route.keyId,+ promptTokens: inputTokens,+ completionTokens: outputTokens,+ });+ }+setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });// #797: remember this turn's thinking trace so the next request fromdiff --git a/server/src/services/cache.ts b/server/src/services/cache.tsindex c8256e62..129d8adf 100644--- a/server/src/services/cache.ts+++ b/server/src/services/cache.ts@@ -280,6 +280,31 @@ interface CacheEntry {// at the end (delete + set), so eviction from the front drops the coldest entry.const store = new Map<string, CacheEntry>();+// Streaming entries live in their own LRU: a stream's replayable artifact is+// the exact SSE frame sequence, which is structurally different from a JSON+// completion body, so the two kinds never share a key across stores.+interface StreamCacheEntry {+ frames: string[]; // verbatim `data: {...}\n\n` (and final `[DONE]`) frames+ platform: string;+ modelId: string;+ keyId: number | null;+ promptTokens: number;+ completionTokens: number;+ hitCount: number;+ createdAtMs: number;+ lastHitAtMs: number | null;+}+const streamStore = new Map<string, StreamCacheEntry>();++function evictToCap<K, V>(map: Map<K, V>): void {+ const cap = cacheMaxEntries();+ while (map.size > cap) {+ const oldest = map.keys().next().value as K | undefined;+ if (oldest === undefined) break;+ map.delete(oldest);+ }+}+// Lifetime-of-process lookup tallies, the denominator behind the dashboard's// hit rate. Entries alone cannot provide it: a cache that is 99% empty is 0%// useful, and a flushed store would otherwise read as a 100% hit rate.@@ -343,11 +368,13 @@ export function __flushPersistenceForTests(): void {* Test-only: drop the in-memory LRU while leaving the SQLite table intact, the* way a process restart does. (clearCache() is the user-facing flush and wipes* both, so it cannot stand in for a restart.) Pending write-through is drained- * first, since a real restart's writes had already landed.+ * first, since a real restart's writes had already landed. Streaming entries+ * are memory-only (never written through), so a restart drops them too.*/export function __resetMemoryForTests(): void {drainPendingWrites();store.clear();+ streamStore.clear();lookupHits = 0;lookupMisses = 0;}@@ -543,6 +570,79 @@ export function loadCacheFromDb(now = Date.now()): void {}}+// ── Streaming entries ──++export interface CachedStreamResponse {+ frames: string[];+ platform: string;+ modelId: string;+ keyId: number | null;+ promptTokens: number;+ completionTokens: number;+}++export interface StoreStreamInput {+ frames: string[];+ platform: string;+ modelId: string;+ keyId: number | null;+ promptTokens: number;+ completionTokens: number;+}++/**+ * Look up a cached stream. Returns null on a miss or when the entry has aged+ * past the TTL. A hit bumps hit_count and moves the entry to MRU.+ */+export function getCachedStreamResponse(cacheKey: string, now = Date.now()): CachedStreamResponse | null {+ const entry = streamStore.get(cacheKey);+ if (!entry) return null;++ if (now - entry.createdAtMs > cacheTtlMs()) {+ streamStore.delete(cacheKey);+ return null;+ }++ entry.hitCount += 1;+ entry.lastHitAtMs = now;+ streamStore.delete(cacheKey);+ streamStore.set(cacheKey, entry);++ return {+ frames: entry.frames,+ platform: entry.platform,+ modelId: entry.modelId,+ keyId: entry.keyId,+ promptTokens: entry.promptTokens,+ completionTokens: entry.completionTokens,+ };+}++/**+ * Store a completed SSE frame sequence for replay. The frames are the verbatim+ * `data: {...}\n\n` lines (including the final `[DONE]`) the client received,+ * so a hit can reproduce the stream byte-for-byte. Best-effort like the JSON+ * store.+ */+export function storeCachedStreamResponse(cacheKey: string, input: StoreStreamInput, now = Date.now()): void {+ if (!Array.isArray(input.frames) || input.frames.length === 0) return;++ streamStore.delete(cacheKey);+ streamStore.set(cacheKey, {+ frames: input.frames,+ platform: input.platform,+ modelId: input.modelId,+ keyId: input.keyId,+ promptTokens: input.promptTokens,+ completionTokens: input.completionTokens,+ hitCount: 0,+ createdAtMs: now,+ lastHitAtMs: null,+ });++ evictToCap(streamStore);+}+// ── Stats / admin ──export interface CacheStats {@@ -580,9 +680,14 @@ export function getCacheStats(): CacheStats {savedPromptTokens += entry.hitCount * entry.promptTokens;savedCompletionTokens += entry.hitCount * entry.completionTokens;}+ for (const entry of streamStore.values()) {+ totalHits += entry.hitCount;+ savedPromptTokens += entry.hitCount * entry.promptTokens;+ savedCompletionTokens += entry.hitCount * entry.completionTokens;+ }const lookups = lookupHits + lookupMisses;return {- entries: store.size,+ entries: store.size + streamStore.size,totalHits,estimatedRequestsSaved: totalHits,savedPromptTokens,@@ -602,8 +707,9 @@ export function getCacheStats(): CacheStats {* rather than deferred: this is an admin route, not the proxy hot path.*/export function clearCache(): number {- const removed = store.size;+ const removed = store.size + streamStore.size;store.clear();+ streamStore.clear();pendingWrites.length = 0;// The lookup tallies describe the cache that just went away; keeping them// would show a 90% hit rate next to zero entries right after a flush.