Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

feat(cache): let stream:true requests hit the response cache (#1099)

suantea (asuan_suan)dd3d1e9666de
diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.ts
index ae566198..89dde4d8 100644
--- a/server/src/__tests__/services/cache.test.ts
+++ b/server/src/__tests__/services/cache.test.ts
@@ -4,6 +4,8 @@ import {
computeCacheKey,
getCachedResponse,
storeCachedResponse,
+ getCachedStreamResponse,
+ storeCachedStreamResponse,
getCacheStats,
clearCache,
loadCacheFromDb,
@@ -358,6 +360,70 @@ describe('response cache', () => {
});
});
+ describe('streaming cache', () => {
+ const frames = (text: string) => [
+ `data: ${JSON.stringify({ choices: [{ delta: { content: text }, finish_reason: null }] })}\n\n`,
+ 'data: [DONE]\n\n',
+ ];
+
+ it('returns null on a streaming miss', () => {
+ expect(getCachedStreamResponse('does-not-exist')).toBeNull();
+ });
+
+ it('replays the exact SSE frames on a streaming hit', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'stream me')] });
+ storeCachedStreamResponse(key, {
+ frames: frames('streamed answer'),
+ platform: 'groq',
+ modelId: 'llama-3.3-70b',
+ keyId: 7,
+ promptTokens: 10,
+ completionTokens: 5,
+ });
+ const hit = getCachedStreamResponse(key);
+ expect(hit).not.toBeNull();
+ expect(hit!.frames).toEqual(frames('streamed answer'));
+ expect(hit!.platform).toBe('groq');
+ expect(hit!.modelId).toBe('llama-3.3-70b');
+ });
+
+ it('keeps JSON and streaming entries separate for the same key', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'both')] });
+ store(key, 'json answer');
+ storeCachedStreamResponse(key, {
+ frames: frames('stream answer'),
+ platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
+ });
+ // The JSON store still serves its entry; the stream store its own.
+ expect((getCachedResponse(key)!.body as any).choices[0].message.content).toBe('json answer');
+ expect(getCachedStreamResponse(key)!.frames).toEqual(frames('stream answer'));
+ });
+
+ it('rejects an empty frame list (nothing to replay)', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'empty')] });
+ storeCachedStreamResponse(key, {
+ frames: [],
+ platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 1, completionTokens: 1,
+ });
+ expect(getCachedStreamResponse(key)).toBeNull();
+ });
+
+ it('counts streaming hits in the aggregated stats and clearCache clears both', () => {
+ const key = computeCacheKey({ model: 'auto', messages: [msg('user', 'stats')] });
+ storeCachedStreamResponse(key, {
+ frames: frames('a'), platform: 'groq', modelId: 'm', keyId: 1, promptTokens: 10, completionTokens: 5,
+ });
+ getCachedStreamResponse(key);
+ getCachedStreamResponse(key);
+ const stats = getCacheStats();
+ expect(stats.entries).toBe(1);
+ expect(stats.totalHits).toBe(2);
+ expect(stats.savedPromptTokens).toBe(20); // 2 hits × 10
+ expect(clearCache()).toBe(1);
+ expect(getCacheStats().entries).toBe(0);
+ });
+ });
+
describe('settings-backed toggle (no restart)', () => {
it('the stored setting enables the cache even with the env var unset', () => {
delete process.env.RESPONSE_CACHE;
diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts
index 8633ca0d..00a46dc8 100644
--- a/server/src/routes/proxy.ts
+++ b/server/src/routes/proxy.ts
@@ -21,7 +21,7 @@ import { isFusionModel, runFusion, fusionConfigSchema, FusionError, FUSION_MODEL
import { isRetryableError, isPaymentRequiredError, isModelNotFoundError, isModelAccessForbiddenError, isClientAbortError, newClientAbortError, newHedgeAbortError, isUpstreamClassificationOutput } from '../lib/error-classify.js';
import { logRequest } from '../lib/request-log.js';
import { observeServedModel } from '../lib/served-model.js';
-import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse } from '../services/cache.js';
+import { parseCacheDirective, cacheActive, isCacheableTemperature, computeCacheKey, getCachedResponse, storeCachedResponse, getCachedStreamResponse, storeCachedStreamResponse } from '../services/cache.js';
import { normalizeIdempotencyKey, hashIdempotencyKey, computeIdempotencyFingerprint, lookupIdempotencyReplay, storeIdempotencyResult } from '../services/idempotency.js';
import { runFallbackLoop, newFallbackState, recordUpstreamSuccess, exhaustedRetryError, setFallbackHeaders, exhaustionErrorPayload, setExhaustionHeaders, type AttemptRecord } from '../lib/fallback-loop.js';
import { routedViaValue, safeHeaderValue } from '../lib/header-value.js';
@@ -1774,7 +1774,10 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// per-request `X-FreeLLM-Cache` header can force or bypass. Off unless enabled
// via the RESPONSE_CACHE env var or the response_cache_enabled setting.
const cacheDirective = parseCacheDirective(req.headers['x-freellm-cache'], req.headers['cache-control']);
- const cacheKey = (!stream && cacheActive(cacheDirective) && isCacheableTemperature(temperature))
+ // Streaming requests participate in the cache too: same canonical key (the
+ // request content is identical), but hits are looked up in the streaming
+ // store and replayed as SSE rather than JSON (see below).
+ const cacheKey = (cacheActive(cacheDirective) && isCacheableTemperature(temperature))
? computeCacheKey({
model: requestedModel, messages, temperature, top_p, max_tokens, tools, tool_choice,
// Normalized stop (providerSafeStop), i.e. what is actually forwarded.
@@ -1800,15 +1803,33 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
})
: null;
if (cacheKey) {
- const hit = getCachedResponse(cacheKey);
- if (hit) {
- // A hit consumes NO provider quota, so recordRequest/recordTokens are
- // deliberately skipped and the reply is not re-logged as provider usage.
- // The savings are reported separately by GET /api/cache/stats.
- res.setHeader('X-Routed-Via', 'cache');
- res.setHeader('X-FreeLLM-Cache', 'HIT');
- res.json(hit.body);
- return;
+ if (stream) {
+ // Streaming hit: replay the captured SSE frame sequence verbatim —
+ // same zero-quota rationale as a JSON hit, with first-byte semantics
+ // preserved (the frames include the leading content chunks, so the
+ // first token arrives immediately).
+ const streamHit = getCachedStreamResponse(cacheKey);
+ if (streamHit) {
+ res.setHeader('Content-Type', 'text/event-stream');
+ res.setHeader('Cache-Control', 'no-cache');
+ res.setHeader('Connection', 'keep-alive');
+ res.setHeader('X-Routed-Via', 'cache');
+ res.setHeader('X-FreeLLM-Cache', 'HIT');
+ for (const frame of streamHit.frames) res.write(frame);
+ res.end();
+ return;
+ }
+ } else {
+ const hit = getCachedResponse(cacheKey);
+ if (hit) {
+ // A hit consumes NO provider quota, so recordRequest/recordTokens are
+ // deliberately skipped and the reply is not re-logged as provider usage.
+ // The savings are reported separately by GET /api/cache/stats.
+ res.setHeader('X-Routed-Via', 'cache');
+ res.setHeader('X-FreeLLM-Cache', 'HIT');
+ res.json(hit.body);
+ return;
+ }
}
}
@@ -2114,6 +2135,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
// Only evidence when a provider serves a different model than routed
// (#534); compared/persisted on success via observeServedModel.
let upstreamModel: string | null = null;
+ // Every `data: ...` frame the client sees, captured for a possible
+ // streaming cache store on success (exact SSE replay on a later hit).
+ const streamFrames: string[] = [];
const flushHeaders = () => {
if (headerSent) return;
@@ -2124,12 +2148,17 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
res.setHeader('Cache-Control', 'no-cache');
res.setHeader('Connection', 'keep-alive');
res.setHeader('X-Routed-Via', routedViaValue(route.platform, route.modelId));
+ res.setHeader('X-FreeLLM-Cache', cacheKey ? 'MISS' : 'OFF');
setFallbackHeaders(res, attempt, attemptLog);
headerSent = true;
// Committed: the answer is on its way, so the retry budget must no
// longer cancel this attempt (it could not fail over now anyway).
ctx.disarmHedge();
- for (const p of preamble) res.write(`data: ${JSON.stringify(p)}\n\n`);
+ for (const p of preamble) {
+ const frame = `data: ${JSON.stringify(p)}\n\n`;
+ streamFrames.push(frame);
+ res.write(frame);
+ }
preamble.length = 0;
};
const mkChunk = (delta: Record<string, unknown>, finish: string | null) => ({
@@ -2139,7 +2168,11 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
model: lastMeta.model ?? route.modelId,
choices: [{ index: 0, delta, finish_reason: finish }],
});
- const writeChunk = (c: unknown) => res.write(`data: ${JSON.stringify(c)}\n\n`);
+ const writeChunk = (c: unknown) => {
+ const frame = `data: ${JSON.stringify(c)}\n\n`;
+ streamFrames.push(frame);
+ res.write(frame);
+ };
try {
const gen = route.provider.streamChatCompletion(
@@ -2444,7 +2477,9 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
},
});
}
- res.write('data: [DONE]\n\n');
+ const doneFrame = 'data: [DONE]\n\n';
+ streamFrames.push(doneFrame);
+ res.write(doneFrame);
res.end();
const upstreamUsage = (usageChunk as { usage?: TokenUsage } | null)?.usage;
@@ -2452,6 +2487,23 @@ proxyRouter.post('/chat/completions', async (req: Request, res: Response) => {
const outputTokens = upstreamUsage?.completion_tokens ?? totalOutputTokens;
const totalTokens = upstreamUsage?.total_tokens ?? (inputTokens + outputTokens);
recordUpstreamSuccess(route, totalTokens);
+
+ // Cache the freshly-generated SSE sequence so an identical later
+ // stream request is replayed without spending another free-tier
+ // slot. A truncated turn (finish 'length') is NOT cached, matching
+ // the JSON cache policy — replaying a cut-off answer would be worse
+ // than regenerating.
+ if (cacheKey && finish !== 'length') {
+ storeCachedStreamResponse(cacheKey, {
+ frames: streamFrames,
+ platform: route.platform,
+ modelId: route.modelId,
+ keyId: route.keyId,
+ promptTokens: inputTokens,
+ completionTokens: outputTokens,
+ });
+ }
+
setStickyModel(messages, route.modelDbId, sessionIdHeader, stickyStrategyKey);
if (handoffMode !== 'off' && sessionKey) recordSuccessfulModel({ sessionKey, modelKey });
// #797: remember this turn's thinking trace so the next request from
diff --git a/server/src/services/cache.ts b/server/src/services/cache.ts
index c8256e62..129d8adf 100644
--- a/server/src/services/cache.ts
+++ b/server/src/services/cache.ts
@@ -280,6 +280,31 @@ interface CacheEntry {
// at the end (delete + set), so eviction from the front drops the coldest entry.
const store = new Map<string, CacheEntry>();
+// Streaming entries live in their own LRU: a stream's replayable artifact is
+// the exact SSE frame sequence, which is structurally different from a JSON
+// completion body, so the two kinds never share a key across stores.
+interface StreamCacheEntry {
+ frames: string[]; // verbatim `data: {...}\n\n` (and final `[DONE]`) frames
+ platform: string;
+ modelId: string;
+ keyId: number | null;
+ promptTokens: number;
+ completionTokens: number;
+ hitCount: number;
+ createdAtMs: number;
+ lastHitAtMs: number | null;
+}
+const streamStore = new Map<string, StreamCacheEntry>();
+
+function evictToCap<K, V>(map: Map<K, V>): void {
+ const cap = cacheMaxEntries();
+ while (map.size > cap) {
+ const oldest = map.keys().next().value as K | undefined;
+ if (oldest === undefined) break;
+ map.delete(oldest);
+ }
+}
+
// Lifetime-of-process lookup tallies, the denominator behind the dashboard's
// hit rate. Entries alone cannot provide it: a cache that is 99% empty is 0%
// useful, and a flushed store would otherwise read as a 100% hit rate.
@@ -343,11 +368,13 @@ export function __flushPersistenceForTests(): void {
* Test-only: drop the in-memory LRU while leaving the SQLite table intact, the
* way a process restart does. (clearCache() is the user-facing flush and wipes
* both, so it cannot stand in for a restart.) Pending write-through is drained
- * first, since a real restart's writes had already landed.
+ * first, since a real restart's writes had already landed. Streaming entries
+ * are memory-only (never written through), so a restart drops them too.
*/
export function __resetMemoryForTests(): void {
drainPendingWrites();
store.clear();
+ streamStore.clear();
lookupHits = 0;
lookupMisses = 0;
}
@@ -543,6 +570,79 @@ export function loadCacheFromDb(now = Date.now()): void {
}
}
+// ── Streaming entries ──
+
+export interface CachedStreamResponse {
+ frames: string[];
+ platform: string;
+ modelId: string;
+ keyId: number | null;
+ promptTokens: number;
+ completionTokens: number;
+}
+
+export interface StoreStreamInput {
+ frames: string[];
+ platform: string;
+ modelId: string;
+ keyId: number | null;
+ promptTokens: number;
+ completionTokens: number;
+}
+
+/**
+ * Look up a cached stream. Returns null on a miss or when the entry has aged
+ * past the TTL. A hit bumps hit_count and moves the entry to MRU.
+ */
+export function getCachedStreamResponse(cacheKey: string, now = Date.now()): CachedStreamResponse | null {
+ const entry = streamStore.get(cacheKey);
+ if (!entry) return null;
+
+ if (now - entry.createdAtMs > cacheTtlMs()) {
+ streamStore.delete(cacheKey);
+ return null;
+ }
+
+ entry.hitCount += 1;
+ entry.lastHitAtMs = now;
+ streamStore.delete(cacheKey);
+ streamStore.set(cacheKey, entry);
+
+ return {
+ frames: entry.frames,
+ platform: entry.platform,
+ modelId: entry.modelId,
+ keyId: entry.keyId,
+ promptTokens: entry.promptTokens,
+ completionTokens: entry.completionTokens,
+ };
+}
+
+/**
+ * Store a completed SSE frame sequence for replay. The frames are the verbatim
+ * `data: {...}\n\n` lines (including the final `[DONE]`) the client received,
+ * so a hit can reproduce the stream byte-for-byte. Best-effort like the JSON
+ * store.
+ */
+export function storeCachedStreamResponse(cacheKey: string, input: StoreStreamInput, now = Date.now()): void {
+ if (!Array.isArray(input.frames) || input.frames.length === 0) return;
+
+ streamStore.delete(cacheKey);
+ streamStore.set(cacheKey, {
+ frames: input.frames,
+ platform: input.platform,
+ modelId: input.modelId,
+ keyId: input.keyId,
+ promptTokens: input.promptTokens,
+ completionTokens: input.completionTokens,
+ hitCount: 0,
+ createdAtMs: now,
+ lastHitAtMs: null,
+ });
+
+ evictToCap(streamStore);
+}
+
// ── Stats / admin ──
export interface CacheStats {
@@ -580,9 +680,14 @@ export function getCacheStats(): CacheStats {
savedPromptTokens += entry.hitCount * entry.promptTokens;
savedCompletionTokens += entry.hitCount * entry.completionTokens;
}
+ for (const entry of streamStore.values()) {
+ totalHits += entry.hitCount;
+ savedPromptTokens += entry.hitCount * entry.promptTokens;
+ savedCompletionTokens += entry.hitCount * entry.completionTokens;
+ }
const lookups = lookupHits + lookupMisses;
return {
- entries: store.size,
+ entries: store.size + streamStore.size,
totalHits,
estimatedRequestsSaved: totalHits,
savedPromptTokens,
@@ -602,8 +707,9 @@ export function getCacheStats(): CacheStats {
* rather than deferred: this is an admin route, not the proxy hot path.
*/
export function clearCache(): number {
- const removed = store.size;
+ const removed = store.size + streamStore.size;
store.clear();
+ streamStore.clear();
pendingWrites.length = 0;
// The lookup tallies describe the cache that just went away; keeping them
// would show a 90% hit rate next to zero entries right after a flush.