7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.
TypeScript
Normalize default-valued sampling params in the cache key (#901)
suantea (asuan_suan)d03021ebff8a
diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.tsindex df867dff..d9d6abc6 100644--- a/server/src/__tests__/services/cache.test.ts+++ b/server/src/__tests__/services/cache.test.ts@@ -87,6 +87,37 @@ describe('response cache', () => {.not.toBe(computeCacheKey({ model: 'auto', messages, temperature: 0.9 }));});+ // Explicit-but-default-valued sampling params must NOT change the key: a+ // client that always serializes top_p:1 / n:1 / presence_penalty:0 /+ // frequency_penalty:0 must hit the same entry as one that omits them.+ it('ignores explicit default-valued sampling params', () => {+ const messages = [msg('user', 'hello')];+ const minimal = computeCacheKey({ model: 'auto', messages, temperature: 0.2 });+ const verbose = computeCacheKey({+ model: 'auto',+ messages,+ temperature: 0.2,+ top_p: 1,+ presence_penalty: 0,+ frequency_penalty: 0,+ n: 1,+ });+ expect(verbose).toBe(minimal);+ });++ it('still keys non-default sampling param values', () => {+ const messages = [msg('user', 'hello')];+ const baseline = computeCacheKey({ model: 'auto', messages, temperature: 0.2 });+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, top_p: 0.9 }))+ .not.toBe(baseline);+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, presence_penalty: 0.5 }))+ .not.toBe(baseline);+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, frequency_penalty: 0.5 }))+ .not.toBe(baseline);+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, n: 2 }))+ .not.toBe(baseline);+ });+it('distinguishes a pinned model from auto', () => {const messages = [msg('user', 'hello')];expect(computeCacheKey({ model: 'gpt-oss-120b', messages }))diff --git a/server/src/services/cache.ts b/server/src/services/cache.tsindex b447e1dd..2d2e62fd 100644--- a/server/src/services/cache.ts+++ b/server/src/services/cache.ts@@ -182,11 +182,11 @@ function normModel(model: string | undefined): string {export function computeCacheKey(input: CacheKeyInput): string {const canonical = stableStringify({- v: 3, // compression fingerprint joined the key+ v: 4, // explicit-but-default-valued sampling params dropped from the keymodel: normModel(input.model),messages: input.messages,temperature: input.temperature,- top_p: input.top_p,+ top_p: defaultableNumber(input.top_p, 1),max_tokens: input.max_tokens,// tools/tool_choice are part of the key so a request with a different tool// set never collides with (or is served) another's cached answer.@@ -196,10 +196,10 @@ export function computeCacheKey(input: CacheKeyInput): string {// by stableStringify, so requests without them keep hashing identically.stop: input.stop,response_format: input.response_format,- n: input.n,+ n: defaultableNumber(input.n, 1),seed: input.seed,- presence_penalty: input.presence_penalty,- frequency_penalty: input.frequency_penalty,+ presence_penalty: defaultableNumber(input.presence_penalty, 0),+ frequency_penalty: defaultableNumber(input.frequency_penalty, 0),logit_bias: input.logit_bias,logprobs: input.logprobs,top_logprobs: input.top_logprobs,@@ -211,6 +211,21 @@ export function computeCacheKey(input: CacheKeyInput): string {return crypto.createHash('sha256').update(canonical).digest('hex');}+// Normalizes a sampling param that was passed explicitly but equals the API+// default down to undefined (stableStringify drops undefined), so a client that+// always serializes the full param set shares a cache entry with one that omits+// the defaults. Without this, top_p:1 / n:1 / presence_penalty:0 /+// frequency_penalty:0 make two otherwise identical requests hash to different+// keys and miss each other. Non-numbers (null, strings, absent) pass through+// unchanged so existing behaviour is preserved.+function defaultableNumber(value: unknown, defaultValue: number): unknown {+ if (typeof value === 'number') {+ if (value === defaultValue) return undefined;+ return value;+ }+ return value;+}+// ── Store ──export interface CachedResponse {