Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript

Normalize default-valued sampling params in the cache key (#901)

suantea (asuan_suan)d03021ebff8a
diff --git a/server/src/__tests__/services/cache.test.ts b/server/src/__tests__/services/cache.test.ts
index df867dff..d9d6abc6 100644
--- a/server/src/__tests__/services/cache.test.ts
+++ b/server/src/__tests__/services/cache.test.ts
@@ -87,6 +87,37 @@ describe('response cache', () => {
.not.toBe(computeCacheKey({ model: 'auto', messages, temperature: 0.9 }));
});
+ // Explicit-but-default-valued sampling params must NOT change the key: a
+ // client that always serializes top_p:1 / n:1 / presence_penalty:0 /
+ // frequency_penalty:0 must hit the same entry as one that omits them.
+ it('ignores explicit default-valued sampling params', () => {
+ const messages = [msg('user', 'hello')];
+ const minimal = computeCacheKey({ model: 'auto', messages, temperature: 0.2 });
+ const verbose = computeCacheKey({
+ model: 'auto',
+ messages,
+ temperature: 0.2,
+ top_p: 1,
+ presence_penalty: 0,
+ frequency_penalty: 0,
+ n: 1,
+ });
+ expect(verbose).toBe(minimal);
+ });
+
+ it('still keys non-default sampling param values', () => {
+ const messages = [msg('user', 'hello')];
+ const baseline = computeCacheKey({ model: 'auto', messages, temperature: 0.2 });
+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, top_p: 0.9 }))
+ .not.toBe(baseline);
+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, presence_penalty: 0.5 }))
+ .not.toBe(baseline);
+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, frequency_penalty: 0.5 }))
+ .not.toBe(baseline);
+ expect(computeCacheKey({ model: 'auto', messages, temperature: 0.2, n: 2 }))
+ .not.toBe(baseline);
+ });
+
it('distinguishes a pinned model from auto', () => {
const messages = [msg('user', 'hello')];
expect(computeCacheKey({ model: 'gpt-oss-120b', messages }))
diff --git a/server/src/services/cache.ts b/server/src/services/cache.ts
index b447e1dd..2d2e62fd 100644
--- a/server/src/services/cache.ts
+++ b/server/src/services/cache.ts
@@ -182,11 +182,11 @@ function normModel(model: string | undefined): string {
export function computeCacheKey(input: CacheKeyInput): string {
const canonical = stableStringify({
- v: 3, // compression fingerprint joined the key
+ v: 4, // explicit-but-default-valued sampling params dropped from the key
model: normModel(input.model),
messages: input.messages,
temperature: input.temperature,
- top_p: input.top_p,
+ top_p: defaultableNumber(input.top_p, 1),
max_tokens: input.max_tokens,
// tools/tool_choice are part of the key so a request with a different tool
// set never collides with (or is served) another's cached answer.
@@ -196,10 +196,10 @@ export function computeCacheKey(input: CacheKeyInput): string {
// by stableStringify, so requests without them keep hashing identically.
stop: input.stop,
response_format: input.response_format,
- n: input.n,
+ n: defaultableNumber(input.n, 1),
seed: input.seed,
- presence_penalty: input.presence_penalty,
- frequency_penalty: input.frequency_penalty,
+ presence_penalty: defaultableNumber(input.presence_penalty, 0),
+ frequency_penalty: defaultableNumber(input.frequency_penalty, 0),
logit_bias: input.logit_bias,
logprobs: input.logprobs,
top_logprobs: input.top_logprobs,
@@ -211,6 +211,21 @@ export function computeCacheKey(input: CacheKeyInput): string {
return crypto.createHash('sha256').update(canonical).digest('hex');
}
+// Normalizes a sampling param that was passed explicitly but equals the API
+// default down to undefined (stableStringify drops undefined), so a client that
+// always serializes the full param set shares a cache entry with one that omits
+// the defaults. Without this, top_p:1 / n:1 / presence_penalty:0 /
+// frequency_penalty:0 make two otherwise identical requests hash to different
+// keys and miss each other. Non-numbers (null, strings, absent) pass through
+// unchanged so existing behaviour is preserved.
+function defaultableNumber(value: unknown, defaultValue: number): unknown {
+ if (typeof value === 'number') {
+ if (value === defaultValue) return undefined;
+ return value;
+ }
+ return value;
+}
+
// ── Store ──
export interface CachedResponse {