Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

refactor(cache): isolate semantic key policy (#1953)

JD Davis740fb9bc16eb
diff --git a/headroom/proxy/semantic_cache.py b/headroom/proxy/semantic_cache.py
index cf14c30a8..1ce8baf48 100644
--- a/headroom/proxy/semantic_cache.py
+++ b/headroom/proxy/semantic_cache.py
@@ -8,8 +8,6 @@ Extracted from server.py for maintainability.
from __future__ import annotations
import asyncio
-import hashlib
-import json
import sys
from collections import OrderedDict
from datetime import datetime
@@ -19,23 +17,9 @@ if TYPE_CHECKING:
from ..memory.tracker import ComponentStats
from headroom.proxy.models import CacheEntry
+from headroom.proxy.semantic_cache_key import compute_semantic_cache_key, strip_cache_control
-
-def _strip_cache_control(obj: Any) -> Any:
- """Recursively drop ``cache_control`` annotations before hashing.
-
- Clients (notably Claude Code) move the ``cache_control`` cache breakpoint to
- the newest content on each call, so the same logical ``system``/``tools``
- payload carries the marker on one call and not the next. Stripping it keeps
- the cache key stable across that movement. Mirrors
- ``helpers._strip_per_call_annotations`` but kept local so the cache module
- stays free of the heavier proxy-helpers import chain.
- """
- if isinstance(obj, dict):
- return {k: _strip_cache_control(v) for k, v in obj.items() if k != "cache_control"}
- if isinstance(obj, list):
- return [_strip_cache_control(item) for item in obj]
- return obj
+_strip_cache_control = strip_cache_control
class SemanticCache:
@@ -66,15 +50,7 @@ class SemanticCache:
untouched). Absent fields don't contribute, so truly-identical requests
still hit.
"""
- normalized = json.dumps(
- {
- "model": model,
- "messages": messages,
- **{k: _strip_cache_control(v) for k, v in key_fields.items()},
- },
- sort_keys=True,
- )
- return hashlib.sha256(normalized.encode()).hexdigest()[:32]
+ return compute_semantic_cache_key(messages, model, **key_fields)
async def get(self, messages: list[dict], model: str, **key_fields: Any) -> CacheEntry | None:
"""Get cached response if exists and not expired."""
diff --git a/headroom/proxy/semantic_cache_key.py b/headroom/proxy/semantic_cache_key.py
new file mode 100644
index 000000000..cbef6dcd6
--- /dev/null
+++ b/headroom/proxy/semantic_cache_key.py
@@ -0,0 +1,33 @@
+"""Pure semantic-cache key construction policy."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from typing import Any
+
+
+def strip_cache_control(obj: Any) -> Any:
+ """Recursively drop ``cache_control`` annotations before hashing."""
+ if isinstance(obj, dict):
+ return {k: strip_cache_control(v) for k, v in obj.items() if k != "cache_control"}
+ if isinstance(obj, list):
+ return [strip_cache_control(item) for item in obj]
+ return obj
+
+
+def compute_semantic_cache_key(
+ messages: list[dict],
+ model: str,
+ **key_fields: Any,
+) -> str:
+ """Compute the proxy semantic-cache key from generation-shaping inputs."""
+ normalized = json.dumps(
+ {
+ "model": model,
+ "messages": messages,
+ **{k: strip_cache_control(v) for k, v in key_fields.items()},
+ },
+ sort_keys=True,
+ )
+ return hashlib.sha256(normalized.encode()).hexdigest()[:32]
diff --git a/tests/test_proxy_semantic_cache_key_policy.py b/tests/test_proxy_semantic_cache_key_policy.py
new file mode 100644
index 000000000..e106779bc
--- /dev/null
+++ b/tests/test_proxy_semantic_cache_key_policy.py
@@ -0,0 +1,43 @@
+"""Tests for pure proxy semantic-cache key policy."""
+
+from __future__ import annotations
+
+from headroom.proxy.semantic_cache_key import (
+ compute_semantic_cache_key,
+ strip_cache_control,
+)
+
+MESSAGES = [{"role": "user", "content": "hello"}]
+MODEL = "claude-haiku-4-5"
+
+
+def test_semantic_cache_key_distinguishes_response_shaping_fields() -> None:
+ assert compute_semantic_cache_key(MESSAGES, MODEL, system="French") != (
+ compute_semantic_cache_key(MESSAGES, MODEL, system="English")
+ )
+ assert compute_semantic_cache_key(MESSAGES, MODEL, temperature=0.0) != (
+ compute_semantic_cache_key(MESSAGES, MODEL, temperature=1.0)
+ )
+
+
+def test_semantic_cache_key_ignores_moved_cache_control() -> None:
+ with_cache_control = [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]
+ without_cache_control = [{"type": "text", "text": "sys"}]
+
+ assert compute_semantic_cache_key(MESSAGES, MODEL, system=with_cache_control) == (
+ compute_semantic_cache_key(MESSAGES, MODEL, system=without_cache_control)
+ )
+
+
+def test_strip_cache_control_recurses_through_dicts_and_lists() -> None:
+ assert strip_cache_control(
+ {
+ "system": [
+ {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}},
+ ],
+ "tools": [{"name": "read", "cache_control": {"type": "ephemeral"}}],
+ }
+ ) == {
+ "system": [{"type": "text", "text": "sys"}],
+ "tools": [{"name": "read"}],
+ }