Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
refactor(cache): isolate semantic key policy (#1953)
JD Davis740fb9bc16eb
diff --git a/headroom/proxy/semantic_cache.py b/headroom/proxy/semantic_cache.pyindex cf14c30a8..1ce8baf48 100644--- a/headroom/proxy/semantic_cache.py+++ b/headroom/proxy/semantic_cache.py@@ -8,8 +8,6 @@ Extracted from server.py for maintainability.from __future__ import annotationsimport asyncio-import hashlib-import jsonimport sysfrom collections import OrderedDictfrom datetime import datetime@@ -19,23 +17,9 @@ if TYPE_CHECKING:from ..memory.tracker import ComponentStatsfrom headroom.proxy.models import CacheEntry+from headroom.proxy.semantic_cache_key import compute_semantic_cache_key, strip_cache_control--def _strip_cache_control(obj: Any) -> Any:- """Recursively drop ``cache_control`` annotations before hashing.-- Clients (notably Claude Code) move the ``cache_control`` cache breakpoint to- the newest content on each call, so the same logical ``system``/``tools``- payload carries the marker on one call and not the next. Stripping it keeps- the cache key stable across that movement. Mirrors- ``helpers._strip_per_call_annotations`` but kept local so the cache module- stays free of the heavier proxy-helpers import chain.- """- if isinstance(obj, dict):- return {k: _strip_cache_control(v) for k, v in obj.items() if k != "cache_control"}- if isinstance(obj, list):- return [_strip_cache_control(item) for item in obj]- return obj+_strip_cache_control = strip_cache_controlclass SemanticCache:@@ -66,15 +50,7 @@ class SemanticCache:untouched). Absent fields don't contribute, so truly-identical requestsstill hit."""- normalized = json.dumps(- {- "model": model,- "messages": messages,- **{k: _strip_cache_control(v) for k, v in key_fields.items()},- },- sort_keys=True,- )- return hashlib.sha256(normalized.encode()).hexdigest()[:32]+ return compute_semantic_cache_key(messages, model, **key_fields)async def get(self, messages: list[dict], model: str, **key_fields: Any) -> CacheEntry | None:"""Get cached response if exists and not expired."""diff --git a/headroom/proxy/semantic_cache_key.py b/headroom/proxy/semantic_cache_key.pynew file mode 100644index 000000000..cbef6dcd6--- /dev/null+++ b/headroom/proxy/semantic_cache_key.py@@ -0,0 +1,33 @@+"""Pure semantic-cache key construction policy."""++from __future__ import annotations++import hashlib+import json+from typing import Any+++def strip_cache_control(obj: Any) -> Any:+ """Recursively drop ``cache_control`` annotations before hashing."""+ if isinstance(obj, dict):+ return {k: strip_cache_control(v) for k, v in obj.items() if k != "cache_control"}+ if isinstance(obj, list):+ return [strip_cache_control(item) for item in obj]+ return obj+++def compute_semantic_cache_key(+ messages: list[dict],+ model: str,+ **key_fields: Any,+) -> str:+ """Compute the proxy semantic-cache key from generation-shaping inputs."""+ normalized = json.dumps(+ {+ "model": model,+ "messages": messages,+ **{k: strip_cache_control(v) for k, v in key_fields.items()},+ },+ sort_keys=True,+ )+ return hashlib.sha256(normalized.encode()).hexdigest()[:32]diff --git a/tests/test_proxy_semantic_cache_key_policy.py b/tests/test_proxy_semantic_cache_key_policy.pynew file mode 100644index 000000000..e106779bc--- /dev/null+++ b/tests/test_proxy_semantic_cache_key_policy.py@@ -0,0 +1,43 @@+"""Tests for pure proxy semantic-cache key policy."""++from __future__ import annotations++from headroom.proxy.semantic_cache_key import (+ compute_semantic_cache_key,+ strip_cache_control,+)++MESSAGES = [{"role": "user", "content": "hello"}]+MODEL = "claude-haiku-4-5"+++def test_semantic_cache_key_distinguishes_response_shaping_fields() -> None:+ assert compute_semantic_cache_key(MESSAGES, MODEL, system="French") != (+ compute_semantic_cache_key(MESSAGES, MODEL, system="English")+ )+ assert compute_semantic_cache_key(MESSAGES, MODEL, temperature=0.0) != (+ compute_semantic_cache_key(MESSAGES, MODEL, temperature=1.0)+ )+++def test_semantic_cache_key_ignores_moved_cache_control() -> None:+ with_cache_control = [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]+ without_cache_control = [{"type": "text", "text": "sys"}]++ assert compute_semantic_cache_key(MESSAGES, MODEL, system=with_cache_control) == (+ compute_semantic_cache_key(MESSAGES, MODEL, system=without_cache_control)+ )+++def test_strip_cache_control_recurses_through_dicts_and_lists() -> None:+ assert strip_cache_control(+ {+ "system": [+ {"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}},+ ],+ "tools": [{"name": "read", "cache_control": {"type": "ephemeral"}}],+ }+ ) == {+ "system": [{"type": "text", "text": "sys"}],+ "tools": [{"name": "read"}],+ }