Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy/cache): strip cache_control from messages in the semantic cache key (#3086)
Abhay Singh2cae0f8eaf62
diff --git a/headroom/proxy/semantic_cache_key.py b/headroom/proxy/semantic_cache_key.pyindex cbef6dcd6..267eddb36 100644--- a/headroom/proxy/semantic_cache_key.py+++ b/headroom/proxy/semantic_cache_key.py@@ -21,11 +21,20 @@ def compute_semantic_cache_key(model: str,**key_fields: Any,) -> str:- """Compute the proxy semantic-cache key from generation-shaping inputs."""+ """Compute the proxy semantic-cache key from generation-shaping inputs.++ ``cache_control`` is stripped from ``messages`` as well as the shaping+ fields: it is a prompt-caching directive for the upstream provider that+ never changes the generated completion, so a moved breakpoint must not+ fragment the key. Messages are the primary key component and, on the+ Anthropic path, the most common place a client (e.g. Claude Code) moves a+ breakpoint between turns, so leaving them un-stripped defeated the strip for+ the field that matters most.+ """normalized = json.dumps({"model": model,- "messages": messages,+ "messages": strip_cache_control(messages),**{k: strip_cache_control(v) for k, v in key_fields.items()},},sort_keys=True,diff --git a/headroom/proxy/semantic_cache_key_policy.py b/headroom/proxy/semantic_cache_key_policy.pyindex 1fef49a7c..3b7d3625d 100644--- a/headroom/proxy/semantic_cache_key_policy.py+++ b/headroom/proxy/semantic_cache_key_policy.py@@ -21,11 +21,20 @@ def compute_semantic_cache_key(model: str,**key_fields: Any,) -> str:- """Compute a stable cache key from request content and shaping fields."""+ """Compute a stable cache key from request content and shaping fields.++ ``cache_control`` is stripped from ``messages`` as well as the shaping+ fields: it is a prompt-caching directive for the upstream provider that+ never changes the generated completion, so a moved breakpoint must not+ fragment the key. Messages are the primary key component and, on the+ Anthropic path, the most common place a client (e.g. Claude Code) moves a+ breakpoint between turns, so leaving them un-stripped defeated the strip for+ the field that matters most.+ """normalized = json.dumps({"model": model,- "messages": messages,+ "messages": strip_cache_control(messages),**{k: strip_cache_control(v) for k, v in key_fields.items()},},sort_keys=True,diff --git a/tests/test_proxy_semantic_cache_key.py b/tests/test_proxy_semantic_cache_key.pyindex e6601cd0d..94db0600b 100644--- a/tests/test_proxy_semantic_cache_key.py+++ b/tests/test_proxy_semantic_cache_key.py@@ -106,6 +106,36 @@ def test_tools_cache_control_ignored():assert _key(cache, tools=tools_cc) == _key(cache, tools=tools_plain)+def test_message_cache_control_breakpoint_move_same_key():+ """A moved cache_control breakpoint on a *message* must not fragment the key.++ Messages are the primary key component and, on the Anthropic path, the most+ common place a client (e.g. Claude Code) moves a breakpoint between turns.+ cache_control is a prompt-caching directive that never changes the+ completion, so the two requests must share a cache entry — previously only+ system/tools breakpoints were stripped, so a message breakpoint missed.+ """+ cache = SemanticCache()+ messages_with_cc = [+ {+ "role": "user",+ "content": [{"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}}],+ }+ ]+ messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]+ assert cache._compute_key(messages_with_cc, MODEL) == cache._compute_key(+ messages_without_cc, MODEL+ )+++def test_message_content_change_still_distinct_key():+ """Stripping cache_control must not collapse genuinely different messages."""+ cache = SemanticCache()+ a = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]+ b = [{"role": "user", "content": [{"type": "text", "text": "goodbye"}]}]+ assert cache._compute_key(a, MODEL) != cache._compute_key(b, MODEL)++# --- behavioral get/set: collision prevented end to end -----------------------diff --git a/tests/test_proxy_semantic_cache_key_policy.py b/tests/test_proxy_semantic_cache_key_policy.pyindex e106779bc..04239d0ac 100644--- a/tests/test_proxy_semantic_cache_key_policy.py+++ b/tests/test_proxy_semantic_cache_key_policy.py@@ -29,6 +29,20 @@ def test_semantic_cache_key_ignores_moved_cache_control() -> None:)+def test_semantic_cache_key_ignores_moved_message_cache_control() -> None:+ """cache_control on a message must not fragment the key either (#327 intent)."""+ messages_with_cc = [+ {+ "role": "user",+ "content": [{"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}}],+ }+ ]+ messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]+ assert compute_semantic_cache_key(messages_with_cc, MODEL) == (+ compute_semantic_cache_key(messages_without_cc, MODEL)+ )++def test_strip_cache_control_recurses_through_dicts_and_lists() -> None:assert strip_cache_control({