Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy/cache): strip cache_control from messages in the semantic cache key (#3086)

Abhay Singh2cae0f8eaf62
diff --git a/headroom/proxy/semantic_cache_key.py b/headroom/proxy/semantic_cache_key.py
index cbef6dcd6..267eddb36 100644
--- a/headroom/proxy/semantic_cache_key.py
+++ b/headroom/proxy/semantic_cache_key.py
@@ -21,11 +21,20 @@ def compute_semantic_cache_key(
model: str,
**key_fields: Any,
) -> str:
- """Compute the proxy semantic-cache key from generation-shaping inputs."""
+ """Compute the proxy semantic-cache key from generation-shaping inputs.
+
+ ``cache_control`` is stripped from ``messages`` as well as the shaping
+ fields: it is a prompt-caching directive for the upstream provider that
+ never changes the generated completion, so a moved breakpoint must not
+ fragment the key. Messages are the primary key component and, on the
+ Anthropic path, the most common place a client (e.g. Claude Code) moves a
+ breakpoint between turns, so leaving them un-stripped defeated the strip for
+ the field that matters most.
+ """
normalized = json.dumps(
{
"model": model,
- "messages": messages,
+ "messages": strip_cache_control(messages),
**{k: strip_cache_control(v) for k, v in key_fields.items()},
},
sort_keys=True,
diff --git a/headroom/proxy/semantic_cache_key_policy.py b/headroom/proxy/semantic_cache_key_policy.py
index 1fef49a7c..3b7d3625d 100644
--- a/headroom/proxy/semantic_cache_key_policy.py
+++ b/headroom/proxy/semantic_cache_key_policy.py
@@ -21,11 +21,20 @@ def compute_semantic_cache_key(
model: str,
**key_fields: Any,
) -> str:
- """Compute a stable cache key from request content and shaping fields."""
+ """Compute a stable cache key from request content and shaping fields.
+
+ ``cache_control`` is stripped from ``messages`` as well as the shaping
+ fields: it is a prompt-caching directive for the upstream provider that
+ never changes the generated completion, so a moved breakpoint must not
+ fragment the key. Messages are the primary key component and, on the
+ Anthropic path, the most common place a client (e.g. Claude Code) moves a
+ breakpoint between turns, so leaving them un-stripped defeated the strip for
+ the field that matters most.
+ """
normalized = json.dumps(
{
"model": model,
- "messages": messages,
+ "messages": strip_cache_control(messages),
**{k: strip_cache_control(v) for k, v in key_fields.items()},
},
sort_keys=True,
diff --git a/tests/test_proxy_semantic_cache_key.py b/tests/test_proxy_semantic_cache_key.py
index e6601cd0d..94db0600b 100644
--- a/tests/test_proxy_semantic_cache_key.py
+++ b/tests/test_proxy_semantic_cache_key.py
@@ -106,6 +106,36 @@ def test_tools_cache_control_ignored():
assert _key(cache, tools=tools_cc) == _key(cache, tools=tools_plain)
+def test_message_cache_control_breakpoint_move_same_key():
+ """A moved cache_control breakpoint on a *message* must not fragment the key.
+
+ Messages are the primary key component and, on the Anthropic path, the most
+ common place a client (e.g. Claude Code) moves a breakpoint between turns.
+ cache_control is a prompt-caching directive that never changes the
+ completion, so the two requests must share a cache entry — previously only
+ system/tools breakpoints were stripped, so a message breakpoint missed.
+ """
+ cache = SemanticCache()
+ messages_with_cc = [
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}}],
+ }
+ ]
+ messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]
+ assert cache._compute_key(messages_with_cc, MODEL) == cache._compute_key(
+ messages_without_cc, MODEL
+ )
+
+
+def test_message_content_change_still_distinct_key():
+ """Stripping cache_control must not collapse genuinely different messages."""
+ cache = SemanticCache()
+ a = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]
+ b = [{"role": "user", "content": [{"type": "text", "text": "goodbye"}]}]
+ assert cache._compute_key(a, MODEL) != cache._compute_key(b, MODEL)
+
+
# --- behavioral get/set: collision prevented end to end -----------------------
diff --git a/tests/test_proxy_semantic_cache_key_policy.py b/tests/test_proxy_semantic_cache_key_policy.py
index e106779bc..04239d0ac 100644
--- a/tests/test_proxy_semantic_cache_key_policy.py
+++ b/tests/test_proxy_semantic_cache_key_policy.py
@@ -29,6 +29,20 @@ def test_semantic_cache_key_ignores_moved_cache_control() -> None:
)
+def test_semantic_cache_key_ignores_moved_message_cache_control() -> None:
+ """cache_control on a message must not fragment the key either (#327 intent)."""
+ messages_with_cc = [
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "hello", "cache_control": {"type": "ephemeral"}}],
+ }
+ ]
+ messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]
+ assert compute_semantic_cache_key(messages_with_cc, MODEL) == (
+ compute_semantic_cache_key(messages_without_cc, MODEL)
+ )
+
+
def test_strip_cache_control_recurses_through_dicts_and_lists() -> None:
assert strip_cache_control(
{