Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): stop prefix transforms from busting the provider cache (#3735)
Tejas Chopraa2bf5ed1ba45
diff --git a/headroom/proxy/system_compaction.py b/headroom/proxy/system_compaction.pyindex 03ac42d3f..6ce99fcbc 100644--- a/headroom/proxy/system_compaction.py+++ b/headroom/proxy/system_compaction.py@@ -41,6 +41,21 @@ def _json_byte_len(value: Any) -> int:return len(json.dumps(value, ensure_ascii=False, default=str, separators=(",", ":")))+def _last_cache_control_index(blocks: list[Any]) -> int:+ """Index of the last block carrying ``cache_control``, or ``-1`` if none.++ Everything at or before this index is inside a prefix the provider has+ already hashed and is billing at the cache-read rate. Mutating those bytes+ busts the entry and re-bills the whole prefix at write rate (1.25x list on+ Anthropic), which costs far more than the compaction saves.+ """+ last = -1+ for i, block in enumerate(blocks):+ if isinstance(block, dict) and block.get("cache_control"):+ last = i+ return last++def _compact_system_blocks(blocks: list[dict[str, Any]],router: Any,@@ -57,11 +72,23 @@ def _compact_system_blocks(modified = Falseupdated: list[dict[str, Any]] = []- for block in blocks:+ # A cache_control marker anywhere in the system array freezes every block at+ # or before it: the provider hashed those exact bytes, so rewriting any of+ # them re-bills the whole prefix as cache creation. Preserving the marker+ # field (as this function does) does NOT preserve the cache entry -- the key+ # is the content, not the marker. Compact only what sits after the last+ # breakpoint. See ``_last_cache_control_index``.+ frozen_upto = _last_cache_control_index(blocks)++ for idx, block in enumerate(blocks):if not isinstance(block, dict) or block.get("type") != "text":updated.append(block)continue+ if idx <= frozen_upto:+ updated.append(block)+ continue+text = block.get("text", "")if not isinstance(text, str) or len(text) < min_chars:updated.append(block)diff --git a/headroom/proxy/tool_schema_compaction.py b/headroom/proxy/tool_schema_compaction.pyindex d43a04cc5..ce2fd9c1a 100644--- a/headroom/proxy/tool_schema_compaction.py+++ b/headroom/proxy/tool_schema_compaction.py@@ -34,11 +34,14 @@ from __future__ import annotationsimport copyimport hashlibimport json+import loggingimport osimport reimport threadingfrom typing import Any+logger = logging.getLogger(__name__)+# Keys that are JSON Schema annotations, not constraints.# Removing them does not change the set of valid inputs.TOOL_SCHEMA_DROP_KEYS: frozenset[str] = frozenset(@@ -392,6 +395,22 @@ def compact_tool_descriptions(if not isinstance(tools, list) or not tools:return payload, False, 0, 0+ # A cache_control marker on any tool means "cache everything up to and+ # including this one". Truncating a description inside that span changes the+ # bytes the provider hashed, so the whole tools prefix -- and every message+ # after it -- re-bills as cache creation. The saving is a few hundred bytes;+ # the bust is the entire conversation. Mirrors the same guard on+ # ``_sort_tools_deterministically`` (handlers/anthropic.py) and+ # ``any_tool_has_cache_control`` in the Rust live zone.+ marked = sum(1 for t in tools if isinstance(t, dict) and t.get("cache_control"))+ if marked:+ logger.info(+ "event=tool_desc_compaction_skipped reason=marker_present tool_count=%d marked=%d",+ len(tools),+ marked,+ )+ return payload, False, 0, 0+strip_sem = strip_semantic_params()key = _cache_key(tools, "L2", max_chars, strip_sem)cached = _cache_get(key)diff --git a/tests/test_anthropic_compaction_transforms.py b/tests/test_anthropic_compaction_transforms.pyindex 676beefe9..d7e5a81f8 100644--- a/tests/test_anthropic_compaction_transforms.py+++ b/tests/test_anthropic_compaction_transforms.py@@ -52,10 +52,16 @@ def _make_anthropic_payload_with_tools() -> dict:def _make_anthropic_payload_with_long_system() -> dict:"""Payload with a long system prompt that qualifies for L3 compaction."""long_text = "x" * 5000 # well above default min_chars+ # Two blocks, mirroring how agentic clients actually shape the system array:+ # a stable block the client has pinned with cache_control (CLAUDE.md, rules)+ # followed by dynamic content. Only the block AFTER the last breakpoint is+ # eligible -- rewriting the pinned one would bust the provider's entry, so+ # compaction must leave it byte-identical.return {"model": "claude-sonnet-4-6","system": [{"type": "text", "text": long_text, "cache_control": {"type": "ephemeral"}},+ {"type": "text", "text": "y" * 5000},],"messages": [{"role": "user", "content": "hello"}],"tools": [],@@ -383,7 +389,11 @@ def test_handler_auxiliary_compaction_respects_optimization_decision(if should_compact:assert "title" not in forwarded["tools"][0]["input_schema"]assert len(forwarded["tools"][0]["description"]) < len(payload["tools"][0]["description"])- assert forwarded["system"][0]["text"] == "short system"+ # Block 0 is pinned by cache_control: byte-identical, marker intact.+ assert forwarded["system"][0]["text"] == payload["system"][0]["text"]+ assert forwarded["system"][0]["cache_control"] == {"type": "ephemeral"}+ # Block 1 sits after the breakpoint and is compacted.+ assert forwarded["system"][1]["text"] == "short system"assert all(label in transforms for label in labels)router.compress.assert_called_once()assert summary["compression"]["total_tokens_removed"] > 0diff --git a/tests/test_cache_prefix_safety.py b/tests/test_cache_prefix_safety.pynew file mode 100644index 000000000..5dbee0268--- /dev/null+++ b/tests/test_cache_prefix_safety.py@@ -0,0 +1,125 @@+"""Transforms must never rewrite bytes the provider has already hashed.++A ``cache_control`` marker means the provider is billing everything up to and+including that point at the cache-read rate (0.1x on Anthropic). Rewriting any+of those bytes busts the entry and re-bills the whole prefix at the write rate+(1.25x) -- a 12.5x swing that dwarfs anything the transform saves.++Preserving the ``cache_control`` *field* does not preserve the *entry*: the+provider's key is the content, not the marker.+"""++from __future__ import annotations++from typing import Any++from headroom.proxy.system_compaction import _compact_system_blocks+from headroom.proxy.tool_schema_compaction import compact_tool_descriptions+++def _text(chars: int, seed: str = "x") -> str:+ return (seed * 10 + " context line. ") * (chars // 25 + 1)+++class _StubRouter:+ """Router whose compression is obvious and always shrinks."""++ def compress(self, text: str, context: str = "") -> Any: # noqa: ARG002+ class _R:+ compressed = "SHRUNK"++ return _R()+++# --------------------------------------------------------------------------+# tool description compaction+# --------------------------------------------------------------------------+++def test_tool_desc_compaction_skips_when_a_tool_is_marked() -> None:+ """A marker on any tool freezes the whole tools prefix."""+ payload = {+ "tools": [+ {"name": "a", "description": _text(4000), "input_schema": {}},+ {+ "name": "b",+ "description": _text(4000, "y"),+ "input_schema": {},+ "cache_control": {"type": "ephemeral"},+ },+ ]+ }+ before = [dict(t) for t in payload["tools"]]++ out, modified, _, _ = compact_tool_descriptions(payload, max_chars=50)++ assert modified is False+ assert out is payload+ assert out["tools"] == before+++def test_tool_desc_compaction_still_runs_when_nothing_is_marked() -> None:+ """The common case -- no markers -- must keep working."""+ payload = {+ "tools": [+ {"name": "a", "description": _text(4000), "input_schema": {}},+ {"name": "b", "description": _text(4000, "y"), "input_schema": {}},+ ]+ }++ out, modified, before_bytes, after_bytes = compact_tool_descriptions(payload, max_chars=50)++ assert modified is True+ assert after_bytes < before_bytes+ assert all(len(t["description"]) <= 80 for t in out["tools"])+++# --------------------------------------------------------------------------+# system prompt compaction+# --------------------------------------------------------------------------+++def test_system_compaction_leaves_blocks_at_or_before_the_marker() -> None:+ """Blocks inside the cached span keep their exact bytes."""+ blocks = [+ {"type": "text", "text": _text(2000, "a")},+ {"type": "text", "text": _text(2000, "b"), "cache_control": {"type": "ephemeral"}},+ {"type": "text", "text": _text(2000, "c")},+ ]+ original = [dict(b) for b in blocks]++ out, modified = _compact_system_blocks(blocks, _StubRouter(), "m", "rid", 500)++ # frozen: identical bytes, marker intact+ assert out[0] == original[0]+ assert out[1] == original[1]+ assert out[1]["cache_control"] == {"type": "ephemeral"}+ # after the breakpoint: free to compact+ assert out[2]["text"] == "SHRUNK"+ assert modified is True+++def test_system_compaction_unrestricted_without_markers() -> None:+ blocks = [+ {"type": "text", "text": _text(2000, "a")},+ {"type": "text", "text": _text(2000, "b")},+ ]++ out, modified = _compact_system_blocks(blocks, _StubRouter(), "m", "rid", 500)++ assert modified is True+ assert [b["text"] for b in out] == ["SHRUNK", "SHRUNK"]+++def test_system_compaction_noop_when_marker_is_last() -> None:+ """Marker on the final block freezes everything."""+ blocks = [+ {"type": "text", "text": _text(2000, "a")},+ {"type": "text", "text": _text(2000, "b"), "cache_control": {"type": "ephemeral"}},+ ]+ original = [dict(b) for b in blocks]++ out, modified = _compact_system_blocks(blocks, _StubRouter(), "m", "rid", 500)++ assert modified is False+ assert out == originaldiff --git a/tests/test_system_compaction.py b/tests/test_system_compaction.pyindex c39c3a888..85daf7455 100644--- a/tests/test_system_compaction.py+++ b/tests/test_system_compaction.py@@ -86,8 +86,13 @@ class TestCompactSystemPromptContentBlocks:assert modified is Falseassert result["system"][0]["text"] == "Short instruction."- def test_preserves_cache_control(self) -> None:- """cache_control must survive compaction."""+ def test_marked_block_is_left_byte_identical(self) -> None:+ """A cached block must not be rewritten, marker or not.++ Preserving the ``cache_control`` field does not preserve the cache+ entry: the provider keys on the content. Rewriting a block it has+ already hashed re-bills the whole prefix at the write rate.+ """payload = {"system": [{@@ -103,10 +108,34 @@ class TestCompactSystemPromptContentBlocks:model="m",request_id="test3",)- assert modified is True+ assert modified is Falseblock = result["system"][0]+ assert block["text"] == "A" * 1000assert block["cache_control"] == {"type": "ephemeral"}+ def test_compacts_after_the_breakpoint_and_keeps_the_marker(self) -> None:+ """Blocks past the last marker are still fair game."""+ payload = {+ "system": [+ {+ "type": "text",+ "text": "A" * 1000,+ "cache_control": {"type": "ephemeral"},+ },+ {"type": "text", "text": "B" * 1000},+ ],+ }+ result, modified, _, _ = compact_system_prompt(+ payload,+ router=_MockRouter(),+ model="m",+ request_id="test3b",+ )+ assert modified is True+ assert result["system"][0]["text"] == "A" * 1000+ assert result["system"][0]["cache_control"] == {"type": "ephemeral"}+ assert result["system"][1]["text"] != "B" * 1000+def test_preserves_non_text_blocks(self) -> None:payload = {"system": [