Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): stop prefix transforms from busting the provider cache (#3735)

Tejas Chopraa2bf5ed1ba45
diff --git a/headroom/proxy/system_compaction.py b/headroom/proxy/system_compaction.py
index 03ac42d3f..6ce99fcbc 100644
--- a/headroom/proxy/system_compaction.py
+++ b/headroom/proxy/system_compaction.py
@@ -41,6 +41,21 @@ def _json_byte_len(value: Any) -> int:
return len(json.dumps(value, ensure_ascii=False, default=str, separators=(",", ":")))
+def _last_cache_control_index(blocks: list[Any]) -> int:
+ """Index of the last block carrying ``cache_control``, or ``-1`` if none.
+
+ Everything at or before this index is inside a prefix the provider has
+ already hashed and is billing at the cache-read rate. Mutating those bytes
+ busts the entry and re-bills the whole prefix at write rate (1.25x list on
+ Anthropic), which costs far more than the compaction saves.
+ """
+ last = -1
+ for i, block in enumerate(blocks):
+ if isinstance(block, dict) and block.get("cache_control"):
+ last = i
+ return last
+
+
def _compact_system_blocks(
blocks: list[dict[str, Any]],
router: Any,
@@ -57,11 +72,23 @@ def _compact_system_blocks(
modified = False
updated: list[dict[str, Any]] = []
- for block in blocks:
+ # A cache_control marker anywhere in the system array freezes every block at
+ # or before it: the provider hashed those exact bytes, so rewriting any of
+ # them re-bills the whole prefix as cache creation. Preserving the marker
+ # field (as this function does) does NOT preserve the cache entry -- the key
+ # is the content, not the marker. Compact only what sits after the last
+ # breakpoint. See ``_last_cache_control_index``.
+ frozen_upto = _last_cache_control_index(blocks)
+
+ for idx, block in enumerate(blocks):
if not isinstance(block, dict) or block.get("type") != "text":
updated.append(block)
continue
+ if idx <= frozen_upto:
+ updated.append(block)
+ continue
+
text = block.get("text", "")
if not isinstance(text, str) or len(text) < min_chars:
updated.append(block)
diff --git a/headroom/proxy/tool_schema_compaction.py b/headroom/proxy/tool_schema_compaction.py
index d43a04cc5..ce2fd9c1a 100644
--- a/headroom/proxy/tool_schema_compaction.py
+++ b/headroom/proxy/tool_schema_compaction.py
@@ -34,11 +34,14 @@ from __future__ import annotations
import copy
import hashlib
import json
+import logging
import os
import re
import threading
from typing import Any
+logger = logging.getLogger(__name__)
+
# Keys that are JSON Schema annotations, not constraints.
# Removing them does not change the set of valid inputs.
TOOL_SCHEMA_DROP_KEYS: frozenset[str] = frozenset(
@@ -392,6 +395,22 @@ def compact_tool_descriptions(
if not isinstance(tools, list) or not tools:
return payload, False, 0, 0
+ # A cache_control marker on any tool means "cache everything up to and
+ # including this one". Truncating a description inside that span changes the
+ # bytes the provider hashed, so the whole tools prefix -- and every message
+ # after it -- re-bills as cache creation. The saving is a few hundred bytes;
+ # the bust is the entire conversation. Mirrors the same guard on
+ # ``_sort_tools_deterministically`` (handlers/anthropic.py) and
+ # ``any_tool_has_cache_control`` in the Rust live zone.
+ marked = sum(1 for t in tools if isinstance(t, dict) and t.get("cache_control"))
+ if marked:
+ logger.info(
+ "event=tool_desc_compaction_skipped reason=marker_present tool_count=%d marked=%d",
+ len(tools),
+ marked,
+ )
+ return payload, False, 0, 0
+
strip_sem = strip_semantic_params()
key = _cache_key(tools, "L2", max_chars, strip_sem)
cached = _cache_get(key)
diff --git a/tests/test_anthropic_compaction_transforms.py b/tests/test_anthropic_compaction_transforms.py
index 676beefe9..d7e5a81f8 100644
--- a/tests/test_anthropic_compaction_transforms.py
+++ b/tests/test_anthropic_compaction_transforms.py
@@ -52,10 +52,16 @@ def _make_anthropic_payload_with_tools() -> dict:
def _make_anthropic_payload_with_long_system() -> dict:
"""Payload with a long system prompt that qualifies for L3 compaction."""
long_text = "x" * 5000 # well above default min_chars
+ # Two blocks, mirroring how agentic clients actually shape the system array:
+ # a stable block the client has pinned with cache_control (CLAUDE.md, rules)
+ # followed by dynamic content. Only the block AFTER the last breakpoint is
+ # eligible -- rewriting the pinned one would bust the provider's entry, so
+ # compaction must leave it byte-identical.
return {
"model": "claude-sonnet-4-6",
"system": [
{"type": "text", "text": long_text, "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": "y" * 5000},
],
"messages": [{"role": "user", "content": "hello"}],
"tools": [],
@@ -383,7 +389,11 @@ def test_handler_auxiliary_compaction_respects_optimization_decision(
if should_compact:
assert "title" not in forwarded["tools"][0]["input_schema"]
assert len(forwarded["tools"][0]["description"]) < len(payload["tools"][0]["description"])
- assert forwarded["system"][0]["text"] == "short system"
+ # Block 0 is pinned by cache_control: byte-identical, marker intact.
+ assert forwarded["system"][0]["text"] == payload["system"][0]["text"]
+ assert forwarded["system"][0]["cache_control"] == {"type": "ephemeral"}
+ # Block 1 sits after the breakpoint and is compacted.
+ assert forwarded["system"][1]["text"] == "short system"
assert all(label in transforms for label in labels)
router.compress.assert_called_once()
assert summary["compression"]["total_tokens_removed"] > 0
diff --git a/tests/test_cache_prefix_safety.py b/tests/test_cache_prefix_safety.py
new file mode 100644
index 000000000..5dbee0268
--- /dev/null
+++ b/tests/test_cache_prefix_safety.py
@@ -0,0 +1,125 @@
+"""Transforms must never rewrite bytes the provider has already hashed.
+
+A ``cache_control`` marker means the provider is billing everything up to and
+including that point at the cache-read rate (0.1x on Anthropic). Rewriting any
+of those bytes busts the entry and re-bills the whole prefix at the write rate
+(1.25x) -- a 12.5x swing that dwarfs anything the transform saves.
+
+Preserving the ``cache_control`` *field* does not preserve the *entry*: the
+provider's key is the content, not the marker.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from headroom.proxy.system_compaction import _compact_system_blocks
+from headroom.proxy.tool_schema_compaction import compact_tool_descriptions
+
+
+def _text(chars: int, seed: str = "x") -> str:
+ return (seed * 10 + " context line. ") * (chars // 25 + 1)
+
+
+class _StubRouter:
+ """Router whose compression is obvious and always shrinks."""
+
+ def compress(self, text: str, context: str = "") -> Any: # noqa: ARG002
+ class _R:
+ compressed = "SHRUNK"
+
+ return _R()
+
+
+# --------------------------------------------------------------------------
+# tool description compaction
+# --------------------------------------------------------------------------
+
+
+def test_tool_desc_compaction_skips_when_a_tool_is_marked() -> None:
+ """A marker on any tool freezes the whole tools prefix."""
+ payload = {
+ "tools": [
+ {"name": "a", "description": _text(4000), "input_schema": {}},
+ {
+ "name": "b",
+ "description": _text(4000, "y"),
+ "input_schema": {},
+ "cache_control": {"type": "ephemeral"},
+ },
+ ]
+ }
+ before = [dict(t) for t in payload["tools"]]
+
+ out, modified, _, _ = compact_tool_descriptions(payload, max_chars=50)
+
+ assert modified is False
+ assert out is payload
+ assert out["tools"] == before
+
+
+def test_tool_desc_compaction_still_runs_when_nothing_is_marked() -> None:
+ """The common case -- no markers -- must keep working."""
+ payload = {
+ "tools": [
+ {"name": "a", "description": _text(4000), "input_schema": {}},
+ {"name": "b", "description": _text(4000, "y"), "input_schema": {}},
+ ]
+ }
+
+ out, modified, before_bytes, after_bytes = compact_tool_descriptions(payload, max_chars=50)
+
+ assert modified is True
+ assert after_bytes < before_bytes
+ assert all(len(t["description"]) <= 80 for t in out["tools"])
+
+
+# --------------------------------------------------------------------------
+# system prompt compaction
+# --------------------------------------------------------------------------
+
+
+def test_system_compaction_leaves_blocks_at_or_before_the_marker() -> None:
+ """Blocks inside the cached span keep their exact bytes."""
+ blocks = [
+ {"type": "text", "text": _text(2000, "a")},
+ {"type": "text", "text": _text(2000, "b"), "cache_control": {"type": "ephemeral"}},
+ {"type": "text", "text": _text(2000, "c")},
+ ]
+ original = [dict(b) for b in blocks]
+
+ out, modified = _compact_system_blocks(blocks, _StubRouter(), "m", "rid", 500)
+
+ # frozen: identical bytes, marker intact
+ assert out[0] == original[0]
+ assert out[1] == original[1]
+ assert out[1]["cache_control"] == {"type": "ephemeral"}
+ # after the breakpoint: free to compact
+ assert out[2]["text"] == "SHRUNK"
+ assert modified is True
+
+
+def test_system_compaction_unrestricted_without_markers() -> None:
+ blocks = [
+ {"type": "text", "text": _text(2000, "a")},
+ {"type": "text", "text": _text(2000, "b")},
+ ]
+
+ out, modified = _compact_system_blocks(blocks, _StubRouter(), "m", "rid", 500)
+
+ assert modified is True
+ assert [b["text"] for b in out] == ["SHRUNK", "SHRUNK"]
+
+
+def test_system_compaction_noop_when_marker_is_last() -> None:
+ """Marker on the final block freezes everything."""
+ blocks = [
+ {"type": "text", "text": _text(2000, "a")},
+ {"type": "text", "text": _text(2000, "b"), "cache_control": {"type": "ephemeral"}},
+ ]
+ original = [dict(b) for b in blocks]
+
+ out, modified = _compact_system_blocks(blocks, _StubRouter(), "m", "rid", 500)
+
+ assert modified is False
+ assert out == original
diff --git a/tests/test_system_compaction.py b/tests/test_system_compaction.py
index c39c3a888..85daf7455 100644
--- a/tests/test_system_compaction.py
+++ b/tests/test_system_compaction.py
@@ -86,8 +86,13 @@ class TestCompactSystemPromptContentBlocks:
assert modified is False
assert result["system"][0]["text"] == "Short instruction."
- def test_preserves_cache_control(self) -> None:
- """cache_control must survive compaction."""
+ def test_marked_block_is_left_byte_identical(self) -> None:
+ """A cached block must not be rewritten, marker or not.
+
+ Preserving the ``cache_control`` field does not preserve the cache
+ entry: the provider keys on the content. Rewriting a block it has
+ already hashed re-bills the whole prefix at the write rate.
+ """
payload = {
"system": [
{
@@ -103,10 +108,34 @@ class TestCompactSystemPromptContentBlocks:
model="m",
request_id="test3",
)
- assert modified is True
+ assert modified is False
block = result["system"][0]
+ assert block["text"] == "A" * 1000
assert block["cache_control"] == {"type": "ephemeral"}
+ def test_compacts_after_the_breakpoint_and_keeps_the_marker(self) -> None:
+ """Blocks past the last marker are still fair game."""
+ payload = {
+ "system": [
+ {
+ "type": "text",
+ "text": "A" * 1000,
+ "cache_control": {"type": "ephemeral"},
+ },
+ {"type": "text", "text": "B" * 1000},
+ ],
+ }
+ result, modified, _, _ = compact_system_prompt(
+ payload,
+ router=_MockRouter(),
+ model="m",
+ request_id="test3b",
+ )
+ assert modified is True
+ assert result["system"][0]["text"] == "A" * 1000
+ assert result["system"][0]["cache_control"] == {"type": "ephemeral"}
+ assert result["system"][1]["text"] != "B" * 1000
+
def test_preserves_non_text_blocks(self) -> None:
payload = {
"system": [