Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): stop skipping tool compaction on a pinned tools array (#3750)
Tejas Chopraa6a9cef9458e
diff --git a/headroom/proxy/tool_schema_compaction.py b/headroom/proxy/tool_schema_compaction.pyindex ce2fd9c1a..f8bad6f30 100644--- a/headroom/proxy/tool_schema_compaction.py+++ b/headroom/proxy/tool_schema_compaction.py@@ -395,21 +395,28 @@ def compact_tool_descriptions(if not isinstance(tools, list) or not tools:return payload, False, 0, 0- # A cache_control marker on any tool means "cache everything up to and- # including this one". Truncating a description inside that span changes the- # bytes the provider hashed, so the whole tools prefix -- and every message- # after it -- re-bills as cache creation. The saving is a few hundred bytes;- # the bust is the entire conversation. Mirrors the same guard on- # ``_sort_tools_deterministically`` (handlers/anthropic.py) and- # ``any_tool_has_cache_control`` in the Rust live zone.- marked = sum(1 for t in tools if isinstance(t, dict) and t.get("cache_control"))- if marked:- logger.info(- "event=tool_desc_compaction_skipped reason=marker_present tool_count=%d marked=%d",- len(tools),- marked,- )- return payload, False, 0, 0+ # NO cache_control guard here, deliberately. The obvious reasoning -- "a+ # marker means the provider hashed these bytes, so rewriting them busts the+ # prefix" -- does not survive contact with the API, because the provider+ # never saw the client's bytes. It only ever sees ours. Compaction is a pure+ # deterministic transform, so turn 2 sends exactly what turn 1 cached and+ # hits. Measured against the real API, 30 pinned tools over 6 turns:+ #+ # always raw write 12,536 then read 12,536 x5 billed 21,938+ # always compacted write 8,726 then read 8,726 x5 billed 15,270 (-30%)+ #+ # Skipping made the cached prefix 30% larger for the life of every session+ # that pins its tools, to avoid a bust that never happened.+ #+ # The one real hazard is INCONSISTENCY -- compacting on some turns and not+ # others, which ``_decision.should_compress`` can produce via a per-request+ # bypass header or a license gate flipping. That is bounded too: Anthropic+ # keeps both prefixes alive, so alternating turns each hit their own entry+ # and the cost is one extra write, once, not a bust per flip.+ #+ # This reasoning is specific to a pure byte transform over a fixed key set.+ # It does NOT extend to system-prompt compaction, whose compressor is+ # pluggable and not established to be deterministic -- that guard stays.strip_sem = strip_semantic_params()key = _cache_key(tools, "L2", max_chars, strip_sem)diff --git a/tests/test_cache_prefix_safety.py b/tests/test_cache_prefix_safety.pyindex 5dbee0268..a444431ac 100644--- a/tests/test_cache_prefix_safety.py+++ b/tests/test_cache_prefix_safety.py@@ -11,10 +11,11 @@ provider's key is the content, not the marker.from __future__ import annotations+import jsonfrom typing import Anyfrom headroom.proxy.system_compaction import _compact_system_blocks-from headroom.proxy.tool_schema_compaction import compact_tool_descriptions+from headroom.proxy.tool_schema_compaction import compact_tool_descriptions, compact_toolsdef _text(chars: int, seed: str = "x") -> str:@@ -36,8 +37,16 @@ class _StubRouter:# ---------------------------------------------------------------------------def test_tool_desc_compaction_skips_when_a_tool_is_marked() -> None:- """A marker on any tool freezes the whole tools prefix."""+def test_tool_desc_compaction_runs_even_when_a_tool_is_marked() -> None:+ """A marker does NOT freeze the tools, and skipping was the expensive choice.++ The provider never saw the client's bytes -- it only ever sees ours -- so+ there is nothing to preserve. Compaction is deterministic, so every turn+ sends the same compacted bytes and the cache hits. Measured against the real+ API with 30 pinned tools over 6 turns, always-compacted billed 15,270 in+ prefix tokens against 21,938 for always-raw: skipping made the cached prefix+ 30% larger for the whole session to avoid a bust that never happened.+ """payload = {"tools": [{"name": "a", "description": _text(4000), "input_schema": {}},@@ -49,13 +58,65 @@ def test_tool_desc_compaction_skips_when_a_tool_is_marked() -> None:},]}- before = [dict(t) for t in payload["tools"]]- out, modified, _, _ = compact_tool_descriptions(payload, max_chars=50)+ out, modified, before_bytes, after_bytes = compact_tool_descriptions(payload, max_chars=50)- assert modified is False- assert out is payload- assert out["tools"] == before+ assert modified is True+ assert after_bytes < before_bytes+ # The marker itself is carried through untouched -- we shrink the schema,+ # we do not take over the client's cache management.+ assert out["tools"][1]["cache_control"] == {"type": "ephemeral"}+++def test_compaction_is_byte_stable_across_calls() -> None:+ """The property the whole argument rests on.++ Cache-safety here is not "we avoided the bytes", it is "we produce the same+ bytes every time". If this ever stops holding, the removed guard has to come+ back -- so it is asserted rather than assumed.+ """++ def _payload() -> dict[str, Any]:+ return {+ "tools": [+ {+ "name": f"t{i}",+ "description": " Lots of whitespace. " * 20,+ "input_schema": {+ "$schema": "https://json-schema.org/draft/2020-12/schema",+ "title": "Args",+ "examples": [{"a": 1}],+ "type": "object",+ "properties": {"a": {"type": "string", "description": " x y "}},+ },+ }+ for i in range(5)+ ]+ }++ first, _, _, _ = compact_tools(_payload())+ second, _, _, _ = compact_tools(_payload())++ assert json.dumps(first["tools"], sort_keys=True) == json.dumps(second["tools"], sort_keys=True)+++def test_a_marked_tools_array_is_compacted_by_layer_one_too() -> None:+ """Layer 1 never had a guard, and now it deliberately never will."""+ payload = {+ "tools": [+ {+ "name": "a",+ "description": "d",+ "input_schema": {"type": "object", "title": "T", "properties": {}},+ "cache_control": {"type": "ephemeral"},+ }+ ]+ }++ out, modified, _, _ = compact_tools(payload)++ assert modified is True+ assert "title" not in out["tools"][0]["input_schema"]def test_tool_desc_compaction_still_runs_when_nothing_is_marked() -> None: