Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): stop skipping tool compaction on a pinned tools array (#3750)

Tejas Chopraa6a9cef9458e
diff --git a/headroom/proxy/tool_schema_compaction.py b/headroom/proxy/tool_schema_compaction.py
index ce2fd9c1a..f8bad6f30 100644
--- a/headroom/proxy/tool_schema_compaction.py
+++ b/headroom/proxy/tool_schema_compaction.py
@@ -395,21 +395,28 @@ def compact_tool_descriptions(
if not isinstance(tools, list) or not tools:
return payload, False, 0, 0
- # A cache_control marker on any tool means "cache everything up to and
- # including this one". Truncating a description inside that span changes the
- # bytes the provider hashed, so the whole tools prefix -- and every message
- # after it -- re-bills as cache creation. The saving is a few hundred bytes;
- # the bust is the entire conversation. Mirrors the same guard on
- # ``_sort_tools_deterministically`` (handlers/anthropic.py) and
- # ``any_tool_has_cache_control`` in the Rust live zone.
- marked = sum(1 for t in tools if isinstance(t, dict) and t.get("cache_control"))
- if marked:
- logger.info(
- "event=tool_desc_compaction_skipped reason=marker_present tool_count=%d marked=%d",
- len(tools),
- marked,
- )
- return payload, False, 0, 0
+ # NO cache_control guard here, deliberately. The obvious reasoning -- "a
+ # marker means the provider hashed these bytes, so rewriting them busts the
+ # prefix" -- does not survive contact with the API, because the provider
+ # never saw the client's bytes. It only ever sees ours. Compaction is a pure
+ # deterministic transform, so turn 2 sends exactly what turn 1 cached and
+ # hits. Measured against the real API, 30 pinned tools over 6 turns:
+ #
+ # always raw write 12,536 then read 12,536 x5 billed 21,938
+ # always compacted write 8,726 then read 8,726 x5 billed 15,270 (-30%)
+ #
+ # Skipping made the cached prefix 30% larger for the life of every session
+ # that pins its tools, to avoid a bust that never happened.
+ #
+ # The one real hazard is INCONSISTENCY -- compacting on some turns and not
+ # others, which ``_decision.should_compress`` can produce via a per-request
+ # bypass header or a license gate flipping. That is bounded too: Anthropic
+ # keeps both prefixes alive, so alternating turns each hit their own entry
+ # and the cost is one extra write, once, not a bust per flip.
+ #
+ # This reasoning is specific to a pure byte transform over a fixed key set.
+ # It does NOT extend to system-prompt compaction, whose compressor is
+ # pluggable and not established to be deterministic -- that guard stays.
strip_sem = strip_semantic_params()
key = _cache_key(tools, "L2", max_chars, strip_sem)
diff --git a/tests/test_cache_prefix_safety.py b/tests/test_cache_prefix_safety.py
index 5dbee0268..a444431ac 100644
--- a/tests/test_cache_prefix_safety.py
+++ b/tests/test_cache_prefix_safety.py
@@ -11,10 +11,11 @@ provider's key is the content, not the marker.
from __future__ import annotations
+import json
from typing import Any
from headroom.proxy.system_compaction import _compact_system_blocks
-from headroom.proxy.tool_schema_compaction import compact_tool_descriptions
+from headroom.proxy.tool_schema_compaction import compact_tool_descriptions, compact_tools
def _text(chars: int, seed: str = "x") -> str:
@@ -36,8 +37,16 @@ class _StubRouter:
# --------------------------------------------------------------------------
-def test_tool_desc_compaction_skips_when_a_tool_is_marked() -> None:
- """A marker on any tool freezes the whole tools prefix."""
+def test_tool_desc_compaction_runs_even_when_a_tool_is_marked() -> None:
+ """A marker does NOT freeze the tools, and skipping was the expensive choice.
+
+ The provider never saw the client's bytes -- it only ever sees ours -- so
+ there is nothing to preserve. Compaction is deterministic, so every turn
+ sends the same compacted bytes and the cache hits. Measured against the real
+ API with 30 pinned tools over 6 turns, always-compacted billed 15,270 in
+ prefix tokens against 21,938 for always-raw: skipping made the cached prefix
+ 30% larger for the whole session to avoid a bust that never happened.
+ """
payload = {
"tools": [
{"name": "a", "description": _text(4000), "input_schema": {}},
@@ -49,13 +58,65 @@ def test_tool_desc_compaction_skips_when_a_tool_is_marked() -> None:
},
]
}
- before = [dict(t) for t in payload["tools"]]
- out, modified, _, _ = compact_tool_descriptions(payload, max_chars=50)
+ out, modified, before_bytes, after_bytes = compact_tool_descriptions(payload, max_chars=50)
- assert modified is False
- assert out is payload
- assert out["tools"] == before
+ assert modified is True
+ assert after_bytes < before_bytes
+ # The marker itself is carried through untouched -- we shrink the schema,
+ # we do not take over the client's cache management.
+ assert out["tools"][1]["cache_control"] == {"type": "ephemeral"}
+
+
+def test_compaction_is_byte_stable_across_calls() -> None:
+ """The property the whole argument rests on.
+
+ Cache-safety here is not "we avoided the bytes", it is "we produce the same
+ bytes every time". If this ever stops holding, the removed guard has to come
+ back -- so it is asserted rather than assumed.
+ """
+
+ def _payload() -> dict[str, Any]:
+ return {
+ "tools": [
+ {
+ "name": f"t{i}",
+ "description": " Lots of whitespace. " * 20,
+ "input_schema": {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "title": "Args",
+ "examples": [{"a": 1}],
+ "type": "object",
+ "properties": {"a": {"type": "string", "description": " x y "}},
+ },
+ }
+ for i in range(5)
+ ]
+ }
+
+ first, _, _, _ = compact_tools(_payload())
+ second, _, _, _ = compact_tools(_payload())
+
+ assert json.dumps(first["tools"], sort_keys=True) == json.dumps(second["tools"], sort_keys=True)
+
+
+def test_a_marked_tools_array_is_compacted_by_layer_one_too() -> None:
+ """Layer 1 never had a guard, and now it deliberately never will."""
+ payload = {
+ "tools": [
+ {
+ "name": "a",
+ "description": "d",
+ "input_schema": {"type": "object", "title": "T", "properties": {}},
+ "cache_control": {"type": "ephemeral"},
+ }
+ ]
+ }
+
+ out, modified, _, _ = compact_tools(payload)
+
+ assert modified is True
+ assert "title" not in out["tools"][0]["input_schema"]
def test_tool_desc_compaction_still_runs_when_nothing_is_marked() -> None: