Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): keep read maturation out of the provider-confirmed prefix (#4522)

gglucass526e06b7e012
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 377413da9..7fb61cc20 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -2645,9 +2645,14 @@ class AnthropicHandlerMixin:
prefix_tracker.restored_maturation_state = None
maturation_mgr.restore_state(state)
_maturation_mgr = maturation_mgr
+ # Never below the floor finalize_turn just replayed: the
+ # pipeline's freeze is clamped to what the local compression
+ # cache can byte-replay, which stops at the first tool_result
+ # it never stored, so on its own it lets a cached Read mature
+ # and re-write the provider cache from that Read onward.
maturation = maturation_mgr.apply(
optimized_messages,
- frozen_message_count=frozen_message_count,
+ frozen_message_count=max(frozen_message_count, _confirmed_frozen),
)
if maturation.replacements_applied or maturation.holding_msg_indices:
optimized_messages = maturation.messages
diff --git a/tests/test_prefix_tracker_restart.py b/tests/test_prefix_tracker_restart.py
index 195a0697e..1c9529528 100644
--- a/tests/test_prefix_tracker_restart.py
+++ b/tests/test_prefix_tracker_restart.py
@@ -137,13 +137,14 @@ _SCENARIOS = {
def _restart_between_turns(
- monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str, **overrides: Any
) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int, dict[str, str]]:
"""Some turns in one proxy, the next turn in a fresh one, one warm
provider cache. Returns (last forwarded before, first forwarded after,
its read hit, the original behind each retrieval reference forwarded
before the restart)."""
build, turns, config, kompress = _SCENARIOS[scenario]
+ config = {**config, **overrides}
monkeypatch.setenv("HEADROOM_CCR_BACKEND", "memory")
# As the desktop app runs it: a cold start forwards the fast pass.
monkeypatch.setenv("HEADROOM_BACKGROUND_COMPRESSION", "1")
@@ -182,8 +183,11 @@ def test_restored_retrieval_references_resolve_to_their_originals(
tracker kept replaying pointed at a hash that no longer resolved."""
monkeypatch.setenv(PREFIX_STATE_FILE_ENV, str(tmp_path / "prefix-state.json"))
+ # Each Read matures as it arrives, before the provider caches it, so
+ # references exist before the restart. A Read that quiesces later is
+ # already cached by then and stays verbatim.
previous, current, _, originals = _restart_between_turns(
- monkeypatch, tmp_path, "read-maturation"
+ monkeypatch, tmp_path, "read-maturation", read_maturation_quiesce_turns=0
)
emitted = _ccr_hashes(current)
diff --git a/tests/test_read_maturation_handler_nobust.py b/tests/test_read_maturation_handler_nobust.py
index f5856b7b2..2ff6cd973 100644
--- a/tests/test_read_maturation_handler_nobust.py
+++ b/tests/test_read_maturation_handler_nobust.py
@@ -183,64 +183,10 @@ def test_held_read_keeps_client_breakpoint_then_matures(monkeypatch):
cost_tracking_enabled=False,
log_requests=False,
)
- app = create_app(config)
- forwarded: list[list[dict]] = []
-
- with TestClient(app) as client:
- proxy = client.app.state.proxy
- original_retry = proxy._retry_request
-
- async def _mock_upstream(method, url, headers, body, stream=False, **kwargs):
- msgs = body.get("messages", []) or []
- forwarded.append(copy.deepcopy(msgs))
- # Simulate Anthropic honestly caching up to the handler's breakpoint
- # (system blocks are cached too), so the prefix tracker advances
- # frozen_message_count as in prod.
- bp = _breakpoint_index(msgs)
- sys_tokens = sum(
- max(1, len(str(b.get("text", ""))) // 4)
- for b in (body.get("system") or [])
- if isinstance(b, dict)
- )
- cached = sys_tokens + (sum(_est_tokens(m) for m in msgs[: bp + 1]) if bp >= 0 else 0)
- return httpx.Response(
- 200,
- json={
- "id": "msg_x",
- "type": "message",
- "role": "assistant",
- "content": [{"type": "text", "text": "ok"}],
- "usage": {
- "input_tokens": 20,
- "output_tokens": 2,
- "cache_read_input_tokens": cached,
- "cache_creation_input_tokens": 0,
- },
- },
- )
-
- proxy._retry_request = _mock_upstream
- try:
- for n in range(0, 7):
- r = client.post(
- "/v1/messages",
- headers={
- "x-api-key": "test-key",
- "anthropic-version": "2023-06-01",
- "x-headroom-session-id": "nobust-1",
- "content-type": "application/json",
- },
- json={
- "model": MODEL,
- "max_tokens": 20,
- "system": SYSTEM,
- "tools": [READ_TOOL],
- "messages": _convo(n),
- },
- )
- assert r.status_code == 200, f"turn {n}: {r.text[:300]}"
- finally:
- proxy._retry_request = original_retry
+ # The provider confirms no cache (as after a lapse), so nothing is frozen
+ # and a quiet Read may mature. Once the provider has cached it, it stays
+ # verbatim instead: test_read_inside_the_cached_prefix_stays_verbatim.
+ forwarded = _drive_session(config, n_turns=7, session_id="nobust-1", cache_hits=False)
assert forwarded, "no requests were forwarded"
@@ -263,7 +209,7 @@ def test_held_read_keeps_client_breakpoint_then_matures(monkeypatch):
# Read, which dropped Claude Code's tail breakpoint for the whole hold and
# re-sent the Read and everything after it uncached on every held turn, to
# save one cache write of the Read. The held Read is now cached with its
- # turn, and maturing it re-writes the cache from the Read onward once.
+ # turn.
moved = [
turn
for turn, msgs in enumerate(forwarded)
@@ -288,11 +234,14 @@ def _drive_session(
session_id: str,
saved_out: list[int] | None = None,
convo=_convo,
+ cache_hits: bool = True,
+ tools: list[dict] | None = None,
) -> list[list[dict]]:
"""Drive ``n_turns`` cumulative turns of ``convo`` through the real handler
with a mocked upstream; return the forwarded message arrays per turn. When
``saved_out`` is given, each turn's emitted ``x-headroom-tokens-saved`` is
- appended to it."""
+ appended to it. The upstream caches up to the handler's breakpoint, or
+ reports no cache at all when ``cache_hits`` is False."""
app = create_app(config)
forwarded: list[list[dict]] = []
with TestClient(app) as client:
@@ -309,6 +258,8 @@ def _drive_session(
if isinstance(b, dict)
)
cached = sys_tokens + (sum(_est_tokens(m) for m in msgs[: bp + 1]) if bp >= 0 else 0)
+ if not cache_hits:
+ cached = 0
return httpx.Response(
200,
json={
@@ -340,7 +291,7 @@ def _drive_session(
"model": MODEL,
"max_tokens": 20,
"system": SYSTEM,
- "tools": [READ_TOOL],
+ "tools": tools or [READ_TOOL],
"messages": convo(n),
},
)
@@ -406,6 +357,121 @@ def test_read_cached_with_its_turn_stays_verbatim(monkeypatch):
assert _first_matured_turn(forwarded) is None, "a cached Read was matured"
+DEFERRED_TOOL = {
+ "name": "mcp__notes__search",
+ "description": "Search notes",
+ "defer_loading": True,
+ "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}},
+}
+
+
+def _tool_search_convo(nquiet: int) -> list[dict]:
+ """``_busy_convo`` after a ToolSearch whose result holds only tool_reference
+ blocks, as Claude Code loads deferred tools. The compression cache reads no
+ text from that result, so its count never passes it, however much of the
+ later tool output compresses."""
+ msgs = _busy_convo(nquiet)
+ search = [
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": "toolu_ts",
+ "name": "ToolSearch",
+ "input": {"query": "notes"},
+ }
+ ],
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_ts",
+ "content": [{"type": "tool_reference", "tool_name": DEFERRED_TOOL["name"]}],
+ }
+ ],
+ },
+ ]
+ return msgs[:1] + search + msgs[1:]
+
+
+def _without_cache_control(messages: list[dict]) -> list[dict]:
+ return json.loads(
+ json.dumps(messages),
+ object_hook=lambda d: {k: v for k, v in d.items() if k != "cache_control"},
+ )
+
+
+def _prefix_rewrites(forwarded: list[list[dict]]) -> list[int]:
+ """Turns whose forwarded messages do not start with the previous turn's.
+ The mock caches each turn through its last message, so any such turn
+ re-writes the provider cache. Breakpoints move every turn and are ignored."""
+ rewrites = []
+ for turn in range(1, len(forwarded)):
+ prev = _without_cache_control(forwarded[turn - 1])
+ if _without_cache_control(forwarded[turn])[: len(prev)] != prev:
+ rewrites.append(turn)
+ return rewrites
+
+
+def _record_pipeline_freeze(monkeypatch) -> list[int]:
+ from headroom.proxy import session_engine
+
+ seen: list[int] = []
+ prepare = session_engine.prepare_turn
+
+ def _recording_prepare(*args, **kwargs):
+ prep = prepare(*args, **kwargs)
+ seen.append(prep.frozen_message_count)
+ return prep
+
+ monkeypatch.setattr(session_engine, "prepare_turn", _recording_prepare)
+ return seen
+
+
+ ("convo", "tools", "pipeline_stop"),
+ [
+ # Nothing compresses, so the compression cache never stores the Read.
+ (_convo, [READ_TOOL], 2),
+ # Tool output compresses, but the count stops at the ToolSearch result.
+ (_tool_search_convo, [READ_TOOL, DEFERRED_TOOL], 2),
+ ],
+ ids=["quiet", "tool-reference"],
+)
+def test_read_inside_the_cached_prefix_stays_verbatim(monkeypatch, convo, tools, pipeline_stop):
+ """By the time the Read's file quiesces the provider has cached it, so
+ maturing it would re-write the cache from the Read onward. The pipeline's
+ own freeze stops below the Read here; maturation must follow the
+ provider-confirmed floor that finalize_turn replays instead."""
+ from headroom.cache.compression_store import reset_compression_store
+
+ monkeypatch.setenv("HEADROOM_CCR_BACKEND", "memory")
+ reset_compression_store()
+ pipeline_freeze = _record_pipeline_freeze(monkeypatch)
+ config = ProxyConfig(
+ optimize=True,
+ read_maturation=True,
+ mode="token",
+ cache_enabled=True,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ )
+
+ forwarded = _drive_session(
+ config, n_turns=8, session_id="cached-read-2", convo=convo, tools=tools
+ )
+
+ assert pipeline_freeze and max(pipeline_freeze) <= pipeline_stop, (
+ f"fixture must keep the pipeline freeze below the Read: {pipeline_freeze}"
+ )
+ assert _first_matured_turn(forwarded) is None, "a cached Read was matured"
+ assert not _prefix_rewrites(forwarded), "the forwarded cached prefix changed"
+
+
def test_quiesce_turns_config_is_honored(monkeypatch):
"""`quiesce_turns` must be runtime-configurable end-to-end: a fresh Read of
/x/foo.py matures `quiesce_turns` quiet turns after it appears (the convo
@@ -428,7 +494,7 @@ def test_quiesce_turns_config_is_honored(monkeypatch):
cost_tracking_enabled=False,
log_requests=False,
)
- forwarded = _drive_session(config, n_turns=4, session_id="quiesce-cfg-1")
+ forwarded = _drive_session(config, n_turns=4, session_id="quiesce-cfg-1", cache_hits=False)
first = _first_matured_turn(forwarded)
assert first == 2, (
f"expected the Read to mature at turn 2 with quiesce_turns=2, "
@@ -503,7 +569,9 @@ def test_replayed_marker_is_not_rebooked_in_the_emitted_savings(monkeypatch):
log_requests=False,
)
saved: list[int] = []
- forwarded = _drive_session(config, n_turns=5, session_id="first-appearance-1", saved_out=saved)
+ forwarded = _drive_session(
+ config, n_turns=5, session_id="first-appearance-1", saved_out=saved, cache_hits=False
+ )
first = _first_matured_turn(forwarded)
assert first is not None, "the Read never matured, so there is nothing to replay"