Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): keep read maturation out of the provider-confirmed prefix (#4522)
gglucass526e06b7e012
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 377413da9..7fb61cc20 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -2645,9 +2645,14 @@ class AnthropicHandlerMixin:prefix_tracker.restored_maturation_state = Nonematuration_mgr.restore_state(state)_maturation_mgr = maturation_mgr+ # Never below the floor finalize_turn just replayed: the+ # pipeline's freeze is clamped to what the local compression+ # cache can byte-replay, which stops at the first tool_result+ # it never stored, so on its own it lets a cached Read mature+ # and re-write the provider cache from that Read onward.maturation = maturation_mgr.apply(optimized_messages,- frozen_message_count=frozen_message_count,+ frozen_message_count=max(frozen_message_count, _confirmed_frozen),)if maturation.replacements_applied or maturation.holding_msg_indices:optimized_messages = maturation.messagesdiff --git a/tests/test_prefix_tracker_restart.py b/tests/test_prefix_tracker_restart.pyindex 195a0697e..1c9529528 100644--- a/tests/test_prefix_tracker_restart.py+++ b/tests/test_prefix_tracker_restart.py@@ -137,13 +137,14 @@ _SCENARIOS = {def _restart_between_turns(- monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, scenario: str, **overrides: Any) -> tuple[list[dict[str, Any]], list[dict[str, Any]], int, dict[str, str]]:"""Some turns in one proxy, the next turn in a fresh one, one warmprovider cache. Returns (last forwarded before, first forwarded after,its read hit, the original behind each retrieval reference forwardedbefore the restart)."""build, turns, config, kompress = _SCENARIOS[scenario]+ config = {**config, **overrides}monkeypatch.setenv("HEADROOM_CCR_BACKEND", "memory")# As the desktop app runs it: a cold start forwards the fast pass.monkeypatch.setenv("HEADROOM_BACKGROUND_COMPRESSION", "1")@@ -182,8 +183,11 @@ def test_restored_retrieval_references_resolve_to_their_originals(tracker kept replaying pointed at a hash that no longer resolved."""monkeypatch.setenv(PREFIX_STATE_FILE_ENV, str(tmp_path / "prefix-state.json"))+ # Each Read matures as it arrives, before the provider caches it, so+ # references exist before the restart. A Read that quiesces later is+ # already cached by then and stays verbatim.previous, current, _, originals = _restart_between_turns(- monkeypatch, tmp_path, "read-maturation"+ monkeypatch, tmp_path, "read-maturation", read_maturation_quiesce_turns=0)emitted = _ccr_hashes(current)diff --git a/tests/test_read_maturation_handler_nobust.py b/tests/test_read_maturation_handler_nobust.pyindex f5856b7b2..2ff6cd973 100644--- a/tests/test_read_maturation_handler_nobust.py+++ b/tests/test_read_maturation_handler_nobust.py@@ -183,64 +183,10 @@ def test_held_read_keeps_client_breakpoint_then_matures(monkeypatch):cost_tracking_enabled=False,log_requests=False,)- app = create_app(config)- forwarded: list[list[dict]] = []-- with TestClient(app) as client:- proxy = client.app.state.proxy- original_retry = proxy._retry_request-- async def _mock_upstream(method, url, headers, body, stream=False, **kwargs):- msgs = body.get("messages", []) or []- forwarded.append(copy.deepcopy(msgs))- # Simulate Anthropic honestly caching up to the handler's breakpoint- # (system blocks are cached too), so the prefix tracker advances- # frozen_message_count as in prod.- bp = _breakpoint_index(msgs)- sys_tokens = sum(- max(1, len(str(b.get("text", ""))) // 4)- for b in (body.get("system") or [])- if isinstance(b, dict)- )- cached = sys_tokens + (sum(_est_tokens(m) for m in msgs[: bp + 1]) if bp >= 0 else 0)- return httpx.Response(- 200,- json={- "id": "msg_x",- "type": "message",- "role": "assistant",- "content": [{"type": "text", "text": "ok"}],- "usage": {- "input_tokens": 20,- "output_tokens": 2,- "cache_read_input_tokens": cached,- "cache_creation_input_tokens": 0,- },- },- )-- proxy._retry_request = _mock_upstream- try:- for n in range(0, 7):- r = client.post(- "/v1/messages",- headers={- "x-api-key": "test-key",- "anthropic-version": "2023-06-01",- "x-headroom-session-id": "nobust-1",- "content-type": "application/json",- },- json={- "model": MODEL,- "max_tokens": 20,- "system": SYSTEM,- "tools": [READ_TOOL],- "messages": _convo(n),- },- )- assert r.status_code == 200, f"turn {n}: {r.text[:300]}"- finally:- proxy._retry_request = original_retry+ # The provider confirms no cache (as after a lapse), so nothing is frozen+ # and a quiet Read may mature. Once the provider has cached it, it stays+ # verbatim instead: test_read_inside_the_cached_prefix_stays_verbatim.+ forwarded = _drive_session(config, n_turns=7, session_id="nobust-1", cache_hits=False)assert forwarded, "no requests were forwarded"@@ -263,7 +209,7 @@ def test_held_read_keeps_client_breakpoint_then_matures(monkeypatch):# Read, which dropped Claude Code's tail breakpoint for the whole hold and# re-sent the Read and everything after it uncached on every held turn, to# save one cache write of the Read. The held Read is now cached with its- # turn, and maturing it re-writes the cache from the Read onward once.+ # turn.moved = [turnfor turn, msgs in enumerate(forwarded)@@ -288,11 +234,14 @@ def _drive_session(session_id: str,saved_out: list[int] | None = None,convo=_convo,+ cache_hits: bool = True,+ tools: list[dict] | None = None,) -> list[list[dict]]:"""Drive ``n_turns`` cumulative turns of ``convo`` through the real handlerwith a mocked upstream; return the forwarded message arrays per turn. When``saved_out`` is given, each turn's emitted ``x-headroom-tokens-saved`` is- appended to it."""+ appended to it. The upstream caches up to the handler's breakpoint, or+ reports no cache at all when ``cache_hits`` is False."""app = create_app(config)forwarded: list[list[dict]] = []with TestClient(app) as client:@@ -309,6 +258,8 @@ def _drive_session(if isinstance(b, dict))cached = sys_tokens + (sum(_est_tokens(m) for m in msgs[: bp + 1]) if bp >= 0 else 0)+ if not cache_hits:+ cached = 0return httpx.Response(200,json={@@ -340,7 +291,7 @@ def _drive_session("model": MODEL,"max_tokens": 20,"system": SYSTEM,- "tools": [READ_TOOL],+ "tools": tools or [READ_TOOL],"messages": convo(n),},)@@ -406,6 +357,121 @@ def test_read_cached_with_its_turn_stays_verbatim(monkeypatch):assert _first_matured_turn(forwarded) is None, "a cached Read was matured"+DEFERRED_TOOL = {+ "name": "mcp__notes__search",+ "description": "Search notes",+ "defer_loading": True,+ "input_schema": {"type": "object", "properties": {"q": {"type": "string"}}},+}+++def _tool_search_convo(nquiet: int) -> list[dict]:+ """``_busy_convo`` after a ToolSearch whose result holds only tool_reference+ blocks, as Claude Code loads deferred tools. The compression cache reads no+ text from that result, so its count never passes it, however much of the+ later tool output compresses."""+ msgs = _busy_convo(nquiet)+ search = [+ {+ "role": "assistant",+ "content": [+ {+ "type": "tool_use",+ "id": "toolu_ts",+ "name": "ToolSearch",+ "input": {"query": "notes"},+ }+ ],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "toolu_ts",+ "content": [{"type": "tool_reference", "tool_name": DEFERRED_TOOL["name"]}],+ }+ ],+ },+ ]+ return msgs[:1] + search + msgs[1:]+++def _without_cache_control(messages: list[dict]) -> list[dict]:+ return json.loads(+ json.dumps(messages),+ object_hook=lambda d: {k: v for k, v in d.items() if k != "cache_control"},+ )+++def _prefix_rewrites(forwarded: list[list[dict]]) -> list[int]:+ """Turns whose forwarded messages do not start with the previous turn's.+ The mock caches each turn through its last message, so any such turn+ re-writes the provider cache. Breakpoints move every turn and are ignored."""+ rewrites = []+ for turn in range(1, len(forwarded)):+ prev = _without_cache_control(forwarded[turn - 1])+ if _without_cache_control(forwarded[turn])[: len(prev)] != prev:+ rewrites.append(turn)+ return rewrites+++def _record_pipeline_freeze(monkeypatch) -> list[int]:+ from headroom.proxy import session_engine++ seen: list[int] = []+ prepare = session_engine.prepare_turn++ def _recording_prepare(*args, **kwargs):+ prep = prepare(*args, **kwargs)+ seen.append(prep.frozen_message_count)+ return prep++ monkeypatch.setattr(session_engine, "prepare_turn", _recording_prepare)+ return seen+++ ("convo", "tools", "pipeline_stop"),+ [+ # Nothing compresses, so the compression cache never stores the Read.+ (_convo, [READ_TOOL], 2),+ # Tool output compresses, but the count stops at the ToolSearch result.+ (_tool_search_convo, [READ_TOOL, DEFERRED_TOOL], 2),+ ],+ ids=["quiet", "tool-reference"],+)+def test_read_inside_the_cached_prefix_stays_verbatim(monkeypatch, convo, tools, pipeline_stop):+ """By the time the Read's file quiesces the provider has cached it, so+ maturing it would re-write the cache from the Read onward. The pipeline's+ own freeze stops below the Read here; maturation must follow the+ provider-confirmed floor that finalize_turn replays instead."""+ from headroom.cache.compression_store import reset_compression_store++ monkeypatch.setenv("HEADROOM_CCR_BACKEND", "memory")+ reset_compression_store()+ pipeline_freeze = _record_pipeline_freeze(monkeypatch)+ config = ProxyConfig(+ optimize=True,+ read_maturation=True,+ mode="token",+ cache_enabled=True,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ )++ forwarded = _drive_session(+ config, n_turns=8, session_id="cached-read-2", convo=convo, tools=tools+ )++ assert pipeline_freeze and max(pipeline_freeze) <= pipeline_stop, (+ f"fixture must keep the pipeline freeze below the Read: {pipeline_freeze}"+ )+ assert _first_matured_turn(forwarded) is None, "a cached Read was matured"+ assert not _prefix_rewrites(forwarded), "the forwarded cached prefix changed"++def test_quiesce_turns_config_is_honored(monkeypatch):"""`quiesce_turns` must be runtime-configurable end-to-end: a fresh Read of/x/foo.py matures `quiesce_turns` quiet turns after it appears (the convo@@ -428,7 +494,7 @@ def test_quiesce_turns_config_is_honored(monkeypatch):cost_tracking_enabled=False,log_requests=False,)- forwarded = _drive_session(config, n_turns=4, session_id="quiesce-cfg-1")+ forwarded = _drive_session(config, n_turns=4, session_id="quiesce-cfg-1", cache_hits=False)first = _first_matured_turn(forwarded)assert first == 2, (f"expected the Read to mature at turn 2 with quiesce_turns=2, "@@ -503,7 +569,9 @@ def test_replayed_marker_is_not_rebooked_in_the_emitted_savings(monkeypatch):log_requests=False,)saved: list[int] = []- forwarded = _drive_session(config, n_turns=5, session_id="first-appearance-1", saved_out=saved)+ forwarded = _drive_session(+ config, n_turns=5, session_id="first-appearance-1", saved_out=saved, cache_hits=False+ )first = _first_matured_turn(forwarded)assert first is not None, "the Read never matured, so there is nothing to replay"