Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): keep proactive expansion out of the cached prefix (#4300)
Matt Van Hornb1f340e0b35d
diff --git a/headroom/ccr/context_tracker.py b/headroom/ccr/context_tracker.pyindex 786eaaf15..ed634e858 100644--- a/headroom/ccr/context_tracker.py+++ b/headroom/ccr/context_tracker.py@@ -31,6 +31,12 @@ from ..cache.compression_store import get_compression_storelogger = logging.getLogger(__name__)+# Wrapper around injected proactive-expansion text. The proxy places this+# block after the last cache breakpoint and strips it from the prefix-tracker+# snapshot so a later overlay/restart never has to replay it (#4287).+PROACTIVE_EXPANSION_OPEN_TAG = "<headroom_proactive_expansion>"+PROACTIVE_EXPANSION_CLOSE_TAG = "</headroom_proactive_expansion>"+def looks_like_claude_code_compact_summary(*texts: str | None) -> bool:"""Return true for Claude Code `/compact` continuation summaries.@@ -575,8 +581,8 @@ class ContextTracker:parts.append("[End Proactive Expansion]")body = "\n".join(parts)# Escape any stray close tag in payload to prevent wrapper boundary forgery- body = body.replace("</headroom_proactive_expansion>", "<\\/headroom_proactive_expansion>")- return f"<headroom_proactive_expansion>\n{body}\n</headroom_proactive_expansion>"+ body = body.replace(PROACTIVE_EXPANSION_CLOSE_TAG, "<\\/headroom_proactive_expansion>")+ return f"{PROACTIVE_EXPANSION_OPEN_TAG}\n{body}\n{PROACTIVE_EXPANSION_CLOSE_TAG}"def get_tracked_hashes(self) -> list[str]:"""Get list of currently tracked hashes."""diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex d77b8ba90..5c21a4786 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -150,6 +150,10 @@ def _is_googleapis_endpoint(value: object) -> bool:return hostname == "googleapis.com" or hostname.endswith(".googleapis.com")+class _ProxyExpansionBlock(dict):+ pass++class _AnthropicTurnHookUsage:"""Usage from hook-triggered Anthropic calls the main response omits.@@ -820,6 +824,7 @@ class AnthropicHandlerMixin:# next turn, so any injection here busts the prefix# cache at this message from then on.and "cache_control" not in block+ and not isinstance(block, _ProxyExpansionBlock)):existing = block.get("text", "")new_content.append({**block, "text": existing + "\n\n" + context_text})@@ -833,6 +838,108 @@ class AnthropicHandlerMixin:return messages+ @staticmethod+ def _is_proactive_expansion_block(block: Any) -> bool:+ return isinstance(block, _ProxyExpansionBlock)++ @classmethod+ def _append_proactive_expansion_after_cached_prefix(+ cls,+ messages: list[dict[str, Any]],+ context_text: str,+ ) -> list[dict[str, Any]]:+ """Place CCR expansion after the last message-level cache breakpoint.++ The extra text is a new content block with no ``cache_control``, so it+ sits in this turn's uncached tail. Existing prefix bytes stay+ identical and role alternation is preserved. The client does not+ resend the block; callers must omit it from the prefix-tracker+ snapshot (#4287).+ """+ if not messages or not context_text:+ return messages++ last_message_index: int | None = None+ last_block_index: int | None = None+ for message_index, message in enumerate(messages):+ if not isinstance(message, dict):+ continue+ content = message.get("content")+ if isinstance(content, list):+ for block_index, block in enumerate(content):+ if isinstance(block, dict) and "cache_control" in block:+ last_message_index = message_index+ last_block_index = block_index+ if "cache_control" in message:+ last_message_index = message_index+ if isinstance(content, list) and content:+ last_block_index = len(content) - 1++ expansion_block = _ProxyExpansionBlock({"type": "text", "text": context_text})+ target_index = last_message_index+ if target_index is None:+ target_index = len(messages) - 1+ while target_index >= 0 and (+ not isinstance(messages[target_index], dict)+ or messages[target_index].get("role") == "system"+ ):+ target_index -= 1+ if target_index < 0 or messages[target_index].get("role") != "user":+ return messages+ last_block_index = None++ message = messages[target_index]+ content = message.get("content")+ updated = list(messages)+ if isinstance(content, list):+ new_content = list(content)+ insert_at = last_block_index + 1 if last_block_index is not None else len(new_content)+ while (+ insert_at < len(new_content)+ and isinstance(new_content[insert_at], dict)+ and new_content[insert_at].get("type") == "tool_result"+ ):+ insert_at += 1+ new_content.insert(insert_at, expansion_block)+ updated[target_index] = {**message, "content": new_content}+ return updated+ if isinstance(content, str):+ cached_block: dict[str, Any] = {"type": "text", "text": content}+ marker = message.get("cache_control")+ if isinstance(marker, dict):+ cached_block["cache_control"] = marker+ updated[target_index] = {+ **{k: v for k, v in message.items() if k != "cache_control"},+ "content": [cached_block, expansion_block],+ }+ return updated+ return messages++ @classmethod+ def _without_proactive_expansion_messages(+ cls,+ messages: list[dict[str, Any]],+ ) -> list[dict[str, Any]]:+ """Drop proxy-injected CCR expansion blocks from a forwarded snapshot."""+ stripped: list[dict[str, Any]] = []+ changed = False+ for message in messages:+ if not isinstance(message, dict):+ stripped.append(message)+ continue+ content = message.get("content")+ if not isinstance(content, list):+ stripped.append(message)+ continue+ kept = [block for block in content if not cls._is_proactive_expansion_block(block)]+ if kept == content:+ stripped.append(message)+ continue+ changed = True+ if kept:+ stripped.append({**message, "content": kept})+ return stripped if changed else list(messages)+@staticmethoddef _strict_previous_turn_frozen_count(messages: list[dict[str, Any]],@@ -2945,11 +3052,15 @@ class AnthropicHandlerMixin:"in cache mode to preserve next-turn prefix stability")else:+ # After the last cache breakpoint, not into the+ # latest user turn: that turn is inside the+ # cached prefix (Claude Code's trailing system+ # marker sits after it). Mutating it makes the+ # next overlay/restart rewrite the conversation.optimized_messages = (- self._append_context_to_latest_non_frozen_user_turn(+ self._append_proactive_expansion_after_cached_prefix(optimized_messages,expansion_text,- frozen_message_count=frozen_message_count,))@@ -4000,7 +4111,10 @@ class AnthropicHandlerMixin:# mode falls back to full unmodified passthrough every# turn instead of compressing the append-only delta.next_original_messages = copy.deepcopy(original_client_messages)- next_forwarded_messages = copy.deepcopy(optimized_messages)+ tracker_forwarded = self._without_proactive_expansion_messages(+ optimized_messages+ )+ next_forwarded_messages = copy.deepcopy(tracker_forwarded)assistant_message = self._assistant_message_from_response_json(backend_response.body)@@ -4010,7 +4124,7 @@ class AnthropicHandlerMixin:if hasattr(prefix_tracker, "classify_cache_miss"):miss = prefix_tracker.classify_cache_miss(cache_read_tokens=cr_tokens,- current_forwarded_messages=optimized_messages,+ current_forwarded_messages=tracker_forwarded,)from headroom.cache.ttl_observations import (record_cache_observation,@@ -5120,9 +5234,14 @@ class AnthropicHandlerMixin:)await self.metrics.record_cache_bust(bust_tokens)- # Update prefix cache tracker for next turn+ # Update prefix cache tracker for next turn. Omit the+ # CCR expansion turn: the client never resends it, so+ # overlay must replay the pre-injection bytes (#4287).next_original_messages = copy.deepcopy(original_client_messages)- next_forwarded_messages = copy.deepcopy(optimized_messages)+ tracker_forwarded = self._without_proactive_expansion_messages(+ optimized_messages+ )+ next_forwarded_messages = copy.deepcopy(tracker_forwarded)assistant_message = self._assistant_message_from_response_json(resp_json)if assistant_message is not None:next_original_messages.append(copy.deepcopy(assistant_message))@@ -5134,15 +5253,15 @@ class AnthropicHandlerMixin:# longer TTL) or the cacheable prefix changed (content shifted).# Classify BEFORE update_from_response, which overwrites the# last-turn state the classifier reads (idle clock, prefix,- # cached-token count). `optimized_messages` is the prefix we- # forwarded this turn; compare it against last turn's.+ # cached-token count). Compare the snapshot we will record,+ # not the expansion turn the client will not resend.# `hasattr` guard: some tests inject a SimpleNamespace stub# tracker that only implements the freeze API, not the full# PrefixCacheTracker surface.if hasattr(prefix_tracker, "classify_cache_miss"):miss = prefix_tracker.classify_cache_miss(cache_read_tokens=cr_tokens,- current_forwarded_messages=optimized_messages,+ current_forwarded_messages=tracker_forwarded,)from headroom.cache.ttl_observations import (record_cache_observation,diff --git a/headroom/proxy/handlers/streaming.py b/headroom/proxy/handlers/streaming.pyindex da3943457..2444a6c4f 100644--- a/headroom/proxy/handlers/streaming.py+++ b/headroom/proxy/handlers/streaming.py@@ -928,6 +928,9 @@ class StreamingMixin:import copy as _copyforwarded_messages = body.get("messages", [])+ strip_expansion = getattr(self, "_without_proactive_expansion_messages", None)+ if callable(strip_expansion) and provider == "anthropic":+ forwarded_messages = strip_expansion(forwarded_messages)if not forwarded_messages and provider == "gemini":# Gemini bodies carry contents[] rather than messages[] (Cloud# Code Assist nests the payload under body["request"]).@@ -2267,6 +2270,9 @@ class StreamingMixin:if optimized_messages is not Noneelse body.get("messages", []))+ strip_expansion = getattr(self, "_without_proactive_expansion_messages", None)+ if callable(strip_expansion) and provider == "anthropic":+ tracker_messages = strip_expansion(tracker_messages)next_forwarded = _copy.deepcopy(tracker_messages)next_original = _copy.deepcopy(original_messages or tracker_messages)if full_sse_bytes:diff --git a/tests/test_cache_prefix_overlay.py b/tests/test_cache_prefix_overlay.pyindex e65ca0798..7d703ccd4 100644--- a/tests/test_cache_prefix_overlay.py+++ b/tests/test_cache_prefix_overlay.py@@ -94,6 +94,27 @@ def test_overlay_never_inflates_forwarded_payload():assert overlay_cached_prefix(optimized, current, previous, inflated_forwarded) == optimized+def test_overlay_declines_an_expansion_baked_into_message_zero_when_floor_is_zero():+ """A first-turn CCR expansion in the cached prefix is larger than the client replay.++ With ``confirmed_frozen_count=0`` overlay refuses the inflated replay, so+ the next turn forwards the unexpanded client bytes and rewrites the cache+ from message 0. Recording the pre-injection snapshot avoids this (#4287).+ """+ original = [M("user", "hello")]+ expanded = [M("user", "hello\n\n<headroom_proactive_expansion>\n" + "x" * 400)]+ current = original + [M("user", "next")]+ optimized = current+ declined = overlay_cached_prefix(+ optimized, current, original, expanded, confirmed_frozen_count=0+ )+ assert declined == optimized+ replayed = overlay_cached_prefix(+ optimized, current, original, original, confirmed_frozen_count=0+ )+ assert replayed[:1] == original++def test_overlay_returns_optimized_when_json_sizing_fails(monkeypatch):optimized = [M("user", "stable"), M("user", "tail")]current = [M("user", "stable"), M("user", "tail")]diff --git a/tests/test_ccr_expansion_conversation_scope.py b/tests/test_ccr_expansion_conversation_scope.pyindex a2bf830ff..2b40b8909 100644--- a/tests/test_ccr_expansion_conversation_scope.py+++ b/tests/test_ccr_expansion_conversation_scope.py@@ -87,7 +87,7 @@ def _track_lead_compression(proxy, cwd: str) -> str: # noqa: ANN001return hash_key-def _forwarded_last_user_text(client: TestClient, cwd: str, messages: list[dict]) -> str:+def _forwarded_request(client: TestClient, cwd: str, messages: list[dict]) -> dict:proxy = client.app.state.proxycaptured: dict = {}@@ -100,7 +100,12 @@ def _forwarded_last_user_text(client: TestClient, cwd: str, messages: list[dict]"type": "message","role": "assistant","content": [{"type": "text", "text": "ok"}],- "usage": {"input_tokens": 1, "output_tokens": 1},+ "usage": {+ "input_tokens": 20,+ "output_tokens": 1,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 20,+ },},)@@ -115,8 +120,34 @@ def _forwarded_last_user_text(client: TestClient, cwd: str, messages: list[dict]json={"model": "claude-sonnet-4-6", "max_tokens": 16, "messages": messages},)assert response.status_code == 200, response.text- content = captured["body"]["messages"][-1]["content"]- return content if isinstance(content, str) else content[0]["text"]+ return captured["body"]+++def _message_blob(message: dict) -> str:+ content = message.get("content")+ if isinstance(content, str):+ return content+ if isinstance(content, list):+ return "\n".join(+ block.get("text", "") if isinstance(block, dict) else str(block) for block in content+ )+ return ""+++def _system_blob(system: object) -> str:+ if isinstance(system, str):+ return system+ if isinstance(system, list):+ return "\n".join(+ block.get("text", "") if isinstance(block, dict) else str(block) for block in system+ )+ return ""+++def _forwarded_blob(body: dict) -> str:+ parts = [_system_blob(body.get("system"))]+ parts.extend(_message_blob(message) for message in body.get("messages") or [])+ return "\n".join(parts)def test_teammate_does_not_receive_lead_tool_output(tmp_path) -> None: # noqa: ANN001@@ -124,12 +155,14 @@ def test_teammate_does_not_receive_lead_tool_output(tmp_path) -> None: # noqa:with _client() as client:_track_lead_compression(client.app.state.proxy, cwd)- forwarded = _forwarded_last_user_text(- client, cwd, [{"role": "user", "content": _TEAMMATE_QUERY}]- )+ body = _forwarded_request(client, cwd, [{"role": "user", "content": _TEAMMATE_QUERY}])+ forwarded = _forwarded_blob(body)assert _EXPANSION_TAG not in forwarded- assert forwarded == _TEAMMATE_QUERY+ last_user = next(+ message for message in reversed(body["messages"]) if message.get("role") == "user"+ )+ assert last_user["content"] == _TEAMMATE_QUERYdef test_conversation_holding_the_marker_still_expands(tmp_path) -> None: # noqa: ANN001@@ -137,32 +170,118 @@ def test_conversation_holding_the_marker_still_expands(tmp_path) -> None: # noqwith _client() as client:hash_key = _track_lead_compression(client.app.state.proxy, cwd)- forwarded = _forwarded_last_user_text(- client,- cwd,- [- {"role": "user", "content": _LEAD_QUERY},- {- "role": "assistant",- "content": [- {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "ls"}}- ],- },- {- "role": "user",- "content": [- {- "type": "tool_result",- "tool_use_id": "t1",- "content": f"[40 items compressed to 4. Retrieve more: hash={hash_key}]",- }- ],- },- {"role": "assistant", "content": "Listed them."},- {"role": "user", "content": _TEAMMATE_QUERY},- ],+ forwarded = _forwarded_blob(+ _forwarded_request(+ client,+ cwd,+ [+ {"role": "user", "content": _LEAD_QUERY},+ {+ "role": "assistant",+ "content": [+ {+ "type": "tool_use",+ "id": "t1",+ "name": "Bash",+ "input": {"command": "ls"},+ }+ ],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "t1",+ "content": (+ f"[40 items compressed to 4. Retrieve more: hash={hash_key}]"+ ),+ }+ ],+ },+ {"role": "assistant", "content": "Listed them."},+ {"role": "user", "content": _TEAMMATE_QUERY},+ ],+ ))assert _EXPANSION_TAG in forwarded# Line 39 is only in the original, never in the compressed preview.assert "src/auth/middleware_39.py" in forwarded+++def test_expansion_stays_out_of_the_cached_prefix_and_tracker_snapshot(+ tmp_path,+) -> None: # noqa: ANN001+ """#4287: expansion after the last cache breakpoint, omitted from overlay state."""+ from headroom.cache.prefix_tracker import overlay_cached_prefix++ cwd = str(tmp_path)+ trailing_system = {+ "role": "system",+ "content": [+ {"type": "text", "text": "# Environment", "cache_control": {"type": "ephemeral"}}+ ],+ }+ history = [+ {"role": "user", "content": _LEAD_QUERY},+ {+ "role": "assistant",+ "content": [+ {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"command": "ls"}}+ ],+ },+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "t1",+ "content": "[40 items compressed to 4. Retrieve more: hash=pending]",+ }+ ],+ },+ {"role": "assistant", "content": "Listed them."},+ {"role": "user", "content": _TEAMMATE_QUERY},+ trailing_system,+ ]++ with _client() as client:+ hash_key = _track_lead_compression(client.app.state.proxy, cwd)+ history[2]["content"][0]["content"] = (+ f"[40 items compressed to 4. Retrieve more: hash={hash_key}]"+ )+ turn1 = _forwarded_request(client, cwd, history)++ forwarded = turn1["messages"]+ assert _EXPANSION_TAG in _forwarded_blob(turn1)+ # The client user turn stays unexpanded; expansion rides after the+ # trailing system's cache_control (relocated to top-level `system`+ # for this model, issue #765).+ assert _EXPANSION_TAG not in _message_blob(forwarded[0])+ last_user = next(+ message for message in reversed(forwarded) if message.get("role") == "user"+ )+ assert last_user["content"] == _TEAMMATE_QUERY+ assert _EXPANSION_TAG in _system_blob(turn1.get("system"))+ assert history[4]["content"] == _TEAMMATE_QUERY+ assert history[5]["content"][0]["text"] == "# Environment"++ trackers = list(client.app.state.proxy.session_tracker_store._trackers.values())+ assert trackers+ snapshot = trackers[0].get_last_forwarded_messages()+ assert snapshot+ assert all(_EXPANSION_TAG not in _message_blob(message) for message in snapshot)+ assert len(snapshot) == len(history) + 1 # client messages plus assistant reply++ next_original = history + [{"role": "user", "content": "follow up"}]+ next_optimized = list(next_original)+ replayed = overlay_cached_prefix(+ next_optimized,+ next_original,+ history,+ snapshot[:-1],+ confirmed_frozen_count=0,+ )+ assert replayed[: len(history)] == snapshot[:-1]+ assert all(_EXPANSION_TAG not in _message_blob(message) for message in replayed)diff --git a/tests/test_proxy_handler_helpers.py b/tests/test_proxy_handler_helpers.pyindex 443615e70..40a9312df 100644--- a/tests/test_proxy_handler_helpers.py+++ b/tests/test_proxy_handler_helpers.py@@ -1265,6 +1265,89 @@ def test_append_context_skips_trailing_system_message() -> None:assert inject(messages, "memory", frozen_message_count=0) is messages+def test_proactive_expansion_appends_after_cached_prefix_without_mutating_it() -> None:+ """CCR expansion must not rewrite a message the provider has already cached (#4287)."""+ from headroom.ccr.context_tracker import (+ PROACTIVE_EXPANSION_CLOSE_TAG,+ PROACTIVE_EXPANSION_OPEN_TAG,+ )++ env_block = {"type": "text", "text": "# Environment", "cache_control": {"type": "ephemeral"}}+ trailing_system = {"role": "system", "content": [env_block]}+ first_user = {+ "role": "user",+ "content": [{"type": "text", "text": "hello from turn 0"}],+ }+ messages = [first_user, trailing_system]+ expansion = f"{PROACTIVE_EXPANSION_OPEN_TAG}\nexpanded\n{PROACTIVE_EXPANSION_CLOSE_TAG}"++ forwarded = AnthropicHandlerMixin._append_proactive_expansion_after_cached_prefix(+ messages, expansion+ )++ assert forwarded[0] == first_user+ assert forwarded[1]["role"] == "system"+ assert forwarded[1]["content"][0] == env_block+ assert forwarded[1]["content"][1] == {"type": "text", "text": expansion}+ assert "cache_control" not in forwarded[1]["content"][1]+ assert AnthropicHandlerMixin._without_proactive_expansion_messages(forwarded) == messages+ assert messages[0]["content"][0]["text"] == "hello from turn 0"+ assert messages[1]["content"] == [env_block]+++def test_proactive_expansion_without_cache_control_stays_on_the_live_user_turn() -> None:+ from headroom.ccr.context_tracker import (+ PROACTIVE_EXPANSION_CLOSE_TAG,+ PROACTIVE_EXPANSION_OPEN_TAG,+ )++ messages = [{"role": "user", "content": "hello"}]+ expansion = f"{PROACTIVE_EXPANSION_OPEN_TAG}\nexpanded\n{PROACTIVE_EXPANSION_CLOSE_TAG}"+ forwarded = AnthropicHandlerMixin._append_proactive_expansion_after_cached_prefix(+ messages, expansion+ )+ assert forwarded[0]["content"][0] == {"type": "text", "text": "hello"}+ assert forwarded[0]["content"][1] == {"type": "text", "text": expansion}+ assert AnthropicHandlerMixin._without_proactive_expansion_messages(forwarded) == [+ {"role": "user", "content": [{"type": "text", "text": "hello"}]}+ ]+ assert messages[0]["content"] == "hello"+++def test_proactive_expansion_keeps_tool_results_together_and_client_text() -> None:+ from headroom.ccr.context_tracker import (+ PROACTIVE_EXPANSION_CLOSE_TAG,+ PROACTIVE_EXPANSION_OPEN_TAG,+ )++ expansion = f"{PROACTIVE_EXPANSION_OPEN_TAG}\nexample\n{PROACTIVE_EXPANSION_CLOSE_TAG}"+ tool_a = {+ "type": "tool_result",+ "tool_use_id": "A",+ "content": "a",+ "cache_control": {"type": "ephemeral"},+ }+ tool_b = {"type": "tool_result", "tool_use_id": "B", "content": "b"}+ messages = [{"role": "user", "content": [tool_a, tool_b]}]+ forwarded = AnthropicHandlerMixin._append_proactive_expansion_after_cached_prefix(+ messages, expansion+ )+ content = forwarded[0]["content"]+ assert [block.get("type") for block in content] == ["tool_result", "tool_result", "text"]+ assert content[0] is tool_a and content[1] is tool_b and "cache_control" not in content[2]+ assert AnthropicHandlerMixin._without_proactive_expansion_messages(forwarded) == messages+ client = [+ {+ "role": "user",+ "content": [+ {"type": "text", "text": "Explain this:"},+ {"type": "text", "text": expansion},+ ],+ }+ ]+ assert AnthropicHandlerMixin._without_proactive_expansion_messages(client) == client++def test_anthropic_image_compression_helper_only_rewrites_latest_eligible_turn() -> None:image_message = {"role": "user",