Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
Add Anthropic cache-mode delta replay
JerrettDavis96fd3d965201
diff --git a/benchmarks/claude_session_mode_benchmark.py b/benchmarks/claude_session_mode_benchmark.pyindex da5c4a89a..135d57dcd 100644--- a/benchmarks/claude_session_mode_benchmark.py+++ b/benchmarks/claude_session_mode_benchmark.py@@ -433,7 +433,7 @@ def resolve_checkpoint_dir(recent_turns_per_session: int | None = None,cache_ttl_minutes: int = DEFAULT_CACHE_TTL_MINUTES,) -> Path:- suffix_parts = ["v3", f"ttl_{cache_ttl_minutes}m"]+ suffix_parts = ["v4", f"ttl_{cache_ttl_minutes}m"]if recent_turns_per_session:suffix_parts.append(f"recent_{recent_turns_per_session}")else:@@ -646,20 +646,36 @@ def _apply_mode_to_messages(model: str,prefix_tracker: PrefixCacheTracker | None,comp_cache: CompressionCache | None,+ previous_original_messages: list[dict[str, Any]] | None = None,+ previous_forwarded_messages: list[dict[str, Any]] | None = None,) -> list[dict[str, Any]]:if mode == "baseline":return copy.deepcopy(messages)- if mode == PROXY_MODE_CACHE:- return copy.deepcopy(messages)assert proxy is not Noneassert prefix_tracker is not None- frozen_message_count = prefix_tracker.get_frozen_message_count()if mode == PROXY_MODE_CACHE:- frozen_message_count = AnthropicHandlerMixin._strict_previous_turn_frozen_count(+ delta = AnthropicHandlerMixin._extract_cache_stable_delta(messages,- frozen_message_count,+ previous_original_messages,+ previous_forwarded_messages,+ )+ if delta is None:+ return copy.deepcopy(messages)+ stable_forwarded_prefix, delta_messages = delta+ if not delta_messages:+ return stable_forwarded_prefix+ context_limit = proxy.anthropic_provider.get_context_limit(model)+ result = proxy.anthropic_pipeline.apply(+ messages=delta_messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(delta_messages),+ frozen_message_count=0,)+ return stable_forwarded_prefix + result.messages++ frozen_message_count = prefix_tracker.get_frozen_message_count()working_messages = copy.deepcopy(messages)if proxy.config.image_optimize and working_messages and _messages_have_images(working_messages):@@ -710,6 +726,7 @@ class _PendingTurn:turn: ReplayTurntokenizer: Anyraw_input_tokens: int+ request_messages: list[dict[str, Any]]forwarded: list[dict[str, Any]]@@ -912,6 +929,8 @@ def _simulate_single_replay_mode(conversation: list[dict[str, Any]] = []conversation_token_total = 0previous_forwarded: list[dict[str, Any]] = []+ previous_original_context: list[dict[str, Any]] | None = None+ previous_forwarded_context: list[dict[str, Any]] | None = Noneprevious_timestamp: datetime | None = Noneprefix_tracker = None if mode == "baseline" else PrefixCacheTracker("anthropic")comp_cache = CompressionCache() if mode == PROXY_MODE_TOKEN else None@@ -928,6 +947,8 @@ def _simulate_single_replay_mode(model=turn.model,prefix_tracker=prefix_tracker,comp_cache=comp_cache,+ previous_original_messages=previous_original_context,+ previous_forwarded_messages=previous_forwarded_context,)if pending is not None:_apply_turn_metrics(@@ -952,6 +973,7 @@ def _simulate_single_replay_mode(cache_write_tokens=0,messages=forwarded,message_token_counts=[tokenizer.count_message(msg) for msg in forwarded],+ original_messages=conversation,)pending = _PendingTurn(@@ -959,12 +981,17 @@ def _simulate_single_replay_mode(turn=turn,tokenizer=tokenizer,raw_input_tokens=raw_input_tokens,+ request_messages=copy.deepcopy(conversation),forwarded=forwarded,)conversation.append(turn.assistant_message)conversation_token_total = raw_input_tokens + tokenizer.count_message(turn.assistant_message)+ previous_original_context = copy.deepcopy(conversation)+ previous_forwarded_context = copy.deepcopy(forwarded) + [+ copy.deepcopy(turn.assistant_message)+ ]if pending is not None:_apply_turn_metrics(diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.pyindex eecb38736..77561ca23 100644--- a/headroom/cache/prefix_tracker.py+++ b/headroom/cache/prefix_tracker.py@@ -16,6 +16,7 @@ pipeline skips them entirely.from __future__ import annotations+import copyimport hashlibimport loggingimport time@@ -88,6 +89,8 @@ class PrefixCacheTracker:self._cached_message_count: int = 0self._turn_number: int = 0self._last_activity: float = time.time()+ self._last_original_messages: list[dict[str, Any]] = []+ self._last_forwarded_messages: list[dict[str, Any]] = []# Statsself._busts_avoided: int = 0@@ -113,6 +116,7 @@ class PrefixCacheTracker:cache_write_tokens: int,messages: list[dict[str, Any]],message_token_counts: list[int] | None = None,+ original_messages: list[dict[str, Any]] | None = None,) -> None:"""Update tracker with cache metrics from the API response.@@ -128,6 +132,8 @@ class PrefixCacheTracker:"""self._last_activity = time.time()self._turn_number += 1+ self._last_original_messages = copy.deepcopy(original_messages or messages)+ self._last_forwarded_messages = copy.deepcopy(messages)# Compute total cached tokens (read + write = what's in cache now)total_cached = cache_read_tokens + cache_write_tokens@@ -167,6 +173,12 @@ class PrefixCacheTracker:cache_write_tokens,)+ def get_last_original_messages(self) -> list[dict[str, Any]]:+ return copy.deepcopy(self._last_original_messages)++ def get_last_forwarded_messages(self) -> list[dict[str, Any]]:+ return copy.deepcopy(self._last_forwarded_messages)+def record_bust_avoided(self, tokens_preserved: int, compression_foregone: int) -> None:"""Record when we chose to preserve cache over compressing."""self._busts_avoided += 1diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 519157660..b1345bf6d 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -155,6 +155,43 @@ class AnthropicHandlerMixin:changed += 1return restored, changed+ @staticmethod+ def _extract_cache_stable_delta(+ current_messages: list[dict[str, Any]],+ previous_original_messages: list[dict[str, Any]] | None,+ previous_forwarded_messages: list[dict[str, Any]] | None,+ ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]] | None:+ """Return (stable_forwarded_prefix, appended_delta_messages) when safe.++ Safe means the prior original request is an exact message-prefix of the+ current original request. This lets us replay the exact forwarded bytes+ for historical context and only transform newly appended message suffixes.+ """+ if not previous_original_messages or previous_forwarded_messages is None:+ return None+ prefix_len = len(previous_original_messages)+ if len(current_messages) < prefix_len:+ return None+ if current_messages[:prefix_len] != previous_original_messages:+ return None+ return (+ copy.deepcopy(previous_forwarded_messages),+ copy.deepcopy(current_messages[prefix_len:]),+ )++ @staticmethod+ def _assistant_message_from_response_json(+ resp_json: dict[str, Any] | None,+ ) -> dict[str, Any] | None:+ if not resp_json:+ return None+ if resp_json.get("role") != "assistant":+ return None+ return {+ "role": "assistant",+ "content": copy.deepcopy(resp_json.get("content", "")),+ }+async def handle_anthropic_messages(self,request: Request,@@ -311,7 +348,7 @@ class AnthropicHandlerMixin:# Hook: pre_compress — let hooks modify messages before compression- if self.config.hooks:+ if self.config.hooks and not is_cache_mode(self.config.mode):from headroom.hooks import CompressContext_hook_ctx = CompressContext(@@ -323,6 +360,8 @@ class AnthropicHandlerMixin:messages = self.config.hooks.pre_compress(messages, _hook_ctx)except Exception as e:logger.debug(f"[{request_id}] pre_compress hook error: {e}")+ else:+ _hook_ctx = None# Apply optimizationtransforms_applied = []@@ -373,7 +412,7 @@ class AnthropicHandlerMixin:result = Nonebiases = (self.config.hooks.compute_biases(messages, _hook_ctx)- if self.config.hooks+ if self.config.hooks and _hook_ctx is not Noneelse None)@@ -436,8 +475,39 @@ class AnthropicHandlerMixin:original_tokens = result.tokens_beforeoptimized_tokens = result.tokens_afterelse:- optimized_messages = messages- optimized_tokens = original_tokens+ previous_original_messages = prefix_tracker.get_last_original_messages()+ previous_forwarded_messages = prefix_tracker.get_last_forwarded_messages()+ delta = self._extract_cache_stable_delta(+ original_client_messages,+ previous_original_messages,+ previous_forwarded_messages,+ )+ if delta is None:+ optimized_messages = messages+ optimized_tokens = original_tokens+ else:+ stable_forwarded_prefix, delta_messages = delta+ if delta_messages:+ result = await asyncio.wait_for(+ asyncio.to_thread(+ lambda: self.anthropic_pipeline.apply(+ messages=delta_messages,+ model=model,+ model_limit=context_limit,+ context=extract_user_query(delta_messages),+ frozen_message_count=0,+ biases=biases,+ )+ ),+ timeout=COMPRESSION_TIMEOUT_SECONDS,+ )+ optimized_messages = stable_forwarded_prefix + result.messages+ transforms_applied = result.transforms_applied+ pipeline_timing = result.timing+ optimized_tokens = tokenizer.count_messages(optimized_messages)+ else:+ optimized_messages = stable_forwarded_prefix+ optimized_tokens = tokenizer.count_messages(optimized_messages)if result and result.waste_signals:waste_signals_dict = result.waste_signals.to_dict()@@ -662,18 +732,6 @@ class AnthropicHandlerMixin:# The echo changes every turn, invalidating the cached prefix.# To re-enable, uncomment and set query_echo_enabled on ProxyConfig.- if is_cache_mode(self.config.mode):- optimized_messages, restored_count = self._restore_frozen_prefix(- original_client_messages,- optimized_messages,- frozen_message_count=frozen_message_count,- )- if restored_count > 0:- logger.warning(- f"[{request_id}] Restored {restored_count} frozen prefix message(s) "- "to preserve cache stability"- )-# Update bodybody["messages"] = optimized_messagesif tools is not None:@@ -1049,10 +1107,17 @@ class AnthropicHandlerMixin:uncached_input_tokens = usage.get("input_tokens", 0)# Update prefix cache tracker for next turn+ next_original_messages = copy.deepcopy(original_client_messages)+ next_forwarded_messages = copy.deepcopy(optimized_messages)+ assistant_message = self._assistant_message_from_response_json(resp_json)+ if assistant_message is not None:+ next_original_messages.append(copy.deepcopy(assistant_message))+ next_forwarded_messages.append(copy.deepcopy(assistant_message))prefix_tracker.update_from_response(cache_read_tokens=cr_tokens,cache_write_tokens=cw_tokens,- messages=optimized_messages,+ messages=next_forwarded_messages,+ original_messages=next_original_messages,)if self.cost_tracker:diff --git a/tests/test_claude_session_mode_benchmark.py b/tests/test_claude_session_mode_benchmark.pyindex b72b822ca..bd4f84644 100644--- a/tests/test_claude_session_mode_benchmark.py+++ b/tests/test_claude_session_mode_benchmark.py@@ -330,7 +330,7 @@ def test_determine_winners_includes_no_cache_counterfactual() -> None:def test_resolve_checkpoint_dir_namespaces_sampling_mode() -> None:base = Path("benchmark_results") / "checkpoints"- assert resolve_checkpoint_dir(base).name == "v3__ttl_5m__full"+ assert resolve_checkpoint_dir(base).name == "v4__ttl_5m__full"assert (- resolve_checkpoint_dir(base, recent_turns_per_session=200).name == "v3__ttl_5m__recent_200"+ resolve_checkpoint_dir(base, recent_turns_per_session=200).name == "v4__ttl_5m__recent_200")diff --git a/tests/test_proxy_anthropic_cache_stability.py b/tests/test_proxy_anthropic_cache_stability.pyindex 22a7535ee..622fc69d7 100644--- a/tests/test_proxy_anthropic_cache_stability.py+++ b/tests/test_proxy_anthropic_cache_stability.py@@ -808,3 +808,92 @@ def test_cache_mode_does_not_forward_latest_turn_rewrites() -> None:assert response.status_code == 200assert captured["body"]["messages"] == original_messages+++def test_cache_mode_reuses_prior_forwarded_prefix_and_compresses_only_new_suffix() -> None:+ captured = {"calls": []}+ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "cache"+ proxy.config.image_optimize = False++ tracker = _FakePrefixTracker(frozen_count=0)+ tracker._last_original_messages = [+ {"role": "user", "content": "turn1"},+ {"role": "assistant", "content": "turn1-assistant"},+ {"role": "user", "content": "turn2"},+ {"role": "assistant", "content": "turn2-assistant"},+ ]+ tracker._last_forwarded_messages = [+ {"role": "user", "content": "turn1"},+ {"role": "assistant", "content": "turn1-assistant"},+ {"role": "user", "content": "COMPRESSED_TURN2"},+ {"role": "assistant", "content": "turn2-assistant"},+ ]+ tracker.get_last_original_messages = lambda: tracker._last_original_messages.copy()+ tracker.get_last_forwarded_messages = lambda: tracker._last_forwarded_messages.copy()++ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (+ "stable-session"+ )+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker++ def _fake_apply(**kwargs):+ captured["calls"].append(kwargs["messages"])+ return SimpleNamespace(+ messages=[{"role": "user", "content": "COMPRESSED_TURN3"}],+ transforms_applied=["fake:delta"],+ timing={},+ tokens_before=40,+ tokens_after=20,+ waste_signals=None,+ )++ proxy.anthropic_pipeline.apply = _fake_apply++ async def _fake_retry(method, url, headers, body, stream=False): # noqa: ANN001+ captured["body"] = body+ return httpx.Response(+ 200,+ json={+ "id": "msg_cache_3",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": "ok"}],+ "usage": {+ "input_tokens": 80,+ "output_tokens": 3,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ },+ },+ )++ proxy._retry_request = _fake_retry++ response = client.post(+ "/v1/messages",+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "messages": [+ {"role": "user", "content": "turn1"},+ {"role": "assistant", "content": "turn1-assistant"},+ {"role": "user", "content": "turn2"},+ {"role": "assistant", "content": "turn2-assistant"},+ {"role": "user", "content": "turn3"},+ ],+ },+ )++ assert response.status_code == 200+ assert captured["calls"] == [[{"role": "user", "content": "turn3"}]]+ assert captured["body"]["messages"] == [+ {"role": "user", "content": "turn1"},+ {"role": "assistant", "content": "turn1-assistant"},+ {"role": "user", "content": "COMPRESSED_TURN2"},+ {"role": "assistant", "content": "turn2-assistant"},+ {"role": "user", "content": "COMPRESSED_TURN3"},+ ]