Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

Add Anthropic cache-mode delta replay

JerrettDavis96fd3d965201
diff --git a/benchmarks/claude_session_mode_benchmark.py b/benchmarks/claude_session_mode_benchmark.py
index da5c4a89a..135d57dcd 100644
--- a/benchmarks/claude_session_mode_benchmark.py
+++ b/benchmarks/claude_session_mode_benchmark.py
@@ -433,7 +433,7 @@ def resolve_checkpoint_dir(
recent_turns_per_session: int | None = None,
cache_ttl_minutes: int = DEFAULT_CACHE_TTL_MINUTES,
) -> Path:
- suffix_parts = ["v3", f"ttl_{cache_ttl_minutes}m"]
+ suffix_parts = ["v4", f"ttl_{cache_ttl_minutes}m"]
if recent_turns_per_session:
suffix_parts.append(f"recent_{recent_turns_per_session}")
else:
@@ -646,20 +646,36 @@ def _apply_mode_to_messages(
model: str,
prefix_tracker: PrefixCacheTracker | None,
comp_cache: CompressionCache | None,
+ previous_original_messages: list[dict[str, Any]] | None = None,
+ previous_forwarded_messages: list[dict[str, Any]] | None = None,
) -> list[dict[str, Any]]:
if mode == "baseline":
return copy.deepcopy(messages)
- if mode == PROXY_MODE_CACHE:
- return copy.deepcopy(messages)
assert proxy is not None
assert prefix_tracker is not None
- frozen_message_count = prefix_tracker.get_frozen_message_count()
if mode == PROXY_MODE_CACHE:
- frozen_message_count = AnthropicHandlerMixin._strict_previous_turn_frozen_count(
+ delta = AnthropicHandlerMixin._extract_cache_stable_delta(
messages,
- frozen_message_count,
+ previous_original_messages,
+ previous_forwarded_messages,
+ )
+ if delta is None:
+ return copy.deepcopy(messages)
+ stable_forwarded_prefix, delta_messages = delta
+ if not delta_messages:
+ return stable_forwarded_prefix
+ context_limit = proxy.anthropic_provider.get_context_limit(model)
+ result = proxy.anthropic_pipeline.apply(
+ messages=delta_messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(delta_messages),
+ frozen_message_count=0,
)
+ return stable_forwarded_prefix + result.messages
+
+ frozen_message_count = prefix_tracker.get_frozen_message_count()
working_messages = copy.deepcopy(messages)
if proxy.config.image_optimize and working_messages and _messages_have_images(working_messages):
@@ -710,6 +726,7 @@ class _PendingTurn:
turn: ReplayTurn
tokenizer: Any
raw_input_tokens: int
+ request_messages: list[dict[str, Any]]
forwarded: list[dict[str, Any]]
@@ -912,6 +929,8 @@ def _simulate_single_replay_mode(
conversation: list[dict[str, Any]] = []
conversation_token_total = 0
previous_forwarded: list[dict[str, Any]] = []
+ previous_original_context: list[dict[str, Any]] | None = None
+ previous_forwarded_context: list[dict[str, Any]] | None = None
previous_timestamp: datetime | None = None
prefix_tracker = None if mode == "baseline" else PrefixCacheTracker("anthropic")
comp_cache = CompressionCache() if mode == PROXY_MODE_TOKEN else None
@@ -928,6 +947,8 @@ def _simulate_single_replay_mode(
model=turn.model,
prefix_tracker=prefix_tracker,
comp_cache=comp_cache,
+ previous_original_messages=previous_original_context,
+ previous_forwarded_messages=previous_forwarded_context,
)
if pending is not None:
_apply_turn_metrics(
@@ -952,6 +973,7 @@ def _simulate_single_replay_mode(
cache_write_tokens=0,
messages=forwarded,
message_token_counts=[tokenizer.count_message(msg) for msg in forwarded],
+ original_messages=conversation,
)
pending = _PendingTurn(
@@ -959,12 +981,17 @@ def _simulate_single_replay_mode(
turn=turn,
tokenizer=tokenizer,
raw_input_tokens=raw_input_tokens,
+ request_messages=copy.deepcopy(conversation),
forwarded=forwarded,
)
conversation.append(turn.assistant_message)
conversation_token_total = raw_input_tokens + tokenizer.count_message(
turn.assistant_message
)
+ previous_original_context = copy.deepcopy(conversation)
+ previous_forwarded_context = copy.deepcopy(forwarded) + [
+ copy.deepcopy(turn.assistant_message)
+ ]
if pending is not None:
_apply_turn_metrics(
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
index eecb38736..77561ca23 100644
--- a/headroom/cache/prefix_tracker.py
+++ b/headroom/cache/prefix_tracker.py
@@ -16,6 +16,7 @@ pipeline skips them entirely.
from __future__ import annotations
+import copy
import hashlib
import logging
import time
@@ -88,6 +89,8 @@ class PrefixCacheTracker:
self._cached_message_count: int = 0
self._turn_number: int = 0
self._last_activity: float = time.time()
+ self._last_original_messages: list[dict[str, Any]] = []
+ self._last_forwarded_messages: list[dict[str, Any]] = []
# Stats
self._busts_avoided: int = 0
@@ -113,6 +116,7 @@ class PrefixCacheTracker:
cache_write_tokens: int,
messages: list[dict[str, Any]],
message_token_counts: list[int] | None = None,
+ original_messages: list[dict[str, Any]] | None = None,
) -> None:
"""Update tracker with cache metrics from the API response.
@@ -128,6 +132,8 @@ class PrefixCacheTracker:
"""
self._last_activity = time.time()
self._turn_number += 1
+ self._last_original_messages = copy.deepcopy(original_messages or messages)
+ self._last_forwarded_messages = copy.deepcopy(messages)
# Compute total cached tokens (read + write = what's in cache now)
total_cached = cache_read_tokens + cache_write_tokens
@@ -167,6 +173,12 @@ class PrefixCacheTracker:
cache_write_tokens,
)
+ def get_last_original_messages(self) -> list[dict[str, Any]]:
+ return copy.deepcopy(self._last_original_messages)
+
+ def get_last_forwarded_messages(self) -> list[dict[str, Any]]:
+ return copy.deepcopy(self._last_forwarded_messages)
+
def record_bust_avoided(self, tokens_preserved: int, compression_foregone: int) -> None:
"""Record when we chose to preserve cache over compressing."""
self._busts_avoided += 1
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 519157660..b1345bf6d 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -155,6 +155,43 @@ class AnthropicHandlerMixin:
changed += 1
return restored, changed
+ @staticmethod
+ def _extract_cache_stable_delta(
+ current_messages: list[dict[str, Any]],
+ previous_original_messages: list[dict[str, Any]] | None,
+ previous_forwarded_messages: list[dict[str, Any]] | None,
+ ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]] | None:
+ """Return (stable_forwarded_prefix, appended_delta_messages) when safe.
+
+ Safe means the prior original request is an exact message-prefix of the
+ current original request. This lets us replay the exact forwarded bytes
+ for historical context and only transform newly appended message suffixes.
+ """
+ if not previous_original_messages or previous_forwarded_messages is None:
+ return None
+ prefix_len = len(previous_original_messages)
+ if len(current_messages) < prefix_len:
+ return None
+ if current_messages[:prefix_len] != previous_original_messages:
+ return None
+ return (
+ copy.deepcopy(previous_forwarded_messages),
+ copy.deepcopy(current_messages[prefix_len:]),
+ )
+
+ @staticmethod
+ def _assistant_message_from_response_json(
+ resp_json: dict[str, Any] | None,
+ ) -> dict[str, Any] | None:
+ if not resp_json:
+ return None
+ if resp_json.get("role") != "assistant":
+ return None
+ return {
+ "role": "assistant",
+ "content": copy.deepcopy(resp_json.get("content", "")),
+ }
+
async def handle_anthropic_messages(
self,
request: Request,
@@ -311,7 +348,7 @@ class AnthropicHandlerMixin:
# Hook: pre_compress — let hooks modify messages before compression
- if self.config.hooks:
+ if self.config.hooks and not is_cache_mode(self.config.mode):
from headroom.hooks import CompressContext
_hook_ctx = CompressContext(
@@ -323,6 +360,8 @@ class AnthropicHandlerMixin:
messages = self.config.hooks.pre_compress(messages, _hook_ctx)
except Exception as e:
logger.debug(f"[{request_id}] pre_compress hook error: {e}")
+ else:
+ _hook_ctx = None
# Apply optimization
transforms_applied = []
@@ -373,7 +412,7 @@ class AnthropicHandlerMixin:
result = None
biases = (
self.config.hooks.compute_biases(messages, _hook_ctx)
- if self.config.hooks
+ if self.config.hooks and _hook_ctx is not None
else None
)
@@ -436,8 +475,39 @@ class AnthropicHandlerMixin:
original_tokens = result.tokens_before
optimized_tokens = result.tokens_after
else:
- optimized_messages = messages
- optimized_tokens = original_tokens
+ previous_original_messages = prefix_tracker.get_last_original_messages()
+ previous_forwarded_messages = prefix_tracker.get_last_forwarded_messages()
+ delta = self._extract_cache_stable_delta(
+ original_client_messages,
+ previous_original_messages,
+ previous_forwarded_messages,
+ )
+ if delta is None:
+ optimized_messages = messages
+ optimized_tokens = original_tokens
+ else:
+ stable_forwarded_prefix, delta_messages = delta
+ if delta_messages:
+ result = await asyncio.wait_for(
+ asyncio.to_thread(
+ lambda: self.anthropic_pipeline.apply(
+ messages=delta_messages,
+ model=model,
+ model_limit=context_limit,
+ context=extract_user_query(delta_messages),
+ frozen_message_count=0,
+ biases=biases,
+ )
+ ),
+ timeout=COMPRESSION_TIMEOUT_SECONDS,
+ )
+ optimized_messages = stable_forwarded_prefix + result.messages
+ transforms_applied = result.transforms_applied
+ pipeline_timing = result.timing
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
+ else:
+ optimized_messages = stable_forwarded_prefix
+ optimized_tokens = tokenizer.count_messages(optimized_messages)
if result and result.waste_signals:
waste_signals_dict = result.waste_signals.to_dict()
@@ -662,18 +732,6 @@ class AnthropicHandlerMixin:
# The echo changes every turn, invalidating the cached prefix.
# To re-enable, uncomment and set query_echo_enabled on ProxyConfig.
- if is_cache_mode(self.config.mode):
- optimized_messages, restored_count = self._restore_frozen_prefix(
- original_client_messages,
- optimized_messages,
- frozen_message_count=frozen_message_count,
- )
- if restored_count > 0:
- logger.warning(
- f"[{request_id}] Restored {restored_count} frozen prefix message(s) "
- "to preserve cache stability"
- )
-
# Update body
body["messages"] = optimized_messages
if tools is not None:
@@ -1049,10 +1107,17 @@ class AnthropicHandlerMixin:
uncached_input_tokens = usage.get("input_tokens", 0)
# Update prefix cache tracker for next turn
+ next_original_messages = copy.deepcopy(original_client_messages)
+ next_forwarded_messages = copy.deepcopy(optimized_messages)
+ assistant_message = self._assistant_message_from_response_json(resp_json)
+ if assistant_message is not None:
+ next_original_messages.append(copy.deepcopy(assistant_message))
+ next_forwarded_messages.append(copy.deepcopy(assistant_message))
prefix_tracker.update_from_response(
cache_read_tokens=cr_tokens,
cache_write_tokens=cw_tokens,
- messages=optimized_messages,
+ messages=next_forwarded_messages,
+ original_messages=next_original_messages,
)
if self.cost_tracker:
diff --git a/tests/test_claude_session_mode_benchmark.py b/tests/test_claude_session_mode_benchmark.py
index b72b822ca..bd4f84644 100644
--- a/tests/test_claude_session_mode_benchmark.py
+++ b/tests/test_claude_session_mode_benchmark.py
@@ -330,7 +330,7 @@ def test_determine_winners_includes_no_cache_counterfactual() -> None:
def test_resolve_checkpoint_dir_namespaces_sampling_mode() -> None:
base = Path("benchmark_results") / "checkpoints"
- assert resolve_checkpoint_dir(base).name == "v3__ttl_5m__full"
+ assert resolve_checkpoint_dir(base).name == "v4__ttl_5m__full"
assert (
- resolve_checkpoint_dir(base, recent_turns_per_session=200).name == "v3__ttl_5m__recent_200"
+ resolve_checkpoint_dir(base, recent_turns_per_session=200).name == "v4__ttl_5m__recent_200"
)
diff --git a/tests/test_proxy_anthropic_cache_stability.py b/tests/test_proxy_anthropic_cache_stability.py
index 22a7535ee..622fc69d7 100644
--- a/tests/test_proxy_anthropic_cache_stability.py
+++ b/tests/test_proxy_anthropic_cache_stability.py
@@ -808,3 +808,92 @@ def test_cache_mode_does_not_forward_latest_turn_rewrites() -> None:
assert response.status_code == 200
assert captured["body"]["messages"] == original_messages
+
+
+def test_cache_mode_reuses_prior_forwarded_prefix_and_compresses_only_new_suffix() -> None:
+ captured = {"calls": []}
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "cache"
+ proxy.config.image_optimize = False
+
+ tracker = _FakePrefixTracker(frozen_count=0)
+ tracker._last_original_messages = [
+ {"role": "user", "content": "turn1"},
+ {"role": "assistant", "content": "turn1-assistant"},
+ {"role": "user", "content": "turn2"},
+ {"role": "assistant", "content": "turn2-assistant"},
+ ]
+ tracker._last_forwarded_messages = [
+ {"role": "user", "content": "turn1"},
+ {"role": "assistant", "content": "turn1-assistant"},
+ {"role": "user", "content": "COMPRESSED_TURN2"},
+ {"role": "assistant", "content": "turn2-assistant"},
+ ]
+ tracker.get_last_original_messages = lambda: tracker._last_original_messages.copy()
+ tracker.get_last_forwarded_messages = lambda: tracker._last_forwarded_messages.copy()
+
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
+ "stable-session"
+ )
+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker
+
+ def _fake_apply(**kwargs):
+ captured["calls"].append(kwargs["messages"])
+ return SimpleNamespace(
+ messages=[{"role": "user", "content": "COMPRESSED_TURN3"}],
+ transforms_applied=["fake:delta"],
+ timing={},
+ tokens_before=40,
+ tokens_after=20,
+ waste_signals=None,
+ )
+
+ proxy.anthropic_pipeline.apply = _fake_apply
+
+ async def _fake_retry(method, url, headers, body, stream=False): # noqa: ANN001
+ captured["body"] = body
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_cache_3",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "ok"}],
+ "usage": {
+ "input_tokens": 80,
+ "output_tokens": 3,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+
+ response = client.post(
+ "/v1/messages",
+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-sonnet-4-6",
+ "max_tokens": 64,
+ "messages": [
+ {"role": "user", "content": "turn1"},
+ {"role": "assistant", "content": "turn1-assistant"},
+ {"role": "user", "content": "turn2"},
+ {"role": "assistant", "content": "turn2-assistant"},
+ {"role": "user", "content": "turn3"},
+ ],
+ },
+ )
+
+ assert response.status_code == 200
+ assert captured["calls"] == [[{"role": "user", "content": "turn3"}]]
+ assert captured["body"]["messages"] == [
+ {"role": "user", "content": "turn1"},
+ {"role": "assistant", "content": "turn1-assistant"},
+ {"role": "user", "content": "COMPRESSED_TURN2"},
+ {"role": "assistant", "content": "turn2-assistant"},
+ {"role": "user", "content": "COMPRESSED_TURN3"},
+ ]