Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache-aligner): hash the frozen conversation prefix so Claude Code cache invalidation is detected (#2085) (#2161)

Rod Boevcc072f0821b7
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 11631d115..ac6bb98c8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## Unreleased
### Fixed
+- **cache aligner:** hash the actual frozen Claude Code prefix instead of only system-message text, so `stable_prefix_hash` / `prefix_changed` now surface prompt-cache churn when a cached tool-result block changes without any system-prompt edit ([#2085](https://github.com/headroomlabs-ai/headroom/issues/2085)).
- **proxy image compression:** run native image compression in a spawned subprocess so an OpenCV or KleidiCV crash now fails open to the original image payload instead of taking down the whole proxy process ([#2107](https://github.com/headroomlabs-ai/headroom/issues/2107)).
- **proxy/windows:** support Windows selector-event-loop startup on uvicorn versions older than 0.36. Newer uvicorn accepts `loop="asyncio:SelectorEventLoop"` as a custom loop-factory import path, but older versions treat it as an unknown built-in loop name and raise `KeyError`. Windows now sets `WindowsSelectorEventLoopPolicy` for those older versions instead of passing an unsupported `loop` value ([#1650](https://github.com/headroomlabs-ai/headroom/issues/1650), [#1621](https://github.com/headroomlabs-ai/headroom/issues/1621)).
- **backends/litellm:** preserve `cache_control` on `tool_result` blocks when converting Anthropic messages for the Bedrock Converse path, and complete streaming cache-stats surfacing in `stream_message`. Complements [#1390](https://github.com/headroomlabs-ai/headroom/pull/1390), which preserves `cache_control` on the system prompt and plain text blocks but explicitly leaves `tool_result` out of scope — in agent loops the moving cache breakpoint lands on the tail `tool_result` far more often than on the system prompt, so that gap left most of the caching benefit on the table. Separately, `stream_message` never requested `stream_options.include_usage`, so LiteLLM/Bedrock never returned a usage chunk over SSE and `cache_read_input_tokens`/`cache_creation_input_tokens` always reported 0 downstream even when the prompt cache was genuinely engaged; the terminal `message_delta` now carries the real cache values captured from the trailing usage chunk once the stream completes.
diff --git a/headroom/transforms/cache_aligner.py b/headroom/transforms/cache_aligner.py
index 2faa9efe7..be481ebfe 100644
--- a/headroom/transforms/cache_aligner.py
+++ b/headroom/transforms/cache_aligner.py
@@ -27,6 +27,7 @@ from __future__ import annotations
import base64
import binascii
+import json
import logging
import uuid as _uuid
from dataclasses import dataclass
@@ -42,6 +43,34 @@ from .base import Transform
logger = logging.getLogger(__name__)
+def _stable_prefix_observability_scope(
+ messages: list[dict[str, Any]],
+ frozen_message_count: int,
+) -> tuple[str, int]:
+ if frozen_message_count <= 0:
+ system_text = "\n---\n".join(
+ (m.get("content") or "")
+ for m in messages
+ if m.get("role") == "system" and isinstance(m.get("content"), str)
+ )
+ return system_text, len(system_text.encode("utf-8"))
+
+ scope = json.dumps(
+ {
+ "frozen_prefix": messages[:frozen_message_count],
+ "trailing_system": [
+ m
+ for i, m in enumerate(messages)
+ if i >= frozen_message_count and m.get("role") == "system"
+ ],
+ },
+ sort_keys=True,
+ ensure_ascii=False,
+ separators=(",", ":"),
+ )
+ return scope, len(scope.encode("utf-8"))
+
+
# Length profile for hex hash detection. Kept as named constants — no magic
# numbers in production code (build constraint #2). MD5 = 32 hex chars,
# SHA1 = 40, SHA256 = 64.
@@ -309,16 +338,12 @@ class CacheAligner(Transform):
warnings.append(msg_text)
logger.warning(msg_text)
- # Compute a stable hash of all system messages for observability.
- # This is just a hash of the (unchanged) bytes — no extraction.
- system_text = "\n---\n".join(
- (m.get("content") or "")
- for m in result_messages
- if m.get("role") == "system" and isinstance(m.get("content"), str)
+ scope_text, prefix_bytes = _stable_prefix_observability_scope(
+ result_messages,
+ frozen_message_count,
)
- stable_hash = compute_short_hash(system_text)
- prefix_bytes = len(system_text.encode("utf-8"))
- prefix_tokens_est = tokenizer.count_text(system_text)
+ stable_hash = compute_short_hash(scope_text)
+ prefix_tokens_est = tokenizer.count_text(scope_text)
prefix_changed = (
self._previous_prefix_hash is not None and self._previous_prefix_hash != stable_hash
)
diff --git a/tests/test_cache_aligner_prefix_stability.py b/tests/test_cache_aligner_prefix_stability.py
new file mode 100644
index 000000000..7b647b5d9
--- /dev/null
+++ b/tests/test_cache_aligner_prefix_stability.py
@@ -0,0 +1,107 @@
+from __future__ import annotations
+
+from copy import deepcopy
+
+from headroom import OpenAIProvider
+from headroom.tokenizer import Tokenizer
+from headroom.transforms.cache_aligner import CacheAligner
+from headroom.utils import compute_short_hash
+
+_provider = OpenAIProvider()
+
+
+def _tokenizer() -> Tokenizer:
+ counter = _provider.get_token_counter("gpt-4o")
+ return Tokenizer(counter, "gpt-4o")
+
+
+def _claude_code_messages(
+ *,
+ cached_tool_output: str = "cached tool output v1",
+ live_tail: str = "latest live turn",
+) -> list[dict[str, object]]:
+ return [
+ {"role": "system", "content": "You are Headroom. Keep the cached prefix stable."},
+ {"role": "user", "content": "Summarize the repo state."},
+ {"role": "assistant", "content": cached_tool_output},
+ {"role": "user", "content": live_tail},
+ ]
+
+
+def test_frozen_prefix_change_flags_prefix_changed() -> None:
+ aligner = CacheAligner()
+ tokenizer = _tokenizer()
+
+ first = _claude_code_messages(cached_tool_output="cached tool output v1")
+ second = _claude_code_messages(cached_tool_output="cached tool output v2")
+
+ result1 = aligner.apply(first, tokenizer, frozen_message_count=3)
+ result2 = aligner.apply(second, tokenizer, frozen_message_count=3)
+
+ assert result1.cache_metrics.prefix_changed is False
+ assert result2.cache_metrics.prefix_changed is True
+ assert result2.cache_metrics.previous_hash == result1.cache_metrics.stable_prefix_hash
+ assert result2.cache_metrics.stable_prefix_hash != result1.cache_metrics.stable_prefix_hash
+
+
+def test_identical_frozen_prefix_is_stable() -> None:
+ aligner = CacheAligner()
+ tokenizer = _tokenizer()
+ messages = _claude_code_messages()
+
+ result1 = aligner.apply(messages, tokenizer, frozen_message_count=3)
+ result2 = aligner.apply(deepcopy(messages), tokenizer, frozen_message_count=3)
+
+ assert result1.cache_metrics.prefix_changed is False
+ assert result2.cache_metrics.prefix_changed is False
+ assert result2.cache_metrics.stable_prefix_hash == result1.cache_metrics.stable_prefix_hash
+
+
+def test_live_tail_change_does_not_flag() -> None:
+ aligner = CacheAligner()
+ tokenizer = _tokenizer()
+
+ first = _claude_code_messages(live_tail="latest live turn")
+ second = _claude_code_messages(live_tail="different live turn")
+
+ aligner.apply(first, tokenizer, frozen_message_count=3)
+ result2 = aligner.apply(second, tokenizer, frozen_message_count=3)
+
+ assert result2.cache_metrics.prefix_changed is False
+
+
+def test_apply_is_byte_equal_deepcopy() -> None:
+ aligner = CacheAligner()
+ tokenizer = _tokenizer()
+ messages = [
+ {
+ "role": "system",
+ "content": "Keep the transcript stable.",
+ "meta": {"source": "test"},
+ },
+ {
+ "role": "user",
+ "content": [{"type": "text", "text": "hello"}],
+ },
+ ]
+
+ result = aligner.apply(messages, tokenizer, frozen_message_count=1)
+
+ assert result.messages == messages
+ assert result.messages is not messages
+ assert result.messages[0] is not messages[0]
+ assert result.messages[1] is not messages[1]
+
+
+def test_first_turn_scope_unchanged() -> None:
+ aligner = CacheAligner()
+ tokenizer = _tokenizer()
+ messages = _claude_code_messages()
+ system_text = messages[0]["content"]
+
+ result = aligner.apply(messages, tokenizer, frozen_message_count=0)
+
+ assert result.cache_metrics.prefix_changed is False
+ assert result.cache_metrics.stable_prefix_hash == compute_short_hash(system_text)
+ assert result.cache_metrics.stable_prefix_bytes == len(str(system_text).encode("utf-8"))
+ assert result.cache_metrics.stable_prefix_tokens_est == tokenizer.count_text(str(system_text))