Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache-aligner): hash the frozen conversation prefix so Claude Code cache invalidation is detected (#2085) (#2161)
Rod Boevcc072f0821b7
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 11631d115..ac6bb98c8 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## Unreleased### Fixed+- **cache aligner:** hash the actual frozen Claude Code prefix instead of only system-message text, so `stable_prefix_hash` / `prefix_changed` now surface prompt-cache churn when a cached tool-result block changes without any system-prompt edit ([#2085](https://github.com/headroomlabs-ai/headroom/issues/2085)).- **proxy image compression:** run native image compression in a spawned subprocess so an OpenCV or KleidiCV crash now fails open to the original image payload instead of taking down the whole proxy process ([#2107](https://github.com/headroomlabs-ai/headroom/issues/2107)).- **proxy/windows:** support Windows selector-event-loop startup on uvicorn versions older than 0.36. Newer uvicorn accepts `loop="asyncio:SelectorEventLoop"` as a custom loop-factory import path, but older versions treat it as an unknown built-in loop name and raise `KeyError`. Windows now sets `WindowsSelectorEventLoopPolicy` for those older versions instead of passing an unsupported `loop` value ([#1650](https://github.com/headroomlabs-ai/headroom/issues/1650), [#1621](https://github.com/headroomlabs-ai/headroom/issues/1621)).- **backends/litellm:** preserve `cache_control` on `tool_result` blocks when converting Anthropic messages for the Bedrock Converse path, and complete streaming cache-stats surfacing in `stream_message`. Complements [#1390](https://github.com/headroomlabs-ai/headroom/pull/1390), which preserves `cache_control` on the system prompt and plain text blocks but explicitly leaves `tool_result` out of scope — in agent loops the moving cache breakpoint lands on the tail `tool_result` far more often than on the system prompt, so that gap left most of the caching benefit on the table. Separately, `stream_message` never requested `stream_options.include_usage`, so LiteLLM/Bedrock never returned a usage chunk over SSE and `cache_read_input_tokens`/`cache_creation_input_tokens` always reported 0 downstream even when the prompt cache was genuinely engaged; the terminal `message_delta` now carries the real cache values captured from the trailing usage chunk once the stream completes.diff --git a/headroom/transforms/cache_aligner.py b/headroom/transforms/cache_aligner.pyindex 2faa9efe7..be481ebfe 100644--- a/headroom/transforms/cache_aligner.py+++ b/headroom/transforms/cache_aligner.py@@ -27,6 +27,7 @@ from __future__ import annotationsimport base64import binascii+import jsonimport loggingimport uuid as _uuidfrom dataclasses import dataclass@@ -42,6 +43,34 @@ from .base import Transformlogger = logging.getLogger(__name__)+def _stable_prefix_observability_scope(+ messages: list[dict[str, Any]],+ frozen_message_count: int,+) -> tuple[str, int]:+ if frozen_message_count <= 0:+ system_text = "\n---\n".join(+ (m.get("content") or "")+ for m in messages+ if m.get("role") == "system" and isinstance(m.get("content"), str)+ )+ return system_text, len(system_text.encode("utf-8"))++ scope = json.dumps(+ {+ "frozen_prefix": messages[:frozen_message_count],+ "trailing_system": [+ m+ for i, m in enumerate(messages)+ if i >= frozen_message_count and m.get("role") == "system"+ ],+ },+ sort_keys=True,+ ensure_ascii=False,+ separators=(",", ":"),+ )+ return scope, len(scope.encode("utf-8"))++# Length profile for hex hash detection. Kept as named constants — no magic# numbers in production code (build constraint #2). MD5 = 32 hex chars,# SHA1 = 40, SHA256 = 64.@@ -309,16 +338,12 @@ class CacheAligner(Transform):warnings.append(msg_text)logger.warning(msg_text)- # Compute a stable hash of all system messages for observability.- # This is just a hash of the (unchanged) bytes — no extraction.- system_text = "\n---\n".join(- (m.get("content") or "")- for m in result_messages- if m.get("role") == "system" and isinstance(m.get("content"), str)+ scope_text, prefix_bytes = _stable_prefix_observability_scope(+ result_messages,+ frozen_message_count,)- stable_hash = compute_short_hash(system_text)- prefix_bytes = len(system_text.encode("utf-8"))- prefix_tokens_est = tokenizer.count_text(system_text)+ stable_hash = compute_short_hash(scope_text)+ prefix_tokens_est = tokenizer.count_text(scope_text)prefix_changed = (self._previous_prefix_hash is not None and self._previous_prefix_hash != stable_hash)diff --git a/tests/test_cache_aligner_prefix_stability.py b/tests/test_cache_aligner_prefix_stability.pynew file mode 100644index 000000000..7b647b5d9--- /dev/null+++ b/tests/test_cache_aligner_prefix_stability.py@@ -0,0 +1,107 @@+from __future__ import annotations++from copy import deepcopy++from headroom import OpenAIProvider+from headroom.tokenizer import Tokenizer+from headroom.transforms.cache_aligner import CacheAligner+from headroom.utils import compute_short_hash++_provider = OpenAIProvider()+++def _tokenizer() -> Tokenizer:+ counter = _provider.get_token_counter("gpt-4o")+ return Tokenizer(counter, "gpt-4o")+++def _claude_code_messages(+ *,+ cached_tool_output: str = "cached tool output v1",+ live_tail: str = "latest live turn",+) -> list[dict[str, object]]:+ return [+ {"role": "system", "content": "You are Headroom. Keep the cached prefix stable."},+ {"role": "user", "content": "Summarize the repo state."},+ {"role": "assistant", "content": cached_tool_output},+ {"role": "user", "content": live_tail},+ ]+++def test_frozen_prefix_change_flags_prefix_changed() -> None:+ aligner = CacheAligner()+ tokenizer = _tokenizer()++ first = _claude_code_messages(cached_tool_output="cached tool output v1")+ second = _claude_code_messages(cached_tool_output="cached tool output v2")++ result1 = aligner.apply(first, tokenizer, frozen_message_count=3)+ result2 = aligner.apply(second, tokenizer, frozen_message_count=3)++ assert result1.cache_metrics.prefix_changed is False+ assert result2.cache_metrics.prefix_changed is True+ assert result2.cache_metrics.previous_hash == result1.cache_metrics.stable_prefix_hash+ assert result2.cache_metrics.stable_prefix_hash != result1.cache_metrics.stable_prefix_hash+++def test_identical_frozen_prefix_is_stable() -> None:+ aligner = CacheAligner()+ tokenizer = _tokenizer()+ messages = _claude_code_messages()++ result1 = aligner.apply(messages, tokenizer, frozen_message_count=3)+ result2 = aligner.apply(deepcopy(messages), tokenizer, frozen_message_count=3)++ assert result1.cache_metrics.prefix_changed is False+ assert result2.cache_metrics.prefix_changed is False+ assert result2.cache_metrics.stable_prefix_hash == result1.cache_metrics.stable_prefix_hash+++def test_live_tail_change_does_not_flag() -> None:+ aligner = CacheAligner()+ tokenizer = _tokenizer()++ first = _claude_code_messages(live_tail="latest live turn")+ second = _claude_code_messages(live_tail="different live turn")++ aligner.apply(first, tokenizer, frozen_message_count=3)+ result2 = aligner.apply(second, tokenizer, frozen_message_count=3)++ assert result2.cache_metrics.prefix_changed is False+++def test_apply_is_byte_equal_deepcopy() -> None:+ aligner = CacheAligner()+ tokenizer = _tokenizer()+ messages = [+ {+ "role": "system",+ "content": "Keep the transcript stable.",+ "meta": {"source": "test"},+ },+ {+ "role": "user",+ "content": [{"type": "text", "text": "hello"}],+ },+ ]++ result = aligner.apply(messages, tokenizer, frozen_message_count=1)++ assert result.messages == messages+ assert result.messages is not messages+ assert result.messages[0] is not messages[0]+ assert result.messages[1] is not messages[1]+++def test_first_turn_scope_unchanged() -> None:+ aligner = CacheAligner()+ tokenizer = _tokenizer()+ messages = _claude_code_messages()+ system_text = messages[0]["content"]++ result = aligner.apply(messages, tokenizer, frozen_message_count=0)++ assert result.cache_metrics.prefix_changed is False+ assert result.cache_metrics.stable_prefix_hash == compute_short_hash(system_text)+ assert result.cache_metrics.stable_prefix_bytes == len(str(system_text).encode("utf-8"))+ assert result.cache_metrics.stable_prefix_tokens_est == tokenizer.count_text(str(system_text))