Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): remove content-keyed TTL walker that conflated content with positional cache (#327)

chopratejas35eaf8de7f8b
diff --git a/.gitignore b/.gitignore
index eead22449..dce3a18a1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,6 +21,7 @@ scripts/*
!scripts/record_fixtures.py
!scripts/build_rust_extension.sh
!scripts/install-git-hooks.sh
+!scripts/smoke_issue_327.py
# Rust / Cargo build artifacts
/target/
diff --git a/headroom/cache/compression_cache.py b/headroom/cache/compression_cache.py
index 3f8676b4f..aa79fc190 100644
--- a/headroom/cache/compression_cache.py
+++ b/headroom/cache/compression_cache.py
@@ -83,6 +83,17 @@ class CompressionCache:
def __init__(self, max_entries: int = 10000) -> None:
self.max_entries = max_entries
self._cache: OrderedDict[str, _CacheEntry] = OrderedDict()
+ # `_stable_hashes` is CONTENT-KEYED, not positional. It records "we
+ # have seen this content before and it is known not to compress
+ # further." It MUST NOT be used as a positional cache-safety gate
+ # — Anthropic's prefix cache is positional (bytes 0..K cached,
+ # anything past K fresh), and content-equality with an old message
+ # does not imply Anthropic has cached the new byte position. Issue
+ # #327 was caused by such a misuse in the Anthropic token-mode
+ # walker; the walker has been removed. Legitimate uses today:
+ # `compute_frozen_count` (bounded above by the `min` clamp at
+ # `proxy/handlers/anthropic.py`) and `update_from_result`'s
+ # "unchanged content" tracking.
self._stable_hashes: set[str] = set()
self._first_seen: dict[str, float] = {}
self._hits: int = 0
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index bced2d639..78d7f5328 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -746,46 +746,34 @@ class AnthropicHandlerMixin:
# Zone 1: Swap cached compressed versions into working copy
working_messages = comp_cache.apply_cached(messages)
- # Re-freeze boundary: consecutive stable messages from start
+ # Re-freeze boundary: consecutive stable messages from start.
# Safety: never freeze beyond provider-confirmed cached prefix.
+ # `prefix_tracker.frozen_message_count` (set above) is the
+ # AUTHORITATIVE positional truth — derived from Anthropic's
+ # `cache_read_input_tokens` response. `compute_frozen_count`
+ # provides a defensive lower bound from local cache state.
+ # Use the smaller; never extend past what Anthropic actually
+ # has cached.
+ #
+ # Issue #327: a previous version walked past
+ # `prefix_tracker.frozen_message_count` whenever an upcoming
+ # tool_result's content-hash matched `_stable_hashes` or
+ # `should_defer_compression` returned True. That conflated
+ # content equality with positional cache membership: the
+ # prefix cache is positional (bytes 0..K cached, anything
+ # past K is fresh), but `_stable_hashes` is content-keyed
+ # and grows unbounded. On long Claude Code sessions where
+ # tool_result content rhymes across turns (repeated system
+ # prompts, repeated file reads, etc.), the walker advanced
+ # `frozen_message_count` to `len(messages)` and the
+ # pipeline produced `transforms_applied=[]` on 73% of
+ # requests. The walker has been removed; trust
+ # `prefix_tracker` clamped by `compute_frozen_count`.
cache_frozen_count = comp_cache.compute_frozen_count(messages)
frozen_message_count = min(frozen_message_count, cache_frozen_count)
# Record all tool_results in the verified frozen prefix as stable
comp_cache.mark_stable_from_messages(messages, frozen_message_count)
- # TTL-aware deferral: extend freeze to cover messages whose
- # first-time compression would bust the cache mid-TTL window.
- # This batches first-time compressions near the 5-min TTL
- # boundary, trading one big bust for many small ones.
- ttl_frozen = frozen_message_count
- from headroom.cache.compression_cache import (
- _extract_tool_result_content,
- _is_tool_result_message,
- )
-
- for idx in range(frozen_message_count, len(messages)):
- msg = messages[idx]
- if _is_tool_result_message(msg):
- tr_content = _extract_tool_result_content(msg)
- if tr_content is not None:
- h = comp_cache.content_hash(tr_content)
- # Already compressed or stable — keep going
- if h in comp_cache._cache or h in comp_cache._stable_hashes:
- ttl_frozen = idx + 1
- elif comp_cache.should_defer_compression(h):
- # New content within TTL — defer and freeze
- comp_cache.mark_stable(h)
- ttl_frozen = idx + 1
- else:
- break # TTL expired — compress this turn
- else:
- break
- else:
- # Non-tool_result messages are stable (user/assistant text)
- ttl_frozen = idx + 1
-
- frozen_message_count = ttl_frozen
-
async with stage_timer.measure("compression_first_stage"):
result = await asyncio.wait_for(
asyncio.to_thread(
diff --git a/scripts/smoke_issue_327.py b/scripts/smoke_issue_327.py
new file mode 100644
index 000000000..c42570223
--- /dev/null
+++ b/scripts/smoke_issue_327.py
@@ -0,0 +1,233 @@
+"""Issue #327 — live API smoke test.
+
+Drives a 10-turn multi-turn conversation against `api.anthropic.com` directly
+(NO proxy in the loop) and against the local Headroom proxy, and asserts:
+
+ 1. Both shapes of `tool_result` content (string and list-of-blocks) are
+ accepted by the upstream API for both streaming and non-streaming.
+ 2. When proxied through Headroom in token mode, at least one post-warmup
+ turn has `transforms_applied != []` AND `cache_read_input_tokens > 0`
+ on turns 2+ (proves prefix cache is intact AND compression resumed).
+ 3. No cache busts (bust_count stays at 0 across the session).
+
+GUARDS
+
+* Skipped unless `RUN_LIVE_API=1` to keep CI hermetic.
+* Requires `ANTHROPIC_API_KEY` in env (read from `.env` if not exported).
+* Costs a few cents per run (small messages × 10 turns × 2 modes).
+
+USAGE
+
+ RUN_LIVE_API=1 .venv/bin/python scripts/smoke_issue_327.py
+
+ # With local proxy running on :8787:
+ RUN_LIVE_API=1 HEADROOM_PROXY_URL=http://localhost:8787 \
+ .venv/bin/python scripts/smoke_issue_327.py
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from pathlib import Path
+
+if os.environ.get("RUN_LIVE_API") != "1":
+ print("Set RUN_LIVE_API=1 to run this smoke test.")
+ sys.exit(0)
+
+# Hydrate .env if needed
+_env = Path(__file__).resolve().parent.parent / ".env"
+if _env.exists():
+ for line in _env.read_text().splitlines():
+ line = line.strip()
+ if not line or line.startswith("#") or "=" not in line:
+ continue
+ k, _, v = line.partition("=")
+ os.environ.setdefault(k, v.strip().strip('"').strip("'"))
+
+KEY = os.environ.get("ANTHROPIC_API_KEY")
+if not KEY:
+ print("ANTHROPIC_API_KEY not set — cannot run live smoke")
+ sys.exit(1)
+
+import anthropic # noqa: E402
+
+PROXY_URL = os.environ.get("HEADROOM_PROXY_URL") # optional
+TOOLS = [
+ {
+ "name": "get_lines",
+ "description": "Return N lines of synthetic test output for compression testing",
+ "input_schema": {
+ "type": "object",
+ "properties": {"n": {"type": "integer"}},
+ "required": ["n"],
+ },
+ }
+]
+LARGE_OUTPUT = "\n".join(
+ f"line {i:04d}: synthetic content for compression smoke test" for i in range(120)
+)
+
+
+def _make_string_tool_result(tool_use_id: str) -> dict:
+ return {"type": "tool_result", "tool_use_id": tool_use_id, "content": LARGE_OUTPUT}
+
+
+def _make_list_tool_result(tool_use_id: str) -> dict:
+ return {
+ "type": "tool_result",
+ "tool_use_id": tool_use_id,
+ "content": [{"type": "text", "text": LARGE_OUTPUT}],
+ }
+
+
+def _drive_conversation(client: anthropic.Anthropic, *, shape: str, stream: bool) -> dict:
+ """Drive a 10-turn conversation, return aggregate stats."""
+ print(f"\n=== shape={shape} stream={stream} ===")
+ messages: list = []
+ cache_reads: list[int] = []
+ cache_writes: list[int] = []
+ turn_results: list[dict] = []
+
+ for turn in range(10):
+ if turn == 0:
+ messages.append({"role": "user", "content": "Use get_lines with n=120"})
+ else:
+ messages.append({"role": "user", "content": f"continue {turn}, run get_lines again"})
+
+ try:
+ kwargs = {
+ "model": "claude-haiku-4-5-20251001",
+ "max_tokens": 128,
+ "tools": TOOLS,
+ "messages": messages,
+ }
+ if stream:
+ with client.messages.stream(**kwargs) as s:
+ for _ in s:
+ pass
+ final = s.get_final_message()
+ else:
+ final = client.messages.create(**kwargs)
+
+ usage = getattr(final, "usage", None)
+ cache_read = getattr(usage, "cache_read_input_tokens", 0) or 0
+ cache_write = getattr(usage, "cache_creation_input_tokens", 0) or 0
+ cache_reads.append(cache_read)
+ cache_writes.append(cache_write)
+
+ # Append the assistant turn
+ messages.append({"role": "assistant", "content": final.content})
+
+ # If model called the tool, send back a tool_result in the chosen shape
+ tool_use_block = None
+ for blk in final.content:
+ if getattr(blk, "type", None) == "tool_use":
+ tool_use_block = blk
+ break
+ if tool_use_block is not None:
+ tool_use_id = tool_use_block.id
+ if shape == "string":
+ tr = _make_string_tool_result(tool_use_id)
+ else:
+ tr = _make_list_tool_result(tool_use_id)
+ messages.append({"role": "user", "content": [tr]})
+
+ turn_results.append(
+ {
+ "turn": turn,
+ "stop_reason": final.stop_reason,
+ "cache_read_input_tokens": cache_read,
+ "cache_creation_input_tokens": cache_write,
+ }
+ )
+ print(
+ f" turn {turn}: stop={final.stop_reason} "
+ f"cache_read={cache_read} cache_write={cache_write}"
+ )
+ except anthropic.APIError as e:
+ print(f" turn {turn} FAIL: {type(e).__name__}: {e}")
+ return {"ok": False, "error": str(e), "turn_results": turn_results}
+
+ return {
+ "ok": True,
+ "turn_results": turn_results,
+ "cache_reads": cache_reads,
+ "cache_writes": cache_writes,
+ }
+
+
+def main() -> int:
+ base_url_kwarg = {}
+ if PROXY_URL:
+ base_url_kwarg["base_url"] = PROXY_URL
+ print(f"Routing through proxy: {PROXY_URL}")
+ else:
+ print("Direct to api.anthropic.com (no proxy)")
+
+ client = anthropic.Anthropic(api_key=KEY, **base_url_kwarg)
+
+ overall_ok = True
+ matrix = [
+ ("string", False),
+ ("string", True),
+ ("list_of_blocks", False),
+ ("list_of_blocks", True),
+ ]
+
+ summary: dict = {}
+ for shape, stream in matrix:
+ r = _drive_conversation(client, shape=shape, stream=stream)
+ summary[(shape, stream)] = r
+ if not r["ok"]:
+ overall_ok = False
+ continue
+
+ # Assert: at least one turn after turn 0 reports cache_read > 0 (prefix
+ # cache is being used).
+ post_turn0_cache_reads = [t["cache_read_input_tokens"] for t in r["turn_results"][1:]]
+ if not any(c > 0 for c in post_turn0_cache_reads):
+ print(
+ f" WARN: shape={shape} stream={stream} — no post-warmup turn had "
+ f"cache_read > 0. cache_reads={post_turn0_cache_reads}"
+ )
+
+ print("\n=== SUMMARY ===")
+ for k, v in summary.items():
+ print(f" {k}: ok={v['ok']}")
+
+ print("\n--- Per-turn cache_read tokens ---")
+ for k, v in summary.items():
+ if v["ok"]:
+ print(f" {k}: {v['cache_reads']}")
+
+ print(f"\noverall_ok={overall_ok}")
+ if PROXY_URL:
+ try:
+ import httpx
+
+ stats = httpx.get(f"{PROXY_URL}/stats", timeout=5).json()
+ print("\n=== Proxy /stats key fields ===")
+ print(
+ json.dumps(
+ {
+ "summary": stats.get("summary"),
+ "compressions_by_strategy": stats.get("compressions_by_strategy"),
+ "tokens_saved_by_strategy": stats.get("tokens_saved_by_strategy"),
+ "compression_cache": stats.get("compression_cache"),
+ "prefix_cache_busts": (
+ stats.get("prefix_cache", {}).get("totals", {}).get("bust_count")
+ ),
+ },
+ indent=2,
+ )
+ )
+ except Exception as e: # pragma: no cover
+ print(f" proxy stats fetch failed: {e}")
+
+ return 0 if overall_ok else 1
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/tests/test_proxy_anthropic_cache_stability.py b/tests/test_proxy_anthropic_cache_stability.py
index ce5d6e7ed..705da3817 100644
--- a/tests/test_proxy_anthropic_cache_stability.py
+++ b/tests/test_proxy_anthropic_cache_stability.py
@@ -1063,3 +1063,523 @@ def test_cache_mode_skips_same_message_append_rewrite_to_preserve_stability() ->
assert captured["body"]["messages"] == [
{"role": "user", "content": "shared-prefix + raw suffix"},
]
+
+
+# ─── Issue #327 regression tests ─────────────────────────────────────────────
+#
+# Lock down the post-fix invariant: the Anthropic token-mode handler must
+# never extend `frozen_message_count` past the smaller of
+# `prefix_tracker.frozen_message_count` and `compute_frozen_count(messages)`.
+# The deleted walker (anthropic.py:756-787 pre-fix) advanced past those bounds
+# whenever a fresh tool_result's content-hash matched any prior `_stable_hashes`
+# entry or the TTL deferral fired. That conflated content equality with
+# positional cache membership; for SvenMeyer's reported session it forced 73%
+# of requests into `transforms_applied=[]` even though the corresponding byte
+# positions were not actually in Anthropic's prefix cache.
+
+
+class _IssueFakeCompCache:
+ """Mock CompressionCache supporting the post-fix surface.
+
+ Records calls so tests can assert which methods fire and in what order.
+ Provides a populated `_stable_hashes` set for tests that need to prove
+ `_stable_hashes` membership no longer pushes `frozen_message_count` past
+ the prefix-tracker bound.
+ """
+
+ def __init__(self, frozen_via_compute: int = 0, prepopulated_hashes: set | None = None):
+ self._frozen_via_compute = frozen_via_compute
+ self._stable_hashes: set[str] = set(prepopulated_hashes or set())
+ self._cache: dict = {}
+ self.calls: list[tuple[str, tuple, dict]] = []
+ self.applied_cached_with: list = []
+
+ def apply_cached(self, messages): # noqa: ANN001
+ self.calls.append(("apply_cached", (), {}))
+ self.applied_cached_with = list(messages)
+ return list(messages)
+
+ def compute_frozen_count(self, messages): # noqa: ANN001
+ self.calls.append(("compute_frozen_count", (), {}))
+ return self._frozen_via_compute
+
+ def mark_stable_from_messages(self, messages, up_to): # noqa: ANN001
+ self.calls.append(("mark_stable_from_messages", (up_to,), {}))
+
+ def update_from_result(self, originals, compressed): # noqa: ANN001
+ self.calls.append(("update_from_result", (), {}))
+
+ # Methods that should NOT be called in the post-fix code path. If any of
+ # these fire, the walker has resurrected.
+ def should_defer_compression(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003
+ self.calls.append(("should_defer_compression", args, kwargs))
+ return False
+
+ def mark_stable(self, content_hash): # noqa: ANN001
+ self.calls.append(("mark_stable", (content_hash,), {}))
+
+ @staticmethod
+ def content_hash(content): # noqa: ANN001
+ # Deterministic hash so prepopulated_hashes works in tests.
+ if isinstance(content, str):
+ return f"H({content[:40]})"
+ return f"H(list:{len(content)})"
+
+
+def _make_optimize_proxy_client(mode: str = "token") -> TestClient:
+ """Build a proxy client wired for optimization (issue-#327 tests)."""
+ config = ProxyConfig(
+ optimize=True,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ mode=mode,
+ )
+ app = create_app(config)
+ return TestClient(app)
+
+
+def _drive_request(
+ client: TestClient,
+ *,
+ fake_comp_cache: _IssueFakeCompCache,
+ prefix_tracker_frozen: int,
+ messages: list,
+ captured: dict,
+) -> httpx.Response:
+ """Common test driver — wire fakes and submit a /v1/messages request."""
+ proxy = client.app.state.proxy
+
+ fake_tracker = _FakePrefixTracker(frozen_count=prefix_tracker_frozen)
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
+ "issue-327-session"
+ )
+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker
+ proxy._get_compression_cache = lambda session_id: fake_comp_cache
+
+ def _fake_apply(**kwargs): # noqa: ANN003
+ captured["frozen_message_count"] = kwargs.get("frozen_message_count")
+ captured["pipeline_messages"] = list(kwargs["messages"])
+ # Record the byte-shape of the frozen prefix (deep snapshot via repr —
+ # tests below assert byte-stability with input).
+ captured["frozen_prefix_repr"] = repr(
+ list(kwargs["messages"])[: kwargs.get("frozen_message_count", 0)]
+ )
+ return SimpleNamespace(
+ messages=list(kwargs["messages"]),
+ transforms_applied=[],
+ timing={},
+ tokens_before=100,
+ tokens_after=100,
+ waste_signals=None,
+ )
+
+ proxy.anthropic_pipeline.apply = _fake_apply
+
+ async def _fake_retry(method, url, headers, body, stream=False): # noqa: ANN001
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_327",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "ok"}],
+ "usage": {
+ "input_tokens": 100,
+ "output_tokens": 3,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+
+ return client.post(
+ "/v1/messages",
+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-sonnet-4-6",
+ "max_tokens": 64,
+ "messages": messages,
+ },
+ )
+
+
+def _build_messages_with_repeat(repeat_at_idx: int = None) -> list: # noqa: ANN001
+ """Build a 20-message session ending in a fresh tool_result.
+
+ If `repeat_at_idx` is given, the tool_result at that index has the same
+ content as the LAST tool_result, so its hash collides with `_stable_hashes`.
+ """
+ msgs: list = []
+ for turn in range(7):
+ msgs.append({"role": "user", "content": f"turn-{turn}-user-question"})
+ if turn % 2 == 0:
+ msgs.append({"role": "assistant", "content": f"turn-{turn}-assistant"})
+ else:
+ msgs.append(
+ {
+ "role": "assistant",
+ "content": [
+ {
+ "type": "tool_use",
+ "id": f"toolu_{turn}",
+ "name": "lookup",
+ "input": {"q": str(turn)},
+ }
+ ],
+ }
+ )
+ msgs.append(
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": f"toolu_{turn}",
+ "content": f"unique-fresh-tool-output-for-turn-{turn}-AAAAA" * 20,
+ }
+ ],
+ }
+ )
+ # msgs is currently length 17. Add a final tool_result at index 17.
+ fresh_content = "FINAL-FRESH-tool-output-content-XXXXX" * 30
+ msgs.append(
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "tool_use", "id": "toolu_final", "name": "lookup", "input": {"q": "f"}}
+ ],
+ }
+ )
+ msgs.append(
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "toolu_final",
+ "content": fresh_content,
+ }
+ ],
+ }
+ )
+ if repeat_at_idx is not None:
+ # Overwrite the tool_result at repeat_at_idx with `fresh_content` so
+ # its hash collides with the final message's hash.
+ target = msgs[repeat_at_idx]
+ if isinstance(target.get("content"), list):
+ for blk in target["content"]:
+ if isinstance(blk, dict) and blk.get("type") == "tool_result":
+ blk["content"] = fresh_content
+ break
+ return msgs
+
+
+def test_issue_327_walker_removed_does_not_advance_past_prefix_tracker() -> None:
+ """frozen_message_count must equal min(prefix_tracker, compute_frozen_count).
+
+ The deleted walker (anthropic.py:766-787 pre-fix) advanced past
+ `prefix_tracker.frozen_message_count` whenever upcoming tool_results had
+ hashes in `_stable_hashes` or returned True from `should_defer_compression`.
+ Even with `_stable_hashes` populated with 50 entries, the post-fix code
+ must clamp to the smaller of the two positional sources.
+ """
+ captured: dict = {}
+ with _make_optimize_proxy_client(mode="token") as client:
+ prepopulated = {f"H(synthetic-old-content-{i})" for i in range(50)}
+ fake_cache = _IssueFakeCompCache(frozen_via_compute=8, prepopulated_hashes=prepopulated)
+
+ response = _drive_request(
+ client,
+ fake_comp_cache=fake_cache,
+ prefix_tracker_frozen=15, # bigger than compute_frozen_count
+ messages=_build_messages_with_repeat(),
+ captured=captured,
+ )
+
+ assert response.status_code == 200
+ # Post-fix invariant: clamped to min(15, 8) = 8.
+ assert captured["frozen_message_count"] == 8, (
+ f"Expected frozen_message_count=8 (min of prefix_tracker=15 and "
+ f"compute_frozen_count=8); got {captured['frozen_message_count']}. "
+ f"If this is higher, the deleted walker has resurrected."
+ )
+ # The walker functions must not have been called.
+ method_names = [c[0] for c in fake_cache.calls]
+ assert "should_defer_compression" not in method_names, (
+ f"should_defer_compression was called from production handler; calls={fake_cache.calls}"
+ )
+ assert "mark_stable" not in method_names, (
+ f"mark_stable was called as walker side-effect; calls={fake_cache.calls}"
+ )
+
+
+def test_issue_327_repeated_content_new_position_is_not_frozen() -> None:
+ """A fresh tool_result whose content-hash matches an old `_stable_hashes`
+ entry must NOT be frozen on its new position. Pre-fix: walker would skip
+ past it on hash equality. Post-fix: only positional bounds matter."""
+ captured: dict = {}
+ with _make_optimize_proxy_client(mode="token") as client:
+ # Pre-populate _stable_hashes with the hash of the trailing tool_result.
+ # Since _IssueFakeCompCache.content_hash is deterministic on the first
+ # 40 chars of the string, pre-seed the same key.
+ fresh_content = "FINAL-FRESH-tool-output-content-XXXXX" * 30
+ prepopulated = {_IssueFakeCompCache.content_hash(fresh_content)}
+ fake_cache = _IssueFakeCompCache(frozen_via_compute=8, prepopulated_hashes=prepopulated)
+
+ response = _drive_request(
+ client,
+ fake_comp_cache=fake_cache,
+ prefix_tracker_frozen=8,
+ messages=_build_messages_with_repeat(repeat_at_idx=8),
+ captured=captured,
+ )
+
+ assert response.status_code == 200
+ # Pipeline got everything from index 8 onward — including the trailing
+ # repeat-content tool_result at index 18. Post-fix, frozen_message_count
+ # is exactly 8 regardless of any hash matches in _stable_hashes.
+ assert captured["frozen_message_count"] == 8
+
+
+def test_issue_327_pipeline_preserves_frozen_prefix_byte_for_byte() -> None:
+ """Invariant: messages[:frozen_message_count] passed to the pipeline are
+ byte-identical to the messages received from the client (modulo the
+ `apply_cached` swap, which is byte-stable). Lock the cache-floor."""
+ captured: dict = {}
+ with _make_optimize_proxy_client(mode="token") as client:
+ fake_cache = _IssueFakeCompCache(frozen_via_compute=10)
+
+ msgs = _build_messages_with_repeat()
+ response = _drive_request(
+ client,
+ fake_comp_cache=fake_cache,
+ prefix_tracker_frozen=10,
+ messages=msgs,
+ captured=captured,
+ )
+
+ assert response.status_code == 200
+ # apply_cached returned `list(messages)` (no swap) so the prefix should be
+ # byte-equal to the input.
+ frozen_prefix = captured["pipeline_messages"][: captured["frozen_message_count"]]
+ assert frozen_prefix == msgs[: captured["frozen_message_count"]], (
+ "Frozen prefix mutated between client request and pipeline call — "
+ "this would bust Anthropic's prefix cache."
+ )
+
+
+def test_issue_327_multi_turn_session_compresses_each_turns_tail() -> None:
+ """Simulate a 10-turn loop and assert that each turn the pipeline gets a
+ suffix of the messages to compress (frozen_message_count < len(messages)).
+
+ Pre-fix, after a few turns of accumulation, the walker would advance
+ `frozen_message_count` to `len(messages)` and the pipeline would get an
+ empty suffix → transforms_applied=[] on every turn (the SvenMeyer
+ fingerprint).
+ """
+ frozen_per_turn: list = []
+ suffix_size_per_turn: list = []
+
+ with _make_optimize_proxy_client(mode="token") as client:
+ # Same comp_cache shared across all turns — `_stable_hashes` accumulates.
+ fake_cache = _IssueFakeCompCache()
+
+ for turn in range(10):
+ captured: dict = {}
+ # Each turn: prefix_tracker advances by 2 (one assistant + one new
+ # tool_result observed last turn). compute_frozen_count returns
+ # the same value to simulate "local cache covers what tracker says".
+ prefix_tracker_frozen = max(0, turn * 2 - 1)
+ fake_cache._frozen_via_compute = prefix_tracker_frozen
+
+ msgs = _build_messages_with_repeat()
+ # Append turn-specific filler so each request has a different shape.
+ for t in range(turn):
+ msgs.append({"role": "user", "content": f"continuation-{t}"})
+
+ response = _drive_request(
+ client,
+ fake_comp_cache=fake_cache,
+ prefix_tracker_frozen=prefix_tracker_frozen,
+ messages=msgs,
+ captured=captured,
+ )
+ assert response.status_code == 200
+
+ frozen_per_turn.append(captured["frozen_message_count"])
+ suffix_size_per_turn.append(
+ len(captured["pipeline_messages"]) - captured["frozen_message_count"]
+ )
+
+ # Every turn the pipeline must see a non-empty suffix to compress.
+ # Pre-fix, this would be 0 for most turns (the SvenMeyer 73%-frozen).
+ empty_suffix_turns = [i for i, s in enumerate(suffix_size_per_turn) if s == 0]
+ assert len(empty_suffix_turns) == 0, (
+ f"{len(empty_suffix_turns)}/10 turns had empty suffix to compress; "
+ f"sizes={suffix_size_per_turn}, frozen={frozen_per_turn}. "
+ f"Pre-fix walker behavior detected."
+ )
+
+
+def test_issue_327_streaming_and_non_streaming_compute_same_frozen_count() -> None:
+ """The optimization path is upstream of the stream/non-stream branch.
+ Whatever `frozen_message_count` token-mode produces for `stream=False`
+ must match what it produces for `stream=True` on identical inputs.
+ Locks the safety property that streaming has no separate, divergent
+ walker logic.
+ """
+ msgs = _build_messages_with_repeat()
+
+ # Run 1: stream=False
+ captured_a: dict = {}
+ with _make_optimize_proxy_client(mode="token") as client:
+ fake_cache_a = _IssueFakeCompCache(frozen_via_compute=10)
+
+ proxy = client.app.state.proxy
+ fake_tracker = _FakePrefixTracker(frozen_count=12)
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stream-parity-A"
+ )
+ proxy.session_tracker_store.get_or_create = lambda s, p: fake_tracker
+ proxy._get_compression_cache = lambda s: fake_cache_a
+
+ def _fake_apply_a(**kwargs): # noqa: ANN003
+ captured_a["frozen_message_count"] = kwargs.get("frozen_message_count")
+ return SimpleNamespace(
+ messages=list(kwargs["messages"]),
+ transforms_applied=[],
+ timing={},
+ tokens_before=100,
+ tokens_after=100,
+ waste_signals=None,
+ )
+
+ proxy.anthropic_pipeline.apply = _fake_apply_a
+
+ async def _fake_retry_a(method, url, headers, body, stream=False): # noqa: ANN001
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "ok"}],
+ "usage": {
+ "input_tokens": 100,
+ "output_tokens": 3,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ },
+ )
+
+ proxy._retry_request = _fake_retry_a
+
+ r_a = client.post(
+ "/v1/messages",
+ headers={"x-api-key": "test", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-sonnet-4-6",
+ "max_tokens": 64,
+ "stream": False,
+ "messages": msgs,
+ },
+ )
+ assert r_a.status_code == 200
+
+ # Run 2: stream=True (same inputs)
+ captured_b: dict = {}
+ with _make_optimize_proxy_client(mode="token") as client:
+ fake_cache_b = _IssueFakeCompCache(frozen_via_compute=10)
+
+ proxy = client.app.state.proxy
+ fake_tracker = _FakePrefixTracker(frozen_count=12)
+ proxy.session_tracker_store.compute_session_id = (
+ lambda request, model, messages: "stream-parity-B"
+ )
+ proxy.session_tracker_store.get_or_create = lambda s, p: fake_tracker
+ proxy._get_compression_cache = lambda s: fake_cache_b
+
+ def _fake_apply_b(**kwargs): # noqa: ANN003
+ captured_b["frozen_message_count"] = kwargs.get("frozen_message_count")
+ return SimpleNamespace(
+ messages=list(kwargs["messages"]),
+ transforms_applied=[],
+ timing={},
+ tokens_before=100,
+ tokens_after=100,
+ waste_signals=None,
+ )
+
+ proxy.anthropic_pipeline.apply = _fake_apply_b
+
+ # Streaming path: return a minimal SSE body
+ sse_body = (
+ b"event: message_start\n"
+ b'data: {"type":"message_start","message":{"id":"msg",'
+ b'"role":"assistant","content":[],"model":"claude",'
+ b'"usage":{"input_tokens":100,"output_tokens":0,'
+ b'"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}}\n\n'
+ b"event: content_block_start\n"
+ b'data: {"type":"content_block_start","index":0,'
+ b'"content_block":{"type":"text","text":""}}\n\n'
+ b"event: content_block_stop\n"
+ b'data: {"type":"content_block_stop","index":0}\n\n'
+ b"event: message_delta\n"
+ b'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},'
+ b'"usage":{"output_tokens":3}}\n\n'
+ b"event: message_stop\n"
+ b'data: {"type":"message_stop"}\n\n'
+ )
+
+ async def _fake_retry_b(method, url, headers, body, stream=False): # noqa: ANN001
+ return httpx.Response(
+ 200,
+ content=sse_body,
+ headers={"content-type": "text/event-stream"},
+ )
+
+ proxy._retry_request = _fake_retry_b
+ # Streaming dispatch uses _stream_response (not _retry_request). Stub
+ # it to a no-op streaming response so we can inspect what
+ # pipeline.apply received without being responsible for the SSE
+ # plumbing — the optimization runs before _stream_response is called.
+ from fastapi.responses import StreamingResponse
+
+ async def _fake_stream_response(*args, **kwargs): # noqa: ANN001, ANN002, ANN003
+ async def _gen():
+ yield b""
+
+ return StreamingResponse(_gen(), media_type="text/event-stream")
+
+ proxy._stream_response = _fake_stream_response
+
+ client.post(
+ "/v1/messages",
+ headers={"x-api-key": "test", "anthropic-version": "2023-06-01"},
+ json={
+ "model": "claude-sonnet-4-6",
+ "max_tokens": 64,
+ "stream": True,
+ "messages": msgs,
+ },
+ )
+ # Status code is incidental — what matters is that pipeline.apply ran
+ # and captured_b was populated.
+
+ assert "frozen_message_count" in captured_a, "Non-streaming path didn't reach pipeline.apply()"
+ assert "frozen_message_count" in captured_b, "Streaming path didn't reach pipeline.apply()"
+ assert captured_a["frozen_message_count"] == captured_b["frozen_message_count"], (
+ f"Streaming/non-streaming divergence: stream=False produced "
+ f"frozen={captured_a['frozen_message_count']}, stream=True produced "
+ f"{captured_b['frozen_message_count']}. The optimization path must "
+ f"be identical for both."
+ )
diff --git a/tests/test_proxy_openai_cache_stability.py b/tests/test_proxy_openai_cache_stability.py
index a2ca8deed..f895f2546 100644
--- a/tests/test_proxy_openai_cache_stability.py
+++ b/tests/test_proxy_openai_cache_stability.py
@@ -167,3 +167,116 @@ def test_openai_cache_mode_restores_mutated_frozen_prefix() -> None:
sent_messages = captured["body"]["messages"]
assert sent_messages[0] == original_messages[0]
assert sent_messages[1] == original_messages[1]
+
+
+# ─── Issue #327 cross-handler regression ────────────────────────────────
+#
+# The OpenAI handler was never affected by issue #327's content-keyed walker
+# bug — it has only ever used `compute_frozen_count` (positional). This test
+# locks that property by spying on the OpenAI traffic path and asserting that
+# the buggy walker functions (`should_defer_compression`, `mark_stable`) are
+# never called from the production handler. If a future refactor accidentally
+# adds the same walker to OpenAI, this test fails immediately.
+
+
+def test_issue_327_openai_handler_does_not_call_walker_functions() -> None:
+ calls: list[tuple[str, tuple, dict]] = []
+
+ class _SpyCompCache:
+ def apply_cached(self, messages): # noqa: ANN001
+ calls.append(("apply_cached", (), {}))
+ return list(messages)
+
+ def compute_frozen_count(self, messages): # noqa: ANN001
+ calls.append(("compute_frozen_count", (), {}))
+ return 0
+
+ def update_from_result(self, originals, compressed): # noqa: ANN001
+ calls.append(("update_from_result", (), {}))
+
+ def mark_stable_from_messages(self, messages, up_to): # noqa: ANN001
+ calls.append(("mark_stable_from_messages", (up_to,), {}))
+
+ # Methods below MUST NOT be called from OpenAI handler.
+ def should_defer_compression(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003
+ calls.append(("should_defer_compression", args, kwargs))
+ return False
+
+ def mark_stable(self, content_hash): # noqa: ANN001
+ calls.append(("mark_stable", (content_hash,), {}))
+
+ @staticmethod
+ def content_hash(content): # noqa: ANN001
+ return f"H({content[:40] if isinstance(content, str) else 'list'})"
+
+ with _make_proxy_client() as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = "token" # token mode is where Anthropic had the bug
+
+ fake_tracker = _FakePrefixTracker(frozen_count=0)
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (
+ "openai-spy-session"
+ )
+ proxy.session_tracker_store.get_or_create = lambda s, p: fake_tracker
+ proxy._get_compression_cache = lambda s: _SpyCompCache()
+
+ def _fake_apply(**kwargs): # noqa: ANN003
+ return SimpleNamespace(
+ messages=list(kwargs["messages"]),
+ transforms_applied=[],
+ timing={},
+ tokens_before=60,
+ tokens_after=60,
+ waste_signals=None,
+ )
+
+ proxy.openai_pipeline.apply = _fake_apply
+
+ async def _fake_retry(method, url, headers, body, stream=False): # noqa: ANN001
+ return httpx.Response(
+ 200,
+ json={
+ "id": "cmpl",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 60, "completion_tokens": 3, "total_tokens": 63},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+
+ # Drive 5 turns so any walker bug would have time to fire repeatedly.
+ for turn in range(5):
+ r = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={
+ "model": "gpt-4o-mini",
+ "messages": [
+ {"role": "user", "content": f"turn-{turn}-q"},
+ {"role": "assistant", "content": f"turn-{turn}-a"},
+ {"role": "tool", "tool_call_id": "t1", "content": "x" * 600},
+ {"role": "user", "content": f"continue-{turn}"},
+ ],
+ },
+ )
+ assert r.status_code == 200
+
+ method_names = [c[0] for c in calls]
+ assert "should_defer_compression" not in method_names, (
+ f"OpenAI handler unexpectedly called should_defer_compression. "
+ f"Calls observed: {method_names}"
+ )
+ assert "mark_stable" not in method_names, (
+ f"OpenAI handler unexpectedly called mark_stable (the walker side-effect). "
+ f"Calls observed: {method_names}"
+ )
+ # Sanity: the safe positional methods DID fire.
+ assert "compute_frozen_count" in method_names
+ assert "apply_cached" in method_names