Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): remove content-keyed TTL walker that conflated content with positional cache (#327)
chopratejas35eaf8de7f8b
diff --git a/.gitignore b/.gitignoreindex eead22449..dce3a18a1 100644--- a/.gitignore+++ b/.gitignore@@ -21,6 +21,7 @@ scripts/*!scripts/record_fixtures.py!scripts/build_rust_extension.sh!scripts/install-git-hooks.sh+!scripts/smoke_issue_327.py# Rust / Cargo build artifacts/target/diff --git a/headroom/cache/compression_cache.py b/headroom/cache/compression_cache.pyindex 3f8676b4f..aa79fc190 100644--- a/headroom/cache/compression_cache.py+++ b/headroom/cache/compression_cache.py@@ -83,6 +83,17 @@ class CompressionCache:def __init__(self, max_entries: int = 10000) -> None:self.max_entries = max_entriesself._cache: OrderedDict[str, _CacheEntry] = OrderedDict()+ # `_stable_hashes` is CONTENT-KEYED, not positional. It records "we+ # have seen this content before and it is known not to compress+ # further." It MUST NOT be used as a positional cache-safety gate+ # — Anthropic's prefix cache is positional (bytes 0..K cached,+ # anything past K fresh), and content-equality with an old message+ # does not imply Anthropic has cached the new byte position. Issue+ # #327 was caused by such a misuse in the Anthropic token-mode+ # walker; the walker has been removed. Legitimate uses today:+ # `compute_frozen_count` (bounded above by the `min` clamp at+ # `proxy/handlers/anthropic.py`) and `update_from_result`'s+ # "unchanged content" tracking.self._stable_hashes: set[str] = set()self._first_seen: dict[str, float] = {}self._hits: int = 0diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex bced2d639..78d7f5328 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -746,46 +746,34 @@ class AnthropicHandlerMixin:# Zone 1: Swap cached compressed versions into working copyworking_messages = comp_cache.apply_cached(messages)- # Re-freeze boundary: consecutive stable messages from start+ # Re-freeze boundary: consecutive stable messages from start.# Safety: never freeze beyond provider-confirmed cached prefix.+ # `prefix_tracker.frozen_message_count` (set above) is the+ # AUTHORITATIVE positional truth — derived from Anthropic's+ # `cache_read_input_tokens` response. `compute_frozen_count`+ # provides a defensive lower bound from local cache state.+ # Use the smaller; never extend past what Anthropic actually+ # has cached.+ #+ # Issue #327: a previous version walked past+ # `prefix_tracker.frozen_message_count` whenever an upcoming+ # tool_result's content-hash matched `_stable_hashes` or+ # `should_defer_compression` returned True. That conflated+ # content equality with positional cache membership: the+ # prefix cache is positional (bytes 0..K cached, anything+ # past K is fresh), but `_stable_hashes` is content-keyed+ # and grows unbounded. On long Claude Code sessions where+ # tool_result content rhymes across turns (repeated system+ # prompts, repeated file reads, etc.), the walker advanced+ # `frozen_message_count` to `len(messages)` and the+ # pipeline produced `transforms_applied=[]` on 73% of+ # requests. The walker has been removed; trust+ # `prefix_tracker` clamped by `compute_frozen_count`.cache_frozen_count = comp_cache.compute_frozen_count(messages)frozen_message_count = min(frozen_message_count, cache_frozen_count)# Record all tool_results in the verified frozen prefix as stablecomp_cache.mark_stable_from_messages(messages, frozen_message_count)- # TTL-aware deferral: extend freeze to cover messages whose- # first-time compression would bust the cache mid-TTL window.- # This batches first-time compressions near the 5-min TTL- # boundary, trading one big bust for many small ones.- ttl_frozen = frozen_message_count- from headroom.cache.compression_cache import (- _extract_tool_result_content,- _is_tool_result_message,- )-- for idx in range(frozen_message_count, len(messages)):- msg = messages[idx]- if _is_tool_result_message(msg):- tr_content = _extract_tool_result_content(msg)- if tr_content is not None:- h = comp_cache.content_hash(tr_content)- # Already compressed or stable — keep going- if h in comp_cache._cache or h in comp_cache._stable_hashes:- ttl_frozen = idx + 1- elif comp_cache.should_defer_compression(h):- # New content within TTL — defer and freeze- comp_cache.mark_stable(h)- ttl_frozen = idx + 1- else:- break # TTL expired — compress this turn- else:- break- else:- # Non-tool_result messages are stable (user/assistant text)- ttl_frozen = idx + 1-- frozen_message_count = ttl_frozen-async with stage_timer.measure("compression_first_stage"):result = await asyncio.wait_for(asyncio.to_thread(diff --git a/scripts/smoke_issue_327.py b/scripts/smoke_issue_327.pynew file mode 100644index 000000000..c42570223--- /dev/null+++ b/scripts/smoke_issue_327.py@@ -0,0 +1,233 @@+"""Issue #327 — live API smoke test.++Drives a 10-turn multi-turn conversation against `api.anthropic.com` directly+(NO proxy in the loop) and against the local Headroom proxy, and asserts:++ 1. Both shapes of `tool_result` content (string and list-of-blocks) are+ accepted by the upstream API for both streaming and non-streaming.+ 2. When proxied through Headroom in token mode, at least one post-warmup+ turn has `transforms_applied != []` AND `cache_read_input_tokens > 0`+ on turns 2+ (proves prefix cache is intact AND compression resumed).+ 3. No cache busts (bust_count stays at 0 across the session).++GUARDS++* Skipped unless `RUN_LIVE_API=1` to keep CI hermetic.+* Requires `ANTHROPIC_API_KEY` in env (read from `.env` if not exported).+* Costs a few cents per run (small messages × 10 turns × 2 modes).++USAGE++ RUN_LIVE_API=1 .venv/bin/python scripts/smoke_issue_327.py++ # With local proxy running on :8787:+ RUN_LIVE_API=1 HEADROOM_PROXY_URL=http://localhost:8787 \+ .venv/bin/python scripts/smoke_issue_327.py+"""++from __future__ import annotations++import json+import os+import sys+from pathlib import Path++if os.environ.get("RUN_LIVE_API") != "1":+ print("Set RUN_LIVE_API=1 to run this smoke test.")+ sys.exit(0)++# Hydrate .env if needed+_env = Path(__file__).resolve().parent.parent / ".env"+if _env.exists():+ for line in _env.read_text().splitlines():+ line = line.strip()+ if not line or line.startswith("#") or "=" not in line:+ continue+ k, _, v = line.partition("=")+ os.environ.setdefault(k, v.strip().strip('"').strip("'"))++KEY = os.environ.get("ANTHROPIC_API_KEY")+if not KEY:+ print("ANTHROPIC_API_KEY not set — cannot run live smoke")+ sys.exit(1)++import anthropic # noqa: E402++PROXY_URL = os.environ.get("HEADROOM_PROXY_URL") # optional+TOOLS = [+ {+ "name": "get_lines",+ "description": "Return N lines of synthetic test output for compression testing",+ "input_schema": {+ "type": "object",+ "properties": {"n": {"type": "integer"}},+ "required": ["n"],+ },+ }+]+LARGE_OUTPUT = "\n".join(+ f"line {i:04d}: synthetic content for compression smoke test" for i in range(120)+)+++def _make_string_tool_result(tool_use_id: str) -> dict:+ return {"type": "tool_result", "tool_use_id": tool_use_id, "content": LARGE_OUTPUT}+++def _make_list_tool_result(tool_use_id: str) -> dict:+ return {+ "type": "tool_result",+ "tool_use_id": tool_use_id,+ "content": [{"type": "text", "text": LARGE_OUTPUT}],+ }+++def _drive_conversation(client: anthropic.Anthropic, *, shape: str, stream: bool) -> dict:+ """Drive a 10-turn conversation, return aggregate stats."""+ print(f"\n=== shape={shape} stream={stream} ===")+ messages: list = []+ cache_reads: list[int] = []+ cache_writes: list[int] = []+ turn_results: list[dict] = []++ for turn in range(10):+ if turn == 0:+ messages.append({"role": "user", "content": "Use get_lines with n=120"})+ else:+ messages.append({"role": "user", "content": f"continue {turn}, run get_lines again"})++ try:+ kwargs = {+ "model": "claude-haiku-4-5-20251001",+ "max_tokens": 128,+ "tools": TOOLS,+ "messages": messages,+ }+ if stream:+ with client.messages.stream(**kwargs) as s:+ for _ in s:+ pass+ final = s.get_final_message()+ else:+ final = client.messages.create(**kwargs)++ usage = getattr(final, "usage", None)+ cache_read = getattr(usage, "cache_read_input_tokens", 0) or 0+ cache_write = getattr(usage, "cache_creation_input_tokens", 0) or 0+ cache_reads.append(cache_read)+ cache_writes.append(cache_write)++ # Append the assistant turn+ messages.append({"role": "assistant", "content": final.content})++ # If model called the tool, send back a tool_result in the chosen shape+ tool_use_block = None+ for blk in final.content:+ if getattr(blk, "type", None) == "tool_use":+ tool_use_block = blk+ break+ if tool_use_block is not None:+ tool_use_id = tool_use_block.id+ if shape == "string":+ tr = _make_string_tool_result(tool_use_id)+ else:+ tr = _make_list_tool_result(tool_use_id)+ messages.append({"role": "user", "content": [tr]})++ turn_results.append(+ {+ "turn": turn,+ "stop_reason": final.stop_reason,+ "cache_read_input_tokens": cache_read,+ "cache_creation_input_tokens": cache_write,+ }+ )+ print(+ f" turn {turn}: stop={final.stop_reason} "+ f"cache_read={cache_read} cache_write={cache_write}"+ )+ except anthropic.APIError as e:+ print(f" turn {turn} FAIL: {type(e).__name__}: {e}")+ return {"ok": False, "error": str(e), "turn_results": turn_results}++ return {+ "ok": True,+ "turn_results": turn_results,+ "cache_reads": cache_reads,+ "cache_writes": cache_writes,+ }+++def main() -> int:+ base_url_kwarg = {}+ if PROXY_URL:+ base_url_kwarg["base_url"] = PROXY_URL+ print(f"Routing through proxy: {PROXY_URL}")+ else:+ print("Direct to api.anthropic.com (no proxy)")++ client = anthropic.Anthropic(api_key=KEY, **base_url_kwarg)++ overall_ok = True+ matrix = [+ ("string", False),+ ("string", True),+ ("list_of_blocks", False),+ ("list_of_blocks", True),+ ]++ summary: dict = {}+ for shape, stream in matrix:+ r = _drive_conversation(client, shape=shape, stream=stream)+ summary[(shape, stream)] = r+ if not r["ok"]:+ overall_ok = False+ continue++ # Assert: at least one turn after turn 0 reports cache_read > 0 (prefix+ # cache is being used).+ post_turn0_cache_reads = [t["cache_read_input_tokens"] for t in r["turn_results"][1:]]+ if not any(c > 0 for c in post_turn0_cache_reads):+ print(+ f" WARN: shape={shape} stream={stream} — no post-warmup turn had "+ f"cache_read > 0. cache_reads={post_turn0_cache_reads}"+ )++ print("\n=== SUMMARY ===")+ for k, v in summary.items():+ print(f" {k}: ok={v['ok']}")++ print("\n--- Per-turn cache_read tokens ---")+ for k, v in summary.items():+ if v["ok"]:+ print(f" {k}: {v['cache_reads']}")++ print(f"\noverall_ok={overall_ok}")+ if PROXY_URL:+ try:+ import httpx++ stats = httpx.get(f"{PROXY_URL}/stats", timeout=5).json()+ print("\n=== Proxy /stats key fields ===")+ print(+ json.dumps(+ {+ "summary": stats.get("summary"),+ "compressions_by_strategy": stats.get("compressions_by_strategy"),+ "tokens_saved_by_strategy": stats.get("tokens_saved_by_strategy"),+ "compression_cache": stats.get("compression_cache"),+ "prefix_cache_busts": (+ stats.get("prefix_cache", {}).get("totals", {}).get("bust_count")+ ),+ },+ indent=2,+ )+ )+ except Exception as e: # pragma: no cover+ print(f" proxy stats fetch failed: {e}")++ return 0 if overall_ok else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/tests/test_proxy_anthropic_cache_stability.py b/tests/test_proxy_anthropic_cache_stability.pyindex ce5d6e7ed..705da3817 100644--- a/tests/test_proxy_anthropic_cache_stability.py+++ b/tests/test_proxy_anthropic_cache_stability.py@@ -1063,3 +1063,523 @@ def test_cache_mode_skips_same_message_append_rewrite_to_preserve_stability() ->assert captured["body"]["messages"] == [{"role": "user", "content": "shared-prefix + raw suffix"},]+++# ─── Issue #327 regression tests ─────────────────────────────────────────────+#+# Lock down the post-fix invariant: the Anthropic token-mode handler must+# never extend `frozen_message_count` past the smaller of+# `prefix_tracker.frozen_message_count` and `compute_frozen_count(messages)`.+# The deleted walker (anthropic.py:756-787 pre-fix) advanced past those bounds+# whenever a fresh tool_result's content-hash matched any prior `_stable_hashes`+# entry or the TTL deferral fired. That conflated content equality with+# positional cache membership; for SvenMeyer's reported session it forced 73%+# of requests into `transforms_applied=[]` even though the corresponding byte+# positions were not actually in Anthropic's prefix cache.+++class _IssueFakeCompCache:+ """Mock CompressionCache supporting the post-fix surface.++ Records calls so tests can assert which methods fire and in what order.+ Provides a populated `_stable_hashes` set for tests that need to prove+ `_stable_hashes` membership no longer pushes `frozen_message_count` past+ the prefix-tracker bound.+ """++ def __init__(self, frozen_via_compute: int = 0, prepopulated_hashes: set | None = None):+ self._frozen_via_compute = frozen_via_compute+ self._stable_hashes: set[str] = set(prepopulated_hashes or set())+ self._cache: dict = {}+ self.calls: list[tuple[str, tuple, dict]] = []+ self.applied_cached_with: list = []++ def apply_cached(self, messages): # noqa: ANN001+ self.calls.append(("apply_cached", (), {}))+ self.applied_cached_with = list(messages)+ return list(messages)++ def compute_frozen_count(self, messages): # noqa: ANN001+ self.calls.append(("compute_frozen_count", (), {}))+ return self._frozen_via_compute++ def mark_stable_from_messages(self, messages, up_to): # noqa: ANN001+ self.calls.append(("mark_stable_from_messages", (up_to,), {}))++ def update_from_result(self, originals, compressed): # noqa: ANN001+ self.calls.append(("update_from_result", (), {}))++ # Methods that should NOT be called in the post-fix code path. If any of+ # these fire, the walker has resurrected.+ def should_defer_compression(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003+ self.calls.append(("should_defer_compression", args, kwargs))+ return False++ def mark_stable(self, content_hash): # noqa: ANN001+ self.calls.append(("mark_stable", (content_hash,), {}))++ @staticmethod+ def content_hash(content): # noqa: ANN001+ # Deterministic hash so prepopulated_hashes works in tests.+ if isinstance(content, str):+ return f"H({content[:40]})"+ return f"H(list:{len(content)})"+++def _make_optimize_proxy_client(mode: str = "token") -> TestClient:+ """Build a proxy client wired for optimization (issue-#327 tests)."""+ config = ProxyConfig(+ optimize=True,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ mode=mode,+ )+ app = create_app(config)+ return TestClient(app)+++def _drive_request(+ client: TestClient,+ *,+ fake_comp_cache: _IssueFakeCompCache,+ prefix_tracker_frozen: int,+ messages: list,+ captured: dict,+) -> httpx.Response:+ """Common test driver — wire fakes and submit a /v1/messages request."""+ proxy = client.app.state.proxy++ fake_tracker = _FakePrefixTracker(frozen_count=prefix_tracker_frozen)+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (+ "issue-327-session"+ )+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: fake_tracker+ proxy._get_compression_cache = lambda session_id: fake_comp_cache++ def _fake_apply(**kwargs): # noqa: ANN003+ captured["frozen_message_count"] = kwargs.get("frozen_message_count")+ captured["pipeline_messages"] = list(kwargs["messages"])+ # Record the byte-shape of the frozen prefix (deep snapshot via repr —+ # tests below assert byte-stability with input).+ captured["frozen_prefix_repr"] = repr(+ list(kwargs["messages"])[: kwargs.get("frozen_message_count", 0)]+ )+ return SimpleNamespace(+ messages=list(kwargs["messages"]),+ transforms_applied=[],+ timing={},+ tokens_before=100,+ tokens_after=100,+ waste_signals=None,+ )++ proxy.anthropic_pipeline.apply = _fake_apply++ async def _fake_retry(method, url, headers, body, stream=False): # noqa: ANN001+ return httpx.Response(+ 200,+ json={+ "id": "msg_327",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": "ok"}],+ "usage": {+ "input_tokens": 100,+ "output_tokens": 3,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ },+ },+ )++ proxy._retry_request = _fake_retry++ return client.post(+ "/v1/messages",+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "messages": messages,+ },+ )+++def _build_messages_with_repeat(repeat_at_idx: int = None) -> list: # noqa: ANN001+ """Build a 20-message session ending in a fresh tool_result.++ If `repeat_at_idx` is given, the tool_result at that index has the same+ content as the LAST tool_result, so its hash collides with `_stable_hashes`.+ """+ msgs: list = []+ for turn in range(7):+ msgs.append({"role": "user", "content": f"turn-{turn}-user-question"})+ if turn % 2 == 0:+ msgs.append({"role": "assistant", "content": f"turn-{turn}-assistant"})+ else:+ msgs.append(+ {+ "role": "assistant",+ "content": [+ {+ "type": "tool_use",+ "id": f"toolu_{turn}",+ "name": "lookup",+ "input": {"q": str(turn)},+ }+ ],+ }+ )+ msgs.append(+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": f"toolu_{turn}",+ "content": f"unique-fresh-tool-output-for-turn-{turn}-AAAAA" * 20,+ }+ ],+ }+ )+ # msgs is currently length 17. Add a final tool_result at index 17.+ fresh_content = "FINAL-FRESH-tool-output-content-XXXXX" * 30+ msgs.append(+ {+ "role": "assistant",+ "content": [+ {"type": "tool_use", "id": "toolu_final", "name": "lookup", "input": {"q": "f"}}+ ],+ }+ )+ msgs.append(+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "toolu_final",+ "content": fresh_content,+ }+ ],+ }+ )+ if repeat_at_idx is not None:+ # Overwrite the tool_result at repeat_at_idx with `fresh_content` so+ # its hash collides with the final message's hash.+ target = msgs[repeat_at_idx]+ if isinstance(target.get("content"), list):+ for blk in target["content"]:+ if isinstance(blk, dict) and blk.get("type") == "tool_result":+ blk["content"] = fresh_content+ break+ return msgs+++def test_issue_327_walker_removed_does_not_advance_past_prefix_tracker() -> None:+ """frozen_message_count must equal min(prefix_tracker, compute_frozen_count).++ The deleted walker (anthropic.py:766-787 pre-fix) advanced past+ `prefix_tracker.frozen_message_count` whenever upcoming tool_results had+ hashes in `_stable_hashes` or returned True from `should_defer_compression`.+ Even with `_stable_hashes` populated with 50 entries, the post-fix code+ must clamp to the smaller of the two positional sources.+ """+ captured: dict = {}+ with _make_optimize_proxy_client(mode="token") as client:+ prepopulated = {f"H(synthetic-old-content-{i})" for i in range(50)}+ fake_cache = _IssueFakeCompCache(frozen_via_compute=8, prepopulated_hashes=prepopulated)++ response = _drive_request(+ client,+ fake_comp_cache=fake_cache,+ prefix_tracker_frozen=15, # bigger than compute_frozen_count+ messages=_build_messages_with_repeat(),+ captured=captured,+ )++ assert response.status_code == 200+ # Post-fix invariant: clamped to min(15, 8) = 8.+ assert captured["frozen_message_count"] == 8, (+ f"Expected frozen_message_count=8 (min of prefix_tracker=15 and "+ f"compute_frozen_count=8); got {captured['frozen_message_count']}. "+ f"If this is higher, the deleted walker has resurrected."+ )+ # The walker functions must not have been called.+ method_names = [c[0] for c in fake_cache.calls]+ assert "should_defer_compression" not in method_names, (+ f"should_defer_compression was called from production handler; calls={fake_cache.calls}"+ )+ assert "mark_stable" not in method_names, (+ f"mark_stable was called as walker side-effect; calls={fake_cache.calls}"+ )+++def test_issue_327_repeated_content_new_position_is_not_frozen() -> None:+ """A fresh tool_result whose content-hash matches an old `_stable_hashes`+ entry must NOT be frozen on its new position. Pre-fix: walker would skip+ past it on hash equality. Post-fix: only positional bounds matter."""+ captured: dict = {}+ with _make_optimize_proxy_client(mode="token") as client:+ # Pre-populate _stable_hashes with the hash of the trailing tool_result.+ # Since _IssueFakeCompCache.content_hash is deterministic on the first+ # 40 chars of the string, pre-seed the same key.+ fresh_content = "FINAL-FRESH-tool-output-content-XXXXX" * 30+ prepopulated = {_IssueFakeCompCache.content_hash(fresh_content)}+ fake_cache = _IssueFakeCompCache(frozen_via_compute=8, prepopulated_hashes=prepopulated)++ response = _drive_request(+ client,+ fake_comp_cache=fake_cache,+ prefix_tracker_frozen=8,+ messages=_build_messages_with_repeat(repeat_at_idx=8),+ captured=captured,+ )++ assert response.status_code == 200+ # Pipeline got everything from index 8 onward — including the trailing+ # repeat-content tool_result at index 18. Post-fix, frozen_message_count+ # is exactly 8 regardless of any hash matches in _stable_hashes.+ assert captured["frozen_message_count"] == 8+++def test_issue_327_pipeline_preserves_frozen_prefix_byte_for_byte() -> None:+ """Invariant: messages[:frozen_message_count] passed to the pipeline are+ byte-identical to the messages received from the client (modulo the+ `apply_cached` swap, which is byte-stable). Lock the cache-floor."""+ captured: dict = {}+ with _make_optimize_proxy_client(mode="token") as client:+ fake_cache = _IssueFakeCompCache(frozen_via_compute=10)++ msgs = _build_messages_with_repeat()+ response = _drive_request(+ client,+ fake_comp_cache=fake_cache,+ prefix_tracker_frozen=10,+ messages=msgs,+ captured=captured,+ )++ assert response.status_code == 200+ # apply_cached returned `list(messages)` (no swap) so the prefix should be+ # byte-equal to the input.+ frozen_prefix = captured["pipeline_messages"][: captured["frozen_message_count"]]+ assert frozen_prefix == msgs[: captured["frozen_message_count"]], (+ "Frozen prefix mutated between client request and pipeline call — "+ "this would bust Anthropic's prefix cache."+ )+++def test_issue_327_multi_turn_session_compresses_each_turns_tail() -> None:+ """Simulate a 10-turn loop and assert that each turn the pipeline gets a+ suffix of the messages to compress (frozen_message_count < len(messages)).++ Pre-fix, after a few turns of accumulation, the walker would advance+ `frozen_message_count` to `len(messages)` and the pipeline would get an+ empty suffix → transforms_applied=[] on every turn (the SvenMeyer+ fingerprint).+ """+ frozen_per_turn: list = []+ suffix_size_per_turn: list = []++ with _make_optimize_proxy_client(mode="token") as client:+ # Same comp_cache shared across all turns — `_stable_hashes` accumulates.+ fake_cache = _IssueFakeCompCache()++ for turn in range(10):+ captured: dict = {}+ # Each turn: prefix_tracker advances by 2 (one assistant + one new+ # tool_result observed last turn). compute_frozen_count returns+ # the same value to simulate "local cache covers what tracker says".+ prefix_tracker_frozen = max(0, turn * 2 - 1)+ fake_cache._frozen_via_compute = prefix_tracker_frozen++ msgs = _build_messages_with_repeat()+ # Append turn-specific filler so each request has a different shape.+ for t in range(turn):+ msgs.append({"role": "user", "content": f"continuation-{t}"})++ response = _drive_request(+ client,+ fake_comp_cache=fake_cache,+ prefix_tracker_frozen=prefix_tracker_frozen,+ messages=msgs,+ captured=captured,+ )+ assert response.status_code == 200++ frozen_per_turn.append(captured["frozen_message_count"])+ suffix_size_per_turn.append(+ len(captured["pipeline_messages"]) - captured["frozen_message_count"]+ )++ # Every turn the pipeline must see a non-empty suffix to compress.+ # Pre-fix, this would be 0 for most turns (the SvenMeyer 73%-frozen).+ empty_suffix_turns = [i for i, s in enumerate(suffix_size_per_turn) if s == 0]+ assert len(empty_suffix_turns) == 0, (+ f"{len(empty_suffix_turns)}/10 turns had empty suffix to compress; "+ f"sizes={suffix_size_per_turn}, frozen={frozen_per_turn}. "+ f"Pre-fix walker behavior detected."+ )+++def test_issue_327_streaming_and_non_streaming_compute_same_frozen_count() -> None:+ """The optimization path is upstream of the stream/non-stream branch.+ Whatever `frozen_message_count` token-mode produces for `stream=False`+ must match what it produces for `stream=True` on identical inputs.+ Locks the safety property that streaming has no separate, divergent+ walker logic.+ """+ msgs = _build_messages_with_repeat()++ # Run 1: stream=False+ captured_a: dict = {}+ with _make_optimize_proxy_client(mode="token") as client:+ fake_cache_a = _IssueFakeCompCache(frozen_via_compute=10)++ proxy = client.app.state.proxy+ fake_tracker = _FakePrefixTracker(frozen_count=12)+ proxy.session_tracker_store.compute_session_id = (+ lambda request, model, messages: "stream-parity-A"+ )+ proxy.session_tracker_store.get_or_create = lambda s, p: fake_tracker+ proxy._get_compression_cache = lambda s: fake_cache_a++ def _fake_apply_a(**kwargs): # noqa: ANN003+ captured_a["frozen_message_count"] = kwargs.get("frozen_message_count")+ return SimpleNamespace(+ messages=list(kwargs["messages"]),+ transforms_applied=[],+ timing={},+ tokens_before=100,+ tokens_after=100,+ waste_signals=None,+ )++ proxy.anthropic_pipeline.apply = _fake_apply_a++ async def _fake_retry_a(method, url, headers, body, stream=False): # noqa: ANN001+ return httpx.Response(+ 200,+ json={+ "id": "msg",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": "ok"}],+ "usage": {+ "input_tokens": 100,+ "output_tokens": 3,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ },+ },+ )++ proxy._retry_request = _fake_retry_a++ r_a = client.post(+ "/v1/messages",+ headers={"x-api-key": "test", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "stream": False,+ "messages": msgs,+ },+ )+ assert r_a.status_code == 200++ # Run 2: stream=True (same inputs)+ captured_b: dict = {}+ with _make_optimize_proxy_client(mode="token") as client:+ fake_cache_b = _IssueFakeCompCache(frozen_via_compute=10)++ proxy = client.app.state.proxy+ fake_tracker = _FakePrefixTracker(frozen_count=12)+ proxy.session_tracker_store.compute_session_id = (+ lambda request, model, messages: "stream-parity-B"+ )+ proxy.session_tracker_store.get_or_create = lambda s, p: fake_tracker+ proxy._get_compression_cache = lambda s: fake_cache_b++ def _fake_apply_b(**kwargs): # noqa: ANN003+ captured_b["frozen_message_count"] = kwargs.get("frozen_message_count")+ return SimpleNamespace(+ messages=list(kwargs["messages"]),+ transforms_applied=[],+ timing={},+ tokens_before=100,+ tokens_after=100,+ waste_signals=None,+ )++ proxy.anthropic_pipeline.apply = _fake_apply_b++ # Streaming path: return a minimal SSE body+ sse_body = (+ b"event: message_start\n"+ b'data: {"type":"message_start","message":{"id":"msg",'+ b'"role":"assistant","content":[],"model":"claude",'+ b'"usage":{"input_tokens":100,"output_tokens":0,'+ b'"cache_read_input_tokens":0,"cache_creation_input_tokens":0}}}\n\n'+ b"event: content_block_start\n"+ b'data: {"type":"content_block_start","index":0,'+ b'"content_block":{"type":"text","text":""}}\n\n'+ b"event: content_block_stop\n"+ b'data: {"type":"content_block_stop","index":0}\n\n'+ b"event: message_delta\n"+ b'data: {"type":"message_delta","delta":{"stop_reason":"end_turn"},'+ b'"usage":{"output_tokens":3}}\n\n'+ b"event: message_stop\n"+ b'data: {"type":"message_stop"}\n\n'+ )++ async def _fake_retry_b(method, url, headers, body, stream=False): # noqa: ANN001+ return httpx.Response(+ 200,+ content=sse_body,+ headers={"content-type": "text/event-stream"},+ )++ proxy._retry_request = _fake_retry_b+ # Streaming dispatch uses _stream_response (not _retry_request). Stub+ # it to a no-op streaming response so we can inspect what+ # pipeline.apply received without being responsible for the SSE+ # plumbing — the optimization runs before _stream_response is called.+ from fastapi.responses import StreamingResponse++ async def _fake_stream_response(*args, **kwargs): # noqa: ANN001, ANN002, ANN003+ async def _gen():+ yield b""++ return StreamingResponse(_gen(), media_type="text/event-stream")++ proxy._stream_response = _fake_stream_response++ client.post(+ "/v1/messages",+ headers={"x-api-key": "test", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "stream": True,+ "messages": msgs,+ },+ )+ # Status code is incidental — what matters is that pipeline.apply ran+ # and captured_b was populated.++ assert "frozen_message_count" in captured_a, "Non-streaming path didn't reach pipeline.apply()"+ assert "frozen_message_count" in captured_b, "Streaming path didn't reach pipeline.apply()"+ assert captured_a["frozen_message_count"] == captured_b["frozen_message_count"], (+ f"Streaming/non-streaming divergence: stream=False produced "+ f"frozen={captured_a['frozen_message_count']}, stream=True produced "+ f"{captured_b['frozen_message_count']}. The optimization path must "+ f"be identical for both."+ )diff --git a/tests/test_proxy_openai_cache_stability.py b/tests/test_proxy_openai_cache_stability.pyindex a2ca8deed..f895f2546 100644--- a/tests/test_proxy_openai_cache_stability.py+++ b/tests/test_proxy_openai_cache_stability.py@@ -167,3 +167,116 @@ def test_openai_cache_mode_restores_mutated_frozen_prefix() -> None:sent_messages = captured["body"]["messages"]assert sent_messages[0] == original_messages[0]assert sent_messages[1] == original_messages[1]+++# ─── Issue #327 cross-handler regression ────────────────────────────────+#+# The OpenAI handler was never affected by issue #327's content-keyed walker+# bug — it has only ever used `compute_frozen_count` (positional). This test+# locks that property by spying on the OpenAI traffic path and asserting that+# the buggy walker functions (`should_defer_compression`, `mark_stable`) are+# never called from the production handler. If a future refactor accidentally+# adds the same walker to OpenAI, this test fails immediately.+++def test_issue_327_openai_handler_does_not_call_walker_functions() -> None:+ calls: list[tuple[str, tuple, dict]] = []++ class _SpyCompCache:+ def apply_cached(self, messages): # noqa: ANN001+ calls.append(("apply_cached", (), {}))+ return list(messages)++ def compute_frozen_count(self, messages): # noqa: ANN001+ calls.append(("compute_frozen_count", (), {}))+ return 0++ def update_from_result(self, originals, compressed): # noqa: ANN001+ calls.append(("update_from_result", (), {}))++ def mark_stable_from_messages(self, messages, up_to): # noqa: ANN001+ calls.append(("mark_stable_from_messages", (up_to,), {}))++ # Methods below MUST NOT be called from OpenAI handler.+ def should_defer_compression(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003+ calls.append(("should_defer_compression", args, kwargs))+ return False++ def mark_stable(self, content_hash): # noqa: ANN001+ calls.append(("mark_stable", (content_hash,), {}))++ @staticmethod+ def content_hash(content): # noqa: ANN001+ return f"H({content[:40] if isinstance(content, str) else 'list'})"++ with _make_proxy_client() as client:+ proxy = client.app.state.proxy+ proxy.config.optimize = True+ proxy.config.mode = "token" # token mode is where Anthropic had the bug++ fake_tracker = _FakePrefixTracker(frozen_count=0)+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: (+ "openai-spy-session"+ )+ proxy.session_tracker_store.get_or_create = lambda s, p: fake_tracker+ proxy._get_compression_cache = lambda s: _SpyCompCache()++ def _fake_apply(**kwargs): # noqa: ANN003+ return SimpleNamespace(+ messages=list(kwargs["messages"]),+ transforms_applied=[],+ timing={},+ tokens_before=60,+ tokens_after=60,+ waste_signals=None,+ )++ proxy.openai_pipeline.apply = _fake_apply++ async def _fake_retry(method, url, headers, body, stream=False): # noqa: ANN001+ return httpx.Response(+ 200,+ json={+ "id": "cmpl",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": "ok"},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 60, "completion_tokens": 3, "total_tokens": 63},+ },+ )++ proxy._retry_request = _fake_retry++ # Drive 5 turns so any walker bug would have time to fire repeatedly.+ for turn in range(5):+ r = client.post(+ "/v1/chat/completions",+ headers={"authorization": "Bearer test-key"},+ json={+ "model": "gpt-4o-mini",+ "messages": [+ {"role": "user", "content": f"turn-{turn}-q"},+ {"role": "assistant", "content": f"turn-{turn}-a"},+ {"role": "tool", "tool_call_id": "t1", "content": "x" * 600},+ {"role": "user", "content": f"continue-{turn}"},+ ],+ },+ )+ assert r.status_code == 200++ method_names = [c[0] for c in calls]+ assert "should_defer_compression" not in method_names, (+ f"OpenAI handler unexpectedly called should_defer_compression. "+ f"Calls observed: {method_names}"+ )+ assert "mark_stable" not in method_names, (+ f"OpenAI handler unexpectedly called mark_stable (the walker side-effect). "+ f"Calls observed: {method_names}"+ )+ # Sanity: the safe positional methods DID fire.+ assert "compute_frozen_count" in method_names+ assert "apply_cached" in method_names