Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(read-maturation): keep the client's tail cache breakpoint on held Reads (#3977)

gglucass205668d95331
diff --git a/docs/content/docs/proxy.mdx b/docs/content/docs/proxy.mdx
index 02ddd60e8..d2459e271 100644
--- a/docs/content/docs/proxy.mdx
+++ b/docs/content/docs/proxy.mdx
@@ -328,7 +328,7 @@ Coding agents re-read the same files repeatedly; these control how stale reads a
| Flag / env | Default | Effect |
|---|---|---|
| `--no-read-lifecycle` | lifecycle on | Stop replacing stale/superseded file reads with CCR markers. |
-| `--read-maturation` / `HEADROOM_READ_MATURATION` | `false` | *(Beta)* Hold freshly-read files out of the prefix cache until the file quiesces. Requires `HEADROOM_ROLLOUT_CHANNEL=beta` (or `dev`). |
+| `--read-maturation` / `HEADROOM_READ_MATURATION` | `false` | *(Beta)* Keep freshly-read files verbatim while in use, then compress them once the file quiesces, unless the provider cache already holds them. Requires `HEADROOM_ROLLOUT_CHANNEL=beta` (or `dev`). |
| `--read-maturation-quiesce-turns` | `5` | Turns of no change before a held read is admitted. |
### Reliability: timeouts, retries, limits
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index f48eb33ef..2ed55fe24 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -770,9 +770,9 @@ def dashboard(port: int, no_open: bool) -> None:
envvar="HEADROOM_READ_MATURATION",
help=(
"EXPERIMENTAL: activity-based read maturation — hold fresh Reads "
- "out of the provider prefix cache and compress them once their "
- "file quiesces. Requires HEADROOM_ROLLOUT_CHANNEL=beta (or dev); "
- "env: HEADROOM_READ_MATURATION=1."
+ "verbatim and compress them once their file quiesces, unless the "
+ "provider cache already holds them. Requires "
+ "HEADROOM_ROLLOUT_CHANNEL=beta (or dev); env: HEADROOM_READ_MATURATION=1."
),
)
@click.option(
diff --git a/headroom/config.py b/headroom/config.py
index f98fad188..e715c11ca 100644
--- a/headroom/config.py
+++ b/headroom/config.py
@@ -652,13 +652,14 @@ class ReadMaturationConfig:
roughly 13x its size. The only cache-safe moment to shrink it is
BEFORE it is ever cache-written.
- Mechanics: a fresh large Read is held out of the provider prefix
- cache (the trailing cache breakpoint is relocated to just before it)
- while its file is ACTIVE, stays verbatim the whole time the model is
- working with it, and matures into a CCR-backed marker once the file
- has been quiet for `quiesce_turns`. Only that final compressed form
- ever enters the cache. No cached byte is ever mutated — there is
- nothing to bust.
+ Mechanics: a fresh large Read is held verbatim while its file is
+ ACTIVE, so the model sees the full text the whole time it is working
+ with it, and matures into a CCR-backed marker once the file has been
+ quiet for `quiesce_turns`, if it is still outside the provider-cached
+ prefix. The held Read is cached with its turn (the client's
+ breakpoints stay in place), so in a warm session it usually stays
+ verbatim: re-writing everything after a cached Read costs more than
+ the Read saves.
Activity-based (not a fixed hold window) because the audit-reads
simulation showed touch gaps are fat-tailed: next-touch p50 is 4
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index ffa45d135..bcc738c34 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -2505,11 +2505,11 @@ class AnthropicHandlerMixin:
# Mechanism B: activity-based read maturation (flag-gated,
# default off). Runs after compression so read_lifecycle
- # markers are respected, and before body assembly so the
- # held-Read breakpoint relocation lands in the forwarded
- # request. Session state (matured markers) rides on the
- # prefix tracker — same affinity and TTL cleanup as the
- # freeze state. Advisory: must never fail the request.
+ # markers are respected, and before body assembly so a
+ # matured marker lands in the forwarded request. Session
+ # state (matured markers) rides on the prefix tracker — same
+ # affinity and TTL cleanup as the freeze state. Advisory:
+ # must never fail the request.
# Bound when maturation runs, so the final accounting step below
# can charge this request's replayed-marker debt. Every earlier
# `tokens_saved` assignment is overwritten by that recount, so the
@@ -2518,10 +2518,7 @@ class AnthropicHandlerMixin:
if self.config.read_maturation and not _bypass:
try:
from headroom.config import ReadMaturationConfig
- from headroom.transforms.read_maturation import (
- ReadMaturationManager,
- relocate_cache_breakpoint,
- )
+ from headroom.transforms.read_maturation import ReadMaturationManager
maturation_mgr = prefix_tracker.read_maturation_manager
if maturation_mgr is None:
@@ -2541,10 +2538,7 @@ class AnthropicHandlerMixin:
frozen_message_count=frozen_message_count,
)
if maturation.replacements_applied or maturation.holding_msg_indices:
- optimized_messages = relocate_cache_breakpoint(
- maturation.messages,
- maturation.holding_msg_indices,
- )
+ optimized_messages = maturation.messages
optimized_tokens = tokenizer.count_messages(optimized_messages)
tokens_saved = max(0, original_tokens - optimized_tokens)
if maturation.newly_matured:
diff --git a/headroom/proxy/models.py b/headroom/proxy/models.py
index d19809a4e..18bb2d364 100644
--- a/headroom/proxy/models.py
+++ b/headroom/proxy/models.py
@@ -312,8 +312,8 @@ class ProxyConfig:
# Read lifecycle management
read_lifecycle: bool = True
- # Mechanism B: activity-based read maturation (hold fresh Reads out of
- # the provider prefix cache; compress once their file quiesces).
+ # Mechanism B: activity-based read maturation (hold fresh Reads
+ # verbatim; compress once their file quiesces, unless already cached).
# Experimental — default off. CLI: --read-maturation;
# env: HEADROOM_READ_MATURATION=1
read_maturation: bool = False
diff --git a/headroom/rollout.py b/headroom/rollout.py
index ed71b113c..ea381ba2f 100644
--- a/headroom/rollout.py
+++ b/headroom/rollout.py
@@ -117,7 +117,7 @@ FEATURES: dict[str, FeatureSpec] = {
name="read_maturation",
available_in=RolloutChannel.BETA,
legacy_env=("HEADROOM_READ_MATURATION",),
- description="Hold-back Read maturation before provider cache entry.",
+ description="Hold-back Read maturation: compress a Read once its file quiesces.",
),
"bedrock_openai_prompt_caching": FeatureSpec(
name="bedrock_openai_prompt_caching",
diff --git a/headroom/transforms/read_maturation.py b/headroom/transforms/read_maturation.py
index bcf637628..9665dd3c3 100644
--- a/headroom/transforms/read_maturation.py
+++ b/headroom/transforms/read_maturation.py
@@ -1,24 +1,32 @@
-"""Mechanism B: hold-back Read maturation — compress before cache entry.
-
-The prefix cache bills you for everything *after* the first changed byte,
-so mutating an already-cached Read is ruinously expensive — but bytes that
-have never been cache-written have no cache entry to bust. This module
-exploits the one safe window: a fresh Read is deliberately held *out* of
-the provider cache (the trailing cache breakpoint is relocated to just
-before it) while its file is active. The model sees the verbatim content
-the whole time it is working with the file. Once the file has been quiet
-for `quiesce_turns`, the content is replaced with a CCR-backed marker —
-and only that final, small form ever enters the cache.
+"""Mechanism B: hold-back Read maturation — compress a Read once it is done.
+
+A fresh Read is held verbatim while its file is active, so the model sees
+the full content the whole time it is working with the file. Once the
+file has been quiet for `quiesce_turns`, the content is replaced with a
+CCR-backed marker, if the Read is still outside the provider-cached
+(frozen) prefix.
+
+The held Read is cached like any other message: the client's cache
+breakpoints are forwarded where the client put them. The prefix cache
+bills everything *after* the first changed byte, so a Read the frozen
+prefix already covers stays verbatim: maturing it would re-write the
+whole tail after it at the cache-write rate to save a cache read of the
+Read. An earlier version kept the verbatim Read out of the cache instead,
+by moving the trailing breakpoint in front of it for the whole hold.
+Claude Code marks the last two messages, so that dropped its tail
+breakpoint, and every held turn re-sent the Read and everything after it
+at the full input rate (one 13-request hold grew its uncached tail from
+~3.9K to ~14.6K tokens per request) to save a single cache write.
Timeline for a Read of file F (quiesce_turns=5):
- turn T: model reads F — verbatim, NOT cached
- T+1..T+k: model edits / re-reads F — read stays verbatim and
- uncached (every touch resets the quiet clock)
- T+k+5: F has been quiet 5 turns → read matures into a marker;
- the breakpoint returns to the tail; the marker form is
- cache-written once
- later turns: marker form read from cache at the provider discount
+ turn T: model reads F — verbatim, cached with the turn
+ T+1..T+k: model edits / re-reads F — read stays verbatim (every
+ touch resets the quiet clock)
+ T+k+5: F has been quiet 5 turns → read matures into a marker
+ if it is still outside the frozen prefix, else it stays
+ verbatim for good
+ later turns: the forwarded form is read from cache
Why activity-based instead of a fixed hold window: the audit-reads
simulation over real traffic showed touch gaps are fat-tailed (next-touch
@@ -31,8 +39,8 @@ still in context. The recovery path is the model's existing behavior.
Two invariants:
-1. **No cached byte is ever mutated.** The verbatim form is never
- cache-written, so maturation invalidates nothing.
+1. **The frozen prefix is never mutated.** A Read only matures while it
+ is outside the provider-confirmed prefix.
2. **Replay is deterministic.** Once matured, the same marker is applied
on every subsequent request (state is session-scoped), so the cached
prefix stays byte-stable for the rest of the session.
@@ -94,8 +102,8 @@ class MaturationResult:
"""Output of one per-request maturation pass."""
messages: list[dict[str, Any]]
- # Message indices that contain still-holding Reads (must stay out of
- # the provider cache this request — feed to relocate_cache_breakpoint).
+ # Message indices that contain still-holding Reads (forwarded verbatim
+ # this request).
holding_msg_indices: list[int] = field(default_factory=list)
holding_reads: int = 0
newly_matured: int = 0
@@ -158,9 +166,9 @@ class ReadMaturationManager:
messages: Conversation messages (Anthropic content-block or
OpenAI role="tool" formats).
frozen_message_count: Provider-cached message count. Reads
- inside the frozen prefix were cache-written verbatim
- before this mechanism saw them (e.g. it was just
- enabled, or state was lost) — they are never touched.
+ inside the frozen prefix are already cache-written
+ verbatim (held Reads are cached with their turn) — they
+ are never touched.
"""
result = MaturationResult(messages=messages)
if not self.config.enabled:
@@ -305,9 +313,8 @@ class ReadMaturationManager:
# NOTE: blocks carrying a client cache_control are NOT skipped —
# Claude Code parks its tail breakpoint on the newest content
# block, which right after a Read is the Read's tool_result
- # itself. Under this mechanism the proxy owns breakpoint
- # placement: relocate_cache_breakpoint() strips/moves breakpoints
- # in the held region after this pass.
+ # itself. The breakpoint stays where the client put it; a
+ # replaced block keeps its cache_control.
if isinstance(content, list):
new_blocks: list[Any] = []
changed = False
@@ -394,71 +401,3 @@ class ReadMaturationManager:
result.replacements_applied += 1
result.bytes_saved += max(0, len(content) - len(marker))
return marker, False
-
-
-def relocate_cache_breakpoint(
- messages: list[dict[str, Any]],
- holding_msg_indices: list[int],
-) -> list[dict[str, Any]]:
- """Park the trailing message-level cache breakpoint before held Reads.
-
- Strips ``cache_control`` from every block at or after the earliest
- holding message, and places one ephemeral breakpoint on the last
- block of the latest *eligible* message before it — so the provider
- caches everything up to (not including) the held Reads. System- and
- tools-level breakpoints are untouched (they live outside messages).
-
- Total breakpoints never increase: at most one is added after one or
- more are removed. Returns the original list unchanged when there is
- nothing to do.
- """
- if not holding_msg_indices:
- return messages
-
- earliest = min(holding_msg_indices)
- out: list[dict[str, Any]] = list(messages)
- stripped_any = False
-
- # 1. Strip breakpoints from the held region [earliest:].
- held_marker: dict[str, Any] | None = None
- for i in range(earliest, len(out)):
- msg = out[i]
- content = msg.get("content")
- if not isinstance(content, list):
- continue
- if any(isinstance(b, dict) and "cache_control" in b for b in content):
- for b in content:
- # Carry the TTL forward: re-anchoring with a bare ephemeral marker
- # would silently downgrade a 1h breakpoint to the 5m default.
- if isinstance(b, dict) and isinstance(b.get("cache_control"), dict):
- held_marker = b["cache_control"]
- out[i] = {
- **msg,
- "content": [
- {k: v for k, v in b.items() if k != "cache_control"}
- if isinstance(b, dict)
- else b
- for b in content
- ],
- }
- stripped_any = True
-
- if not stripped_any:
- # No client breakpoint in the held region — nothing was going to
- # cache the held Reads this request; leave placement alone.
- return out
-
- # 2. Re-anchor: ephemeral breakpoint on the last block of the latest
- # block-style message before the held region.
- for i in range(earliest - 1, -1, -1):
- content = out[i].get("content")
- if isinstance(content, list) and content and isinstance(content[-1], dict):
- new_content = list(content)
- new_content[-1] = {
- **new_content[-1],
- "cache_control": dict(held_marker) if held_marker else {"type": "ephemeral"},
- }
- out[i] = {**out[i], "content": new_content}
- break
-
- return out
diff --git a/tests/test_cache_ttl_preserved.py b/tests/test_cache_ttl_preserved.py
index 4aeabb4ce..0b5c997e1 100644
--- a/tests/test_cache_ttl_preserved.py
+++ b/tests/test_cache_ttl_preserved.py
@@ -2,8 +2,8 @@
``normalize_message_cache_control`` deliberately preserves an explicit
``cache_control.ttl`` so a client on Anthropic's 1h cache isn't silently
-downgraded to the 5-minute default (#2375). Two other sites also strip a
-breakpoint and re-place it, and both used to hardcode a bare ephemeral marker —
+downgraded to the 5-minute default (#2375). Tool-search deferral also strips a
+breakpoint and re-places it, and used to hardcode a bare ephemeral marker —
undoing that guarantee. A downgrade is invisible (the request still succeeds)
and costs a full prefix re-write on every gap past 5 minutes, so it needs a test
rather than a comment.
@@ -12,34 +12,10 @@ rather than a comment.
from typing import Any
from headroom.proxy.helpers import inject_tool_search_deferral
-from headroom.transforms.read_maturation import relocate_cache_breakpoint
TTL_1H = {"type": "ephemeral", "ttl": "1h"}
-def _markers(blocks: list[dict[str, Any]]) -> list[dict[str, Any]]:
- return [b["cache_control"] for b in blocks if isinstance(b, dict) and "cache_control" in b]
-
-
-def _held(marker: dict[str, Any]) -> list[dict[str, Any]]:
- return [
- {"role": "user", "content": [{"type": "text", "text": "keep"}]},
- {"role": "user", "content": [{"type": "text", "text": "held", "cache_control": marker}]},
- ]
-
-
-def test_read_maturation_reanchor_keeps_ttl() -> None:
- # Breakpoint sits inside the held-Read region, so it is moved back before it.
- out = relocate_cache_breakpoint(_held(TTL_1H), holding_msg_indices=[1])
- assert _markers(out[0]["content"]) == [TTL_1H], "re-anchored breakpoint lost the 1h ttl"
- assert _markers(out[1]["content"]) == [], "held region should carry no breakpoint"
-
-
-def test_read_maturation_reanchor_defaults_to_5m() -> None:
- out = relocate_cache_breakpoint(_held({"type": "ephemeral"}), holding_msg_indices=[1])
- assert _markers(out[0]["content"]) == [{"type": "ephemeral"}]
-
-
def _tools(marker: dict[str, Any] | None) -> list[dict[str, Any]]:
# Needs >= _TOOL_SEARCH_MIN_TOOLS (12) to trigger, with one core tool resident
# and the tools-array breakpoint riding on a tool that will be deferred.
diff --git a/tests/test_live/test_live_maturation.py b/tests/test_live/test_live_maturation.py
index 0b8a792d2..ae8f09590 100644
--- a/tests/test_live/test_live_maturation.py
+++ b/tests/test_live/test_live_maturation.py
@@ -1,17 +1,14 @@
-"""Live validation of Mechanism B's no-bust invariant against the real API.
+"""Live validation of Mechanism B's cache behaviour against the real API.
-The design's central claim, tested empirically:
+Tested empirically:
-1. Request A holds a fresh Read out of the cache (breakpoint relocated
- to just before it) → the provider's cache_creation must NOT include
- the read content.
-2. Request B (one turn later, read matured into a marker, breakpoint
- back at the tail) → the provider must report a cache READ covering
- request A's cached prefix — proving the prefix survived the read's
- replacement, i.e. nothing was busted.
-
-If assertion 2 fails, breakpoint relocation breaks prefix matching and
-the mechanism needs redesign before it is enabled anywhere.
+1. Request A holds a fresh Read verbatim and forwards the client's tail
+ breakpoint on it → the provider's cache_creation includes the read
+ content (the held Read is cached with its turn).
+2. Request B (one turn later, file quiet, but the Read is now inside the
+ provider-confirmed prefix) → the Read is NOT matured, and the provider
+ reports a cache READ covering request A's cached prefix — nothing was
+ busted.
Skipped without ANTHROPIC_API_KEY. Costs ~15K haiku tokens per run.
"""
@@ -24,10 +21,7 @@ import httpx
import pytest
from headroom.config import ReadMaturationConfig
-from headroom.transforms.read_maturation import (
- ReadMaturationManager,
- relocate_cache_breakpoint,
-)
+from headroom.transforms.read_maturation import ReadMaturationManager
pytestmark = pytest.mark.skipif(
not os.environ.get("ANTHROPIC_API_KEY"),
@@ -113,29 +107,25 @@ def conv_base() -> list[dict]:
class TestNoBustInvariantLive:
- def test_hold_then_mature_preserves_cache(self):
+ def test_held_read_is_cached_and_not_rewritten(self):
mgr = ReadMaturationManager(ReadMaturationConfig(enabled=True, quiesce_turns=1))
- # ── Request A: fresh read → held, breakpoint relocated before it.
+ # ── Request A: fresh read → held verbatim, client breakpoint kept.
msgs_a = conv_base()
res_a = mgr.apply(msgs_a)
assert res_a.holding_msg_indices == [2], "fixture must trigger holding"
- fwd_a = relocate_cache_breakpoint(res_a.messages, res_a.holding_msg_indices)
- assert "cache_control" not in fwd_a[2]["content"][0]
+ fwd_a = res_a.messages
+ assert "cache_control" in fwd_a[2]["content"][0]
usage_a = call(fwd_a)
created_a = usage_a.get("cache_creation_input_tokens", 0)
input_a = usage_a.get("input_tokens", 0)
- # The held read (~4K tokens of input) must NOT be in the cache
- # write. created_a covers system pad + first two messages only.
- assert created_a > 0, f"prefix did not cache at all: {usage_a}"
- assert input_a > 3000, f"read content missing from input: {usage_a}"
- total_a = created_a + input_a + usage_a.get("cache_read_input_tokens", 0)
- assert created_a < total_a * 0.7, (
- f"cache write covered the held read — hold failed: {usage_a}"
- )
+ # The held read (~4K tokens) is in the cache write, not the
+ # uncached input: the client's tail breakpoint sits on it.
+ assert created_a > 3000, f"held read was not cache-written: {usage_a}"
+ assert input_a < 1000, f"held read was sent uncached: {usage_a}"
- # ── Request B: one assistant turn later, file quiet → matured.
+ # ── Request B: one assistant turn later, file quiet, Read cached.
msgs_b = [
*conv_base(),
{"role": "assistant", "content": [{"type": "text", "text": "Read it."}]},
@@ -154,26 +144,23 @@ class TestNoBustInvariantLive:
# no longer carries one.
del msgs_b[2]["content"][0]["cache_control"]
- res_b = mgr.apply(msgs_b)
- assert res_b.newly_matured == 1, "read must mature after quiesce"
- fwd_b = relocate_cache_breakpoint(res_b.messages, res_b.holding_msg_indices)
- marker = fwd_b[2]["content"][0]["content"]
- assert "Retrieve original: hash=" in marker
+ # Request A cached messages 0-2, so the provider-confirmed prefix
+ # now covers the Read: it stays verbatim instead of maturing.
+ res_b = mgr.apply(msgs_b, frozen_message_count=3)
+ assert res_b.newly_matured == 0, "a cached read must not mature"
+ fwd_b = res_b.messages
+ assert fwd_b[2]["content"][0]["content"] == FILE_CONTENT
usage_b = call(fwd_b)
read_b = usage_b.get("cache_read_input_tokens", 0)
- # THE invariant: request A's cached prefix must still be valid —
- # the matured read sat outside it, so replacing it busts nothing.
+ # Request A's cached prefix, Read included, must still be valid.
assert read_b >= created_a * 0.9, (
- f"NO-BUST INVARIANT FAILED: request B read {read_b} cached tokens "
- f"but request A created {created_a} — breakpoint relocation broke "
- f"prefix matching. A={usage_a} B={usage_b}"
+ f"request B read {read_b} cached tokens but request A created "
+ f"{created_a}: the cached prefix was busted. A={usage_a} B={usage_b}"
)
- # And the matured form is small: B's uncached input should be far
- # below the read size (marker + two short turns, not 4K tokens).
assert usage_b.get("input_tokens", 0) < 2500, (
- f"matured request still carried heavy uncached input: {usage_b}"
+ f"request B carried heavy uncached input: {usage_b}"
)
diff --git a/tests/test_read_maturation.py b/tests/test_read_maturation.py
index db37c7e04..a3604ca4a 100644
--- a/tests/test_read_maturation.py
+++ b/tests/test_read_maturation.py
@@ -1,8 +1,8 @@
"""Tests for Mechanism B: activity-based hold-back Read maturation.
The invariants under test, beyond decision behavior:
-1. No cached byte is ever mutated — frozen-prefix content and content
- carrying a client cache_control breakpoint are untouched.
+1. The provider-cached (frozen) prefix is never mutated, and a client
+ cache_control breakpoint is never moved or dropped.
2. Replay is deterministic — once matured, the same marker is applied on
every subsequent request, byte-identical.
3. Holding is derived from the conversation (file activity), so the
@@ -14,10 +14,7 @@ from __future__ import annotations
import pytest
from headroom.config import ReadMaturationConfig
-from headroom.transforms.read_maturation import (
- ReadMaturationManager,
- relocate_cache_breakpoint,
-)
+from headroom.transforms.read_maturation import ReadMaturationManager
CONTENT = " 1\tdef foo():\n 2\t return 42\n" * 60 # > 2048B
SMALL = " 1\tok\n"
@@ -177,6 +174,26 @@ class TestActivityDecision:
assert res.holding_msg_indices == []
assert read_content(res) == CONTENT
+ def test_held_read_matures_only_outside_frozen_prefix(self):
+ """A held Read is cached with its turn. At quiesce it matures if it
+ is still outside the frozen prefix (e.g. a cold prefix); once the
+ frozen prefix covers it, it stays verbatim — maturing it would
+ re-write everything after it at the cache-write rate."""
+ held = base_conv()
+ msgs = [*held, *quiet(5)]
+
+ m = manager(quiesce_turns=5)
+ assert m.apply(held).holding_msg_indices == [2]
+ res = m.apply(msgs, frozen_message_count=2) # Read in the live zone
+ assert res.newly_matured == 1
+ assert "compressed after use" in read_content(res)
+
+ m = manager(quiesce_turns=5)
+ assert m.apply(held).holding_msg_indices == [2]
+ res = m.apply(msgs, frozen_message_count=3) # frozen prefix covers it
+ assert res.newly_matured == 0
+ assert res.messages is msgs
+
def test_respects_lifecycle_markers(self):
marker = "[Read content stale: /x/foo.py ... Retrieve original: hash=abc123]" + " " * 2048
msgs = [
@@ -188,21 +205,21 @@ class TestActivityDecision:
assert res.holding_msg_indices == []
assert res.newly_matured == 0
- def test_client_breakpoint_on_fresh_read_is_held_and_relocated(self):
+ def test_client_breakpoint_on_fresh_read_is_held_and_kept(self):
"""Claude Code parks its tail breakpoint on the newest block —
right after a Read, that's the Read result itself. The read must
- still be held (verbatim) and relocation must move the breakpoint
- off it, otherwise the verbatim form gets cache-written."""
+ still be held (verbatim) and the breakpoint must stay on it, so
+ the turn is cached; it also survives the block's maturation."""
msgs = base_conv()
msgs[2]["content"][0]["cache_control"] = {"type": "ephemeral"}
- res = manager().apply(msgs)
+ m = manager(quiesce_turns=5)
+ res = m.apply(msgs)
assert res.holding_msg_indices == [2]
- assert res.messages[2]["content"][0]["content"] == CONTENT
+ assert res.messages[2]["content"][0] == msgs[2]["content"][0]
- out = relocate_cache_breakpoint(res.messages, res.holding_msg_indices)
- # Breakpoint stripped from the held read, re-anchored before it.
- assert "cache_control" not in out[2]["content"][0]
- assert out[1]["content"][-1].get("cache_control") == {"type": "ephemeral"}
+ res = m.apply([*msgs, *quiet(5)])
+ assert res.newly_matured == 1
+ assert res.messages[2]["content"][0]["cache_control"] == {"type": "ephemeral"}
def test_openai_format(self):
msgs = [{"role": "user", "content": "look"}, *openai_read("r1", "/x/foo.py", CONTENT)]
@@ -267,15 +284,12 @@ class TestProxyWiring:
tracker carries matured markers across requests."""
from headroom.cache.prefix_tracker import PrefixCacheTracker
from headroom.config import ReadMaturationConfig
- from headroom.transforms.read_maturation import (
- ReadMaturationManager,
- relocate_cache_breakpoint,
- )
+ from headroom.transforms.read_maturation import ReadMaturationManager
tracker = PrefixCacheTracker("anthropic")
assert tracker.read_maturation_manager is None
- # Request 1: fresh read — held; breakpoint relocated.
+ # Request 1: fresh read — held.
tracker.read_maturation_manager = ReadMaturationManager(
ReadMaturationConfig(enabled=True, quiesce_turns=5)
)
@@ -285,8 +299,6 @@ class TestProxyWiring:
}
res = tracker.read_maturation_manager.apply(msgs)
assert res.holding_msg_indices == [2]
- out = relocate_cache_breakpoint(res.messages, res.holding_msg_indices)
- assert len(out) == len(msgs)
# Request N (file quiet): same manager matures and replays.
later = [*base_conv(), *quiet(5)]
@@ -297,58 +309,6 @@ class TestProxyWiring:
assert replay.newly_matured == 0
-class TestBreakpointRelocation:
- def _msgs_with_tail_breakpoint(self) -> list[dict]:
- msgs = [
- {"role": "user", "content": [{"type": "text", "text": "earlier turn"}]},
- *anthropic_read("r1", "/x/foo.py", CONTENT),
- ]
- msgs[-1]["content"][-1] = {
- **msgs[-1]["content"][-1],
- "cache_control": {"type": "ephemeral"},
- }
- return msgs
-
- @staticmethod
- def _breakpoint_indices(msgs: list[dict]) -> list[int]:
- return [
- i
- for i, m in enumerate(msgs)
- if isinstance(m.get("content"), list)
- and any(isinstance(b, dict) and "cache_control" in b for b in m["content"])
- ]
-
- def test_noop_without_holds(self):
- msgs = self._msgs_with_tail_breakpoint()
- assert relocate_cache_breakpoint(msgs, []) is msgs
-
- def test_relocates_before_held_read(self):
- msgs = self._msgs_with_tail_breakpoint()
- out = relocate_cache_breakpoint(msgs, [2])
-
- # Held region [2:] carries no breakpoint; re-anchored on the
- # latest eligible message before it (index 1 — the assistant
- # tool_use message), so everything up to but excluding the held
- # Read still gets cached.
- assert self._breakpoint_indices(out) == [1]
- assert out[1]["content"][-1]["cache_control"] == {"type": "ephemeral"}
- assert len(self._breakpoint_indices(out)) <= len(self._breakpoint_indices(msgs))
-
- def test_noop_when_no_breakpoint_in_held_region(self):
- msgs = [
- {"role": "user", "content": [{"type": "text", "text": "x"}]},
- *anthropic_read("r1", "/x/foo.py", CONTENT),
- ]
- out = relocate_cache_breakpoint(msgs, [2])
- assert self._breakpoint_indices(out) == []
-
- def test_originals_not_mutated(self):
- msgs = self._msgs_with_tail_breakpoint()
- before = [str(m) for m in msgs]
- relocate_cache_breakpoint(msgs, [2])
- assert [str(m) for m in msgs] == before
-
-
if __name__ == "__main__":
pytest.main([__file__, "-v"])
diff --git a/tests/test_read_maturation_handler_nobust.py b/tests/test_read_maturation_handler_nobust.py
index 79c2029fb..f5856b7b2 100644
--- a/tests/test_read_maturation_handler_nobust.py
+++ b/tests/test_read_maturation_handler_nobust.py
@@ -1,10 +1,11 @@
-"""Integration: Mechanism B (read maturation) no-bust invariant, through the
+"""Integration: Mechanism B (read maturation) cache behaviour, through the
REAL Anthropic handler, across a multi-turn session.
-The design's central claim is that the verbatim Read is held *out* of the
-provider prefix cache until it matures, so "no cached byte is ever mutated."
-The unit tests in ``test_read_maturation.py`` call the manager in isolation
-with ``frozen_message_count=0``; the live test in ``test_live/`` does a
+The held Read is cached with its turn: the client's breakpoints are forwarded
+where the client put them, and the Read matures once its file quiesces even
+though the provider-confirmed prefix has moved past it by then. The unit
+tests in ``test_read_maturation.py`` call the manager in isolation with
+``frozen_message_count=0``; the live test in ``test_live/`` does a
2-request hold->mature with no intermediate turns. Neither exercises the
realistic path where a held Read sits across several turns while the prefix
tracker advances ``frozen_message_count`` from the provider's reported cache
@@ -16,12 +17,10 @@ everything up to the breakpoint the handler chose, system blocks included),
so the prefix tracker advances exactly as in production. It then asserts,
directly on the FORWARDED bytes:
-1. no-bust: the verbatim Read is never forwarded inside the cached prefix
- (at or before the last cache_control breakpoint) — if it were, maturing it
- later would mutate a cached byte and bust the prefix;
-2. the mechanism actually engages: the Read is held verbatim (out of cache)
- while the file is active, then matures into a CCR marker once it quiesces,
- in that order.
+1. the client's breakpoint is forwarded where the client put it on every
+ turn, including the turns that hold the Read;
+2. the mechanism actually engages: the Read is held verbatim while the file
+ is active, then matures into a CCR marker once it quiesces, in that order.
Note on cache-state isolation: the CCR store is persistent (SQLite at
~/.headroom/ccr_store.db by default) and shared across processes, so stale
@@ -32,6 +31,7 @@ store for deterministic results.
from __future__ import annotations
import copy
+import json
import pytest
@@ -104,6 +104,35 @@ def _convo(nquiet: int) -> list[dict]:
return msgs
+def _busy_convo(nquiet: int) -> list[dict]:
+ """``_convo`` with a large JSON Bash output before the Read and on every
+ quiet turn, as in a real coding session. The router compresses those, and
+ a compression run is what marks the held Read as locally replayable, so
+ the frozen count follows the provider's cache past it."""
+
+ def bash(tid: str) -> list[dict]:
+ out = json.dumps([{"path": f"src/{tid}_{j}.py", "size": j} for j in range(300)])
+ return [
+ {
+ "role": "assistant",
+ "content": [
+ {"type": "tool_use", "id": tid, "name": "Bash", "input": {"command": "ls"}}
+ ],
+ },
+ {
+ "role": "user",
+ "content": [{"type": "tool_result", "tool_use_id": tid, "content": out}],
+ },
+ ]
+
+ msgs: list[dict] = [{"role": "user", "content": [{"type": "text", "text": "Read /x/foo.py"}]}]
+ msgs += bash("toolu_b0") + _read_pair(tail=False)
+ for i in range(1, nquiet + 1):
+ msgs += bash(f"toolu_b{i}")
+ msgs[-1]["content"][-1]["cache_control"] = {"type": "ephemeral"}
+ return msgs
+
+
def _breakpoint_index(messages: list[dict]) -> int:
"""Index of the last message carrying a cache_control block (-1 if none).
Anthropic caches everything up to AND INCLUDING this message."""
@@ -132,7 +161,7 @@ def _est_tokens(message: dict) -> int:
return max(1, len(str(message.get("content", ""))) // 4)
-def test_verbatim_read_never_cache_written_before_maturation(monkeypatch):
+def test_held_read_keeps_client_breakpoint_then_matures(monkeypatch):
# Isolate the CCR store: it is persistent (SQLite) and shared across
# processes by default, so stale entries from other runs would perturb
# maturation timing and make this test non-deterministic. The in-memory
@@ -216,31 +245,36 @@ def test_verbatim_read_never_cache_written_before_maturation(monkeypatch):
assert forwarded, "no requests were forwarded"
# Per-turn classification of the Read's forwarded form.
- held_verbatim = [] # turns where the verbatim Read is OUTSIDE the cache prefix (correct hold)
- cached_verbatim = [] # turns where the verbatim Read is INSIDE the cache prefix (bust risk)
+ held_verbatim = [] # turns where the Read is forwarded verbatim (held)
matured = [] # turns where the Read has become a CCR marker
for turn, msgs in enumerate(forwarded):
- bp = _breakpoint_index(msgs)
- for i, m in enumerate(msgs):
+ for m in msgs:
content = _read_result_content(m)
if content is None:
continue
if content == BIG:
- (cached_verbatim if i <= bp else held_verbatim).append(turn)
+ held_verbatim.append(turn)
elif "Retrieve original: hash=" in content:
matured.append(turn)
- # INVARIANT 1 (no-bust): the verbatim Read must never be forwarded inside
- # the cached prefix. If it is, maturing it later mutates a cached byte.
- assert not cached_verbatim, (
- "no-bust invariant violated: verbatim Read was cache-written before "
- f"maturation on turn(s) {cached_verbatim}. Maturing it later busts the cache."
- )
+ # INVARIANT 1: the client's breakpoint is forwarded where the client put
+ # it. This used to be the opposite ("the verbatim Read is never
+ # cache-written"): the handler moved the breakpoint in front of the held
+ # Read, which dropped Claude Code's tail breakpoint for the whole hold and
+ # re-sent the Read and everything after it uncached on every held turn, to
+ # save one cache write of the Read. The held Read is now cached with its
+ # turn, and maturing it re-writes the cache from the Read onward once.
+ moved = [
+ turn
+ for turn, msgs in enumerate(forwarded)
+ if _breakpoint_index(msgs) != _breakpoint_index(_convo(turn))
+ ]
+ assert not moved, f"client cache breakpoint moved or dropped on turn(s) {moved}"
# INVARIANT 2 (mechanism actually engages): the Read is held verbatim while
# the file is active, then matures once it quiesces. Guards against a
# vacuous pass where maturation silently no-ops.
- assert held_verbatim, "expected the fresh Read to be held verbatim out of cache on early turns"
+ assert held_verbatim, "expected the fresh Read to be held verbatim on early turns"
assert matured, "expected the Read to mature into a CCR marker after quiescing"
# The matured marker only appears AFTER the verbatim hold (ordering).
assert min(matured) > max(held_verbatim), (
@@ -253,10 +287,12 @@ def _drive_session(
n_turns: int,
session_id: str,
saved_out: list[int] | None = None,
+ convo=_convo,
) -> list[list[dict]]:
- """Drive ``n_turns`` cumulative turns through the real handler with a mocked
- upstream; return the forwarded message arrays per turn. When ``saved_out``
- is given, each turn's emitted ``x-headroom-tokens-saved`` is appended to it."""
+ """Drive ``n_turns`` cumulative turns of ``convo`` through the real handler
+ with a mocked upstream; return the forwarded message arrays per turn. When
+ ``saved_out`` is given, each turn's emitted ``x-headroom-tokens-saved`` is
+ appended to it."""
app = create_app(config)
forwarded: list[list[dict]] = []
with TestClient(app) as client:
@@ -305,7 +341,7 @@ def _drive_session(
"max_tokens": 20,
"system": SYSTEM,
"tools": [READ_TOOL],
- "messages": _convo(n),
+ "messages": convo(n),
},
)
assert r.status_code == 200, f"turn {n}: {r.text[:300]}"
@@ -326,6 +362,50 @@ def _first_matured_turn(forwarded: list[list[dict]]) -> int | None:
return None
+def test_read_cached_with_its_turn_stays_verbatim(monkeypatch):
+ """In a session whose tool output compresses, the frozen count follows the
+ provider's cache past the held Read from the next request on. The Read is
+ then left verbatim for good: maturing it would re-write everything after it
+ at the cache-write rate, which costs more than the Read saves."""
+ from headroom.cache.compression_store import reset_compression_store
+ from headroom.transforms.read_maturation import ReadMaturationManager
+
+ monkeypatch.setenv("HEADROOM_CCR_BACKEND", "memory")
+ reset_compression_store()
+
+ frozen_seen: list[int] = []
+ apply = ReadMaturationManager.apply
+
+ def _recording_apply(self, messages, frozen_message_count=0):
+ frozen_seen.append(frozen_message_count)
+ return apply(self, messages, frozen_message_count=frozen_message_count)
+
+ monkeypatch.setattr(ReadMaturationManager, "apply", _recording_apply)
+
+ config = ProxyConfig(
+ optimize=True,
+ read_maturation=True,
+ mode="token",
+ cache_enabled=True,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ )
+ forwarded = _drive_session(config, n_turns=7, session_id="cached-read-1", convo=_busy_convo)
+
+ moved = [
+ turn
+ for turn, msgs in enumerate(forwarded)
+ if _breakpoint_index(msgs) != _breakpoint_index(_busy_convo(turn))
+ ]
+ assert not moved, f"client cache breakpoint moved or dropped on turn(s) {moved}"
+ read_at = 4 # the Read's tool_result message in _busy_convo
+ assert all(f > read_at for f in frozen_seen[1:]), (
+ f"fixture must cache the held Read: frozen counts {frozen_seen}"
+ )
+ assert _first_matured_turn(forwarded) is None, "a cached Read was matured"
+
+
def test_quiesce_turns_config_is_honored(monkeypatch):
"""`quiesce_turns` must be runtime-configurable end-to-end: a fresh Read of
/x/foo.py matures `quiesce_turns` quiet turns after it appears (the convo