Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): keep prefix lineage across a replaced system tail (#3934)

gglucass16eaec1e95cd
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
index 83ed16334..b29107485 100644
--- a/headroom/cache/prefix_tracker.py
+++ b/headroom/cache/prefix_tracker.py
@@ -367,6 +367,81 @@ def _classify_history_canonical(
)
+_TRANSIENT_SYSTEM_LINEAGE_ENV = "HEADROOM_TRANSIENT_SYSTEM_LINEAGE"
+
+
+def _transient_system_lineage_enabled() -> bool:
+ return os.environ.get(_TRANSIENT_SYSTEM_LINEAGE_ENV, "").strip().lower() not in (
+ "0",
+ "false",
+ "no",
+ "off",
+ )
+
+
+def _extends_past_system_tail(current_messages: list[Any], previous_messages: list[Any]) -> bool:
+ """Whether ``current`` continues ``previous`` once its trailing system turn is set aside.
+
+ Claude Code ends some requests with a ``role:"system"`` reminder and
+ replaces it on the next turn, so the recorded history is never a prefix of
+ the new one. Only that last message is excluded: everything before it
+ must survive unchanged and include real user/assistant history, so a
+ changed leading or historical system instruction still diverges, and the
+ new history must be longer, so a sibling that only swaps the final
+ instruction is not a continuation.
+ """
+ if len(previous_messages) < 2 or len(current_messages) <= len(previous_messages):
+ return False
+ tail = previous_messages[-1]
+ if not isinstance(tail, dict) or tail.get("role") != "system":
+ return False
+ stable = previous_messages[:-1]
+ return current_messages[: len(stable)] == stable and any(
+ isinstance(message, dict) and message.get("role") in ("user", "assistant")
+ for message in stable
+ )
+
+
+def _replaced_system_tail_delta(
+ current_messages: list[dict[str, Any]],
+ previous_original_messages: list[dict[str, Any]],
+ previous_forwarded_messages: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]] | None:
+ """Cache-stable split for a request that replaced the previous trailing system turn.
+
+ The handlers record the model's reply after the request it answered, so the
+ replaced reminder is the last recorded message or sits right before that
+ reply. It is left out of the replayed prefix (it is not in this request);
+ every other recorded message must match ``current`` position for position.
+ """
+ if not _transient_system_lineage_enabled() or len(previous_forwarded_messages) != len(
+ previous_original_messages
+ ):
+ return None
+ current = _lineage_snapshot(_canonicalize_for_prefix_compare(current_messages))
+ previous = _lineage_snapshot(_canonicalize_for_prefix_compare(previous_original_messages))
+ if len(current) != len(current_messages) or len(previous) != len(previous_original_messages):
+ # The projection dropped a directive-only message; raw indices would shift.
+ return None
+ for request_len in (len(previous), len(previous) - 1):
+ reply = previous[request_len:]
+ if reply and not (isinstance(reply[0], dict) and reply[0].get("role") == "assistant"):
+ continue
+ reminder = request_len - 1
+ if (
+ _extends_past_system_tail(current, previous[:request_len])
+ and current[reminder : reminder + len(reply)] == reply
+ ):
+ return (
+ copy.deepcopy(
+ previous_forwarded_messages[:reminder]
+ + previous_forwarded_messages[request_len:]
+ ),
+ copy.deepcopy(current_messages[reminder + len(reply) :]),
+ )
+ return None
+
+
def classify_history_relation(
current_messages: list[dict[str, Any]],
previous_messages: list[dict[str, Any]],
@@ -427,10 +502,18 @@ def extract_cache_stable_delta(
This is a COMPARISON + slice only: the returned prefix is the previously-forwarded
bytes verbatim and the delta is the raw appended messages — never a rebuild from the
canonical projection — so the projection dropping non-semantic fields is safe.
+
+ A previous request that ended in a ``role:"system"`` reminder the client has since
+ replaced also qualifies: its forwarded bytes are replayed without that reminder
+ (``HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0`` disables this).
"""
if not previous_original_messages or previous_forwarded_messages is None:
return None
relation = classify_history_relation(current_messages, previous_original_messages)
+ if relation.kind == RELATION_DIVERGED:
+ return _replaced_system_tail_delta(
+ current_messages, previous_original_messages, previous_forwarded_messages
+ )
if relation.kind not in (RELATION_EXACT, RELATION_MESSAGE_APPEND):
# A same-message block append needs a block-level splice in
# ``overlay_cached_prefix``; slicing only whole messages would silently
@@ -1433,7 +1516,10 @@ class SessionTrackerStore:
where a large leading run and two-block identity suffix survive while
the middle tail is regenerated; all other rewrites start a fresh
lineage. This keeps #2671's stable cache boundary attached without
- merging unrelated parallel sub-calls.
+ merging unrelated parallel sub-calls. When nothing else matches, an
+ Anthropic chain whose trailing ``role:"system"`` reminder the client
+ replaced is matched on the history before that reminder
+ (``HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0`` disables this).
Byte-identical histories (templated fan-outs before they diverge)
intentionally share a tracker: their provider cache line is identical
too, so sharing is harmless.
@@ -1527,6 +1613,28 @@ class SessionTrackerStore:
len(rewrite_candidates) == 1 or rewrite_candidates[0][0] != rewrite_candidates[1][0]
):
best_key = rewrite_candidates[0][1]
+ elif (
+ not rewrite_candidates
+ and provider == "anthropic"
+ and _transient_system_lineage_enabled()
+ ):
+ # Nothing matched as recorded. A chain whose only change is
+ # its replaced trailing system reminder is still this
+ # conversation; without it every such turn starts a cold
+ # tracker, the confirmed prefix is not replayed, and any
+ # recompressed history re-writes the provider cache. Only a
+ # unique longest chain qualifies, so equal-length siblings
+ # that differ in their reminder stay separate.
+ tail_candidates = [
+ (len(chain), key)
+ for key, chain in by_length
+ if self._lineage_affinities.get(key) == cache_affinity
+ and _extends_past_system_tail(snap, chain)
+ ]
+ if tail_candidates and (
+ len(tail_candidates) == 1 or tail_candidates[0][0] != tail_candidates[1][0]
+ ):
+ best_key = tail_candidates[0][1]
if best_key is None:
cap = self._default_config.max_lineages_per_session
diff --git a/tests/test_cache/test_prefix_tracker.py b/tests/test_cache/test_prefix_tracker.py
index 69f39e59e..cc2504c94 100644
--- a/tests/test_cache/test_prefix_tracker.py
+++ b/tests/test_cache/test_prefix_tracker.py
@@ -13,6 +13,8 @@ from headroom.cache.prefix_tracker import (
PrefixCacheTracker,
PrefixFreezeConfig,
SessionTrackerStore,
+ extract_cache_stable_delta,
+ overlay_cached_prefix,
)
@@ -775,6 +777,149 @@ class TestConversationLineageResolution:
fresh = store.resolve_tracker("sid", "anthropic", messages=self._history("A", 2))
assert fresh._turn_number == 0
+ @staticmethod
+ def _reminder_turns() -> tuple[list[dict], list[dict]]:
+ """Claude Code shape: a trailing system reminder replaced by the next turn."""
+ history = [
+ {"role": "user", "content": "Inspect example.py."},
+ {"role": "assistant", "content": "Reading the file."},
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": "read-1",
+ "content": "original source line\n" * 400,
+ }
+ ],
+ },
+ ]
+ reminder = {"role": "system", "content": "Batch independent tools this turn."}
+ previous = [*history, reminder]
+ current = [
+ *history,
+ {"role": "assistant", "content": "Now run the checks."},
+ {"role": "user", "content": "The checks passed."},
+ dict(reminder),
+ ]
+ return previous, current
+
+ def test_replaced_system_tail_keeps_confirmed_tracker(self, store):
+ """The previous request ended in a system reminder the client swapped
+ for new turns. Its unchanged history must keep the tracker and its
+ confirmed frozen prefix instead of starting a cold lineage."""
+ previous, current = self._reminder_turns()
+ forwarded = [dict(m) for m in previous]
+ forwarded[2] = {**previous[2], "content": "previously cached compressed result"}
+ tracker = store.resolve_tracker("sid", "anthropic", previous, cache_affinity="a")
+ tracker.update_from_response(
+ cache_read_tokens=0,
+ cache_write_tokens=1_000_000,
+ messages=forwarded,
+ original_messages=previous,
+ message_token_counts=[100] * len(previous),
+ )
+ frozen = tracker.get_frozen_message_count()
+ assert frozen > 0
+
+ assert store.resolve_tracker("sid", "anthropic", current, cache_affinity="a") is tracker
+ assert tracker.get_frozen_message_count() == frozen
+ # The existing replay restores the unchanged prefix and stops at the
+ # replaced reminder; this turn's tail is forwarded as processed.
+ processed = [dict(m) for m in current]
+ processed[2] = {**current[2], "content": "a different compressed result"}
+ replayed = overlay_cached_prefix(
+ processed, current, previous, forwarded, confirmed_frozen_count=frozen
+ )
+ assert replayed[:3] == forwarded[:3]
+ assert replayed[3:] == processed[3:]
+
+ def test_moving_system_tail_records_full_snapshots(self, store):
+ """The reminder moves and changes every turn; each turn stays on one
+ tracker and the store records the full current history."""
+ previous, current = self._reminder_turns()
+ tracker = store.resolve_tracker("sid", "anthropic", previous)
+ for i in range(4):
+ current[-1] = {"role": "system", "content": f"reminder {i}"}
+ assert store.resolve_tracker("sid", "anthropic", current) is tracker
+ assert store._lineages["sid"]["sid"][-1]["content"][0]["text"] == f"reminder {i}"
+ assert len(store._lineages["sid"]["sid"]) == len(current)
+ current = [
+ *current[:-1],
+ {"role": "assistant", "content": f"reply {i}"},
+ {"role": "user", "content": f"next {i}"},
+ current[-1],
+ ]
+
+ def test_system_tail_fallback_respects_affinity_and_provider(self):
+ previous, current = self._reminder_turns()
+ for provider, affinity in (("anthropic", "other-tools"), ("openai", "a")):
+ store = SessionTrackerStore()
+ tracker = store.resolve_tracker("sid", provider, previous, cache_affinity="a")
+ resolved = store.resolve_tracker("sid", provider, current, cache_affinity=affinity)
+ assert resolved is not tracker
+
+ @pytest.mark.parametrize(
+ ("index", "role"),
+ [(0, "user"), (0, "system"), (2, "system")],
+ ids=["changed-history", "changed-leading-system", "changed-historical-system"],
+ )
+ def test_system_tail_fallback_requires_unchanged_history(self, store, index, role):
+ """Only the trailing reminder is set aside: any other change, including
+ a changed leading or historical system instruction, still diverges."""
+ previous, current = self._reminder_turns()
+ previous.insert(index, {"role": role, "content": "original"})
+ current.insert(index, {"role": role, "content": "changed"})
+ tracker = store.resolve_tracker("sid", "anthropic", previous)
+ assert store.resolve_tracker("sid", "anthropic", current) is not tracker
+
+ def test_swapped_final_instruction_is_not_a_continuation(self, store):
+ """Same-length siblings that differ only in their final system message
+ stay separate: the new history must extend past the old reminder."""
+ first = [{"role": "user", "content": "task"}, {"role": "system", "content": "A"}]
+ second = [{"role": "user", "content": "task"}, {"role": "system", "content": "B"}]
+ tracker = store.resolve_tracker("sid", "anthropic", first)
+ assert store.resolve_tracker("sid", "anthropic", second) is not tracker
+ assert store.resolve_tracker("sid", "anthropic", first) is tracker
+
+ def test_ambiguous_system_tail_siblings_are_not_merged(self, store, monkeypatch):
+ """Two lineages that differ only in their reminder (recorded here with
+ the fallback disabled) are an ambiguous match: start a fresh lineage
+ instead of guessing, and an exact match still wins."""
+ previous, current = self._reminder_turns()
+ sibling = [*previous[:-1], {"role": "system", "content": "other reminder"}]
+ monkeypatch.setenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE", "0")
+ tracker = store.resolve_tracker("sid", "anthropic", previous)
+ other = store.resolve_tracker("sid", "anthropic", sibling)
+ assert other is not tracker
+ monkeypatch.delenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE")
+ assert store.resolve_tracker("sid", "anthropic", current) not in (tracker, other)
+ assert store.resolve_tracker("sid", "anthropic", sibling) is other
+
+ def test_cache_delta_replays_prefix_past_replaced_system_tail(self, monkeypatch):
+ """Cache mode's delta split leaves the replaced reminder out of the
+ replayed prefix, with or without the model reply the handlers record
+ after the request, and compresses everything after it as new."""
+ previous, current = self._reminder_turns()
+ reply = current[3]
+ forwarded = [*previous[:2], {**previous[2], "content": "compressed"}, previous[3], reply]
+ for recorded in ([*previous, reply], previous):
+ replay = forwarded[: len(recorded)]
+ prefix, delta = extract_cache_stable_delta(current, recorded, replay)
+ assert prefix == replay[:3] + replay[4:]
+ assert delta == current[len(prefix) :]
+
+ # A system message followed by a client turn is history, not a reminder.
+ changed = [*previous, {"role": "user", "content": "Continue."}]
+ dropped = [*previous[:3], changed[4], *current[3:]]
+ assert extract_cache_stable_delta(dropped, changed, changed) is None
+ # The client's copy of the reply must match the recorded one.
+ other = [*previous, {"role": "assistant", "content": "Something else."}]
+ assert extract_cache_stable_delta(current, other, other) is None
+
+ monkeypatch.setenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE", "0")
+ assert extract_cache_stable_delta(current, [*previous, reply], forwarded) is None
+
def test_shared_session_id_is_not_rotated(self, store):
"""Composition guard: lineage resolution must not leak into session-id
derivation — beta stickiness and the compression cache key on it."""
diff --git a/tests/test_cache_mode_cold_start.py b/tests/test_cache_mode_cold_start.py
index 4b85c7479..c3b9ab2c2 100644
--- a/tests/test_cache_mode_cold_start.py
+++ b/tests/test_cache_mode_cold_start.py
@@ -235,3 +235,101 @@ def test_cache_mode_prefix_mismatch_passes_through_with_tag(monkeypatch):
assert forwarded[0]["content"][0]["content"] == original_text
# ...but now visibly tagged instead of silent.
assert handler.recorded_tags["passthrough_reason"] == "cache_mode_prefix_mismatch"
+
+
+def _assistant(turn: int) -> dict:
+ return {
+ "role": "assistant",
+ "content": [{"type": "tool_use", "id": f"toolu_{turn}", "name": "read", "input": {}}],
+ }
+
+
+def _tool_result(turn: int) -> dict:
+ return {
+ "role": "user",
+ "content": [
+ {
+ "type": "tool_result",
+ "tool_use_id": f"toolu_{turn}",
+ "content": f"verbose tool output {turn} " * 200,
+ }
+ ],
+ }
+
+
+def _compress_unfrozen(messages, model, frozen_message_count=0, **kwargs):
+ """Like ``_fake_pipeline_apply`` but leaves frozen messages alone, as the
+ real pipeline does."""
+ result = _fake_pipeline_apply(messages[frozen_message_count:], model)
+ return TransformResult(
+ messages=list(messages[:frozen_message_count]) + result.messages,
+ tokens_before=result.tokens_before,
+ tokens_after=result.tokens_after,
+ transforms_applied=result.transforms_applied,
+ )
+
+
+class _ConversationHandler(_DummyAnthropicHandler):
+ """Real session tracker store; the upstream answers with the next tool call."""
+
+ def __init__(self) -> None:
+ from headroom.cache.prefix_tracker import SessionTrackerStore
+
+ super().__init__()
+ self.session_tracker_store = SessionTrackerStore()
+ self.anthropic_pipeline = SimpleNamespace(apply=MagicMock(side_effect=_compress_unfrozen))
+ self.forwarded: list[list[dict]] = []
+
+ async def _retry_request(self, method, url, headers, body, **_kwargs):
+ self.forwarded.append(json.loads(json.dumps(body["messages"])))
+ reply = {
+ "id": f"msg_{len(self.forwarded)}",
+ "type": "message",
+ "role": "assistant",
+ "content": _assistant(len(self.forwarded) + 1)["content"],
+ "usage": {"input_tokens": 1, "output_tokens": 1},
+ }
+ return SimpleNamespace(
+ status_code=200, headers={}, content=json.dumps(reply).encode(), json=lambda: reply
+ )
+
+
+def test_cache_mode_replaced_system_tail_keeps_compressed_prefix(monkeypatch):
+ """Claude Code replaces a trailing system reminder every turn. Cache mode
+ must replay last turn's compressed prefix without the old reminder and
+ compress only the new messages, not fall back to raw forwarding."""
+ import headroom.tokenizers as _tk
+
+ monkeypatch.setattr(_tk, "get_tokenizer", lambda model: _DummyTokenizer())
+ handler = _ConversationHandler()
+ model = "claude-fable-5"
+ history = [{"role": "user", "content": "Read the files."}, _assistant(1), _tool_result(1)]
+ turns = [
+ history,
+ [*history, _assistant(2), _tool_result(2), {"role": "system", "content": "Reminder A"}],
+ [
+ *history,
+ _assistant(2),
+ _tool_result(2),
+ _assistant(3),
+ _tool_result(3),
+ {"role": "system", "content": "Reminder B"},
+ ],
+ ]
+ for messages in turns:
+ request = _build_request({"model": model, "max_tokens": 16, "messages": messages})
+ anyio.run(handler.handle_anthropic_messages, request)
+
+ from headroom.cache.prefix_tracker import _strip_cache_control
+
+ # The proxy re-places cache_control markers every turn; compare the bytes under them.
+ second, third = (_strip_cache_control(m) for m in handler.forwarded[1:])
+ assert [m["role"] for m in third] == [m["role"] for m in turns[2]]
+ # Everything before the replaced reminder is last turn's forwarded bytes...
+ assert third[:5] == second[:5]
+ assert second[2]["content"][0]["content"] == _COMPRESSED_TEXT
+ assert second[4]["content"][0]["content"] == _COMPRESSED_TEXT
+ # ...the new tool result is compressed, and the current reminder is sent as is.
+ assert third[6]["content"][0]["content"] == _COMPRESSED_TEXT
+ assert third[-1] == {"role": "system", "content": "Reminder B"}
+ assert "passthrough_reason" not in handler.recorded_tags