Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): keep prefix lineage across a replaced system tail (#3934)
gglucass16eaec1e95cd
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.pyindex 83ed16334..b29107485 100644--- a/headroom/cache/prefix_tracker.py+++ b/headroom/cache/prefix_tracker.py@@ -367,6 +367,81 @@ def _classify_history_canonical()+_TRANSIENT_SYSTEM_LINEAGE_ENV = "HEADROOM_TRANSIENT_SYSTEM_LINEAGE"+++def _transient_system_lineage_enabled() -> bool:+ return os.environ.get(_TRANSIENT_SYSTEM_LINEAGE_ENV, "").strip().lower() not in (+ "0",+ "false",+ "no",+ "off",+ )+++def _extends_past_system_tail(current_messages: list[Any], previous_messages: list[Any]) -> bool:+ """Whether ``current`` continues ``previous`` once its trailing system turn is set aside.++ Claude Code ends some requests with a ``role:"system"`` reminder and+ replaces it on the next turn, so the recorded history is never a prefix of+ the new one. Only that last message is excluded: everything before it+ must survive unchanged and include real user/assistant history, so a+ changed leading or historical system instruction still diverges, and the+ new history must be longer, so a sibling that only swaps the final+ instruction is not a continuation.+ """+ if len(previous_messages) < 2 or len(current_messages) <= len(previous_messages):+ return False+ tail = previous_messages[-1]+ if not isinstance(tail, dict) or tail.get("role") != "system":+ return False+ stable = previous_messages[:-1]+ return current_messages[: len(stable)] == stable and any(+ isinstance(message, dict) and message.get("role") in ("user", "assistant")+ for message in stable+ )+++def _replaced_system_tail_delta(+ current_messages: list[dict[str, Any]],+ previous_original_messages: list[dict[str, Any]],+ previous_forwarded_messages: list[dict[str, Any]],+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]] | None:+ """Cache-stable split for a request that replaced the previous trailing system turn.++ The handlers record the model's reply after the request it answered, so the+ replaced reminder is the last recorded message or sits right before that+ reply. It is left out of the replayed prefix (it is not in this request);+ every other recorded message must match ``current`` position for position.+ """+ if not _transient_system_lineage_enabled() or len(previous_forwarded_messages) != len(+ previous_original_messages+ ):+ return None+ current = _lineage_snapshot(_canonicalize_for_prefix_compare(current_messages))+ previous = _lineage_snapshot(_canonicalize_for_prefix_compare(previous_original_messages))+ if len(current) != len(current_messages) or len(previous) != len(previous_original_messages):+ # The projection dropped a directive-only message; raw indices would shift.+ return None+ for request_len in (len(previous), len(previous) - 1):+ reply = previous[request_len:]+ if reply and not (isinstance(reply[0], dict) and reply[0].get("role") == "assistant"):+ continue+ reminder = request_len - 1+ if (+ _extends_past_system_tail(current, previous[:request_len])+ and current[reminder : reminder + len(reply)] == reply+ ):+ return (+ copy.deepcopy(+ previous_forwarded_messages[:reminder]+ + previous_forwarded_messages[request_len:]+ ),+ copy.deepcopy(current_messages[reminder + len(reply) :]),+ )+ return None++def classify_history_relation(current_messages: list[dict[str, Any]],previous_messages: list[dict[str, Any]],@@ -427,10 +502,18 @@ def extract_cache_stable_delta(This is a COMPARISON + slice only: the returned prefix is the previously-forwardedbytes verbatim and the delta is the raw appended messages — never a rebuild from thecanonical projection — so the projection dropping non-semantic fields is safe.++ A previous request that ended in a ``role:"system"`` reminder the client has since+ replaced also qualifies: its forwarded bytes are replayed without that reminder+ (``HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0`` disables this)."""if not previous_original_messages or previous_forwarded_messages is None:return Nonerelation = classify_history_relation(current_messages, previous_original_messages)+ if relation.kind == RELATION_DIVERGED:+ return _replaced_system_tail_delta(+ current_messages, previous_original_messages, previous_forwarded_messages+ )if relation.kind not in (RELATION_EXACT, RELATION_MESSAGE_APPEND):# A same-message block append needs a block-level splice in# ``overlay_cached_prefix``; slicing only whole messages would silently@@ -1433,7 +1516,10 @@ class SessionTrackerStore:where a large leading run and two-block identity suffix survive whilethe middle tail is regenerated; all other rewrites start a freshlineage. This keeps #2671's stable cache boundary attached without- merging unrelated parallel sub-calls.+ merging unrelated parallel sub-calls. When nothing else matches, an+ Anthropic chain whose trailing ``role:"system"`` reminder the client+ replaced is matched on the history before that reminder+ (``HEADROOM_TRANSIENT_SYSTEM_LINEAGE=0`` disables this).Byte-identical histories (templated fan-outs before they diverge)intentionally share a tracker: their provider cache line is identicaltoo, so sharing is harmless.@@ -1527,6 +1613,28 @@ class SessionTrackerStore:len(rewrite_candidates) == 1 or rewrite_candidates[0][0] != rewrite_candidates[1][0]):best_key = rewrite_candidates[0][1]+ elif (+ not rewrite_candidates+ and provider == "anthropic"+ and _transient_system_lineage_enabled()+ ):+ # Nothing matched as recorded. A chain whose only change is+ # its replaced trailing system reminder is still this+ # conversation; without it every such turn starts a cold+ # tracker, the confirmed prefix is not replayed, and any+ # recompressed history re-writes the provider cache. Only a+ # unique longest chain qualifies, so equal-length siblings+ # that differ in their reminder stay separate.+ tail_candidates = [+ (len(chain), key)+ for key, chain in by_length+ if self._lineage_affinities.get(key) == cache_affinity+ and _extends_past_system_tail(snap, chain)+ ]+ if tail_candidates and (+ len(tail_candidates) == 1 or tail_candidates[0][0] != tail_candidates[1][0]+ ):+ best_key = tail_candidates[0][1]if best_key is None:cap = self._default_config.max_lineages_per_sessiondiff --git a/tests/test_cache/test_prefix_tracker.py b/tests/test_cache/test_prefix_tracker.pyindex 69f39e59e..cc2504c94 100644--- a/tests/test_cache/test_prefix_tracker.py+++ b/tests/test_cache/test_prefix_tracker.py@@ -13,6 +13,8 @@ from headroom.cache.prefix_tracker import (PrefixCacheTracker,PrefixFreezeConfig,SessionTrackerStore,+ extract_cache_stable_delta,+ overlay_cached_prefix,)@@ -775,6 +777,149 @@ class TestConversationLineageResolution:fresh = store.resolve_tracker("sid", "anthropic", messages=self._history("A", 2))assert fresh._turn_number == 0+ @staticmethod+ def _reminder_turns() -> tuple[list[dict], list[dict]]:+ """Claude Code shape: a trailing system reminder replaced by the next turn."""+ history = [+ {"role": "user", "content": "Inspect example.py."},+ {"role": "assistant", "content": "Reading the file."},+ {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": "read-1",+ "content": "original source line\n" * 400,+ }+ ],+ },+ ]+ reminder = {"role": "system", "content": "Batch independent tools this turn."}+ previous = [*history, reminder]+ current = [+ *history,+ {"role": "assistant", "content": "Now run the checks."},+ {"role": "user", "content": "The checks passed."},+ dict(reminder),+ ]+ return previous, current++ def test_replaced_system_tail_keeps_confirmed_tracker(self, store):+ """The previous request ended in a system reminder the client swapped+ for new turns. Its unchanged history must keep the tracker and its+ confirmed frozen prefix instead of starting a cold lineage."""+ previous, current = self._reminder_turns()+ forwarded = [dict(m) for m in previous]+ forwarded[2] = {**previous[2], "content": "previously cached compressed result"}+ tracker = store.resolve_tracker("sid", "anthropic", previous, cache_affinity="a")+ tracker.update_from_response(+ cache_read_tokens=0,+ cache_write_tokens=1_000_000,+ messages=forwarded,+ original_messages=previous,+ message_token_counts=[100] * len(previous),+ )+ frozen = tracker.get_frozen_message_count()+ assert frozen > 0++ assert store.resolve_tracker("sid", "anthropic", current, cache_affinity="a") is tracker+ assert tracker.get_frozen_message_count() == frozen+ # The existing replay restores the unchanged prefix and stops at the+ # replaced reminder; this turn's tail is forwarded as processed.+ processed = [dict(m) for m in current]+ processed[2] = {**current[2], "content": "a different compressed result"}+ replayed = overlay_cached_prefix(+ processed, current, previous, forwarded, confirmed_frozen_count=frozen+ )+ assert replayed[:3] == forwarded[:3]+ assert replayed[3:] == processed[3:]++ def test_moving_system_tail_records_full_snapshots(self, store):+ """The reminder moves and changes every turn; each turn stays on one+ tracker and the store records the full current history."""+ previous, current = self._reminder_turns()+ tracker = store.resolve_tracker("sid", "anthropic", previous)+ for i in range(4):+ current[-1] = {"role": "system", "content": f"reminder {i}"}+ assert store.resolve_tracker("sid", "anthropic", current) is tracker+ assert store._lineages["sid"]["sid"][-1]["content"][0]["text"] == f"reminder {i}"+ assert len(store._lineages["sid"]["sid"]) == len(current)+ current = [+ *current[:-1],+ {"role": "assistant", "content": f"reply {i}"},+ {"role": "user", "content": f"next {i}"},+ current[-1],+ ]++ def test_system_tail_fallback_respects_affinity_and_provider(self):+ previous, current = self._reminder_turns()+ for provider, affinity in (("anthropic", "other-tools"), ("openai", "a")):+ store = SessionTrackerStore()+ tracker = store.resolve_tracker("sid", provider, previous, cache_affinity="a")+ resolved = store.resolve_tracker("sid", provider, current, cache_affinity=affinity)+ assert resolved is not tracker++ @pytest.mark.parametrize(+ ("index", "role"),+ [(0, "user"), (0, "system"), (2, "system")],+ ids=["changed-history", "changed-leading-system", "changed-historical-system"],+ )+ def test_system_tail_fallback_requires_unchanged_history(self, store, index, role):+ """Only the trailing reminder is set aside: any other change, including+ a changed leading or historical system instruction, still diverges."""+ previous, current = self._reminder_turns()+ previous.insert(index, {"role": role, "content": "original"})+ current.insert(index, {"role": role, "content": "changed"})+ tracker = store.resolve_tracker("sid", "anthropic", previous)+ assert store.resolve_tracker("sid", "anthropic", current) is not tracker++ def test_swapped_final_instruction_is_not_a_continuation(self, store):+ """Same-length siblings that differ only in their final system message+ stay separate: the new history must extend past the old reminder."""+ first = [{"role": "user", "content": "task"}, {"role": "system", "content": "A"}]+ second = [{"role": "user", "content": "task"}, {"role": "system", "content": "B"}]+ tracker = store.resolve_tracker("sid", "anthropic", first)+ assert store.resolve_tracker("sid", "anthropic", second) is not tracker+ assert store.resolve_tracker("sid", "anthropic", first) is tracker++ def test_ambiguous_system_tail_siblings_are_not_merged(self, store, monkeypatch):+ """Two lineages that differ only in their reminder (recorded here with+ the fallback disabled) are an ambiguous match: start a fresh lineage+ instead of guessing, and an exact match still wins."""+ previous, current = self._reminder_turns()+ sibling = [*previous[:-1], {"role": "system", "content": "other reminder"}]+ monkeypatch.setenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE", "0")+ tracker = store.resolve_tracker("sid", "anthropic", previous)+ other = store.resolve_tracker("sid", "anthropic", sibling)+ assert other is not tracker+ monkeypatch.delenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE")+ assert store.resolve_tracker("sid", "anthropic", current) not in (tracker, other)+ assert store.resolve_tracker("sid", "anthropic", sibling) is other++ def test_cache_delta_replays_prefix_past_replaced_system_tail(self, monkeypatch):+ """Cache mode's delta split leaves the replaced reminder out of the+ replayed prefix, with or without the model reply the handlers record+ after the request, and compresses everything after it as new."""+ previous, current = self._reminder_turns()+ reply = current[3]+ forwarded = [*previous[:2], {**previous[2], "content": "compressed"}, previous[3], reply]+ for recorded in ([*previous, reply], previous):+ replay = forwarded[: len(recorded)]+ prefix, delta = extract_cache_stable_delta(current, recorded, replay)+ assert prefix == replay[:3] + replay[4:]+ assert delta == current[len(prefix) :]++ # A system message followed by a client turn is history, not a reminder.+ changed = [*previous, {"role": "user", "content": "Continue."}]+ dropped = [*previous[:3], changed[4], *current[3:]]+ assert extract_cache_stable_delta(dropped, changed, changed) is None+ # The client's copy of the reply must match the recorded one.+ other = [*previous, {"role": "assistant", "content": "Something else."}]+ assert extract_cache_stable_delta(current, other, other) is None++ monkeypatch.setenv("HEADROOM_TRANSIENT_SYSTEM_LINEAGE", "0")+ assert extract_cache_stable_delta(current, [*previous, reply], forwarded) is None+def test_shared_session_id_is_not_rotated(self, store):"""Composition guard: lineage resolution must not leak into session-idderivation — beta stickiness and the compression cache key on it."""diff --git a/tests/test_cache_mode_cold_start.py b/tests/test_cache_mode_cold_start.pyindex 4b85c7479..c3b9ab2c2 100644--- a/tests/test_cache_mode_cold_start.py+++ b/tests/test_cache_mode_cold_start.py@@ -235,3 +235,101 @@ def test_cache_mode_prefix_mismatch_passes_through_with_tag(monkeypatch):assert forwarded[0]["content"][0]["content"] == original_text# ...but now visibly tagged instead of silent.assert handler.recorded_tags["passthrough_reason"] == "cache_mode_prefix_mismatch"+++def _assistant(turn: int) -> dict:+ return {+ "role": "assistant",+ "content": [{"type": "tool_use", "id": f"toolu_{turn}", "name": "read", "input": {}}],+ }+++def _tool_result(turn: int) -> dict:+ return {+ "role": "user",+ "content": [+ {+ "type": "tool_result",+ "tool_use_id": f"toolu_{turn}",+ "content": f"verbose tool output {turn} " * 200,+ }+ ],+ }+++def _compress_unfrozen(messages, model, frozen_message_count=0, **kwargs):+ """Like ``_fake_pipeline_apply`` but leaves frozen messages alone, as the+ real pipeline does."""+ result = _fake_pipeline_apply(messages[frozen_message_count:], model)+ return TransformResult(+ messages=list(messages[:frozen_message_count]) + result.messages,+ tokens_before=result.tokens_before,+ tokens_after=result.tokens_after,+ transforms_applied=result.transforms_applied,+ )+++class _ConversationHandler(_DummyAnthropicHandler):+ """Real session tracker store; the upstream answers with the next tool call."""++ def __init__(self) -> None:+ from headroom.cache.prefix_tracker import SessionTrackerStore++ super().__init__()+ self.session_tracker_store = SessionTrackerStore()+ self.anthropic_pipeline = SimpleNamespace(apply=MagicMock(side_effect=_compress_unfrozen))+ self.forwarded: list[list[dict]] = []++ async def _retry_request(self, method, url, headers, body, **_kwargs):+ self.forwarded.append(json.loads(json.dumps(body["messages"])))+ reply = {+ "id": f"msg_{len(self.forwarded)}",+ "type": "message",+ "role": "assistant",+ "content": _assistant(len(self.forwarded) + 1)["content"],+ "usage": {"input_tokens": 1, "output_tokens": 1},+ }+ return SimpleNamespace(+ status_code=200, headers={}, content=json.dumps(reply).encode(), json=lambda: reply+ )+++def test_cache_mode_replaced_system_tail_keeps_compressed_prefix(monkeypatch):+ """Claude Code replaces a trailing system reminder every turn. Cache mode+ must replay last turn's compressed prefix without the old reminder and+ compress only the new messages, not fall back to raw forwarding."""+ import headroom.tokenizers as _tk++ monkeypatch.setattr(_tk, "get_tokenizer", lambda model: _DummyTokenizer())+ handler = _ConversationHandler()+ model = "claude-fable-5"+ history = [{"role": "user", "content": "Read the files."}, _assistant(1), _tool_result(1)]+ turns = [+ history,+ [*history, _assistant(2), _tool_result(2), {"role": "system", "content": "Reminder A"}],+ [+ *history,+ _assistant(2),+ _tool_result(2),+ _assistant(3),+ _tool_result(3),+ {"role": "system", "content": "Reminder B"},+ ],+ ]+ for messages in turns:+ request = _build_request({"model": model, "max_tokens": 16, "messages": messages})+ anyio.run(handler.handle_anthropic_messages, request)++ from headroom.cache.prefix_tracker import _strip_cache_control++ # The proxy re-places cache_control markers every turn; compare the bytes under them.+ second, third = (_strip_cache_control(m) for m in handler.forwarded[1:])+ assert [m["role"] for m in third] == [m["role"] for m in turns[2]]+ # Everything before the replaced reminder is last turn's forwarded bytes...+ assert third[:5] == second[:5]+ assert second[2]["content"][0]["content"] == _COMPRESSED_TEXT+ assert second[4]["content"][0]["content"] == _COMPRESSED_TEXT+ # ...the new tool result is compressed, and the current reminder is sent as is.+ assert third[6]["content"][0]["content"] == _COMPRESSED_TEXT+ assert third[-1] == {"role": "system", "content": "Reminder B"}+ assert "passthrough_reason" not in handler.recorded_tags