Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): reconcile Chat content-block cache markers after prefix replay (#4333)

tbartik35ca9b835616
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
index fd0ad9fdf..4a5d286d8 100644
--- a/headroom/cache/prefix_tracker.py
+++ b/headroom/cache/prefix_tracker.py
@@ -1105,33 +1105,37 @@ def mirror_client_message_cache_control(
messages: list[dict[str, Any]],
client_messages: list[dict[str, Any]] | None,
) -> list[dict[str, Any]]:
- """Keep ``cache_control`` outside content blocks exactly where the client put it.
+ """Keep Chat Completions ``cache_control`` exactly where the client put it.
OpenAI-compatible clients that talk to Anthropic models through a gateway
- (OpenCode via ``@ai-sdk/openai-compatible``, then LiteLLM) do not mark
- content blocks. They mark the MESSAGE dict — ``{"role": "tool", "content":
- "...", "cache_control": {...}}`` — and, on an assistant turn, the entries of
- its ``tool_calls`` list, which the gateway turns into ``tool_use`` blocks.
- ``normalize_message_cache_control`` only sees block markers
- (``_client_marker_positions`` reads list content), so after
- ``overlay_cached_prefix`` / ``finalize_turn`` replay an earlier turn's
- forwarded messages, these markers ride along untouched. A client that marks
- three places per request (system, the newest tool call, the newest tool
- result) ends up forwarding four, then five and more, and the gateway's
- translation to Anthropic blocks crosses the limit of four (``A maximum of 4
- blocks with cache_control may be provided. Found 6``).
-
- The client's current request is the authority for these markers, the same
- rule the block-level normalizer applies: on each message and on each of its
- ``tool_calls``, drop the marker unless the client put one there in this
- request, in which case copy the client's own value. Content — and any
- block-level marker in it — is left alone, so the replayed bytes the provider
- cached stay byte-identical.
-
- Index alignment is the invariant (the pipeline preserves message count and
- each message's tool calls); where it does not hold, that message or list is
- returned unchanged rather than guessing, and the breakpoint budget guard
- still bounds the total.
+ (OpenCode via ``@ai-sdk/openai-compatible``, then LiteLLM) mark three kinds
+ of holder: the MESSAGE dict — ``{"role": "tool", "content": "...",
+ "cache_control": {...}}`` —, the entries of an assistant's ``tool_calls``
+ list, which the gateway turns into ``tool_use`` blocks, and the content
+ blocks of list-form content (OpenCode V2 sends system, user and tool
+ content as text blocks). After ``overlay_cached_prefix`` /
+ ``finalize_turn`` replay an earlier turn's forwarded messages, last turn's
+ markers ride along with the replayed bytes. A client that moves its
+ breakpoint forward each step then forwards one marker more per step, and
+ the gateway's translation to Anthropic blocks crosses the limit of four
+ (``A maximum of 4 blocks with cache_control may be provided. Found 5``).
+
+ The client's current request is the authority for these markers: on each
+ message, each of its ``tool_calls`` and each of its content blocks, drop the
+ marker unless the client put one there in this request, in which case copy
+ the client's own value (an explicit ``ttl`` survives per position). Only the
+ ``cache_control`` keys move; every other byte of the replayed content stays
+ what the provider cached.
+
+ Unlike ``normalize_message_cache_control`` this never creates a marker and
+ never moves one to a different block: in Chat the system prompt is part of
+ ``messages``, so a consolidating fallback would also strip the client's
+ system breakpoints. Index alignment is the invariant (the pipeline preserves
+ message count and each message's tool calls); where it does not hold, that
+ message or list is left unchanged rather than guessing. A replay may also
+ bring back an earlier turn's block-form content where the client now sends
+ a string (OpenCode does this for older user turns): the client marks no
+ block there, so the replayed block markers are dropped.
"""
if not isinstance(client_messages, list) or len(client_messages) != len(messages):
return messages
@@ -1153,11 +1157,41 @@ def mirror_client_message_cache_control(
if any(c for _, c in mirrored):
new_msg = {**new_msg, "tool_calls": [c for c, _ in mirrored]}
msg_changed = True
+ blocks = new_msg.get("content")
+ client_blocks = client.get("content") if isinstance(client, dict) else None
+ if isinstance(blocks, list):
+ mirrored = _mirror_block_markers(blocks, client_blocks)
+ if mirrored is not blocks:
+ new_msg = {**new_msg, "content": mirrored}
+ msg_changed = True
out.append(new_msg)
changed = changed or msg_changed
return out if changed else messages
+def _mirror_block_markers(blocks: list[Any], client_content: Any) -> list[Any]:
+ """Give each forwarded content block the client's block marker (or none).
+
+ A client message that marks no block (string content, which a replay may
+ bring back in an earlier turn's block form, or unmarked blocks) leaves no
+ block marker to keep, so every one is dropped. When the client does mark
+ blocks, positions map 1:1 only if the block counts match; otherwise the
+ blocks are returned unchanged rather than guessing.
+ """
+ client_blocks = client_content if isinstance(client_content, list) else []
+ client_marks = any(
+ isinstance(b, dict) and isinstance(b.get("cache_control"), dict) for b in client_blocks
+ )
+ if not client_marks:
+ client_blocks = [None] * len(blocks)
+ elif len(client_blocks) != len(blocks):
+ return blocks
+ mirrored = [_mirror_marker(b, cb) for b, cb in zip(blocks, client_blocks)]
+ if not any(c for _, c in mirrored):
+ return blocks
+ return [b for b, _ in mirrored]
+
+
def _mirror_marker(holder: Any, client_holder: Any) -> tuple[Any, bool]:
"""Give ``holder`` the client's own ``cache_control`` (or none); report change."""
if not isinstance(holder, dict):
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 9caf9f858..14484c3fa 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4323,11 +4323,10 @@ class OpenAIHandlerMixin:
# Both replays above forward earlier turns' messages byte-identical,
# including the cache_control each carried back then. Chat Completions
- # clients mark the message dict and an assistant's tool_calls, which the
- # block-level normalizer does not see, so without this the markers
- # accumulate turn over turn and an Anthropic-backed gateway rejects the
- # request (more than four breakpoints). Content is untouched: only the
- # markers move.
+ # clients mark the message dict, an assistant's tool_calls and content
+ # blocks, so without this the markers accumulate turn over turn and an
+ # Anthropic-backed gateway rejects the request (more than four
+ # breakpoints). Only the markers move; every other byte stays replayed.
from headroom.cache.prefix_tracker import mirror_client_message_cache_control
optimized_messages = mirror_client_message_cache_control(
diff --git a/tests/test_openai_block_cache_control_replay.py b/tests/test_openai_block_cache_control_replay.py
new file mode 100644
index 000000000..936125915
--- /dev/null
+++ b/tests/test_openai_block_cache_control_replay.py
@@ -0,0 +1,397 @@
+"""Chat Completions content-block ``cache_control`` must not accumulate across replays.
+
+OpenCode V2 talking to a Claude model through an OpenAI-compatible gateway marks
+content BLOCKS in Chat messages (``{"role": "user", "content": [{"type": "text",
+..., "cache_control": {...}}]}``), not only the message dict and ``tool_calls``.
+The prefix replay forwards earlier turns' messages byte-identical, so last turn's
+block marker came back next to the client's moved one: the client sent four, the
+proxy forwarded five, and the gateway's Anthropic translation failed with
+``A maximum of 4 blocks with cache_control may be provided. Found 5``.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient
+
+from headroom.cache.prefix_tracker import mirror_client_message_cache_control
+from headroom.proxy.server import ProxyConfig, create_app
+
+CC = {"type": "ephemeral"}
+CC_1H = {"type": "ephemeral", "ttl": "1h"}
+
+
+def _text(text: str, marker: dict | None = None) -> dict:
+ block = {"type": "text", "text": text}
+ return {**block, "cache_control": dict(marker)} if marker else block
+
+
+def _marker_paths(value, path: str = "") -> list[str]: # noqa: ANN001
+ found: list[str] = []
+ if isinstance(value, dict):
+ for key, item in value.items():
+ if key == "cache_control":
+ found.append(path or ".")
+ else:
+ found.extend(_marker_paths(item, f"{path}.{key}" if path else key))
+ elif isinstance(value, list):
+ for index, item in enumerate(value):
+ found.extend(_marker_paths(item, f"{path}.{index}" if path else str(index)))
+ return found
+
+
+def _strip_markers(value): # noqa: ANN001, ANN201
+ if isinstance(value, dict):
+ return {k: _strip_markers(v) for k, v in value.items() if k != "cache_control"}
+ if isinstance(value, list):
+ return [_strip_markers(v) for v in value]
+ return value
+
+
+def _system(first: dict | None = CC, last: dict | None = CC) -> dict:
+ return {
+ "role": "system",
+ "content": [
+ _text("You are an agent.", first),
+ _text("Preserve records."),
+ _text("Use the tool."),
+ _text("Answer briefly.", last),
+ ],
+ }
+
+
+def _call(cid: str = "c1") -> dict:
+ return {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"id": cid, "type": "function", "function": {"name": "read", "arguments": "{}"}}
+ ],
+ }
+
+
+# ── the helper ──────────────────────────────────────────────────────────────
+
+
+def test_moved_user_breakpoint_drops_the_replayed_block_marker() -> None:
+ client = [
+ _system(),
+ {"role": "user", "content": [_text("q")]},
+ _call(),
+ {"role": "tool", "tool_call_id": "c1", "content": [_text("result", CC)]},
+ ]
+ replayed = copy.deepcopy(client)
+ replayed[1]["content"][0]["cache_control"] = dict(CC) # last turn's breakpoint
+ before = copy.deepcopy(replayed)
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert (
+ _marker_paths(out) == _marker_paths(client) == ["0.content.0", "0.content.3", "3.content.0"]
+ )
+ assert replayed == before # input not mutated
+ assert _strip_markers(out) == _strip_markers(replayed)
+
+
+def test_replayed_block_content_stays_byte_identical() -> None:
+ client = [
+ {"role": "tool", "tool_call_id": "c1", "content": [_text("raw " * 50)]},
+ {"role": "user", "content": [_text("next", CC)]},
+ ]
+ replayed = [
+ {"role": "tool", "tool_call_id": "c1", "content": [_text("<compressed>", CC)]},
+ copy.deepcopy(client[1]),
+ ]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert out[0]["content"] == [_text("<compressed>")]
+ assert out[1] is replayed[1]
+
+
+def test_explicit_mid_message_positions_and_ttls_are_copied_verbatim() -> None:
+ client = [
+ _system(first=CC_1H, last=CC),
+ {"role": "user", "content": [_text("a"), _text("b", CC_1H), _text("c")]},
+ ]
+ replayed = [
+ _system(first=CC, last=CC),
+ {"role": "user", "content": [_text("a"), _text("b"), _text("c", CC)]},
+ ]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert out[0]["content"][0]["cache_control"] == CC_1H
+ assert out[0]["content"][3]["cache_control"] == CC
+ assert [("cache_control" in b) for b in out[1]["content"]] == [False, True, False]
+ assert out[1]["content"][1]["cache_control"] == CC_1H
+
+
+def test_client_without_block_markers_gets_none_back_and_none_created() -> None:
+ client = [_system(first=None, last=None), {"role": "user", "content": [_text("q")]}]
+ replayed = [_system(), {"role": "user", "content": [_text("q", CC)]}]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert _marker_paths(out) == []
+
+
+def test_client_without_any_markers_changes_nothing() -> None:
+ client = [
+ {"role": "system", "content": [_text("s")]},
+ {"role": "user", "content": [_text("q")]},
+ ]
+ messages = copy.deepcopy(client)
+ assert mirror_client_message_cache_control(messages, client) is messages
+
+
+def test_mixed_block_message_and_tool_call_markers() -> None:
+ call = _call()
+ marked_call = {**call, "tool_calls": [{**call["tool_calls"][0], "cache_control": dict(CC)}]}
+ client = [
+ _system(),
+ {"role": "user", "content": [_text("q")]},
+ marked_call,
+ {"role": "tool", "tool_call_id": "c1", "content": "out", "cache_control": dict(CC)},
+ ]
+ replayed = [
+ _system(),
+ {"role": "user", "content": [_text("q", CC)], "cache_control": dict(CC)},
+ call,
+ {"role": "tool", "tool_call_id": "c1", "content": "out"},
+ ]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert (
+ _marker_paths(out)
+ == _marker_paths(client)
+ == ["0.content.0", "0.content.3", "2.tool_calls.0", "3"]
+ )
+
+
+def test_block_count_mismatch_leaves_that_message_unchanged() -> None:
+ client = [
+ {"role": "user", "content": [_text("a"), _text("b", CC)]},
+ {"role": "user", "content": [_text("c", CC)]},
+ ]
+ # The pipeline merged two blocks; there is no safe 1:1 mapping, so no guessing.
+ replayed = [
+ {"role": "user", "content": [_text("ab", CC)]},
+ {"role": "user", "content": [_text("c")]},
+ ]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert out[0] is replayed[0]
+ assert out[1]["content"][0]["cache_control"] == CC
+
+
+def test_replayed_blocks_for_a_string_client_message_lose_their_markers() -> None:
+ # OpenCode V2 sends an older user turn as a string; the replay brings back
+ # the block form it forwarded when that turn was the newest, marker included.
+ client = [{"role": "user", "content": "q"}]
+ replayed = [{"role": "user", "content": [_text("q", CC)]}]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert out[0]["content"] == [_text("q")]
+
+
+def test_block_count_mismatch_without_client_block_markers_only_drops_markers() -> None:
+ client = [{"role": "user", "content": [_text("a"), _text("b")]}]
+ replayed = [{"role": "user", "content": [_text("ab", CC)]}]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert out[0]["content"] == [_text("ab")]
+
+
+def test_literal_cache_control_in_tool_text_and_arguments_is_user_data() -> None:
+ payload = json.dumps({"cache_control": {"type": "ephemeral"}})
+ call = {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"id": "c1", "type": "function", "function": {"name": "write", "arguments": payload}}
+ ],
+ }
+ client = [call, {"role": "tool", "tool_call_id": "c1", "content": [_text(payload)]}]
+ replayed = copy.deepcopy(client)
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert out is replayed
+ assert out[1]["content"][0]["text"] == payload
+
+
+def test_reconciliation_is_idempotent() -> None:
+ client = [
+ _system(),
+ {"role": "user", "content": [_text("q")]},
+ {"role": "user", "content": [_text("r", CC)]},
+ ]
+ replayed = [
+ _system(),
+ {"role": "user", "content": [_text("q", CC)]},
+ {"role": "user", "content": [_text("r", CC)]},
+ ]
+
+ once = mirror_client_message_cache_control(replayed, client)
+ twice = mirror_client_message_cache_control(once, client)
+
+ assert twice is once
+
+
+# ── through the Chat Completions handler ────────────────────────────────────
+
+BIG = "synthetic result " * 400
+
+
+class _Tracker:
+ """Session tracker with last turn recorded; records what this turn stores."""
+
+ def __init__(self, previous_original, previous_forwarded, frozen: int): # noqa: ANN001
+ self._original = previous_original
+ self._forwarded = previous_forwarded
+ self._frozen = frozen
+ self.recorded: list[dict] | None = None
+
+ def get_frozen_message_count(self) -> int:
+ return self._frozen
+
+ def get_last_original_messages(self): # noqa: ANN201
+ return copy.deepcopy(self._original)
+
+ def get_last_forwarded_messages(self): # noqa: ANN201
+ return copy.deepcopy(self._forwarded)
+
+ def update_from_response(self, **kwargs): # noqa: ANN003
+ self.recorded = copy.deepcopy(kwargs.get("messages"))
+
+
+def _opencode_turns(string_user: bool = False) -> tuple[list[dict], list[dict], list[dict]]:
+ """The block-marked shape OpenCode V2 sends to Claude over Chat, two steps in.
+
+ Last step the client marked two system blocks and the user block, and the
+ proxy forwarded that. This step the client moved the moving breakpoint to the
+ tool result and left the user block unmarked. Live OpenCode V2 also resends
+ that older user turn as a plain string (``string_user``), so the replay
+ brings back a block form the client no longer sends.
+ """
+ user = {
+ "role": "user",
+ "content": "Read fixture.json" if string_user else [_text("Read fixture.json")],
+ }
+ tool = {"role": "tool", "tool_call_id": "c1", "content": [_text(BIG)]}
+ previous = [_system(), {"role": "user", "content": [_text("Read fixture.json", CC)]}]
+ current = [_system(), user, _call(), {**tool, "content": [_text(BIG, CC)]}]
+ return copy.deepcopy(previous), copy.deepcopy(previous), current
+
+
+def _app_config() -> ProxyConfig:
+ return ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+
+
+def _send(mode: str, frozen: int, stream: bool, turns=None): # noqa: ANN001, ANN202
+ previous_original, previous_forwarded, current = turns or _opencode_turns()
+ captured: dict = {}
+ tracker = _Tracker(previous_original, previous_forwarded, frozen)
+ with TestClient(create_app(_app_config())) as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = mode
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "s"
+ proxy.session_tracker_store.get_or_create = (
+ lambda session_id, provider, cache_ttl_seconds=None: tracker
+ )
+ proxy.openai_pipeline.apply = lambda **kwargs: SimpleNamespace(
+ messages=kwargs["messages"],
+ transforms_applied=[],
+ timing={},
+ tokens_before=100,
+ tokens_after=100,
+ waste_signals=None,
+ )
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ captured["body"] = json.loads(body) if isinstance(body, (bytes, str)) else body
+ return httpx.Response(
+ 200,
+ json={
+ "id": "c",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 100, "completion_tokens": 1, "total_tokens": 101},
+ },
+ )
+
+ async def _fake_stream(url, headers, body, *args, **kwargs): # noqa: ANN001, ANN002
+ captured["body"] = body
+ return httpx.Response(200, text="data: [DONE]\n\n")
+
+ proxy._retry_request = _fake_retry
+ proxy._stream_response = _fake_stream
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "claude-haiku-4-5", "stream": stream, "messages": current},
+ )
+ assert response.status_code == 200, response.text
+ return captured["body"], current, tracker
+
+
[email protected]("string_user", [False, True], ids=["block-user", "string-user"])
[email protected]("stream", [False, True], ids=["buffered", "streaming"])
[email protected]("mode,frozen", [("cache", 0), ("token", 2)])
+def test_forwarded_block_markers_follow_the_client(
+ mode: str, frozen: int, stream: bool, string_user: bool
+) -> None:
+ forwarded, current, _ = _send(mode, frozen, stream, _opencode_turns(string_user))
+
+ paths = _marker_paths(forwarded["messages"])
+ assert paths == _marker_paths(current) == ["0.content.0", "0.content.3", "3.content.0"]
+ if not string_user:
+ assert _strip_markers(forwarded["messages"]) == _strip_markers(current)
+ else:
+ # The replayed block form of the old user turn is what the provider cached.
+ assert forwarded["messages"][1]["content"] == [_text("Read fixture.json")]
+ assert _strip_markers(forwarded["messages"][2:]) == _strip_markers(current[2:])
+
+
+def test_tracker_records_the_reconciled_layout_so_turn_three_stays_bounded() -> None:
+ forwarded, current, tracker = _send("cache", 0, False)
+ assert tracker.recorded is not None
+ assert _marker_paths(tracker.recorded) == _marker_paths(forwarded["messages"])
+
+ # Turn 3 replays what turn 2 recorded; the moving breakpoint moves again.
+ tool2 = {"role": "tool", "tool_call_id": "c2", "content": [_text("second", CC)]}
+ turn3 = copy.deepcopy(current)
+ turn3[3]["content"][0].pop("cache_control")
+ turn3 += [_call("c2"), tool2]
+ forwarded3, _, _ = _send("cache", 0, False, (current, tracker.recorded, turn3))
+
+ assert _marker_paths(forwarded3["messages"]) == ["0.content.0", "0.content.3", "5.content.0"]
diff --git a/tests/test_openai_message_level_cache_control.py b/tests/test_openai_message_level_cache_control.py
index 86d6eeab4..418c3ebfe 100644
--- a/tests/test_openai_message_level_cache_control.py
+++ b/tests/test_openai_message_level_cache_control.py
@@ -117,7 +117,7 @@ def test_nothing_to_do_returns_the_same_list() -> None:
assert mirror_client_message_cache_control(messages, client) is messages
-def test_block_level_markers_are_left_to_the_block_normalizer() -> None:
+def test_block_markers_already_where_the_client_put_them_are_left_alone() -> None:
block = {"type": "text", "text": "x", "cache_control": CC}
client = [{"role": "user", "content": [dict(block)]}]
messages = [{"role": "user", "content": [dict(block)]}]