Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): keep cache_control outside content blocks where the Chat Completions client put it (#3895)

afernandez-stratioa6d6c14cf781
diff --git a/headroom/cache/prefix_tracker.py b/headroom/cache/prefix_tracker.py
index c9907bb4c..83ed16334 100644
--- a/headroom/cache/prefix_tracker.py
+++ b/headroom/cache/prefix_tracker.py
@@ -870,6 +870,77 @@ def normalize_message_cache_control(
return out if changed else messages
+def mirror_client_message_cache_control(
+ messages: list[dict[str, Any]],
+ client_messages: list[dict[str, Any]] | None,
+) -> list[dict[str, Any]]:
+ """Keep ``cache_control`` outside content blocks exactly where the client put it.
+
+ OpenAI-compatible clients that talk to Anthropic models through a gateway
+ (OpenCode via ``@ai-sdk/openai-compatible``, then LiteLLM) do not mark
+ content blocks. They mark the MESSAGE dict — ``{"role": "tool", "content":
+ "...", "cache_control": {...}}`` — and, on an assistant turn, the entries of
+ its ``tool_calls`` list, which the gateway turns into ``tool_use`` blocks.
+ ``normalize_message_cache_control`` only sees block markers
+ (``_client_marker_positions`` reads list content), so after
+ ``overlay_cached_prefix`` / ``finalize_turn`` replay an earlier turn's
+ forwarded messages, these markers ride along untouched. A client that marks
+ three places per request (system, the newest tool call, the newest tool
+ result) ends up forwarding four, then five and more, and the gateway's
+ translation to Anthropic blocks crosses the limit of four (``A maximum of 4
+ blocks with cache_control may be provided. Found 6``).
+
+ The client's current request is the authority for these markers, the same
+ rule the block-level normalizer applies: on each message and on each of its
+ ``tool_calls``, drop the marker unless the client put one there in this
+ request, in which case copy the client's own value. Content — and any
+ block-level marker in it — is left alone, so the replayed bytes the provider
+ cached stay byte-identical.
+
+ Index alignment is the invariant (the pipeline preserves message count and
+ each message's tool calls); where it does not hold, that message or list is
+ returned unchanged rather than guessing, and the breakpoint budget guard
+ still bounds the total.
+ """
+ if not isinstance(client_messages, list) or len(client_messages) != len(messages):
+ return messages
+ changed = False
+ out: list[dict[str, Any]] = []
+ for msg, client in zip(messages, client_messages):
+ if not isinstance(msg, dict):
+ out.append(msg)
+ continue
+ new_msg, msg_changed = _mirror_marker(msg, client)
+ calls = new_msg.get("tool_calls")
+ client_calls = client.get("tool_calls") if isinstance(client, dict) else None
+ if (
+ isinstance(calls, list)
+ and isinstance(client_calls, list)
+ and len(calls) == len(client_calls)
+ ):
+ mirrored = [_mirror_marker(c, cc) for c, cc in zip(calls, client_calls)]
+ if any(c for _, c in mirrored):
+ new_msg = {**new_msg, "tool_calls": [c for c, _ in mirrored]}
+ msg_changed = True
+ out.append(new_msg)
+ changed = changed or msg_changed
+ return out if changed else messages
+
+
+def _mirror_marker(holder: Any, client_holder: Any) -> tuple[Any, bool]:
+ """Give ``holder`` the client's own ``cache_control`` (or none); report change."""
+ if not isinstance(holder, dict):
+ return holder, False
+ client_marker = client_holder.get("cache_control") if isinstance(client_holder, dict) else None
+ if isinstance(client_marker, dict):
+ if holder.get("cache_control") == client_marker:
+ return holder, False
+ return {**holder, "cache_control": dict(client_marker)}, True
+ if "cache_control" in holder:
+ return {k: v for k, v in holder.items() if k != "cache_control"}, True
+ return holder, False
+
+
class PrefixCacheTracker:
"""Tracks provider prefix cache state across turns in a session.
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index d1c30db8c..525dc7dd1 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4350,6 +4350,19 @@ class OpenAIHandlerMixin:
confirmed_frozen_count=_openai_confirmed_frozen,
).messages
+ # Both replays above forward earlier turns' messages byte-identical,
+ # including the cache_control each carried back then. Chat Completions
+ # clients mark the message dict and an assistant's tool_calls, which the
+ # block-level normalizer does not see, so without this the markers
+ # accumulate turn over turn and an Anthropic-backed gateway rejects the
+ # request (more than four breakpoints). Content is untouched: only the
+ # markers move.
+ from headroom.cache.prefix_tracker import mirror_client_message_cache_control
+
+ optimized_messages = mirror_client_message_cache_control(
+ optimized_messages, original_client_messages
+ )
+
# Memory: inject context and tools for OpenAI requests.
#
# PR-A3 follow-up to A2: memory context now routes exclusively to
diff --git a/tests/test_openai_message_level_cache_control.py b/tests/test_openai_message_level_cache_control.py
new file mode 100644
index 000000000..d137f3b97
--- /dev/null
+++ b/tests/test_openai_message_level_cache_control.py
@@ -0,0 +1,266 @@
+"""``cache_control`` outside content blocks must not accumulate across replayed turns.
+
+Chat Completions clients that reach Anthropic models through a gateway (OpenCode
+via ``@ai-sdk/openai-compatible``, then LiteLLM) put breakpoints on the message
+dict and on an assistant's ``tool_calls`` entries, not on a content block, and
+move them to the newest messages on every call.
+The prefix replay (``finalize_turn`` / ``overlay_cached_prefix``) forwards earlier
+turns' messages byte-identical, so the markers they carried back then came along
+and piled up behind the client's current ones. Two markers from the client became
+four and more on the wire, the gateway translated them into Anthropic blocks, and the
+request failed with ``A maximum of 4 blocks with cache_control may be provided``.
+"""
+
+from __future__ import annotations
+
+import copy
+import json
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient
+
+from headroom.cache.prefix_tracker import mirror_client_message_cache_control
+from headroom.proxy.server import ProxyConfig, create_app
+
+CC = {"type": "ephemeral"}
+
+
+def _marked(msg: dict, marker: dict | None = CC) -> dict:
+ return {**msg, "cache_control": dict(marker)} if marker else dict(msg)
+
+
+def _message_markers(messages: list[dict]) -> list[int]:
+ return [i for i, m in enumerate(messages) if isinstance(m, dict) and "cache_control" in m]
+
+
+def _all_markers(messages: list[dict]) -> list[str]:
+ """Every marker outside content blocks: ``"i"`` on a message, ``"i.j"`` on a tool call."""
+ found = []
+ for i, m in enumerate(messages):
+ if "cache_control" in m:
+ found.append(str(i))
+ for j, call in enumerate(m.get("tool_calls") or []):
+ if "cache_control" in call:
+ found.append(f"{i}.{j}")
+ return found
+
+
+# ── the helper ──────────────────────────────────────────────────────────────
+
+
+def test_replayed_markers_are_dropped_and_the_clients_are_kept() -> None:
+ client = [
+ _marked({"role": "system", "content": "sys"}),
+ {"role": "user", "content": "q"},
+ {"role": "tool", "tool_call_id": "t3", "content": "old output"},
+ _marked({"role": "tool", "tool_call_id": "t5", "content": "new output"}),
+ ]
+ # What a replay produced: last turn's markers on user and tool 3 came along.
+ replayed = [
+ _marked({"role": "system", "content": "sys"}),
+ _marked({"role": "user", "content": "q"}),
+ _marked({"role": "tool", "tool_call_id": "t3", "content": "old <compressed>"}),
+ {"role": "tool", "tool_call_id": "t5", "content": "new output"},
+ ]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert _message_markers(out) == [0, 3]
+ # Only the marker moves: the replayed (compressed) content is what the
+ # provider cached, and it must stay byte-identical.
+ assert [m["content"] for m in out] == [m["content"] for m in replayed]
+
+
+def test_replayed_tool_call_markers_are_dropped_and_the_clients_are_kept() -> None:
+ def call(cid: str, marker: bool) -> dict:
+ c = {"id": cid, "type": "function", "function": {"name": "bash", "arguments": "{}"}}
+ return _marked(c) if marker else c
+
+ client = [
+ {"role": "assistant", "content": "", "tool_calls": [call("t3", False)]},
+ {"role": "assistant", "content": "", "tool_calls": [call("t5", True)]},
+ ]
+ # Last step's newest call (t3) carried the marker; the replay brought it back.
+ replayed = [
+ {"role": "assistant", "content": "", "tool_calls": [call("t3", True)]},
+ {"role": "assistant", "content": "", "tool_calls": [call("t5", True)]},
+ ]
+
+ out = mirror_client_message_cache_control(replayed, client)
+
+ assert _all_markers(out) == ["1.0"]
+ assert out[1] is replayed[1] # already right: left as-is
+
+
+def test_tool_calls_of_different_length_are_left_alone() -> None:
+ c = {"id": "t", "type": "function", "function": {"name": "bash", "arguments": "{}"}}
+ client = [{"role": "assistant", "content": "", "tool_calls": [c]}]
+ messages = [{"role": "assistant", "content": "", "tool_calls": [_marked(c), _marked(c)]}]
+ assert mirror_client_message_cache_control(messages, client) is messages
+
+
+def test_the_clients_marker_value_is_copied_verbatim() -> None:
+ ttl = {"type": "ephemeral", "ttl": "1h"}
+ client = [_marked({"role": "user", "content": "q"}, ttl)]
+ out = mirror_client_message_cache_control([_marked({"role": "user", "content": "q"})], client)
+ assert out[0]["cache_control"] == ttl
+
+
+def test_nothing_to_do_returns_the_same_list() -> None:
+ client = [_marked({"role": "system", "content": "sys"}), {"role": "user", "content": "q"}]
+ messages = copy.deepcopy(client)
+ assert mirror_client_message_cache_control(messages, client) is messages
+
+
+def test_block_level_markers_are_left_to_the_block_normalizer() -> None:
+ block = {"type": "text", "text": "x", "cache_control": CC}
+ client = [{"role": "user", "content": [dict(block)]}]
+ messages = [{"role": "user", "content": [dict(block)]}]
+ out = mirror_client_message_cache_control(messages, client)
+ assert out is messages
+ assert out[0]["content"][0]["cache_control"] == CC
+
+
+def test_misaligned_lists_are_returned_unchanged() -> None:
+ messages = [
+ _marked({"role": "user", "content": "a"}),
+ _marked({"role": "user", "content": "b"}),
+ ]
+ assert mirror_client_message_cache_control(messages, messages[:1]) is messages
+ assert mirror_client_message_cache_control(messages, None) is messages
+
+
+# ── through the Chat Completions handler ────────────────────────────────────
+
+BIG = "row," * 4000
+
+
+class _Tracker:
+ """Stands in for the session tracker with last turn already recorded."""
+
+ def __init__(self, previous_original: list[dict], previous_forwarded: list[dict]):
+ self._original = previous_original
+ self._forwarded = previous_forwarded
+
+ def get_frozen_message_count(self) -> int:
+ return 0
+
+ def get_last_original_messages(self): # noqa: ANN201
+ return copy.deepcopy(self._original)
+
+ def get_last_forwarded_messages(self): # noqa: ANN201
+ return copy.deepcopy(self._forwarded)
+
+ def update_from_response(self, **kwargs): # noqa: ANN003
+ return None
+
+
+def _turns() -> tuple[list[dict], list[dict], list[dict]]:
+ """The shape OpenCode sends to a Claude model, three steps into one turn.
+
+ Last step the client marked the system prompt, the newest tool call (t3) and
+ its result (tool 3), and headroom forwarded tool 3 compressed. This step the
+ client moved both markers to t5 and tool 5 and left t3 and tool 3 unmarked.
+ """
+ system = {"role": "system", "content": "You are an agent. " * 50}
+ user = {"role": "user", "content": "run three commands"}
+ call3 = {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"id": "t3", "type": "function", "function": {"name": "bash", "arguments": "{}"}}
+ ],
+ }
+ tool3 = {"role": "tool", "tool_call_id": "t3", "content": BIG}
+ call5 = {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {"id": "t5", "type": "function", "function": {"name": "bash", "arguments": "{}"}}
+ ],
+ }
+ tool5 = {"role": "tool", "tool_call_id": "t5", "content": "30"}
+
+ def with_marked_call(assistant: dict) -> dict:
+ return {**assistant, "tool_calls": [_marked(assistant["tool_calls"][0])]}
+
+ previous_original = [_marked(system), user, with_marked_call(call3), _marked(tool3)]
+ previous_forwarded = [
+ _marked(system),
+ user,
+ with_marked_call(call3),
+ _marked({**tool3, "content": "row,<compressed: 4000 rows>"}),
+ ]
+ current = [_marked(system), user, call3, tool3, with_marked_call(call5), _marked(tool5)]
+ return previous_original, previous_forwarded, current
+
+
[email protected]("mode", ["cache", "token"])
+def test_forwarded_markers_follow_the_client_not_the_replay(mode: str) -> None:
+ previous_original, previous_forwarded, current = _turns()
+ captured: dict = {}
+
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+ proxy.config.optimize = True
+ proxy.config.mode = mode
+ tracker = _Tracker(previous_original, previous_forwarded)
+ proxy.session_tracker_store.compute_session_id = lambda request, model, messages: "s"
+ proxy.session_tracker_store.get_or_create = lambda session_id, provider: tracker
+ proxy.openai_pipeline.apply = lambda **kwargs: SimpleNamespace(
+ messages=kwargs["messages"],
+ transforms_applied=[],
+ timing={},
+ tokens_before=100,
+ tokens_after=100,
+ waste_signals=None,
+ )
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ captured["body"] = json.loads(body) if isinstance(body, (bytes, str)) else body
+ return httpx.Response(
+ 200,
+ json={
+ "id": "c",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "ok"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 100, "completion_tokens": 1, "total_tokens": 101},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+
+ response = client.post(
+ "/v1/chat/completions",
+ headers={"authorization": "Bearer test-key"},
+ json={"model": "claude-sonnet-4-6", "messages": current},
+ )
+
+ assert response.status_code == 200, response.text
+ forwarded = captured["body"]["messages"]
+ # Precondition: the replay was live — tool 3 went out compressed.
+ assert forwarded[3]["content"] != BIG
+ # The client marked the system prompt, t5 and tool 5; last step's markers on
+ # t3 and tool 3 must not ride along with the replayed bytes.
+ assert _all_markers(forwarded) == _all_markers(current) == ["0", "4.0", "5"]