Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): cap cache_control breakpoints on OpenAI chat path (#4471) (#4491)

Parideboy55602fc1e207
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index b1cb4e61f..9caf9f858 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -4334,6 +4334,24 @@ class OpenAIHandlerMixin:
optimized_messages, original_client_messages
)
+ from headroom.proxy.helpers import enforce_cache_breakpoint_budget
+
+ # Bypass (x-headroom-bypass: true, _bypass computed at ~L3458) opts out
+ # of ALL compression: never mutate those requests here.
+ # client_messages=None: passing the client list runs
+ # normalize_message_cache_control, whose block-level legacy fallback can
+ # add a marker to a list-content user message on chat requests and then
+ # cost a real system marker. Without it only the oldest-first drop runs
+ # — stale replayed markers are oldest, so they go first.
+ if not _bypass:
+ _, optimized_messages, tools, _ = enforce_cache_breakpoint_budget(
+ None,
+ optimized_messages,
+ tools,
+ client_messages=None,
+ request_id=request_id,
+ )
+
# Memory: inject context and tools for OpenAI requests.
#
# PR-A3 follow-up to A2: memory context now routes exclusively to
diff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.py
index 69e72df59..d4c09bb82 100644
--- a/headroom/proxy/helpers.py
+++ b/headroom/proxy/helpers.py
@@ -458,6 +458,12 @@ def count_cache_breakpoints(
if not isinstance(msg, dict):
continue
found = 1 if "cache_control" in msg else 0
+ # OpenAI chat shape: markers on an assistant's tool_calls entries.
+ tool_calls = msg.get("tool_calls")
+ if isinstance(tool_calls, list):
+ found += sum(
+ 1 for tc in tool_calls if isinstance(tc, dict) and "cache_control" in tc
+ )
content = msg.get("content")
if isinstance(content, list):
for block in content:
@@ -847,6 +853,16 @@ def enforce_cache_breakpoint_budget(
budget["left"] -= 1
new_blocks.append(block)
msg = {**msg, "content": new_blocks}
+ tool_calls = msg.get("tool_calls")
+ if isinstance(tool_calls, list) and budget["left"] > 0:
+ new_calls = []
+ for tc in tool_calls:
+ if budget["left"] > 0 and isinstance(tc, dict) and "cache_control" in tc:
+ tc = {k: v for k, v in tc.items() if k != "cache_control"}
+ budget["left"] -= 1
+ new_calls.append(tc)
+ if new_calls != tool_calls:
+ msg = {**msg, "tool_calls": new_calls}
if budget["left"] > 0 and "cache_control" in msg:
msg = {k: v for k, v in msg.items() if k != "cache_control"}
budget["left"] -= 1
diff --git a/tests/gateway/test_cache_breakpoint_budget.py b/tests/gateway/test_cache_breakpoint_budget.py
index 0c1b43aeb..d1f9830a2 100644
--- a/tests/gateway/test_cache_breakpoint_budget.py
+++ b/tests/gateway/test_cache_breakpoint_budget.py
@@ -107,3 +107,72 @@ def test_gateway_body_is_held_to_four_markers(headroom_client) -> None:
)
assert data["messages"] == out["messages"]
assert "cache_breakpoint_budget" in data["transforms_applied"]
+
+
+def test_openai_chat_replayed_markers_capped_oldest_first() -> None:
+ # Issue #4471: a replayed frozen prefix keeps an old message-level marker
+ # and the message count differs from the client's, so the mirror step
+ # bails; the budget guard must still cap OpenAI chat shapes at four.
+ client = [
+ {"role": "system", "content": "s1", "cache_control": CC},
+ {"role": "system", "content": "s2", "cache_control": CC},
+ {"role": "user", "content": "read f1"},
+ {
+ "role": "assistant",
+ "content": None,
+ "tool_calls": [
+ {"id": "a", "type": "function", "function": {"name": "r", "arguments": "{}"}}
+ ],
+ },
+ {"role": "tool", "tool_call_id": "a", "content": "f1 body", "cache_control": CC},
+ ]
+ tools = [{"type": "function", "function": {"name": "r"}, "cache_control": CC}]
+ replayed = [dict(m) for m in client]
+ replayed[3] = {
+ **replayed[3],
+ "tool_calls": [{**replayed[3]["tool_calls"][0], "cache_control": CC}],
+ }
+ replayed.insert(2, {"role": "user", "content": "earlier", "cache_control": CC})
+
+ assert count_cache_breakpoints(None, replayed, tools)["total"] == 6
+ _, messages, tools_out, stats = enforce_cache_breakpoint_budget(
+ None, replayed, tools, client_messages=None
+ )
+ assert stats["repaired"]
+ assert count_cache_breakpoints(None, messages, tools_out)["total"] <= 4
+ assert "cache_control" in messages[-1]
+
+
+def test_openai_chat_budget_drops_stale_tool_marker_keeps_systems() -> None:
+ # Issue #4471: on the OpenAI chat path client_messages=None so
+ # normalize_message_cache_control's block-level legacy fallback cannot add
+ # a marker to a list-content user message and then cost a real system
+ # marker. Only the oldest-first drop runs, so stale replayed markers go
+ # first while system markers and the latest tool anchor are kept.
+ messages = [
+ # older replayed tool result carrying a marker: oldest, dropped first
+ {"role": "tool", "tool_call_id": "t1", "content": "stale", "cache_control": CC},
+ {"role": "system", "content": "system one", "cache_control": CC},
+ {"role": "system", "content": "system two", "cache_control": CC},
+ {"role": "user", "content": [{"type": "text", "text": "hi"}]},
+ # latest tool result carrying a marker: kept
+ {"role": "tool", "tool_call_id": "t2", "content": "fresh", "cache_control": CC},
+ ]
+ tools = [{"type": "function", "function": {"name": "r"}, "cache_control": CC}]
+
+ assert count_cache_breakpoints(None, messages, tools)["total"] == 5
+ _, out, tools_out, stats = enforce_cache_breakpoint_budget(
+ None, messages, tools, client_messages=None
+ )
+ assert stats["repaired"]
+ assert count_cache_breakpoints(None, out, tools_out)["total"] <= 4
+ assert "cache_control" in out[1]
+ assert "cache_control" in out[2]
+ assert "cache_control" in out[-1]
+ assert "cache_control" not in out[0]
+ assert out[3] == {"role": "user", "content": [{"type": "text", "text": "hi"}]}
+
+
+def test_count_includes_tool_calls_markers() -> None:
+ msg = {"role": "assistant", "tool_calls": [{"id": "a", "cache_control": CC}]}
+ assert count_cache_breakpoints(None, [msg], None)["total"] == 1