Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): cap cache_control breakpoints on OpenAI chat path (#4471) (#4491)
Parideboy55602fc1e207
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex b1cb4e61f..9caf9f858 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -4334,6 +4334,24 @@ class OpenAIHandlerMixin:optimized_messages, original_client_messages)+ from headroom.proxy.helpers import enforce_cache_breakpoint_budget++ # Bypass (x-headroom-bypass: true, _bypass computed at ~L3458) opts out+ # of ALL compression: never mutate those requests here.+ # client_messages=None: passing the client list runs+ # normalize_message_cache_control, whose block-level legacy fallback can+ # add a marker to a list-content user message on chat requests and then+ # cost a real system marker. Without it only the oldest-first drop runs+ # — stale replayed markers are oldest, so they go first.+ if not _bypass:+ _, optimized_messages, tools, _ = enforce_cache_breakpoint_budget(+ None,+ optimized_messages,+ tools,+ client_messages=None,+ request_id=request_id,+ )+# Memory: inject context and tools for OpenAI requests.## PR-A3 follow-up to A2: memory context now routes exclusively todiff --git a/headroom/proxy/helpers.py b/headroom/proxy/helpers.pyindex 69e72df59..d4c09bb82 100644--- a/headroom/proxy/helpers.py+++ b/headroom/proxy/helpers.py@@ -458,6 +458,12 @@ def count_cache_breakpoints(if not isinstance(msg, dict):continuefound = 1 if "cache_control" in msg else 0+ # OpenAI chat shape: markers on an assistant's tool_calls entries.+ tool_calls = msg.get("tool_calls")+ if isinstance(tool_calls, list):+ found += sum(+ 1 for tc in tool_calls if isinstance(tc, dict) and "cache_control" in tc+ )content = msg.get("content")if isinstance(content, list):for block in content:@@ -847,6 +853,16 @@ def enforce_cache_breakpoint_budget(budget["left"] -= 1new_blocks.append(block)msg = {**msg, "content": new_blocks}+ tool_calls = msg.get("tool_calls")+ if isinstance(tool_calls, list) and budget["left"] > 0:+ new_calls = []+ for tc in tool_calls:+ if budget["left"] > 0 and isinstance(tc, dict) and "cache_control" in tc:+ tc = {k: v for k, v in tc.items() if k != "cache_control"}+ budget["left"] -= 1+ new_calls.append(tc)+ if new_calls != tool_calls:+ msg = {**msg, "tool_calls": new_calls}if budget["left"] > 0 and "cache_control" in msg:msg = {k: v for k, v in msg.items() if k != "cache_control"}budget["left"] -= 1diff --git a/tests/gateway/test_cache_breakpoint_budget.py b/tests/gateway/test_cache_breakpoint_budget.pyindex 0c1b43aeb..d1f9830a2 100644--- a/tests/gateway/test_cache_breakpoint_budget.py+++ b/tests/gateway/test_cache_breakpoint_budget.py@@ -107,3 +107,72 @@ def test_gateway_body_is_held_to_four_markers(headroom_client) -> None:)assert data["messages"] == out["messages"]assert "cache_breakpoint_budget" in data["transforms_applied"]+++def test_openai_chat_replayed_markers_capped_oldest_first() -> None:+ # Issue #4471: a replayed frozen prefix keeps an old message-level marker+ # and the message count differs from the client's, so the mirror step+ # bails; the budget guard must still cap OpenAI chat shapes at four.+ client = [+ {"role": "system", "content": "s1", "cache_control": CC},+ {"role": "system", "content": "s2", "cache_control": CC},+ {"role": "user", "content": "read f1"},+ {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ {"id": "a", "type": "function", "function": {"name": "r", "arguments": "{}"}}+ ],+ },+ {"role": "tool", "tool_call_id": "a", "content": "f1 body", "cache_control": CC},+ ]+ tools = [{"type": "function", "function": {"name": "r"}, "cache_control": CC}]+ replayed = [dict(m) for m in client]+ replayed[3] = {+ **replayed[3],+ "tool_calls": [{**replayed[3]["tool_calls"][0], "cache_control": CC}],+ }+ replayed.insert(2, {"role": "user", "content": "earlier", "cache_control": CC})++ assert count_cache_breakpoints(None, replayed, tools)["total"] == 6+ _, messages, tools_out, stats = enforce_cache_breakpoint_budget(+ None, replayed, tools, client_messages=None+ )+ assert stats["repaired"]+ assert count_cache_breakpoints(None, messages, tools_out)["total"] <= 4+ assert "cache_control" in messages[-1]+++def test_openai_chat_budget_drops_stale_tool_marker_keeps_systems() -> None:+ # Issue #4471: on the OpenAI chat path client_messages=None so+ # normalize_message_cache_control's block-level legacy fallback cannot add+ # a marker to a list-content user message and then cost a real system+ # marker. Only the oldest-first drop runs, so stale replayed markers go+ # first while system markers and the latest tool anchor are kept.+ messages = [+ # older replayed tool result carrying a marker: oldest, dropped first+ {"role": "tool", "tool_call_id": "t1", "content": "stale", "cache_control": CC},+ {"role": "system", "content": "system one", "cache_control": CC},+ {"role": "system", "content": "system two", "cache_control": CC},+ {"role": "user", "content": [{"type": "text", "text": "hi"}]},+ # latest tool result carrying a marker: kept+ {"role": "tool", "tool_call_id": "t2", "content": "fresh", "cache_control": CC},+ ]+ tools = [{"type": "function", "function": {"name": "r"}, "cache_control": CC}]++ assert count_cache_breakpoints(None, messages, tools)["total"] == 5+ _, out, tools_out, stats = enforce_cache_breakpoint_budget(+ None, messages, tools, client_messages=None+ )+ assert stats["repaired"]+ assert count_cache_breakpoints(None, out, tools_out)["total"] <= 4+ assert "cache_control" in out[1]+ assert "cache_control" in out[2]+ assert "cache_control" in out[-1]+ assert "cache_control" not in out[0]+ assert out[3] == {"role": "user", "content": [{"type": "text", "text": "hi"}]}+++def test_count_includes_tool_calls_markers() -> None:+ msg = {"role": "assistant", "tool_calls": [{"id": "a", "cache_control": CC}]}+ assert count_cache_breakpoints(None, [msg], None)["total"] == 1