Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ccr): serve headroom_retrieve when the model calls it alongside client tools (#4034)
gglucass01d919485423
diff --git a/headroom/ccr/response_handler.py b/headroom/ccr/response_handler.pyindex 417141dcd..f9bcab143 100644--- a/headroom/ccr/response_handler.py+++ b/headroom/ccr/response_handler.py@@ -24,8 +24,10 @@ from headroom.proxy.anthropic_wire import AnthropicSSEEnvelopefrom ..cache.compression_store import format_retrieval_miss_detail, get_compression_storefrom .tool_calls import (CCRToolCall,+ drop_tool_calls,extract_tool_calls,has_ccr_tool_calls,+ is_ccr_tool_call,parse_ccr_tool_calls,)from .tool_injection import CCR_TOOL_NAME@@ -476,18 +478,41 @@ class CCRResponseHandler:# No CCR tool calls, we're donebreak- # If the model called CCR alongside non-CCR tools, we cannot build- # a valid continuation — every tool_use in the assistant message- # requires a matching tool_result, but we only have CCR results.- # Skip CCR handling and let the client resolve all tool calls.+ # The model called CCR alongside client tools. The client has no+ # headroom_retrieve (Claude Code answers "No such tool available:+ # headroom_retrieve", so the model never gets the content), and a+ # continuation needs a tool_result for every tool_use. The client+ # calls have not run yet, so drop them and serve the retrieval now;+ # the model decides again, content in hand, in the continuation.+ # That continuation replaces this turn whatever it does: re-issue+ # the calls, change them or drop them. Only a failed continuation+ # hands this turn back (below), since the model made no newer+ # decision. A CCR-named call without a valid hash, or a provider+ # where dropping a sibling is not safe (see drop_tool_calls), keeps+ # the turn as it is for the client to resolve.+ mixed_turn: dict[str, Any] | None = Noneif other_calls:- logger.warning(- "CCR: Skipping CCR handling — model called %d non-CCR tool(s) "- "alongside headroom_retrieve. Cannot create a valid continuation "- "without results for the other tools. Client must handle all tool calls.",+ trimmed = (+ current_response+ if any(is_ccr_tool_call(c) for c in other_calls)+ else drop_tool_calls(current_response, provider, other_calls)+ )+ if trimmed is current_response:+ logger.warning(+ "CCR: Skipping CCR handling — model called %d non-CCR tool(s) "+ "alongside headroom_retrieve. Cannot create a valid continuation "+ "without results for the other tools. Client must handle all tool calls.",+ len(other_calls),+ )+ break+ logger.info(+ "CCR: model called headroom_retrieve alongside %d client tool(s); "+ "serving the retrieval and dropping the unrun client call(s) "+ "for the model to re-issue",len(other_calls),)- break+ mixed_turn = current_response+ current_response = trimmedrounds += 1with self._retrieval_count_lock:@@ -540,7 +565,11 @@ class CCRResponseHandler:# entirely (#3129).logger.error("CCR: Continuation API call failed: %s: %r", type(e).__name__, e)# Return the response we had (with unhandled CCR calls)- # The client will see the tool_use and might handle it differently+ # The client will see the tool_use and might handle it differently.+ # For a mixed turn that is the model's own turn, client calls+ # included (#839), never the trimmed one that lost them.+ if mixed_turn is not None:+ current_response = mixed_turnbreakif rounds >= self.config.max_retrieval_rounds:diff --git a/headroom/ccr/tool_calls.py b/headroom/ccr/tool_calls.pyindex 44d75ff34..8318b0564 100644--- a/headroom/ccr/tool_calls.py+++ b/headroom/ccr/tool_calls.py@@ -70,6 +70,36 @@ def extract_tool_calls(response: dict[str, Any], provider: str) -> list[dict[strreturn []+def drop_tool_calls(+ response: dict[str, Any],+ provider: str,+ calls: list[dict[str, Any]],+) -> dict[str, Any]:+ """Return a shallow copy of ``response`` without ``calls``.++ ``calls`` are objects ``extract_tool_calls`` returned for this response and+ are matched by identity. Anthropic and OpenAI chat only: a Responses+ ``function_call`` can be the item a ``reasoning`` item requires next, and a+ Gemini ``functionCall`` part can carry the turn's thought signature, so+ removing one there can make the continuation invalid. Other providers get+ ``response`` back unchanged.+ """+ drop = {id(call) for call in calls}+ if provider == "anthropic":+ content = response.get("content")+ if isinstance(content, list):+ return {**response, "content": [b for b in content if id(b) not in drop]}+ elif provider == "openai":+ choices = response.get("choices")+ if isinstance(choices, list) and choices and isinstance(choices[0], dict):+ message = choices[0].get("message")+ if isinstance(message, dict) and isinstance(message.get("tool_calls"), list):+ tool_calls = [c for c in message["tool_calls"] if id(c) not in drop]+ first = {**choices[0], "message": {**message, "tool_calls": tool_calls}}+ return {**response, "choices": [first, *choices[1:]]}+ return response++def is_ccr_tool_call(tool_call: dict[str, Any]) -> bool:"""Return true when a provider-native tool call names the CCR retrieval tool."""return (diff --git a/tests/test_ccr_response_handler.py b/tests/test_ccr_response_handler.pyindex 9621efacc..60798e663 100644--- a/tests/test_ccr_response_handler.py+++ b/tests/test_ccr_response_handler.py@@ -523,14 +523,13 @@ class TestCCRResponseHandling:assert result == response@pytest.mark.asyncio- async def test_handle_response_mixed_tools_skips_ccr(self):- """When CCR and non-CCR tools are called together, skip CCR.-- Building a valid continuation is impossible without results for the- non-CCR tools (Anthropic requires every tool_use to have a- tool_result). Skipping CCR avoids a wasted 400 API call and returns- the original response immediately so the client can resolve all- tool calls itself.+ async def test_handle_response_mixed_tools_serves_retrieval(self):+ """CCR called alongside a client tool: serve the retrieval now.++ The client has no headroom_retrieve, so handing the turn back made+ Claude Code answer "No such tool available: headroom_retrieve" and the+ model never got the content. The client tool has not run yet, so it is+ dropped from the continuation and the model re-issues it there."""store = get_compression_store()hash_key = store.store(original="[1,2,3]", compressed="[]")@@ -539,6 +538,7 @@ class TestCCRResponseHandling:mixed_response = {"content": [+ {"type": "text", "text": "Reading both."},{"type": "tool_use","id": "ccr_call",@@ -551,21 +551,269 @@ class TestCCRResponseHandling:"name": "read_file","input": {"path": "/etc/config"},},+ ],+ "stop_reason": "tool_use",+ }+ reissued = {+ "content": [+ {+ "type": "tool_use",+ "id": "user_call_2",+ "name": "read_file",+ "input": {"path": "/etc/config"},+ }+ ],+ "stop_reason": "tool_use",+ }+ sent = []++ async def mock_api_call(messages, tools):+ sent.append(messages)+ return reissued++ result = await handler.handle_response(mixed_response, [], None, mock_api_call, "anthropic")++ assert len(sent) == 1+ assistant, tool_result = sent[0]+ assert [b.get("id") for b in assistant["content"]] == [None, "ccr_call"]+ assert tool_result["content"][0]["tool_use_id"] == "ccr_call"+ assert "[1,2,3]" in tool_result["content"][0]["content"]+ # The client gets the re-issued client tool, never headroom_retrieve.+ assert result is reissued+ assert handler.residual_ccr_status(result, "anthropic") == "resolved"+ # The model's own response is not mutated.+ assert len(mixed_response["content"]) == 3++ @pytest.mark.asyncio+ async def test_handle_response_mixed_tools_serves_retrieval_openai(self):+ store = get_compression_store()+ hash_key = store.store(original="[1,2,3]", compressed="[]")++ handler = CCRResponseHandler()++ def call(call_id, name, arguments):+ return {+ "id": call_id,+ "type": "function",+ "function": {"name": name, "arguments": arguments},+ }++ mixed_response = {+ "choices": [+ {+ "message": {+ "role": "assistant",+ "content": None,+ "tool_calls": [+ call("call_user", "read_file", '{"path": "/etc/config"}'),+ call("call_ccr", CCR_TOOL_NAME, json.dumps({"hash": hash_key})),+ ],+ },+ "finish_reason": "tool_calls",+ }]}+ final = {+ "choices": [+ {+ "message": {+ "role": "assistant",+ "content": None,+ "tool_calls": [call("call_user_2", "read_file", '{"path":"/etc/config"}')],+ },+ "finish_reason": "tool_calls",+ }+ ]+ }+ sent = []- api_call_count = 0+ async def mock_api_call(messages, tools):+ sent.append(messages)+ return final++ result = await handler.handle_response(mixed_response, [], None, mock_api_call, "openai")++ assert result is final+ assistant, tool_result = sent[0]+ assert [c["id"] for c in assistant["tool_calls"]] == ["call_ccr"]+ assert tool_result["tool_call_id"] == "call_ccr"++ @staticmethod+ def _mixed_turn(hash_key, client_input):+ return {+ "content": [+ {+ "type": "tool_use",+ "id": "ccr_call",+ "name": CCR_TOOL_NAME,+ "input": {"hash": hash_key},+ },+ {+ "type": "tool_use",+ "id": "user_call",+ "name": "write_file",+ "input": client_input,+ },+ ],+ "stop_reason": "tool_use",+ }++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "continuation",+ [+ # Withdraws the client call once it has the content.+ {+ "content": [{"type": "text", "text": "Do not overwrite old.txt."}],+ "stop_reason": "end_turn",+ },+ # Changes it.+ {+ "content": [+ {+ "type": "tool_use",+ "id": "user_call_2",+ "name": "write_file",+ "input": {"path": "new.txt", "content": "safe output"},+ }+ ],+ "stop_reason": "tool_use",+ },+ # Re-issues it unchanged.+ {+ "content": [+ {+ "type": "tool_use",+ "id": "user_call_2",+ "name": "write_file",+ "input": {"path": "old.txt", "content": "overwrite"},+ }+ ],+ "stop_reason": "tool_use",+ },+ ],+ ids=["withdrawn", "changed", "reissued"],+ )+ async def test_handle_response_mixed_tools_returns_the_continuation(self, continuation):+ """The continuation is the model's newer decision, made with the+ retrieved content: the client gets it, never the dropped call."""+ store = get_compression_store()+ hash_key = store.store(original="[1,2,3]", compressed="[]")+ mixed_response = self._mixed_turn(hash_key, {"path": "old.txt", "content": "overwrite"})async def mock_api_call(messages, tools):- nonlocal api_call_count- api_call_count += 1- return {"content": [{"type": "text", "text": "continuation"}]}+ return continuation- result = await handler.handle_response(mixed_response, [], None, mock_api_call, "anthropic")+ result = await CCRResponseHandler().handle_response(+ mixed_response, [], None, mock_api_call, "anthropic"+ )++ assert result is continuation+ assert "user_call" not in [b.get("id") for b in result["content"]]++ @pytest.mark.asyncio+ async def test_handle_response_mixed_tools_failed_continuation_returns_turn(self):+ """No newer decision: the client gets the model's own turn, client+ call included, as before (#839)."""+ store = get_compression_store()+ hash_key = store.store(original="[1,2,3]", compressed="[]")+ mixed_response = self._mixed_turn(hash_key, {"path": "/tmp/out"})++ async def mock_api_call(messages, tools):+ raise RuntimeError("upstream down")++ result = await CCRResponseHandler().handle_response(+ mixed_response, [], None, mock_api_call, "anthropic"+ )++ assert result is mixed_response++ @pytest.mark.asyncio+ async def test_handle_response_mixed_tools_later_failure_keeps_the_continuation(self):+ """A continuation that succeeded supersedes the mixed turn even when a+ later retrieval round fails."""+ store = get_compression_store()+ hash_key = store.store(original="[1,2,3]", compressed="[]")+ mixed_response = self._mixed_turn(hash_key, {"path": "/tmp/out"})+ retrieve_again = {+ "content": [+ {+ "type": "tool_use",+ "id": "ccr_call_2",+ "name": CCR_TOOL_NAME,+ "input": {"hash": hash_key},+ }+ ],+ "stop_reason": "tool_use",+ }+ calls = 0++ async def mock_api_call(messages, tools):+ nonlocal calls+ calls += 1+ if calls > 1:+ raise RuntimeError("upstream down")+ return retrieve_again++ result = await CCRResponseHandler().handle_response(+ mixed_response, [], None, mock_api_call, "anthropic"+ )++ assert calls == 2+ assert result is retrieve_again++ @pytest.mark.asyncio+ async def test_handle_response_malformed_ccr_call_keeps_turn(self):+ """A headroom_retrieve without a valid hash is not a client call to+ drop: the turn is handed back unchanged."""+ store = get_compression_store()+ hash_key = store.store(original="[1,2,3]", compressed="[]")+ mixed_response = self._mixed_turn(hash_key, {})+ mixed_response["content"][1]["name"] = CCR_TOOL_NAME++ async def mock_api_call(messages, tools):+ raise AssertionError("no continuation when a CCR call is malformed")++ result = await CCRResponseHandler().handle_response(+ mixed_response, [], None, mock_api_call, "anthropic"+ )++ assert result is mixed_response++ @pytest.mark.asyncio+ async def test_handle_response_mixed_tools_responses_still_skips(self):+ """Responses API: a function_call can be the item a reasoning item+ requires next, so a sibling is not dropped there; the turn is handed+ back unchanged as before (#839)."""+ store = get_compression_store()+ hash_key = store.store(original="[1,2,3]", compressed="[]")++ handler = CCRResponseHandler()+ mixed_response = {+ "output": [+ {"type": "reasoning", "id": "rs_1", "summary": []},+ {+ "type": "function_call",+ "call_id": "call_user",+ "name": "read_file",+ "arguments": "{}",+ },+ {+ "type": "function_call",+ "call_id": "call_ccr",+ "name": CCR_TOOL_NAME,+ "arguments": json.dumps({"hash": hash_key}),+ },+ ]+ }++ async def mock_api_call(messages, tools):+ raise AssertionError("no continuation for a Responses mixed turn")++ result = await handler.handle_response(+ mixed_response, [], None, mock_api_call, "openai_responses"+ )- # CCR skipped — no continuation call made (avoids the 400 API round-trip)- assert api_call_count == 0, "should not attempt continuation with mixed tools"- # Original response returned unchanged so client can handle all tool callsassert result is mixed_responsediff --git a/tests/test_ccr_tool_calls.py b/tests/test_ccr_tool_calls.pyindex bf2522b42..b469f5dfe 100644--- a/tests/test_ccr_tool_calls.py+++ b/tests/test_ccr_tool_calls.py@@ -2,6 +2,7 @@ from __future__ import annotationsfrom headroom.ccr.tool_calls import (CCRToolCall,+ drop_tool_calls,extract_tool_calls,has_ccr_tool_calls,parse_ccr_tool_calls,@@ -114,3 +115,33 @@ def test_tool_call_id_for_provider_models_matching_result_ids() -> None:== "call_1")assert tool_call_id_for_provider({"id": "tool_1"}, "anthropic") == "tool_1"+++def test_drop_tool_calls_removes_only_the_given_calls() -> None:+ keep = {"type": "tool_use", "id": "r", "name": CCR_TOOL_NAME}+ drop = {"type": "tool_use", "id": "b", "name": "Read"}+ anthropic = {"content": [{"type": "text", "text": "hi"}, keep, drop], "stop_reason": "x"}+ out = drop_tool_calls(anthropic, "anthropic", [drop])+ assert out["content"] == [{"type": "text", "text": "hi"}, keep]+ assert out["stop_reason"] == "x"+ assert len(anthropic["content"]) == 3++ call = {"id": "c", "function": {"name": "Read", "arguments": "{}"}}+ openai = {"choices": [{"message": {"tool_calls": [call]}}, {"index": 1}]}+ out = drop_tool_calls(openai, "openai", [call])+ assert out["choices"] == [{"message": {"tool_calls": []}}, {"index": 1}]+ assert openai["choices"][0]["message"]["tool_calls"] == [call]+++def test_drop_tool_calls_returns_unrecognized_shapes_unchanged() -> None:+ """Callers detect "cannot drop here" by identity."""+ for response, provider in [+ ({"content": "text"}, "anthropic"),+ ({"choices": []}, "openai"),+ ({"choices": [None]}, "openai"),+ ({"choices": [{"message": None}]}, "openai"),+ ({"choices": [{"message": {"content": "no tools"}}]}, "openai"),+ ({"output": []}, "openai_responses"),+ ({"candidates": []}, "google"),+ ]:+ assert drop_tool_calls(response, provider, []) is responsediff --git a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.pyindex e36e2ab09..2bf8110e5 100644--- a/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py+++ b/tests/test_proxy/test_anthropic_streaming_ccr_retrieve.py@@ -84,8 +84,9 @@ def _message_response(content: list[dict], *, stop_reason: str = "end_turn") ->class _ContinuationClient:- def __init__(self, response_json: dict) -> None:+ def __init__(self, response_json: dict, status_code: int = 200) -> None:self.response_json = response_json+ self.status_code = status_codeself.post_calls: list[dict] = []async def post(self, url, *, content=None, headers=None, timeout=None): # noqa: ANN001@@ -97,7 +98,7 @@ class _ContinuationClient:"timeout": timeout,})- return httpx.Response(200, json=self.response_json)+ return httpx.Response(self.status_code, json=self.response_json)async def aclose(self) -> None:return None@@ -257,20 +258,26 @@ def test_streaming_with_headroom_retrieve_available_but_unused_returns_sse() ->proxy._stream_response.assert_not_awaited()-def test_mixed_ccr_and_client_tool_streams_both_blocks_as_sse() -> None:- """LEGAL mixed turn (#839, #2089): headroom_retrieve emitted alongside a- client tool. The proxy cannot synthesize the client tool_result, so it must- hand the turn back for the client to resolve — a 200 SSE stream preserving- BOTH tool_use blocks, matching the non-streaming path. It must NOT 502 and- must NOT issue a continuation request."""+def test_mixed_ccr_and_client_tool_serves_retrieval_then_streams_reissued_tool() -> None:+ """headroom_retrieve emitted alongside a client tool (#839, #2089). The+ client has no headroom_retrieve (Claude Code answers "No such tool+ available"), so the proxy serves the retrieval in a continuation without+ the unrun client call, and streams the continuation, where the model+ re-issues it. No headroom_retrieve reaches the client and nothing 502s."""config = _make_config()+ store = get_compression_store()+ hash_key = store.store(+ original=json.dumps({"secret": "retrieved answer"}),+ compressed="{}",+ original_item_count=1,+ )initial_response = _message_response([{"type": "tool_use","id": "toolu_ccr","name": "headroom_retrieve",- "input": {"hash": "abc123"},+ "input": {"hash": hash_key},},{"type": "tool_use",@@ -281,6 +288,17 @@ def test_mixed_ccr_and_client_tool_streams_both_blocks_as_sse() -> None:],stop_reason="tool_use",)+ reissued = _message_response(+ [+ {+ "type": "tool_use",+ "id": "toolu_client_2",+ "name": "client_tool",+ "input": {"value": 1},+ }+ ],+ stop_reason="tool_use",+ )with patch("headroom.proxy.server.AnyLLMBackend"):app = create_app(config)@@ -289,7 +307,7 @@ def test_mixed_ccr_and_client_tool_streams_both_blocks_as_sse() -> None:proxy._stream_response = AsyncMock(side_effect=AssertionError("live streaming path should not be used"))- continuation_client = _ContinuationClient(_message_response([]))+ continuation_client = _ContinuationClient(reissued)proxy.http_client = continuation_clientasync def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001@@ -319,14 +337,97 @@ def test_mixed_ccr_and_client_tool_streams_both_blocks_as_sse() -> None:assert resp.status_code == 200, resp.textassert "text/event-stream" in resp.headers["content-type"]- # Both tool_use blocks are preserved for the client to resolve.- assert "headroom_retrieve" in resp.text- assert "client_tool" in resp.text- assert "toolu_ccr" in resp.text- assert "toolu_client" in resp.text+ assert "toolu_client_2" in resp.text+ assert "headroom_retrieve" not in resp.text+ assert 'toolu_client"' not in resp.textassert "Unable to safely complete streamed CCR retrieval" not in resp.text- # No continuation is issued — the client resolves all tool calls.- assert continuation_client.post_calls == []+ assert len(continuation_client.post_calls) == 1+ body = json.loads(continuation_client.post_calls[0]["content"].decode())+ assistant, tool_result = body["messages"][-2:]+ assert [b["id"] for b in assistant["content"]] == ["toolu_ccr"]+ assert tool_result["content"][0]["tool_use_id"] == "toolu_ccr"+ assert "retrieved answer" in json.dumps(tool_result)++[email protected]("status_code", [200, 500], ids=["withdrawn", "failed"])+def test_mixed_turn_streams_the_continuation_or_the_turn_on_failure(status_code: int) -> None:+ """LEGAL mixed turn (#839, #2089). A continuation that withdraws the client+ tool is what the client gets: the dropped call never reaches it. If the+ continuation fails, the model made no newer decision, so the client gets+ the model's own turn as a 200 SSE stream with BOTH tool_use blocks. No 502."""+ config = _make_config()+ store = get_compression_store()+ hash_key = store.store(+ original=json.dumps({"secret": "retrieved answer"}),+ compressed="{}",+ original_item_count=1,+ )+ initial_response = _message_response(+ [+ {+ "type": "tool_use",+ "id": "toolu_ccr",+ "name": "headroom_retrieve",+ "input": {"hash": hash_key},+ },+ {+ "type": "tool_use",+ "id": "toolu_client",+ "name": "client_tool",+ "input": {"value": 1},+ },+ ],+ stop_reason="tool_use",+ )++ with patch("headroom.proxy.server.AnyLLMBackend"):+ app = create_app(config)+ with TestClient(app) as client:+ proxy = client.app.state.proxy+ proxy._stream_response = AsyncMock(+ side_effect=AssertionError("live streaming path should not be used")+ )+ continuation_client = _ContinuationClient(+ _message_response([{"type": "text", "text": "all done"}]), status_code+ )+ proxy.http_client = continuation_client++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ assert body["stream"] is False+ return httpx.Response(200, json=initial_response)++ proxy._retry_request = _fake_retry # type: ignore[assignment]++ resp = client.post(+ "/v1/messages",+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "stream": True,+ "tools": [+ create_ccr_tool_definition("anthropic"),+ {+ "name": "client_tool",+ "description": "Client-owned tool",+ "input_schema": {"type": "object", "properties": {}},+ },+ ],+ "messages": [{"role": "user", "content": _buffered("use tools")}],+ },+ )++ assert resp.status_code == 200, resp.text+ assert "text/event-stream" in resp.headers["content-type"]+ if status_code == 200:+ assert "all done" in resp.text+ assert "toolu_client" not in resp.text+ assert "headroom_retrieve" not in resp.text+ else:+ assert "toolu_ccr" in resp.text+ assert "toolu_client" in resp.text+ assert "Unable to safely complete streamed CCR retrieval" not in resp.text+ assert len(continuation_client.post_calls) == 1def test_unresolved_ccr_only_streams_through_as_200() -> None: