Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): don't send a memory continuation with unanswered client tool calls (#4009) (#4013)

Parideboy87e05b68b8aa
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index a20f52645..272aa7648 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -4731,18 +4731,62 @@ class AnthropicHandlerMixin:
request_context=memory_request_ctx,
)
- if tool_results:
+ turn_content = resp_json.get("content") or []
+ answered = {result.get("tool_use_id") for result in tool_results}
+ unanswered = [
+ block
+ for block in turn_content
+ if isinstance(block, dict)
+ and block.get("type") == "tool_use"
+ and block.get("id") not in answered
+ ]
+ if tool_results and unanswered:
+ # The continuation replays this whole turn, and
+ # Anthropic rejects any tool_use without a result
+ # (#4009). Only the client can answer its own
+ # tools, so the turn goes back to it, minus the
+ # memory calls it never declared and that already
+ # ran here (same rule as the streaming path, #3947).
+ logger.info(
+ f"[{request_id}] Memory: Turn also called a client "
+ "tool; returning the turn to the client"
+ )
+ resp_json = {
+ **resp_json,
+ "content": [
+ block
+ for block in turn_content
+ if not (
+ isinstance(block, dict)
+ and block.get("type") == "tool_use"
+ and block.get("name") in server_memory_tool_names
+ )
+ ],
+ }
+ response = httpx.Response(
+ status_code=200,
+ content=json.dumps(resp_json).encode(),
+ headers={
+ key: value
+ for key, value in response.headers.items()
+ if key.lower()
+ not in ("content-encoding", "content-length")
+ },
+ )
+ elif tool_results:
# Create continuation messages
assistant_msg = {
"role": "assistant",
- "content": resp_json.get("content", []),
+ "content": turn_content,
}
user_msg = {
"role": "user",
"content": tool_results,
}
- continuation_messages = optimized_messages + [
+ # body["messages"], not optimized_messages: it is
+ # what this turn actually sent upstream.
+ continuation_messages = body["messages"] + [
assistant_msg,
user_msg,
]
@@ -4763,9 +4807,16 @@ class AnthropicHandlerMixin:
# Update response with continuation
resp_json = cont_response.json()
response = cont_response
- logger.info(
- f"[{request_id}] Memory: Tool calls handled, continuation complete"
- )
+ if cont_response.status_code >= 400:
+ logger.warning(
+ f"[{request_id}] Memory: Continuation failed with "
+ f"upstream status {cont_response.status_code}"
+ )
+ else:
+ logger.info(
+ f"[{request_id}] Memory: Tool calls handled, "
+ "continuation complete"
+ )
except Exception as e:
logger.warning(
diff --git a/tests/test_anthropic_buffered_memory_continuation.py b/tests/test_anthropic_buffered_memory_continuation.py
new file mode 100644
index 000000000..d3403b361
--- /dev/null
+++ b/tests/test_anthropic_buffered_memory_continuation.py
@@ -0,0 +1,161 @@
+"""Buffered memory continuations answer every tool_use they replay (GH #4009).
+
+On a buffered Anthropic turn the proxy runs its injected memory tools and sends
+the model a continuation. That continuation replays the whole assistant turn,
+so a client tool called alongside a memory tool used to go upstream with no
+``tool_result`` and the provider rejected it with a 400.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+import httpx
+import pytest
+from fastapi.testclient import TestClient
+
+from headroom.proxy.server import ProxyConfig, create_app
+
+MEMORY_TOOLS = ("memory_save", "memory_search")
+
+
+class _MemoryHandler:
+ def __init__(self) -> None:
+ self.config = type(
+ "MemoryConfig",
+ (),
+ {"inject_context": False, "inject_tools": True, "project_root_override": ""},
+ )()
+ self.initialized = False
+ self.backend = None
+ self.executed: list[str] = []
+
+ def compute_memory_tool_definitions(self, provider: str) -> list[dict[str, Any]]:
+ return [
+ {"name": name, "description": name, "input_schema": {"type": "object"}}
+ for name in MEMORY_TOOLS
+ ]
+
+ def get_beta_headers(self) -> dict[str, str]:
+ return {}
+
+ def has_memory_tool_calls(self, response: dict[str, Any], provider: str) -> bool:
+ return bool(self._memory_calls(response))
+
+ async def handle_memory_tool_calls(
+ self, response: dict[str, Any], user_id: str, provider: str, **kwargs: Any
+ ) -> list[dict[str, Any]]:
+ # Like the real handler: results only for memory tools.
+ results = []
+ for block in self._memory_calls(response):
+ self.executed.append(block["name"])
+ results.append({"type": "tool_result", "tool_use_id": block["id"], "content": "ok"})
+ return results
+
+ @staticmethod
+ def _memory_calls(response: dict[str, Any]) -> list[dict[str, Any]]:
+ return [
+ block
+ for block in response.get("content") or []
+ if block.get("type") == "tool_use" and block.get("name") in MEMORY_TOOLS
+ ]
+
+
+def _message(content: list[dict[str, Any]], stop_reason: str) -> dict[str, Any]:
+ return {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "model": "claude-sonnet-4-6",
+ "content": content,
+ "stop_reason": stop_reason,
+ "usage": {"input_tokens": 10, "output_tokens": 5},
+ }
+
+
+MEMORY_CALL = {
+ "type": "tool_use",
+ "id": "toolu_mem",
+ "name": "memory_search",
+ "input": {"query": "deploy region"},
+}
+CLIENT_CALL = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}
+
+
+def _run(upstream: list[httpx.Response]) -> tuple[httpx.Response, list[dict], _MemoryHandler]:
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=False,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+ sent: list[dict] = []
+ handler = _MemoryHandler()
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+ proxy.memory_handler = handler
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ sent.append(json.loads(json.dumps(body)))
+ return upstream[len(sent) - 1]
+
+ proxy._retry_request = _fake_retry # type: ignore[assignment]
+ response = client.post(
+ "/v1/messages",
+ headers={
+ "x-api-key": "test-key",
+ "anthropic-version": "2023-06-01",
+ "x-headroom-user-id": "u1",
+ },
+ json={
+ "model": "claude-sonnet-4-6",
+ "max_tokens": 64,
+ "tools": [{"name": "Bash", "input_schema": {"type": "object"}}],
+ "messages": [{"role": "user", "content": "check the deploy region"}],
+ },
+ )
+ return response, sent, handler
+
+
+def test_mixed_turn_returns_client_tool_without_continuation() -> None:
+ first = _message([MEMORY_CALL, CLIENT_CALL], "tool_use")
+ response, sent, handler = _run([httpx.Response(200, json=first)])
+
+ assert response.status_code == 200, response.text
+ # No continuation: it could not answer the Bash call.
+ assert len(sent) == 1
+ assert handler.executed == ["memory_search"]
+ # The client gets its own tool call back, not the proxy's.
+ assert [block["name"] for block in response.json()["content"]] == ["Bash"]
+
+
+def test_memory_only_turn_continues_with_a_result_for_every_call() -> None:
+ first = _message([MEMORY_CALL], "tool_use")
+ final = _message([{"type": "text", "text": "us-east-1"}], "end_turn")
+ response, sent, _ = _run([httpx.Response(200, json=first), httpx.Response(200, json=final)])
+
+ assert len(sent) == 2
+ assistant, user = sent[1]["messages"][-2:]
+ assert assistant == {"role": "assistant", "content": [MEMORY_CALL]}
+ assert [r["tool_use_id"] for r in user["content"]] == ["toolu_mem"]
+ assert sent[1]["messages"][:-2] == sent[0]["messages"]
+ assert response.json()["content"] == [{"type": "text", "text": "us-east-1"}]
+
+
+def test_failed_continuation_is_not_logged_as_complete(caplog: pytest.LogCaptureFixture) -> None:
+ first = _message([MEMORY_CALL], "tool_use")
+ rejected = httpx.Response(
+ 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "no"}}
+ )
+ with caplog.at_level(logging.INFO, logger="headroom.proxy"):
+ _run([httpx.Response(200, json=first), rejected])
+
+ assert "Memory: Continuation failed with upstream status 400" in caplog.text
+ assert "continuation complete" not in caplog.text