Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): don't send a memory continuation with unanswered client tool calls (#4009) (#4013)
Parideboy87e05b68b8aa
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex a20f52645..272aa7648 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -4731,18 +4731,62 @@ class AnthropicHandlerMixin:request_context=memory_request_ctx,)- if tool_results:+ turn_content = resp_json.get("content") or []+ answered = {result.get("tool_use_id") for result in tool_results}+ unanswered = [+ block+ for block in turn_content+ if isinstance(block, dict)+ and block.get("type") == "tool_use"+ and block.get("id") not in answered+ ]+ if tool_results and unanswered:+ # The continuation replays this whole turn, and+ # Anthropic rejects any tool_use without a result+ # (#4009). Only the client can answer its own+ # tools, so the turn goes back to it, minus the+ # memory calls it never declared and that already+ # ran here (same rule as the streaming path, #3947).+ logger.info(+ f"[{request_id}] Memory: Turn also called a client "+ "tool; returning the turn to the client"+ )+ resp_json = {+ **resp_json,+ "content": [+ block+ for block in turn_content+ if not (+ isinstance(block, dict)+ and block.get("type") == "tool_use"+ and block.get("name") in server_memory_tool_names+ )+ ],+ }+ response = httpx.Response(+ status_code=200,+ content=json.dumps(resp_json).encode(),+ headers={+ key: value+ for key, value in response.headers.items()+ if key.lower()+ not in ("content-encoding", "content-length")+ },+ )+ elif tool_results:# Create continuation messagesassistant_msg = {"role": "assistant",- "content": resp_json.get("content", []),+ "content": turn_content,}user_msg = {"role": "user","content": tool_results,}- continuation_messages = optimized_messages + [+ # body["messages"], not optimized_messages: it is+ # what this turn actually sent upstream.+ continuation_messages = body["messages"] + [assistant_msg,user_msg,]@@ -4763,9 +4807,16 @@ class AnthropicHandlerMixin:# Update response with continuationresp_json = cont_response.json()response = cont_response- logger.info(- f"[{request_id}] Memory: Tool calls handled, continuation complete"- )+ if cont_response.status_code >= 400:+ logger.warning(+ f"[{request_id}] Memory: Continuation failed with "+ f"upstream status {cont_response.status_code}"+ )+ else:+ logger.info(+ f"[{request_id}] Memory: Tool calls handled, "+ "continuation complete"+ )except Exception as e:logger.warning(diff --git a/tests/test_anthropic_buffered_memory_continuation.py b/tests/test_anthropic_buffered_memory_continuation.pynew file mode 100644index 000000000..d3403b361--- /dev/null+++ b/tests/test_anthropic_buffered_memory_continuation.py@@ -0,0 +1,161 @@+"""Buffered memory continuations answer every tool_use they replay (GH #4009).++On a buffered Anthropic turn the proxy runs its injected memory tools and sends+the model a continuation. That continuation replays the whole assistant turn,+so a client tool called alongside a memory tool used to go upstream with no+``tool_result`` and the provider rejected it with a 400.+"""++from __future__ import annotations++import json+import logging+from typing import Any++import httpx+import pytest+from fastapi.testclient import TestClient++from headroom.proxy.server import ProxyConfig, create_app++MEMORY_TOOLS = ("memory_save", "memory_search")+++class _MemoryHandler:+ def __init__(self) -> None:+ self.config = type(+ "MemoryConfig",+ (),+ {"inject_context": False, "inject_tools": True, "project_root_override": ""},+ )()+ self.initialized = False+ self.backend = None+ self.executed: list[str] = []++ def compute_memory_tool_definitions(self, provider: str) -> list[dict[str, Any]]:+ return [+ {"name": name, "description": name, "input_schema": {"type": "object"}}+ for name in MEMORY_TOOLS+ ]++ def get_beta_headers(self) -> dict[str, str]:+ return {}++ def has_memory_tool_calls(self, response: dict[str, Any], provider: str) -> bool:+ return bool(self._memory_calls(response))++ async def handle_memory_tool_calls(+ self, response: dict[str, Any], user_id: str, provider: str, **kwargs: Any+ ) -> list[dict[str, Any]]:+ # Like the real handler: results only for memory tools.+ results = []+ for block in self._memory_calls(response):+ self.executed.append(block["name"])+ results.append({"type": "tool_result", "tool_use_id": block["id"], "content": "ok"})+ return results++ @staticmethod+ def _memory_calls(response: dict[str, Any]) -> list[dict[str, Any]]:+ return [+ block+ for block in response.get("content") or []+ if block.get("type") == "tool_use" and block.get("name") in MEMORY_TOOLS+ ]+++def _message(content: list[dict[str, Any]], stop_reason: str) -> dict[str, Any]:+ return {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-6",+ "content": content,+ "stop_reason": stop_reason,+ "usage": {"input_tokens": 10, "output_tokens": 5},+ }+++MEMORY_CALL = {+ "type": "tool_use",+ "id": "toolu_mem",+ "name": "memory_search",+ "input": {"query": "deploy region"},+}+CLIENT_CALL = {"type": "tool_use", "id": "toolu_bash", "name": "Bash", "input": {"command": "ls"}}+++def _run(upstream: list[httpx.Response]) -> tuple[httpx.Response, list[dict], _MemoryHandler]:+ config = ProxyConfig(+ optimize=False,+ cache_enabled=False,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )+ sent: list[dict] = []+ handler = _MemoryHandler()+ with TestClient(create_app(config)) as client:+ proxy = client.app.state.proxy+ proxy.memory_handler = handler++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ sent.append(json.loads(json.dumps(body)))+ return upstream[len(sent) - 1]++ proxy._retry_request = _fake_retry # type: ignore[assignment]+ response = client.post(+ "/v1/messages",+ headers={+ "x-api-key": "test-key",+ "anthropic-version": "2023-06-01",+ "x-headroom-user-id": "u1",+ },+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "tools": [{"name": "Bash", "input_schema": {"type": "object"}}],+ "messages": [{"role": "user", "content": "check the deploy region"}],+ },+ )+ return response, sent, handler+++def test_mixed_turn_returns_client_tool_without_continuation() -> None:+ first = _message([MEMORY_CALL, CLIENT_CALL], "tool_use")+ response, sent, handler = _run([httpx.Response(200, json=first)])++ assert response.status_code == 200, response.text+ # No continuation: it could not answer the Bash call.+ assert len(sent) == 1+ assert handler.executed == ["memory_search"]+ # The client gets its own tool call back, not the proxy's.+ assert [block["name"] for block in response.json()["content"]] == ["Bash"]+++def test_memory_only_turn_continues_with_a_result_for_every_call() -> None:+ first = _message([MEMORY_CALL], "tool_use")+ final = _message([{"type": "text", "text": "us-east-1"}], "end_turn")+ response, sent, _ = _run([httpx.Response(200, json=first), httpx.Response(200, json=final)])++ assert len(sent) == 2+ assistant, user = sent[1]["messages"][-2:]+ assert assistant == {"role": "assistant", "content": [MEMORY_CALL]}+ assert [r["tool_use_id"] for r in user["content"]] == ["toolu_mem"]+ assert sent[1]["messages"][:-2] == sent[0]["messages"]+ assert response.json()["content"] == [{"type": "text", "text": "us-east-1"}]+++def test_failed_continuation_is_not_logged_as_complete(caplog: pytest.LogCaptureFixture) -> None:+ first = _message([MEMORY_CALL], "tool_use")+ rejected = httpx.Response(+ 400, json={"type": "error", "error": {"type": "invalid_request_error", "message": "no"}}+ )+ with caplog.at_level(logging.INFO, logger="headroom.proxy"):+ _run([httpx.Response(200, json=first), rejected])++ assert "Memory: Continuation failed with upstream status 400" in caplog.text+ assert "continuation complete" not in caplog.text