Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(anthropic): preserve failed CCR continuations (#3843)
AxelRay94bb055810dc
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 9424c7935..db39101dc 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -4396,6 +4396,11 @@ class AnthropicHandlerMixin:f"CCR: Got response status={cont_response.status_code}, "f"content-encoding={cont_response.headers.get('content-encoding')}")+ if not 200 <= cont_response.status_code < 300:+ raise RuntimeError(+ "CCR continuation returned non-success status "+ f"{cont_response.status_code}"+ )result: dict[str, Any] = cont_response.json()logger.info("CCR: Parsed JSON successfully")return result@@ -4419,52 +4424,83 @@ class AnthropicHandlerMixin:api_call_fn,provider="anthropic",)- final_resp_json = preserve_opaque_response_fields(- resp_json, final_resp_json- )- # Update response content with final response- resp_json = final_resp_json- # Remove encoding headers since content is now uncompressed JSON- ccr_response_headers = {- k: v- for k, v in response.headers.items()- if k.lower() not in ("content-encoding", "content-length")- }- try:- ccr_content = json.dumps(final_resp_json).encode()- except (TypeError, ValueError) as json_err:- logger.warning(- f"[{request_id}] CCR: JSON serialization failed: {json_err}"+ if final_resp_json is resp_json:+ # The handler intentionally returns the same+ # object for both a failed continuation and+ # an intentional mixed-tool skip. Identity is+ # still the right signal for whether to+ # re-serialize, but not for the diagnostic+ # message; classify the residual tools first.+ from headroom.ccr.response_handler import (+ RESIDUAL_CCR_ERROR,+ RESIDUAL_CCR_SKIPPED_MIXED,)- ccr_content = json.dumps(resp_json).encode()- response = httpx.Response(- status_code=200,- content=ccr_content,- headers=ccr_response_headers,- )- # Only claim success when no headroom_retrieve remains.- # On an intentional mixed-tool skip (#839) the response- # still carries headroom_retrieve for the client to- # resolve — logging "handled successfully" there is- # misleading. Classify via the shared, provider-generic- # residual-CCR signal.- from headroom.ccr.response_handler import (- RESIDUAL_CCR_SKIPPED_MIXED,- )- residual_status = self.ccr_response_handler.residual_ccr_status(- final_resp_json, "anthropic"- )- if residual_status == RESIDUAL_CCR_SKIPPED_MIXED:- logger.info(- f"[{request_id}] CCR: Skipped retrieval — "- "headroom_retrieve returned alongside a client "- "tool for the client to resolve"+ residual_status = self.ccr_response_handler.residual_ccr_status(+ final_resp_json, "anthropic")+ if residual_status == RESIDUAL_CCR_SKIPPED_MIXED:+ logger.info(+ f"[{request_id}] CCR: Skipped retrieval — "+ "headroom_retrieve returned alongside a client "+ "tool for the client to resolve"+ )+ elif residual_status == RESIDUAL_CCR_ERROR:+ logger.info(+ f"[{request_id}] CCR: Continuation failed; "+ "forwarding the original upstream response"+ )+ else:+ logger.info(+ f"[{request_id}] CCR: Retrieval handled successfully"+ )else:- logger.info(- f"[{request_id}] CCR: Retrieval handled successfully"+ final_resp_json = preserve_opaque_response_fields(+ resp_json, final_resp_json+ )+ # Update response content with final response+ resp_json = final_resp_json+ # Remove encoding headers since content is now uncompressed JSON+ ccr_response_headers = {+ k: v+ for k, v in response.headers.items()+ if k.lower() not in ("content-encoding", "content-length")+ }+ try:+ ccr_content = json.dumps(final_resp_json).encode()+ except (TypeError, ValueError) as json_err:+ logger.warning(+ f"[{request_id}] CCR: JSON serialization failed: {json_err}"+ )+ ccr_content = json.dumps(resp_json).encode()+ response = httpx.Response(+ status_code=200,+ content=ccr_content,+ headers=ccr_response_headers,+ )+ # Only claim success when no headroom_retrieve remains.+ # On an intentional mixed-tool skip (#839) the response+ # still carries headroom_retrieve for the client to+ # resolve — logging "handled successfully" there is+ # misleading. Classify via the shared, provider-generic+ # residual-CCR signal.+ from headroom.ccr.response_handler import (+ RESIDUAL_CCR_SKIPPED_MIXED,+ )++ residual_status = self.ccr_response_handler.residual_ccr_status(+ final_resp_json, "anthropic")+ if residual_status == RESIDUAL_CCR_SKIPPED_MIXED:+ logger.info(+ f"[{request_id}] CCR: Skipped retrieval — "+ "headroom_retrieve returned alongside a client "+ "tool for the client to resolve"+ )+ else:+ logger.info(+ f"[{request_id}] CCR: Retrieval handled successfully"+ )except Exception as e:import tracebackdiff --git a/tests/test_proxy/test_anthropic_ccr_raise.py b/tests/test_proxy/test_anthropic_ccr_raise.pyindex 339a45c0a..7e0f050f5 100644--- a/tests/test_proxy/test_anthropic_ccr_raise.py+++ b/tests/test_proxy/test_anthropic_ccr_raise.py@@ -10,6 +10,7 @@ the raw ``headroom_retrieve`` tool-call body to the client.from __future__ import annotations+import jsonfrom unittest.mock import AsyncMock, MagicMock, patchimport pytest@@ -19,6 +20,7 @@ httpx = pytest.importorskip("httpx")from fastapi.testclient import TestClient # noqa: E402+from headroom.cache.compression_store import get_compression_store # noqa: E402from headroom.proxy.server import ProxyConfig, create_app # noqa: E402@@ -36,7 +38,7 @@ def _make_config() -> ProxyConfig:)-def _tool_call_response() -> dict:+def _tool_call_response(hash_key: str = "badhash") -> dict:"""Anthropic-shaped response containing a headroom_retrieve tool use block."""return {"id": "msg_ccr_test",@@ -47,7 +49,7 @@ def _tool_call_response() -> dict:"type": "tool_use","id": "toolu_bad","name": "headroom_retrieve",- "input": {"hash": "badhash"},+ "input": {"hash": hash_key},}],"stop_reason": "tool_use",@@ -110,3 +112,62 @@ def test_anthropic_ccr_exception_reraises_not_swallowed():"proxy silently forwarded the raw CCR tool call — silent fallback not fixed")assert "error" in body++[email protected]("status", [401, 429, 500])+def test_anthropic_ccr_non_2xx_continuation_keeps_original_response(status: int):+ """A failed CCR continuation must not become a synthetic HTTP 200 error."""+ config = _make_config()+ hash_key = get_compression_store().store(+ original="retrieved content",+ compressed="compressed content",+ original_item_count=1,+ )+ initial_response = _tool_call_response(hash_key)+ initial_bytes = json.dumps(initial_response, separators=(",", ":")).encode()+ continuation_error = {+ "type": "error",+ "error": {"type": "authentication_error", "message": "invalid api key"},+ }++ class _ContinuationClient:+ def __init__(self) -> None:+ self.post_calls: list[dict] = []++ async def post(self, url, *, content=None, headers=None, timeout=None): # noqa: ANN001+ self.post_calls.append({"url": url, "content": content, "headers": headers})+ return httpx.Response(status, json=continuation_error)++ async def aclose(self) -> None:+ return None++ with patch("headroom.proxy.server.AnyLLMBackend"):+ app = create_app(config)+ with TestClient(app) as client:+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ return httpx.Response(+ 200,+ content=initial_bytes,+ headers={"content-type": "application/json"},+ )++ continuation_client = _ContinuationClient()+ proxy._retry_request = _fake_retry+ proxy.http_client = continuation_client++ response = client.post(+ "/v1/messages",+ headers={"x-api-key": "test-key", "anthropic-version": "2023-06-01"},+ json={+ "model": "claude-sonnet-4-6",+ "max_tokens": 64,+ "messages": [{"role": "user", "content": "retrieve it"}],+ },+ )++ assert len(continuation_client.post_calls) == 1+ assert response.status_code == 200+ assert response.json() == initial_response+ assert response.content == initial_bytes