Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): do not cache error replies delivered as http 200 (#3930)
gglucass1132a54b2148
diff --git a/headroom/proxy/semantic_cache.py b/headroom/proxy/semantic_cache.pyindex 998e8f3c4..dee64cf55 100644--- a/headroom/proxy/semantic_cache.py+++ b/headroom/proxy/semantic_cache.py@@ -8,6 +8,7 @@ Extracted from server.py for maintainability.from __future__ import annotationsimport asyncio+import jsonimport sysfrom collections import OrderedDictfrom datetime import datetime@@ -22,6 +23,31 @@ from headroom.proxy.semantic_cache_key_policy import compute_semantic_cache_key,_strip_cache_control = strip_cache_control+def _is_cacheable_reply(response_body: bytes) -> bool:+ """Whether a 200 body is a reply the cache may replay.++ The status code alone does not say so: an Anthropic- or OpenAI-compatible+ gateway can answer 200 with an error object, or with nothing. Stored, that+ body is replayed to every matching request for the full TTL and the+ upstream is never asked again. Refuse an empty body, and JSON that is not a+ non-empty object or that carries an error (Anthropic ``"type": "error"``,+ OpenAI ``"error": {...}``). Bytes that are not JSON pass: the cache is+ format-agnostic, and both handlers only store a reply they have decoded.+ """+ if not response_body:+ return False+ try:+ reply = json.loads(response_body)+ except ValueError:+ return True+ return (+ isinstance(reply, dict)+ and bool(reply)+ and reply.get("type") != "error"+ and reply.get("error") is None+ )++class SemanticCache:"""Simple semantic cache based on message content hash.@@ -84,6 +110,9 @@ class SemanticCache:"""Cache a response."""key = self._compute_key(messages, model, **key_fields)+ if not _is_cacheable_reply(response_body):+ return+async with self._lock:# If key already exists, remove it first to update positionif key in self._cache:diff --git a/tests/test_proxy_response_cache_store_guard.py b/tests/test_proxy_response_cache_store_guard.pynew file mode 100644index 000000000..0a1cf6017--- /dev/null+++ b/tests/test_proxy_response_cache_store_guard.py@@ -0,0 +1,122 @@+"""The response cache must not store an error delivered as HTTP 200.++Both handlers gate ``SemanticCache.set`` on ``status_code == 200``. Anthropic- and+OpenAI-compatible gateways can answer 200 with an error object (or an empty+object), and once stored that body is replayed to every matching non-streaming+request for the full TTL without the upstream ever being asked again.+"""++from __future__ import annotations++import json++import httpx+import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient++from headroom.proxy.semantic_cache import _is_cacheable_reply+from headroom.proxy.server import ProxyConfig, create_app++MESSAGES = [{"role": "user", "content": "Say hi."}]++ANTHROPIC_ERROR = {"type": "error", "error": {"type": "api_error", "message": "boom"}}+OPENAI_ERROR = {"error": {"message": "boom", "type": "server_error", "code": None}}++ANTHROPIC_OK = {+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": "Hello"}],+ "usage": {"input_tokens": 10, "output_tokens": 3},+}+OPENAI_OK = {+ "id": "chatcmpl_1",+ "object": "chat.completion",+ "choices": [+ {"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}+ ],+ "usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13},+}+++ "body,cacheable",+ [+ (json.dumps(ANTHROPIC_OK).encode(), True),+ (json.dumps(OPENAI_OK).encode(), True),+ (json.dumps({**OPENAI_OK, "error": None}).encode(), True),+ (json.dumps(ANTHROPIC_ERROR).encode(), False),+ (json.dumps(OPENAI_ERROR).encode(), False),+ (b"", False),+ (b"opaque-bytes", True),+ (b"{}", False),+ (b"null", False),+ (b"[]", False),+ ],+)+def test_is_cacheable_reply(body: bytes, cacheable: bool) -> None:+ assert _is_cacheable_reply(body) is cacheable+++def _client() -> TestClient:+ config = ProxyConfig(+ optimize=False,+ cache_enabled=True,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )+ return TestClient(create_app(config))+++ "path,headers,model,error_body,ok_body",+ [+ (+ "/v1/messages",+ {"x-api-key": "test-key", "anthropic-version": "2023-06-01"},+ "claude-haiku-4-5",+ ANTHROPIC_ERROR,+ ANTHROPIC_OK,+ ),+ (+ "/v1/chat/completions",+ {"authorization": "Bearer test-key"},+ "gpt-4o-mini",+ OPENAI_ERROR,+ OPENAI_OK,+ ),+ ],+)+def test_error_200_is_not_replayed_from_cache(path, headers, model, error_body, ok_body) -> None:+ calls = {"n": 0}++ with _client() as client:++ async def _fake_retry(method, url, req_headers, body, stream=False, **kwargs): # noqa: ANN001+ calls["n"] += 1+ return httpx.Response(200, json=error_body if calls["n"] == 1 else ok_body)++ client.app.state.proxy._retry_request = _fake_retry+ request = {"model": model, "max_tokens": 64, "messages": MESSAGES, "stream": False}++ first = client.post(path, headers=headers, json=request)+ assert first.json() == error_body++ # Before the fix the error body was cached: this came back from the+ # cache with the upstream never called a second time.+ second = client.post(path, headers=headers, json=request)+ assert calls["n"] == 2+ assert second.json() == ok_body++ # A real reply is still cached.+ third = client.post(path, headers=headers, json=request)+ assert calls["n"] == 2+ assert third.json() == ok_body