Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): do not cache error replies delivered as http 200 (#3930)

gglucass1132a54b2148
diff --git a/headroom/proxy/semantic_cache.py b/headroom/proxy/semantic_cache.py
index 998e8f3c4..dee64cf55 100644
--- a/headroom/proxy/semantic_cache.py
+++ b/headroom/proxy/semantic_cache.py
@@ -8,6 +8,7 @@ Extracted from server.py for maintainability.
from __future__ import annotations
import asyncio
+import json
import sys
from collections import OrderedDict
from datetime import datetime
@@ -22,6 +23,31 @@ from headroom.proxy.semantic_cache_key_policy import compute_semantic_cache_key,
_strip_cache_control = strip_cache_control
+def _is_cacheable_reply(response_body: bytes) -> bool:
+ """Whether a 200 body is a reply the cache may replay.
+
+ The status code alone does not say so: an Anthropic- or OpenAI-compatible
+ gateway can answer 200 with an error object, or with nothing. Stored, that
+ body is replayed to every matching request for the full TTL and the
+ upstream is never asked again. Refuse an empty body, and JSON that is not a
+ non-empty object or that carries an error (Anthropic ``"type": "error"``,
+ OpenAI ``"error": {...}``). Bytes that are not JSON pass: the cache is
+ format-agnostic, and both handlers only store a reply they have decoded.
+ """
+ if not response_body:
+ return False
+ try:
+ reply = json.loads(response_body)
+ except ValueError:
+ return True
+ return (
+ isinstance(reply, dict)
+ and bool(reply)
+ and reply.get("type") != "error"
+ and reply.get("error") is None
+ )
+
+
class SemanticCache:
"""Simple semantic cache based on message content hash.
@@ -84,6 +110,9 @@ class SemanticCache:
"""Cache a response."""
key = self._compute_key(messages, model, **key_fields)
+ if not _is_cacheable_reply(response_body):
+ return
+
async with self._lock:
# If key already exists, remove it first to update position
if key in self._cache:
diff --git a/tests/test_proxy_response_cache_store_guard.py b/tests/test_proxy_response_cache_store_guard.py
new file mode 100644
index 000000000..0a1cf6017
--- /dev/null
+++ b/tests/test_proxy_response_cache_store_guard.py
@@ -0,0 +1,122 @@
+"""The response cache must not store an error delivered as HTTP 200.
+
+Both handlers gate ``SemanticCache.set`` on ``status_code == 200``. Anthropic- and
+OpenAI-compatible gateways can answer 200 with an error object (or an empty
+object), and once stored that body is replayed to every matching non-streaming
+request for the full TTL without the upstream ever being asked again.
+"""
+
+from __future__ import annotations
+
+import json
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient
+
+from headroom.proxy.semantic_cache import _is_cacheable_reply
+from headroom.proxy.server import ProxyConfig, create_app
+
+MESSAGES = [{"role": "user", "content": "Say hi."}]
+
+ANTHROPIC_ERROR = {"type": "error", "error": {"type": "api_error", "message": "boom"}}
+OPENAI_ERROR = {"error": {"message": "boom", "type": "server_error", "code": None}}
+
+ANTHROPIC_OK = {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "Hello"}],
+ "usage": {"input_tokens": 10, "output_tokens": 3},
+}
+OPENAI_OK = {
+ "id": "chatcmpl_1",
+ "object": "chat.completion",
+ "choices": [
+ {"index": 0, "message": {"role": "assistant", "content": "Hello"}, "finish_reason": "stop"}
+ ],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13},
+}
+
+
+ "body,cacheable",
+ [
+ (json.dumps(ANTHROPIC_OK).encode(), True),
+ (json.dumps(OPENAI_OK).encode(), True),
+ (json.dumps({**OPENAI_OK, "error": None}).encode(), True),
+ (json.dumps(ANTHROPIC_ERROR).encode(), False),
+ (json.dumps(OPENAI_ERROR).encode(), False),
+ (b"", False),
+ (b"opaque-bytes", True),
+ (b"{}", False),
+ (b"null", False),
+ (b"[]", False),
+ ],
+)
+def test_is_cacheable_reply(body: bytes, cacheable: bool) -> None:
+ assert _is_cacheable_reply(body) is cacheable
+
+
+def _client() -> TestClient:
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=True,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+ return TestClient(create_app(config))
+
+
+ "path,headers,model,error_body,ok_body",
+ [
+ (
+ "/v1/messages",
+ {"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
+ "claude-haiku-4-5",
+ ANTHROPIC_ERROR,
+ ANTHROPIC_OK,
+ ),
+ (
+ "/v1/chat/completions",
+ {"authorization": "Bearer test-key"},
+ "gpt-4o-mini",
+ OPENAI_ERROR,
+ OPENAI_OK,
+ ),
+ ],
+)
+def test_error_200_is_not_replayed_from_cache(path, headers, model, error_body, ok_body) -> None:
+ calls = {"n": 0}
+
+ with _client() as client:
+
+ async def _fake_retry(method, url, req_headers, body, stream=False, **kwargs): # noqa: ANN001
+ calls["n"] += 1
+ return httpx.Response(200, json=error_body if calls["n"] == 1 else ok_body)
+
+ client.app.state.proxy._retry_request = _fake_retry
+ request = {"model": model, "max_tokens": 64, "messages": MESSAGES, "stream": False}
+
+ first = client.post(path, headers=headers, json=request)
+ assert first.json() == error_body
+
+ # Before the fix the error body was cached: this came back from the
+ # cache with the upstream never called a second time.
+ second = client.post(path, headers=headers, json=request)
+ assert calls["n"] == 2
+ assert second.json() == ok_body
+
+ # A real reply is still cached.
+ third = client.post(path, headers=headers, json=request)
+ assert calls["n"] == 2
+ assert third.json() == ok_body