Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): partition response cache by upstream (#3349)

C0sm0catsdfdc72511ef4
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 7d67ce66f..492439992 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -1336,6 +1336,10 @@ class AnthropicHandlerMixin:
# semantics (#1473 review). Non-generation metadata (metadata,
# service_tier) is intentionally excluded.
cache_key_fields = {
+ # A per-request x-headroom-base-url selects a different response
+ # producer. Without it in the key, two gateways serving the same
+ # model/messages can return each other's cached response (#3346).
+ "upstream_base_url": upstream_base_url,
"system": body.get("system"),
"tools": body.get("tools"),
"tool_choice": body.get("tool_choice"),
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index efeecbd53..466e2283d 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3598,6 +3598,10 @@ class OpenAIHandlerMixin:
# review). Transport/metadata fields (stream, store, user, service_tier)
# and the deprecated functions API are intentionally excluded.
cache_key_fields = {
+ # The resolved upstream is request-local when x-headroom-base-url
+ # is present. Responses from separate gateways are not equivalent
+ # even when their model/messages fields match (#3346).
+ "upstream_base_url": upstream_base_url,
"tools": body.get("tools"),
"tool_choice": body.get("tool_choice"),
"response_format": body.get("response_format"),
diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.py
index bd62070f7..acc40342b 100644
--- a/tests/test_ccr_buffered_stream_signed_thinking.py
+++ b/tests/test_ccr_buffered_stream_signed_thinking.py
@@ -314,6 +314,7 @@ def test_cache_hit_never_replays_a_foreign_content_type() -> None:
key = proxy.cache._compute_key(
body["messages"],
body["model"],
+ upstream_base_url=None,
system=None,
tools=None,
tool_choice=None,
diff --git a/tests/test_proxy_openai_cache_key_integration.py b/tests/test_proxy_openai_cache_key_integration.py
index e41a02a24..b4f201fe3 100644
--- a/tests/test_proxy_openai_cache_key_integration.py
+++ b/tests/test_proxy_openai_cache_key_integration.py
@@ -114,3 +114,59 @@ def test_openai_differing_field_not_served_from_cache(field, a, b) -> None:
assert ra2.status_code == 200
assert _content(ra2) == "resp-1"
assert calls["n"] == 2
+
+
+def test_openai_different_custom_upstream_not_served_from_cache(monkeypatch) -> None:
+ """Two OpenAI-compatible gateways must not share a response entry (#3346)."""
+ calls: list[str] = []
+
+ monkeypatch.setattr(
+ "headroom.proxy.handlers.openai._resolve_openai_upstream_base",
+ lambda headers: headers.get("x-headroom-base-url"),
+ )
+
+ with _make_cached_proxy_client() as client:
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ calls.append(str(url))
+ content = "gateway-A" if "gateway-a" in str(url) else "gateway-B"
+ return httpx.Response(
+ 200,
+ json={
+ "id": f"chatcmpl_{len(calls)}",
+ "object": "chat.completion",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": content},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13},
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+ headers = {"authorization": "Bearer test-key"}
+ body = _body()
+
+ first = client.post(
+ "/v1/chat/completions",
+ headers={**headers, "x-headroom-base-url": "https://gateway-a.example"},
+ json=body,
+ )
+ second = client.post(
+ "/v1/chat/completions",
+ headers={**headers, "x-headroom-base-url": "https://gateway-b.example"},
+ json=body,
+ )
+
+ assert first.status_code == 200
+ assert second.status_code == 200
+ assert _content(first) == "gateway-A"
+ assert _content(second) == "gateway-B"
+ assert calls == [
+ "https://gateway-a.example/v1/chat/completions",
+ "https://gateway-b.example/v1/chat/completions",
+ ]
diff --git a/tests/test_proxy_semantic_cache_key_integration.py b/tests/test_proxy_semantic_cache_key_integration.py
index c7aa3c46e..b77364428 100644
--- a/tests/test_proxy_semantic_cache_key_integration.py
+++ b/tests/test_proxy_semantic_cache_key_integration.py
@@ -167,3 +167,62 @@ def test_different_thinking_not_served_from_cache() -> None:
assert a2.status_code == 200
assert _text(a2) == "resp-1"
assert calls["n"] == 2
+
+
+def test_different_custom_upstream_not_served_from_cache(monkeypatch) -> None:
+ """The response producer is part of cache identity (#3346)."""
+ calls: list[str] = []
+
+ async def allow_example_upstream(_url: str) -> bool:
+ return True
+
+ monkeypatch.setattr(
+ "headroom.providers.proxy_routes.is_safe_upstream_url_async",
+ allow_example_upstream,
+ )
+
+ with _make_cached_proxy_client() as client:
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ calls.append(str(url))
+ text = "gateway-A" if "gateway-a" in str(url) else "gateway-B"
+ return httpx.Response(
+ 200,
+ json={
+ "id": f"msg_{len(calls)}",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": text}],
+ "usage": {
+ "input_tokens": 10,
+ "output_tokens": 3,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ },
+ )
+
+ proxy._retry_request = _fake_retry
+ headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}
+ body = _body("Answer with the gateway identity.")
+
+ first = client.post(
+ "/v1/messages",
+ headers={**headers, "x-headroom-base-url": "https://gateway-a.example"},
+ json=body,
+ )
+ second = client.post(
+ "/v1/messages",
+ headers={**headers, "x-headroom-base-url": "https://gateway-b.example"},
+ json=body,
+ )
+
+ assert first.status_code == 200
+ assert second.status_code == 200
+ assert _text(first) == "gateway-A"
+ assert _text(second) == "gateway-B"
+ assert calls == [
+ "https://gateway-a.example/v1/messages",
+ "https://gateway-b.example/v1/messages",
+ ]