Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(proxy): partition response cache by upstream (#3349)
C0sm0catsdfdc72511ef4
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 7d67ce66f..492439992 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -1336,6 +1336,10 @@ class AnthropicHandlerMixin:# semantics (#1473 review). Non-generation metadata (metadata,# service_tier) is intentionally excluded.cache_key_fields = {+ # A per-request x-headroom-base-url selects a different response+ # producer. Without it in the key, two gateways serving the same+ # model/messages can return each other's cached response (#3346).+ "upstream_base_url": upstream_base_url,"system": body.get("system"),"tools": body.get("tools"),"tool_choice": body.get("tool_choice"),diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex efeecbd53..466e2283d 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -3598,6 +3598,10 @@ class OpenAIHandlerMixin:# review). Transport/metadata fields (stream, store, user, service_tier)# and the deprecated functions API are intentionally excluded.cache_key_fields = {+ # The resolved upstream is request-local when x-headroom-base-url+ # is present. Responses from separate gateways are not equivalent+ # even when their model/messages fields match (#3346).+ "upstream_base_url": upstream_base_url,"tools": body.get("tools"),"tool_choice": body.get("tool_choice"),"response_format": body.get("response_format"),diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.pyindex bd62070f7..acc40342b 100644--- a/tests/test_ccr_buffered_stream_signed_thinking.py+++ b/tests/test_ccr_buffered_stream_signed_thinking.py@@ -314,6 +314,7 @@ def test_cache_hit_never_replays_a_foreign_content_type() -> None:key = proxy.cache._compute_key(body["messages"],body["model"],+ upstream_base_url=None,system=None,tools=None,tool_choice=None,diff --git a/tests/test_proxy_openai_cache_key_integration.py b/tests/test_proxy_openai_cache_key_integration.pyindex e41a02a24..b4f201fe3 100644--- a/tests/test_proxy_openai_cache_key_integration.py+++ b/tests/test_proxy_openai_cache_key_integration.py@@ -114,3 +114,59 @@ def test_openai_differing_field_not_served_from_cache(field, a, b) -> None:assert ra2.status_code == 200assert _content(ra2) == "resp-1"assert calls["n"] == 2+++def test_openai_different_custom_upstream_not_served_from_cache(monkeypatch) -> None:+ """Two OpenAI-compatible gateways must not share a response entry (#3346)."""+ calls: list[str] = []++ monkeypatch.setattr(+ "headroom.proxy.handlers.openai._resolve_openai_upstream_base",+ lambda headers: headers.get("x-headroom-base-url"),+ )++ with _make_cached_proxy_client() as client:+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ calls.append(str(url))+ content = "gateway-A" if "gateway-a" in str(url) else "gateway-B"+ return httpx.Response(+ 200,+ json={+ "id": f"chatcmpl_{len(calls)}",+ "object": "chat.completion",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": content},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13},+ },+ )++ proxy._retry_request = _fake_retry+ headers = {"authorization": "Bearer test-key"}+ body = _body()++ first = client.post(+ "/v1/chat/completions",+ headers={**headers, "x-headroom-base-url": "https://gateway-a.example"},+ json=body,+ )+ second = client.post(+ "/v1/chat/completions",+ headers={**headers, "x-headroom-base-url": "https://gateway-b.example"},+ json=body,+ )++ assert first.status_code == 200+ assert second.status_code == 200+ assert _content(first) == "gateway-A"+ assert _content(second) == "gateway-B"+ assert calls == [+ "https://gateway-a.example/v1/chat/completions",+ "https://gateway-b.example/v1/chat/completions",+ ]diff --git a/tests/test_proxy_semantic_cache_key_integration.py b/tests/test_proxy_semantic_cache_key_integration.pyindex c7aa3c46e..b77364428 100644--- a/tests/test_proxy_semantic_cache_key_integration.py+++ b/tests/test_proxy_semantic_cache_key_integration.py@@ -167,3 +167,62 @@ def test_different_thinking_not_served_from_cache() -> None:assert a2.status_code == 200assert _text(a2) == "resp-1"assert calls["n"] == 2+++def test_different_custom_upstream_not_served_from_cache(monkeypatch) -> None:+ """The response producer is part of cache identity (#3346)."""+ calls: list[str] = []++ async def allow_example_upstream(_url: str) -> bool:+ return True++ monkeypatch.setattr(+ "headroom.providers.proxy_routes.is_safe_upstream_url_async",+ allow_example_upstream,+ )++ with _make_cached_proxy_client() as client:+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ calls.append(str(url))+ text = "gateway-A" if "gateway-a" in str(url) else "gateway-B"+ return httpx.Response(+ 200,+ json={+ "id": f"msg_{len(calls)}",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": text}],+ "usage": {+ "input_tokens": 10,+ "output_tokens": 3,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ },+ },+ )++ proxy._retry_request = _fake_retry+ headers = {"x-api-key": "test-key", "anthropic-version": "2023-06-01"}+ body = _body("Answer with the gateway identity.")++ first = client.post(+ "/v1/messages",+ headers={**headers, "x-headroom-base-url": "https://gateway-a.example"},+ json=body,+ )+ second = client.post(+ "/v1/messages",+ headers={**headers, "x-headroom-base-url": "https://gateway-b.example"},+ json=body,+ )++ assert first.status_code == 200+ assert second.status_code == 200+ assert _text(first) == "gateway-A"+ assert _text(second) == "gateway-B"+ assert calls == [+ "https://gateway-a.example/v1/messages",+ "https://gateway-b.example/v1/messages",+ ]