Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(proxy): preserve response cache keys across in-place hooks (#4457)

lorenzozane03590cd61e17
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index ac90f2a4b..377413da9 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -1544,7 +1544,9 @@ class AnthropicHandlerMixin:
# it was looked up by — the cache would never hit and would fill with
# unreachable entries. Reuse this raw snapshot verbatim at cache.set
# (the same reason cache_key_fields is snapshotted here, #327).
- cache_lookup_messages = messages
+ cache_lookup_messages = (
+ copy.deepcopy(messages) if self.cache and not stream else messages
+ )
# Response-cache partition: a cached response is only ever replayed to a
# caller presenting the same provider credentials and principal (01-F15).
# Snapshotted with the key fields so lookup and store agree. None means
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index 4bd8f48f0..b1cb4e61f 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -3701,7 +3701,7 @@ class OpenAIHandlerMixin:
# `messages`, but it runs before this snapshot, so its output is already
# captured — keep this snapshot after image compression, or a reorder
# silently reintroduces the drift.
- cache_lookup_messages = messages
+ cache_lookup_messages = copy.deepcopy(messages) if self.cache and not stream else messages
# Response-cache partition: a cached response is only ever replayed to a
# caller presenting the same provider credentials and principal (01-F15).
# Snapshotted with the key fields so lookup and store agree. None means
diff --git a/tests/test_response_cache_in_place_hook.py b/tests/test_response_cache_in_place_hook.py
new file mode 100644
index 000000000..a30a43e88
--- /dev/null
+++ b/tests/test_response_cache_in_place_hook.py
@@ -0,0 +1,101 @@
+"""Response-cache keys must survive in-place pre_compress mutations."""
+
+from __future__ import annotations
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient # noqa: E402
+
+from headroom.hooks import CompressionHooks # noqa: E402
+from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
+
+
+class _InPlaceHooks(CompressionHooks):
+ def pre_compress(self, messages, ctx):
+ for message in messages:
+ if message.get("role") == "user":
+ message["content"] = "rewritten in place"
+ return messages
+
+
+ ("provider", "path", "headers", "body", "response"),
+ [
+ (
+ "openai",
+ "/v1/chat/completions",
+ {"authorization": "Bearer test-key"},
+ {"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "original"}]},
+ {
+ "id": "chatcmpl_1",
+ "object": "chat.completion",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": "reply"},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
+ },
+ ),
+ (
+ "anthropic",
+ "/v1/messages",
+ {"x-api-key": "test-key", "anthropic-version": "2023-06-01"},
+ {
+ "model": "claude-haiku-4-5",
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "original"}],
+ },
+ {
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": "reply"}],
+ "usage": {
+ "input_tokens": 5,
+ "output_tokens": 1,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ },
+ ),
+ ],
+)
+def test_in_place_pre_compress_mutation_keeps_response_cache_hit(
+ provider: str, path: str, headers: dict, body: dict, response: dict
+) -> None:
+ calls = {"n": 0}
+ config = ProxyConfig(
+ optimize=False,
+ cache_enabled=True,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ hooks=_InPlaceHooks(),
+ )
+
+ with TestClient(create_app(config)) as client:
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, request_headers, request_body, stream=False, **kwargs):
+ calls["n"] += 1
+ result = dict(response)
+ result["id"] = f"{provider}_{calls['n']}"
+ return httpx.Response(200, json=result)
+
+ proxy._retry_request = _fake_retry
+ first = client.post(path, headers=headers, json=body)
+ second = client.post(path, headers=headers, json=body)
+
+ assert first.status_code == second.status_code == 200
+ assert calls["n"] == 1
+ assert first.json() == second.json()