Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(security): partition the response cache by credential and principal (#3862)
Tejas Chopraee6cfcccf6a8
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.pyindex 0b5477d0a..6f97dda20 100644--- a/headroom/proxy/handlers/anthropic.py+++ b/headroom/proxy/handlers/anthropic.py@@ -66,6 +66,7 @@ from headroom.proxy.nonstream_sse_policy import should_recover_sse_replyfrom headroom.proxy.outcome import RequestOutcomefrom headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_forfrom headroom.proxy.rate_limit_identity import rate_limit_identity+from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partitionfrom headroom.proxy.tenant_key import resolve_tenant_key, set_request_tenant_keyfrom headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokensfrom headroom.utils import format_exception_message@@ -1430,10 +1431,21 @@ class AnthropicHandlerMixin:# unreachable entries. Reuse this raw snapshot verbatim at cache.set# (the same reason cache_key_fields is snapshotted here, #327).cache_lookup_messages = messages+ # Response-cache partition: a cached response is only ever replayed to a+ # caller presenting the same provider credentials and principal (01-F15).+ # Snapshotted with the key fields so lookup and store agree. None means+ # the principal could not be established: skip the cache entirely.+ # Only resolved when the cache can be used, so streaming and+ # cache-disabled requests never pay for identity resolution.+ cache_partition = (+ compute_request_cache_partition(request) if self.cache and not stream else None+ )# Check cache (non-streaming only)cache_hit = False- if self.cache and not stream:- cached = await self.cache.get(messages, model, **cache_key_fields)+ if self.cache and not stream and cache_partition is not None:+ cached = await self.cache.get(+ messages, model, partition=cache_partition, **cache_key_fields+ )if cached:cache_hit = Trueself.pipeline_extensions.emit(@@ -4835,6 +4847,7 @@ class AnthropicHandlerMixin:if (self.cacheand not stream+ and cache_partition is not Noneand response.status_code == 200and resp_json is not None):@@ -4844,6 +4857,7 @@ class AnthropicHandlerMixin:response.content,dict(response.headers),tokens_saved=tokens_saved,+ partition=cache_partition,**cache_key_fields,)diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.pyindex da5291faa..bc980af5a 100644--- a/headroom/proxy/handlers/openai.py+++ b/headroom/proxy/handlers/openai.py@@ -37,6 +37,7 @@ from headroom.proxy.identity import resolve_memory_identityfrom headroom.proxy.loopback_guard import is_loopback_hostfrom headroom.proxy.modes import is_cache_modefrom headroom.proxy.rate_limit_identity import rate_limit_identity+from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partitionfrom headroom.proxy.stage_timer import StageTimer, emit_stage_timings_logfrom headroom.proxy.upstream_guard import is_safe_upstream_urlfrom headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS@@ -3915,9 +3916,20 @@ class OpenAIHandlerMixin:# captured — keep this snapshot after image compression, or a reorder# silently reintroduces the drift.cache_lookup_messages = messages+ # Response-cache partition: a cached response is only ever replayed to a+ # caller presenting the same provider credentials and principal (01-F15).+ # Snapshotted with the key fields so lookup and store agree. None means+ # the principal could not be established: skip the cache entirely.+ # Only resolved when the cache can be used, so streaming and+ # cache-disabled requests never pay for identity resolution.+ cache_partition = (+ compute_request_cache_partition(request) if self.cache and not stream else None+ )# Check cache- if self.cache and not stream:- cached = await self.cache.get(messages, model, **cache_key_fields)+ if self.cache and not stream and cache_partition is not None:+ cached = await self.cache.get(+ messages, model, partition=cache_partition, **cache_key_fields+ )if cached:self.pipeline_extensions.emit(PipelineStage.INPUT_CACHED,@@ -5802,13 +5814,19 @@ class OpenAIHandlerMixin:# site, which let a response built for a stream:true request# answer a later non-streaming caller (#3019). Stating the# invariant keeps that from being reintroduced silently.- if self.cache and not stream and response.status_code == 200:+ if (+ self.cache+ and not stream+ and cache_partition is not None+ and response.status_code == 200+ ):await self.cache.set(cache_lookup_messages,model,response.content,dict(response.headers),tokens_saved,+ partition=cache_partition,**cache_key_fields,)diff --git a/headroom/proxy/identity.py b/headroom/proxy/identity.pyindex 18b9bed42..4ca4d56e1 100644--- a/headroom/proxy/identity.py+++ b/headroom/proxy/identity.py@@ -30,12 +30,39 @@ class IdentityResolver(Protocol):_resolver: IdentityResolver | None = None+class UnresolvedPrincipalError(LookupError):+ """An installed identity resolver could not establish a principal."""++def set_identity_resolver(resolver: IdentityResolver | None) -> None:"""Install (or clear) a custom identity resolver — the enterprise hook."""global _resolver_resolver = resolver+def resolve_authenticated_principal(request: Any) -> str | None:+ """Return the authenticated principal for ``request``, or ``None``.++ Only a registered custom resolver (see :func:`set_identity_resolver`)+ establishes a real per-caller identity. The OSS default binds every+ network caller to the same proxy-token identity and lets loopback callers+ *choose* a partition by header, so neither is an authenticated principal+ and both return ``None`` here. Callers that must isolate data between+ principals (e.g. the response cache) combine this with the caller's own+ provider credential rather than trusting the default identity.++ An installed resolver that raises, or returns no principal, means the+ principal is unknown rather than absent: this raises so the caller fails+ closed instead of silently sharing data across principals.+ """+ if _resolver is None:+ return None+ principal = _resolver(request, default="")+ if not principal:+ raise UnresolvedPrincipalError("identity resolver returned no principal")+ return principal++def _default_os_user() -> str:return os.environ.get("USER", os.environ.get("USERNAME", "default"))diff --git a/headroom/proxy/semantic_cache.py b/headroom/proxy/semantic_cache.pyindex dee64cf55..9cd8aeb1a 100644--- a/headroom/proxy/semantic_cache.py+++ b/headroom/proxy/semantic_cache.py@@ -61,7 +61,9 @@ class SemanticCache:self._cache: OrderedDict[str, CacheEntry] = OrderedDict()self._lock = asyncio.Lock()- def _compute_key(self, messages: list[dict], model: str, **key_fields: Any) -> str:+ def _compute_key(+ self, messages: list[dict], model: str, *, partition: str, **key_fields: Any+ ) -> str:"""Compute cache key from messages, model, and response-shaping fields.``key_fields`` carries every request field that changes generation,@@ -75,12 +77,19 @@ class SemanticCache:on ``system``/``tools`` does not fragment the key (scalars pass throughuntouched). Absent fields don't contribute, so truly-identical requestsstill hit.++ ``partition`` is the caller's cache partition+ (:func:`headroom.proxy.semantic_cache_key_policy.compute_request_cache_partition`)+ and is required: on a shared proxy, a response is only ever replayed to a+ caller presenting the same provider credentials and principal."""- return compute_semantic_cache_key(messages, model, **key_fields)+ return compute_semantic_cache_key(messages, model, partition=partition, **key_fields)- async def get(self, messages: list[dict], model: str, **key_fields: Any) -> CacheEntry | None:+ async def get(+ self, messages: list[dict], model: str, *, partition: str, **key_fields: Any+ ) -> CacheEntry | None:"""Get cached response if exists and not expired."""- key = self._compute_key(messages, model, **key_fields)+ key = self._compute_key(messages, model, partition=partition, **key_fields)async with self._lock:entry = self._cache.get(key)@@ -105,10 +114,12 @@ class SemanticCache:response_body: bytes,response_headers: dict[str, str],tokens_saved: int = 0,+ *,+ partition: str,**key_fields: Any,):"""Cache a response."""- key = self._compute_key(messages, model, **key_fields)+ key = self._compute_key(messages, model, partition=partition, **key_fields)if not _is_cacheable_reply(response_body):returndiff --git a/headroom/proxy/semantic_cache_key_policy.py b/headroom/proxy/semantic_cache_key_policy.pyindex 45b9818c1..65009bd89 100644--- a/headroom/proxy/semantic_cache_key_policy.py+++ b/headroom/proxy/semantic_cache_key_policy.py@@ -3,9 +3,104 @@from __future__ import annotationsimport hashlib+import hmacimport json+import logging+import re+import secrets+from collections.abc import Mappingfrom typing import Any+from headroom.proxy.internal_header_policy import INTERNAL_HEADER_PREFIX, is_credential_header++logger = logging.getLogger(__name__)++# Per-process key for the cache partition HMAC. The semantic response cache is+# an in-memory, per-process structure, so the partition only has to be stable+# for this process's lifetime; a fresh random key means a partition id can+# never be computed (or brute-forced from a guessed credential) outside it.+_PARTITION_KEY = secrets.token_bytes(32)++# A header carries caller credentials or selects the billed account when the+# shared credential-header rule matches it (``authorization``,+# ``proxy-authorization``, ``cookie``, ``x-api-key``, ...) or its name ends in+# one of these tokens (``x-goog-api-key``, ``chatgpt-account-id``,+# ``openai-organization``, ``openai-project``, ...). Every such header the+# handlers forward upstream must be in the partition. Matching by shape rather+# than by a fixed provider list keeps new providers partitioned by default.+_CREDENTIAL_HEADER_RE = re.compile(+ r"(^|[-_])(api[-_]?key|key|token|secret|account|account[-_]id|organization|project)$"+)+# Per-request nonces that match the credential shape but identify nothing: a+# fresh value per call would make every request a unique partition.+_NON_CREDENTIAL_HEADERS = frozenset({"idempotency-key", "x-idempotency-key"})+ANONYMOUS_PARTITION = "anon"+++def _is_credential_header(name: str) -> bool:+ # ``x-headroom-*`` headers (including the proxy's own token, which the+ # security gate has already removed) never reach the upstream, so they+ # never identify the upstream account.+ lowered = name.lower()+ if lowered.startswith(INTERNAL_HEADER_PREFIX) or lowered in _NON_CREDENTIAL_HEADERS:+ return False+ return is_credential_header(lowered) or bool(_CREDENTIAL_HEADER_RE.search(lowered))+++def compute_cache_partition(+ headers: Mapping[str, str] | Any,+ *,+ principal: str | None = None,+) -> str:+ """Return the response-cache partition for one caller.++ Two requests may share a cached response only when they present the same+ provider credentials / account selectors and the same authenticated+ principal. The partition is an HMAC under a per-process random key, never a+ plain hash, so it cannot be used to confirm a guessed credential.++ Requests with no caller credential and no principal (the proxy supplies the+ operator's key) share the ``anon`` partition: they are billed to, and+ answered by, the same account.+ """+ material: list[tuple[str, str]] = []+ items: list[tuple[Any, Any]] = list(headers.items()) if hasattr(headers, "items") else []+ for name, value in items:+ if value and _is_credential_header(str(name)):+ material.append(("h:" + str(name).lower(), str(value).strip()))+ if principal:+ material.append(("principal", principal))+ if not material:+ return ANONYMOUS_PARTITION+ material.sort()+ digest = hmac.new(+ _PARTITION_KEY, json.dumps(material, separators=(",", ":")).encode(), hashlib.sha256+ ).hexdigest()+ return "p_" + digest[:32]+++def compute_request_cache_partition(request: Any) -> str | None:+ """Partition for a live proxy request: credentials + authenticated principal.++ Returns ``None`` when an installed identity resolver raises or returns no+ principal. The caller must then bypass the response cache (no lookup, no+ store): falling back to the credential-only partition would let tenants+ sharing one operator key read each other's cached responses.+ """+ from headroom.proxy.identity import resolve_authenticated_principal++ try:+ principal = resolve_authenticated_principal(request)+ except Exception:+ logger.warning(+ "Identity resolver could not establish a principal; "+ "bypassing the response cache for this request",+ exc_info=True,+ )+ return None+ headers = getattr(request, "headers", None) or {}+ return compute_cache_partition(headers, principal=principal)+def strip_cache_control(obj: Any) -> Any:"""Recursively drop prompt-cache annotations, preserving schema properties."""@@ -33,6 +128,8 @@ def _strip_cache_control(obj: Any, *, preserve_key: bool) -> Any:def compute_semantic_cache_key(messages: list[dict],model: str,+ *,+ partition: str,**key_fields: Any,) -> str:"""Compute a stable cache key from request content and shaping fields.@@ -44,9 +141,15 @@ def compute_semantic_cache_key(Anthropic path, the most common place a client (e.g. Claude Code) moves abreakpoint between turns, so leaving them un-stripped defeated the strip forthe field that matters most.++ ``partition`` (see :func:`compute_cache_partition`) is mandatory: a shared+ proxy must never answer one caller from another caller's cached response."""+ if not partition:+ raise ValueError("semantic cache key requires a caller partition")normalized = json.dumps({+ "partition": partition,"model": model,"messages": strip_cache_control(messages),**{k: strip_cache_control(v) for k, v in key_fields.items()},diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.pyindex acc40342b..8130d0145 100644--- a/tests/test_ccr_buffered_stream_signed_thinking.py+++ b/tests/test_ccr_buffered_stream_signed_thinking.py@@ -27,6 +27,7 @@ httpx = pytest.importorskip("httpx")from fastapi.testclient import TestClient # noqa: E402from headroom.proxy.models import CacheEntry # noqa: E402+from headroom.proxy.semantic_cache_key_policy import compute_cache_partition # noqa: E402from headroom.proxy.server import ProxyConfig, create_app # noqa: E402RETRIEVE_TOOL = {@@ -314,6 +315,8 @@ def test_cache_hit_never_replays_a_foreign_content_type() -> None:key = proxy.cache._compute_key(body["messages"],body["model"],+ # Same partition the handler derives for this caller (01-F15).+ partition=compute_cache_partition(_headers()),upstream_base_url=None,system=None,tools=None,diff --git a/tests/test_proxy_response_cache_partition.py b/tests/test_proxy_response_cache_partition.pynew file mode 100644index 000000000..564fccddd--- /dev/null+++ b/tests/test_proxy_response_cache_partition.py@@ -0,0 +1,390 @@+"""01-F15: the response cache must never answer one caller from another's entry.++On a shared proxy (several principals behind one Headroom), two callers sending+the identical request under different provider credentials used to share one+semantic-cache entry: caller B was served the completion generated for caller A,+under A's key. The partition is now a keyed HMAC of the caller's credential+headers (and authenticated principal, when an identity resolver is installed),+threaded through every cache lookup and store.+"""++from __future__ import annotations++import hashlib+from types import SimpleNamespace++import httpx+import pytest++pytest.importorskip("fastapi")++from fastapi.testclient import TestClient++from headroom.proxy import identity+from headroom.proxy.semantic_cache import SemanticCache+from headroom.proxy.semantic_cache_key_policy import (+ ANONYMOUS_PARTITION,+ compute_cache_partition,+ compute_request_cache_partition,+)+from headroom.proxy.server import ProxyConfig, create_app++# ---------------------------------------------------------------------------+# Partition function+# ---------------------------------------------------------------------------+++def test_partition_differs_per_credential_and_is_stable_per_credential() -> None:+ a = compute_cache_partition({"x-api-key": "sk-ant-A"})+ b = compute_cache_partition({"x-api-key": "sk-ant-B"})+ assert a != b+ assert a == compute_cache_partition({"X-Api-Key": "sk-ant-A"})+++ "header",+ [+ "authorization",+ "proxy-authorization",+ "cookie",+ "x-api-key",+ "api-key",+ "x-goog-api-key",+ "chatgpt-account-id",+ "openai-organization",+ "openai-project",+ ],+)+def test_every_credential_or_account_header_partitions(header: str) -> None:+ assert compute_cache_partition({header: "one"}) != compute_cache_partition({header: "two"})+++def test_non_credential_headers_do_not_fragment_the_cache() -> None:+ base = compute_cache_partition({"x-api-key": "k"})+ noisy = {+ "x-api-key": "k",+ "idempotency-key": "per-request-nonce",+ "x-headroom-proxy-token": "proxy-secret",+ "user-agent": "sdk/1.0",+ "anthropic-version": "2023-06-01",+ "content-type": "application/json",+ }+ assert compute_cache_partition(noisy) == base+++def test_no_credential_and_no_principal_is_the_shared_operator_partition() -> None:+ assert compute_cache_partition({"user-agent": "x"}) == ANONYMOUS_PARTITION+++def test_partition_is_keyed_not_a_plain_hash_of_the_credential() -> None:+ part = compute_cache_partition({"x-api-key": "sk-ant-A"})+ for plain in (+ hashlib.sha256(b"sk-ant-A").hexdigest()[:32],+ hashlib.sha256(b"x-api-key:sk-ant-A").hexdigest()[:32],+ ):+ assert plain not in part+++def test_authenticated_principal_partitions_callers_sharing_a_credential() -> None:+ same_key = {"authorization": "Bearer operator-key"}+ assert compute_cache_partition(same_key, principal="alice") != compute_cache_partition(+ same_key, principal="bob"+ )+++def test_semantic_cache_refuses_calls_without_a_partition() -> None:+ cache = SemanticCache()+ with pytest.raises(TypeError):+ cache._compute_key([{"role": "user", "content": "x"}], "m") # type: ignore[call-arg]+++async def test_semantic_cache_isolates_partitions() -> None:+ cache = SemanticCache()+ msgs = [{"role": "user", "content": "Say hi."}]+ await cache.set(msgs, "m", b"A-body", {}, partition="p_a")+ assert await cache.get(msgs, "m", partition="p_b") is None+ hit = await cache.get(msgs, "m", partition="p_a")+ assert hit is not None and hit.response_body == b"A-body"+++# ---------------------------------------------------------------------------+# End to end through the real handlers+# ---------------------------------------------------------------------------+++def _client(*, cache_enabled: bool = True) -> TestClient:+ return TestClient(+ create_app(+ ProxyConfig(+ optimize=False,+ cache_enabled=cache_enabled,+ rate_limit_enabled=False,+ cost_tracking_enabled=False,+ log_requests=False,+ ccr_inject_tool=False,+ ccr_handle_responses=False,+ ccr_context_tracking=False,+ image_optimize=False,+ )+ )+ )+++def _anthropic_reply(text: str) -> httpx.Response:+ return httpx.Response(+ 200,+ json={+ "id": "msg_1",+ "type": "message",+ "role": "assistant",+ "content": [{"type": "text", "text": text}],+ "usage": {+ "input_tokens": 10,+ "output_tokens": 3,+ "cache_read_input_tokens": 0,+ "cache_creation_input_tokens": 0,+ },+ },+ )+++def _openai_reply(text: str) -> httpx.Response:+ return httpx.Response(+ 200,+ json={+ "id": "chatcmpl_1",+ "object": "chat.completion",+ "choices": [+ {+ "index": 0,+ "message": {"role": "assistant", "content": text},+ "finish_reason": "stop",+ }+ ],+ "usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13},+ },+ )+++ANTHROPIC_BODY = {+ "model": "claude-haiku-4-5",+ "max_tokens": 64,+ "messages": [{"role": "user", "content": "What is my account's secret plan?"}],+ "stream": False,+}+OPENAI_BODY = {+ "model": "gpt-4o-mini",+ "max_tokens": 64,+ "messages": [{"role": "user", "content": "What is my account's secret plan?"}],+ "stream": False,+}+++def _drive(path: str, body: dict, reply, header: str, fmt: str): # noqa: ANN001+ """Send A, B (different credential), A again; return bodies and upstream owners."""+ seen: list[str] = []+ out: list[str] = []+ with _client() as client:+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ lowered = {k.lower(): v for k, v in headers.items()}+ owner = "A" if "key-A" in lowered.get(header, "") else "B"+ seen.append(owner)+ return reply(f"private answer for {owner}")++ proxy._retry_request = _fake_retry+ extra = {"anthropic-version": "2023-06-01"} if path == "/v1/messages" else {}+ for who in ("A", "B", "A"):+ r = client.post(path, headers={header: fmt.format(who), **extra}, json=body)+ assert r.status_code == 200, r.text+ out.append(r.text)+ return out, seen+++ "path,body,reply,header,fmt",+ [+ ("/v1/messages", ANTHROPIC_BODY, _anthropic_reply, "x-api-key", "key-{}"),+ ("/v1/chat/completions", OPENAI_BODY, _openai_reply, "authorization", "Bearer key-{}"),+ # Cookie and Proxy-Authorization are forwarded upstream, so a+ # cookie-authenticated gateway must not let two sessions share a reply.+ ("/v1/messages", ANTHROPIC_BODY, _anthropic_reply, "cookie", "session=key-{}"),+ ("/v1/chat/completions", OPENAI_BODY, _openai_reply, "cookie", "session=key-{}"),+ (+ "/v1/messages",+ ANTHROPIC_BODY,+ _anthropic_reply,+ "proxy-authorization",+ "Basic key-{}",+ ),+ (+ "/v1/chat/completions",+ OPENAI_BODY,+ _openai_reply,+ "proxy-authorization",+ "Basic key-{}",+ ),+ ],+ ids=[+ "anthropic",+ "openai",+ "anthropic-cookie",+ "openai-cookie",+ "anthropic-proxy-authorization",+ "openai-proxy-authorization",+ ],+)+def test_second_caller_is_never_served_first_callers_cached_response(+ path, body, reply, header, fmt+) -> None:+ (a, b, a_again), upstream = _drive(path, body, reply, header, fmt)+ # B must reach the upstream under its own key and never see A's answer.+ assert "private answer for A" not in b+ assert "private answer for B" in b+ # A's repeat is still a cache hit: single-credential hit rate is unchanged.+ assert "private answer for A" in a_again+ assert upstream == ["A", "B"]+++def test_installed_identity_resolver_partitions_callers_on_one_operator_key() -> None:+ identity.set_identity_resolver(+ lambda request, *, default: request.headers.get("x-test-user", "")+ )+ try:+ with _client() as client:+ proxy = client.app.state.proxy+ calls: list[str] = []++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ calls.append("x")+ return _anthropic_reply(f"answer {len(calls)}")++ proxy._retry_request = _fake_retry+ common = {"x-api-key": "shared-operator-key", "anthropic-version": "2023-06-01"}+ r1 = client.post(+ "/v1/messages", headers={**common, "x-test-user": "alice"}, json=ANTHROPIC_BODY+ )+ r2 = client.post(+ "/v1/messages", headers={**common, "x-test-user": "bob"}, json=ANTHROPIC_BODY+ )+ assert "answer 1" in r1.text+ assert "answer 2" in r2.text+ assert len(calls) == 2+ finally:+ identity.set_identity_resolver(None)+++def _failing_resolver(request, *, default): # noqa: ANN001+ raise RuntimeError("identity backend unavailable")+++def _empty_resolver(request, *, default): # noqa: ANN001+ # Installed, but cannot establish a principal for this caller.+ return ""+++UNRESOLVED_RESOLVERS = pytest.mark.parametrize(+ "resolver", [_failing_resolver, _empty_resolver], ids=["raises", "empty"]+)+++@UNRESOLVED_RESOLVERS+def test_failing_identity_resolver_yields_no_partition(resolver) -> None: # noqa: ANN001+ identity.set_identity_resolver(resolver)+ try:+ request = SimpleNamespace(headers={"x-api-key": "shared-operator-key"})+ assert compute_request_cache_partition(request) is None+ finally:+ identity.set_identity_resolver(None)+++def test_no_installed_resolver_keeps_the_credential_partition() -> None:+ request = SimpleNamespace(headers={"x-api-key": "shared-operator-key"})+ assert compute_request_cache_partition(request) == compute_cache_partition(request.headers)+++ "path,body,reply,headers",+ [+ (+ "/v1/messages",+ ANTHROPIC_BODY,+ _anthropic_reply,+ {"x-api-key": "shared-operator-key", "anthropic-version": "2023-06-01"},+ ),+ (+ "/v1/chat/completions",+ OPENAI_BODY,+ _openai_reply,+ {"authorization": "Bearer shared-operator-key"},+ ),+ ],+ ids=["anthropic", "openai"],+)+@UNRESOLVED_RESOLVERS+def test_failing_identity_resolver_bypasses_the_response_cache(+ resolver, # noqa: ANN001+ path, # noqa: ANN001+ body, # noqa: ANN001+ reply, # noqa: ANN001+ headers, # noqa: ANN001+) -> None:+ # A resolver that raises or returns no principal must not collapse tenants+ # on one operator key into the credential-only partition: the cache is+ # neither read nor written.+ identity.set_identity_resolver(resolver)+ try:+ with _client() as client:+ proxy = client.app.state.proxy+ calls: list[str] = []++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ calls.append("x")+ return reply(f"answer {len(calls)}")++ proxy._retry_request = _fake_retry+ texts = [+ client.post(path, headers={**headers, "x-test-user": who}, json=body).text+ for who in ("alice", "bob", "alice")+ ]+ assert "answer 1" in texts[0]+ assert "answer 1" not in texts[1] and "answer 2" in texts[1]+ assert "answer 3" in texts[2]+ assert len(calls) == 3+ finally:+ identity.set_identity_resolver(None)+++ "path,body,reply,headers",+ [+ ("/v1/messages", ANTHROPIC_BODY, _anthropic_reply, {"x-api-key": "k"}),+ ("/v1/chat/completions", OPENAI_BODY, _openai_reply, {"authorization": "Bearer k"}),+ ],+ ids=["anthropic", "openai"],+)+def test_cache_disabled_requests_skip_identity_resolution(path, body, reply, headers) -> None:+ # No response cache means no partition is needed, so the (possibly+ # remote) identity resolver must not be consulted for it.+ resolved: list[str] = []++ def _counting_resolver(request, *, default): # noqa: ANN001+ resolved.append("x")+ return "alice"++ identity.set_identity_resolver(_counting_resolver)+ try:+ with _client(cache_enabled=False) as client:+ proxy = client.app.state.proxy++ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001+ return reply("ok")++ proxy._retry_request = _fake_retry+ extra = {"anthropic-version": "2023-06-01"} if path == "/v1/messages" else {}+ assert client.post(path, headers={**headers, **extra}, json=body).status_code == 200+ assert resolved == []+ finally:+ identity.set_identity_resolver(None)diff --git a/tests/test_proxy_semantic_cache_key.py b/tests/test_proxy_semantic_cache_key.pyindex 7d1c9c628..67e1c35c6 100644--- a/tests/test_proxy_semantic_cache_key.py+++ b/tests/test_proxy_semantic_cache_key.py@@ -22,7 +22,7 @@ MODEL = "claude-haiku-4-5"def _key(cache: SemanticCache, **kw) -> str:- return cache._compute_key(MESSAGES, MODEL, **kw)+ return cache._compute_key(MESSAGES, MODEL, partition="p_test", **kw)# --- too loose: different inputs must NOT collide (the bug) --------------------@@ -145,8 +145,8 @@ def test_message_cache_control_breakpoint_move_same_key():}]messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]- assert cache._compute_key(messages_with_cc, MODEL) == cache._compute_key(- messages_without_cc, MODEL+ assert cache._compute_key(messages_with_cc, MODEL, partition="p_test") == cache._compute_key(+ messages_without_cc, MODEL, partition="p_test")@@ -155,7 +155,9 @@ def test_message_content_change_still_distinct_key():cache = SemanticCache()a = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]b = [{"role": "user", "content": [{"type": "text", "text": "goodbye"}]}]- assert cache._compute_key(a, MODEL) != cache._compute_key(b, MODEL)+ assert cache._compute_key(a, MODEL, partition="p_test") != cache._compute_key(+ b, MODEL, partition="p_test"+ )# --- behavioral get/set: collision prevented end to end -----------------------@@ -165,11 +167,13 @@ async def test_get_set_collision_prevented():"""Store under system A; fetching with system B is a MISS (no contamination),fetching with system A is a HIT."""cache = SemanticCache()- await cache.set(MESSAGES, MODEL, b"french-body", {}, system="Answer only in French.")+ await cache.set(+ MESSAGES, MODEL, b"french-body", {}, partition="p_test", system="Answer only in French."+ )- miss = await cache.get(MESSAGES, MODEL, system="Answer only in English.")+ miss = await cache.get(MESSAGES, MODEL, partition="p_test", system="Answer only in English.")assert miss is None- hit = await cache.get(MESSAGES, MODEL, system="Answer only in French.")+ hit = await cache.get(MESSAGES, MODEL, partition="p_test", system="Answer only in French.")assert hit is not Noneassert hit.response_body == b"french-body"diff --git a/tests/test_proxy_semantic_cache_key_policy.py b/tests/test_proxy_semantic_cache_key_policy.pyindex 09a817e90..07e8a2a27 100644--- a/tests/test_proxy_semantic_cache_key_policy.py+++ b/tests/test_proxy_semantic_cache_key_policy.py@@ -12,11 +12,11 @@ MODEL = "claude-haiku-4-5"def test_semantic_cache_key_distinguishes_response_shaping_fields() -> None:- assert compute_semantic_cache_key(MESSAGES, MODEL, system="French") != (- compute_semantic_cache_key(MESSAGES, MODEL, system="English")+ assert compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", system="French") != (+ compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", system="English"))- assert compute_semantic_cache_key(MESSAGES, MODEL, temperature=0.0) != (- compute_semantic_cache_key(MESSAGES, MODEL, temperature=1.0)+ assert compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", temperature=0.0) != (+ compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", temperature=1.0))@@ -24,8 +24,12 @@ def test_semantic_cache_key_ignores_moved_cache_control() -> None:with_cache_control = [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]without_cache_control = [{"type": "text", "text": "sys"}]- assert compute_semantic_cache_key(MESSAGES, MODEL, system=with_cache_control) == (- compute_semantic_cache_key(MESSAGES, MODEL, system=without_cache_control)+ assert compute_semantic_cache_key(+ MESSAGES, MODEL, partition="p_test", system=with_cache_control+ ) == (+ compute_semantic_cache_key(+ MESSAGES, MODEL, partition="p_test", system=without_cache_control+ ))@@ -38,8 +42,8 @@ def test_semantic_cache_key_ignores_moved_message_cache_control() -> None:}]messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]- assert compute_semantic_cache_key(messages_with_cc, MODEL) == (- compute_semantic_cache_key(messages_without_cc, MODEL)+ assert compute_semantic_cache_key(messages_with_cc, MODEL, partition="p_test") == (+ compute_semantic_cache_key(messages_without_cc, MODEL, partition="p_test"))diff --git a/tests/test_semantic_cache_key_policy.py b/tests/test_semantic_cache_key_policy.pyindex 44797b857..330989764 100644--- a/tests/test_semantic_cache_key_policy.py+++ b/tests/test_semantic_cache_key_policy.py@@ -30,19 +30,23 @@ def test_strip_cache_control_recurses_through_dicts_and_lists() -> None:def test_compute_semantic_cache_key_is_stable_for_identical_inputs() -> None:kwargs = {"system": "sys", "tools": [{"name": "read"}], "temperature": 0.2}- assert compute_semantic_cache_key(MESSAGES, MODEL, **kwargs) == compute_semantic_cache_key(+ assert compute_semantic_cache_key(+ MESSAGES, MODEL, partition="p_test", **kwargs+ ) == compute_semantic_cache_key(MESSAGES,MODEL,+ partition="p_test",**kwargs,)def test_compute_semantic_cache_key_distinguishes_response_shaping_fields() -> None:assert compute_semantic_cache_key(- MESSAGES, MODEL, temperature=0.0+ MESSAGES, MODEL, partition="p_test", temperature=0.0) != compute_semantic_cache_key(MESSAGES,MODEL,+ partition="p_test",temperature=1.0,)@@ -53,10 +57,12 @@ def test_compute_semantic_cache_key_ignores_moved_cache_control_breakpoints() ->assert compute_semantic_cache_key(MESSAGES,MODEL,+ partition="p_test",system=with_breakpoint,) == compute_semantic_cache_key(MESSAGES,MODEL,+ partition="p_test",system=without_breakpoint,)@@ -64,8 +70,11 @@ def test_compute_semantic_cache_key_ignores_moved_cache_control_breakpoints() ->def test_semantic_cache_private_key_wrapper_delegates_to_policy() -> None:cache = SemanticCache()kwargs = {"system": "sys", "tools": [{"name": "read"}], "temperature": 0.2}- assert cache._compute_key(MESSAGES, MODEL, **kwargs) == compute_semantic_cache_key(+ assert cache._compute_key(+ MESSAGES, MODEL, partition="p_test", **kwargs+ ) == compute_semantic_cache_key(MESSAGES,MODEL,+ partition="p_test",**kwargs,)