Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(security): partition the response cache by credential and principal (#3862)

Tejas Chopraee6cfcccf6a8
diff --git a/headroom/proxy/handlers/anthropic.py b/headroom/proxy/handlers/anthropic.py
index 0b5477d0a..6f97dda20 100644
--- a/headroom/proxy/handlers/anthropic.py
+++ b/headroom/proxy/handlers/anthropic.py
@@ -66,6 +66,7 @@ from headroom.proxy.nonstream_sse_policy import should_recover_sse_reply
from headroom.proxy.outcome import RequestOutcome
from headroom.proxy.output_shaper import shaper_enabled_for, steering_allowed_for
from headroom.proxy.rate_limit_identity import rate_limit_identity
+from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.tenant_key import resolve_tenant_key, set_request_tenant_key
from headroom.proxy.thinking_tokens import ThinkingTokens, extract_thinking_tokens
from headroom.utils import format_exception_message
@@ -1430,10 +1431,21 @@ class AnthropicHandlerMixin:
# unreachable entries. Reuse this raw snapshot verbatim at cache.set
# (the same reason cache_key_fields is snapshotted here, #327).
cache_lookup_messages = messages
+ # Response-cache partition: a cached response is only ever replayed to a
+ # caller presenting the same provider credentials and principal (01-F15).
+ # Snapshotted with the key fields so lookup and store agree. None means
+ # the principal could not be established: skip the cache entirely.
+ # Only resolved when the cache can be used, so streaming and
+ # cache-disabled requests never pay for identity resolution.
+ cache_partition = (
+ compute_request_cache_partition(request) if self.cache and not stream else None
+ )
# Check cache (non-streaming only)
cache_hit = False
- if self.cache and not stream:
- cached = await self.cache.get(messages, model, **cache_key_fields)
+ if self.cache and not stream and cache_partition is not None:
+ cached = await self.cache.get(
+ messages, model, partition=cache_partition, **cache_key_fields
+ )
if cached:
cache_hit = True
self.pipeline_extensions.emit(
@@ -4835,6 +4847,7 @@ class AnthropicHandlerMixin:
if (
self.cache
and not stream
+ and cache_partition is not None
and response.status_code == 200
and resp_json is not None
):
@@ -4844,6 +4857,7 @@ class AnthropicHandlerMixin:
response.content,
dict(response.headers),
tokens_saved=tokens_saved,
+ partition=cache_partition,
**cache_key_fields,
)
diff --git a/headroom/proxy/handlers/openai.py b/headroom/proxy/handlers/openai.py
index da5291faa..bc980af5a 100644
--- a/headroom/proxy/handlers/openai.py
+++ b/headroom/proxy/handlers/openai.py
@@ -37,6 +37,7 @@ from headroom.proxy.identity import resolve_memory_identity
from headroom.proxy.loopback_guard import is_loopback_host
from headroom.proxy.modes import is_cache_mode
from headroom.proxy.rate_limit_identity import rate_limit_identity
+from headroom.proxy.semantic_cache_key_policy import compute_request_cache_partition
from headroom.proxy.stage_timer import StageTimer, emit_stage_timings_log
from headroom.proxy.upstream_guard import is_safe_upstream_url
from headroom.proxy.ws_headers import WS_HOP_BY_HOP_HEADERS
@@ -3915,9 +3916,20 @@ class OpenAIHandlerMixin:
# captured — keep this snapshot after image compression, or a reorder
# silently reintroduces the drift.
cache_lookup_messages = messages
+ # Response-cache partition: a cached response is only ever replayed to a
+ # caller presenting the same provider credentials and principal (01-F15).
+ # Snapshotted with the key fields so lookup and store agree. None means
+ # the principal could not be established: skip the cache entirely.
+ # Only resolved when the cache can be used, so streaming and
+ # cache-disabled requests never pay for identity resolution.
+ cache_partition = (
+ compute_request_cache_partition(request) if self.cache and not stream else None
+ )
# Check cache
- if self.cache and not stream:
- cached = await self.cache.get(messages, model, **cache_key_fields)
+ if self.cache and not stream and cache_partition is not None:
+ cached = await self.cache.get(
+ messages, model, partition=cache_partition, **cache_key_fields
+ )
if cached:
self.pipeline_extensions.emit(
PipelineStage.INPUT_CACHED,
@@ -5802,13 +5814,19 @@ class OpenAIHandlerMixin:
# site, which let a response built for a stream:true request
# answer a later non-streaming caller (#3019). Stating the
# invariant keeps that from being reintroduced silently.
- if self.cache and not stream and response.status_code == 200:
+ if (
+ self.cache
+ and not stream
+ and cache_partition is not None
+ and response.status_code == 200
+ ):
await self.cache.set(
cache_lookup_messages,
model,
response.content,
dict(response.headers),
tokens_saved,
+ partition=cache_partition,
**cache_key_fields,
)
diff --git a/headroom/proxy/identity.py b/headroom/proxy/identity.py
index 18b9bed42..4ca4d56e1 100644
--- a/headroom/proxy/identity.py
+++ b/headroom/proxy/identity.py
@@ -30,12 +30,39 @@ class IdentityResolver(Protocol):
_resolver: IdentityResolver | None = None
+class UnresolvedPrincipalError(LookupError):
+ """An installed identity resolver could not establish a principal."""
+
+
def set_identity_resolver(resolver: IdentityResolver | None) -> None:
"""Install (or clear) a custom identity resolver — the enterprise hook."""
global _resolver
_resolver = resolver
+def resolve_authenticated_principal(request: Any) -> str | None:
+ """Return the authenticated principal for ``request``, or ``None``.
+
+ Only a registered custom resolver (see :func:`set_identity_resolver`)
+ establishes a real per-caller identity. The OSS default binds every
+ network caller to the same proxy-token identity and lets loopback callers
+ *choose* a partition by header, so neither is an authenticated principal
+ and both return ``None`` here. Callers that must isolate data between
+ principals (e.g. the response cache) combine this with the caller's own
+ provider credential rather than trusting the default identity.
+
+ An installed resolver that raises, or returns no principal, means the
+ principal is unknown rather than absent: this raises so the caller fails
+ closed instead of silently sharing data across principals.
+ """
+ if _resolver is None:
+ return None
+ principal = _resolver(request, default="")
+ if not principal:
+ raise UnresolvedPrincipalError("identity resolver returned no principal")
+ return principal
+
+
def _default_os_user() -> str:
return os.environ.get("USER", os.environ.get("USERNAME", "default"))
diff --git a/headroom/proxy/semantic_cache.py b/headroom/proxy/semantic_cache.py
index dee64cf55..9cd8aeb1a 100644
--- a/headroom/proxy/semantic_cache.py
+++ b/headroom/proxy/semantic_cache.py
@@ -61,7 +61,9 @@ class SemanticCache:
self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
self._lock = asyncio.Lock()
- def _compute_key(self, messages: list[dict], model: str, **key_fields: Any) -> str:
+ def _compute_key(
+ self, messages: list[dict], model: str, *, partition: str, **key_fields: Any
+ ) -> str:
"""Compute cache key from messages, model, and response-shaping fields.
``key_fields`` carries every request field that changes generation,
@@ -75,12 +77,19 @@ class SemanticCache:
on ``system``/``tools`` does not fragment the key (scalars pass through
untouched). Absent fields don't contribute, so truly-identical requests
still hit.
+
+ ``partition`` is the caller's cache partition
+ (:func:`headroom.proxy.semantic_cache_key_policy.compute_request_cache_partition`)
+ and is required: on a shared proxy, a response is only ever replayed to a
+ caller presenting the same provider credentials and principal.
"""
- return compute_semantic_cache_key(messages, model, **key_fields)
+ return compute_semantic_cache_key(messages, model, partition=partition, **key_fields)
- async def get(self, messages: list[dict], model: str, **key_fields: Any) -> CacheEntry | None:
+ async def get(
+ self, messages: list[dict], model: str, *, partition: str, **key_fields: Any
+ ) -> CacheEntry | None:
"""Get cached response if exists and not expired."""
- key = self._compute_key(messages, model, **key_fields)
+ key = self._compute_key(messages, model, partition=partition, **key_fields)
async with self._lock:
entry = self._cache.get(key)
@@ -105,10 +114,12 @@ class SemanticCache:
response_body: bytes,
response_headers: dict[str, str],
tokens_saved: int = 0,
+ *,
+ partition: str,
**key_fields: Any,
):
"""Cache a response."""
- key = self._compute_key(messages, model, **key_fields)
+ key = self._compute_key(messages, model, partition=partition, **key_fields)
if not _is_cacheable_reply(response_body):
return
diff --git a/headroom/proxy/semantic_cache_key_policy.py b/headroom/proxy/semantic_cache_key_policy.py
index 45b9818c1..65009bd89 100644
--- a/headroom/proxy/semantic_cache_key_policy.py
+++ b/headroom/proxy/semantic_cache_key_policy.py
@@ -3,9 +3,104 @@
from __future__ import annotations
import hashlib
+import hmac
import json
+import logging
+import re
+import secrets
+from collections.abc import Mapping
from typing import Any
+from headroom.proxy.internal_header_policy import INTERNAL_HEADER_PREFIX, is_credential_header
+
+logger = logging.getLogger(__name__)
+
+# Per-process key for the cache partition HMAC. The semantic response cache is
+# an in-memory, per-process structure, so the partition only has to be stable
+# for this process's lifetime; a fresh random key means a partition id can
+# never be computed (or brute-forced from a guessed credential) outside it.
+_PARTITION_KEY = secrets.token_bytes(32)
+
+# A header carries caller credentials or selects the billed account when the
+# shared credential-header rule matches it (``authorization``,
+# ``proxy-authorization``, ``cookie``, ``x-api-key``, ...) or its name ends in
+# one of these tokens (``x-goog-api-key``, ``chatgpt-account-id``,
+# ``openai-organization``, ``openai-project``, ...). Every such header the
+# handlers forward upstream must be in the partition. Matching by shape rather
+# than by a fixed provider list keeps new providers partitioned by default.
+_CREDENTIAL_HEADER_RE = re.compile(
+ r"(^|[-_])(api[-_]?key|key|token|secret|account|account[-_]id|organization|project)$"
+)
+# Per-request nonces that match the credential shape but identify nothing: a
+# fresh value per call would make every request a unique partition.
+_NON_CREDENTIAL_HEADERS = frozenset({"idempotency-key", "x-idempotency-key"})
+ANONYMOUS_PARTITION = "anon"
+
+
+def _is_credential_header(name: str) -> bool:
+ # ``x-headroom-*`` headers (including the proxy's own token, which the
+ # security gate has already removed) never reach the upstream, so they
+ # never identify the upstream account.
+ lowered = name.lower()
+ if lowered.startswith(INTERNAL_HEADER_PREFIX) or lowered in _NON_CREDENTIAL_HEADERS:
+ return False
+ return is_credential_header(lowered) or bool(_CREDENTIAL_HEADER_RE.search(lowered))
+
+
+def compute_cache_partition(
+ headers: Mapping[str, str] | Any,
+ *,
+ principal: str | None = None,
+) -> str:
+ """Return the response-cache partition for one caller.
+
+ Two requests may share a cached response only when they present the same
+ provider credentials / account selectors and the same authenticated
+ principal. The partition is an HMAC under a per-process random key, never a
+ plain hash, so it cannot be used to confirm a guessed credential.
+
+ Requests with no caller credential and no principal (the proxy supplies the
+ operator's key) share the ``anon`` partition: they are billed to, and
+ answered by, the same account.
+ """
+ material: list[tuple[str, str]] = []
+ items: list[tuple[Any, Any]] = list(headers.items()) if hasattr(headers, "items") else []
+ for name, value in items:
+ if value and _is_credential_header(str(name)):
+ material.append(("h:" + str(name).lower(), str(value).strip()))
+ if principal:
+ material.append(("principal", principal))
+ if not material:
+ return ANONYMOUS_PARTITION
+ material.sort()
+ digest = hmac.new(
+ _PARTITION_KEY, json.dumps(material, separators=(",", ":")).encode(), hashlib.sha256
+ ).hexdigest()
+ return "p_" + digest[:32]
+
+
+def compute_request_cache_partition(request: Any) -> str | None:
+ """Partition for a live proxy request: credentials + authenticated principal.
+
+ Returns ``None`` when an installed identity resolver raises or returns no
+ principal. The caller must then bypass the response cache (no lookup, no
+ store): falling back to the credential-only partition would let tenants
+ sharing one operator key read each other's cached responses.
+ """
+ from headroom.proxy.identity import resolve_authenticated_principal
+
+ try:
+ principal = resolve_authenticated_principal(request)
+ except Exception:
+ logger.warning(
+ "Identity resolver could not establish a principal; "
+ "bypassing the response cache for this request",
+ exc_info=True,
+ )
+ return None
+ headers = getattr(request, "headers", None) or {}
+ return compute_cache_partition(headers, principal=principal)
+
def strip_cache_control(obj: Any) -> Any:
"""Recursively drop prompt-cache annotations, preserving schema properties."""
@@ -33,6 +128,8 @@ def _strip_cache_control(obj: Any, *, preserve_key: bool) -> Any:
def compute_semantic_cache_key(
messages: list[dict],
model: str,
+ *,
+ partition: str,
**key_fields: Any,
) -> str:
"""Compute a stable cache key from request content and shaping fields.
@@ -44,9 +141,15 @@ def compute_semantic_cache_key(
Anthropic path, the most common place a client (e.g. Claude Code) moves a
breakpoint between turns, so leaving them un-stripped defeated the strip for
the field that matters most.
+
+ ``partition`` (see :func:`compute_cache_partition`) is mandatory: a shared
+ proxy must never answer one caller from another caller's cached response.
"""
+ if not partition:
+ raise ValueError("semantic cache key requires a caller partition")
normalized = json.dumps(
{
+ "partition": partition,
"model": model,
"messages": strip_cache_control(messages),
**{k: strip_cache_control(v) for k, v in key_fields.items()},
diff --git a/tests/test_ccr_buffered_stream_signed_thinking.py b/tests/test_ccr_buffered_stream_signed_thinking.py
index acc40342b..8130d0145 100644
--- a/tests/test_ccr_buffered_stream_signed_thinking.py
+++ b/tests/test_ccr_buffered_stream_signed_thinking.py
@@ -27,6 +27,7 @@ httpx = pytest.importorskip("httpx")
from fastapi.testclient import TestClient # noqa: E402
from headroom.proxy.models import CacheEntry # noqa: E402
+from headroom.proxy.semantic_cache_key_policy import compute_cache_partition # noqa: E402
from headroom.proxy.server import ProxyConfig, create_app # noqa: E402
RETRIEVE_TOOL = {
@@ -314,6 +315,8 @@ def test_cache_hit_never_replays_a_foreign_content_type() -> None:
key = proxy.cache._compute_key(
body["messages"],
body["model"],
+ # Same partition the handler derives for this caller (01-F15).
+ partition=compute_cache_partition(_headers()),
upstream_base_url=None,
system=None,
tools=None,
diff --git a/tests/test_proxy_response_cache_partition.py b/tests/test_proxy_response_cache_partition.py
new file mode 100644
index 000000000..564fccddd
--- /dev/null
+++ b/tests/test_proxy_response_cache_partition.py
@@ -0,0 +1,390 @@
+"""01-F15: the response cache must never answer one caller from another's entry.
+
+On a shared proxy (several principals behind one Headroom), two callers sending
+the identical request under different provider credentials used to share one
+semantic-cache entry: caller B was served the completion generated for caller A,
+under A's key. The partition is now a keyed HMAC of the caller's credential
+headers (and authenticated principal, when an identity resolver is installed),
+threaded through every cache lookup and store.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from types import SimpleNamespace
+
+import httpx
+import pytest
+
+pytest.importorskip("fastapi")
+
+from fastapi.testclient import TestClient
+
+from headroom.proxy import identity
+from headroom.proxy.semantic_cache import SemanticCache
+from headroom.proxy.semantic_cache_key_policy import (
+ ANONYMOUS_PARTITION,
+ compute_cache_partition,
+ compute_request_cache_partition,
+)
+from headroom.proxy.server import ProxyConfig, create_app
+
+# ---------------------------------------------------------------------------
+# Partition function
+# ---------------------------------------------------------------------------
+
+
+def test_partition_differs_per_credential_and_is_stable_per_credential() -> None:
+ a = compute_cache_partition({"x-api-key": "sk-ant-A"})
+ b = compute_cache_partition({"x-api-key": "sk-ant-B"})
+ assert a != b
+ assert a == compute_cache_partition({"X-Api-Key": "sk-ant-A"})
+
+
+ "header",
+ [
+ "authorization",
+ "proxy-authorization",
+ "cookie",
+ "x-api-key",
+ "api-key",
+ "x-goog-api-key",
+ "chatgpt-account-id",
+ "openai-organization",
+ "openai-project",
+ ],
+)
+def test_every_credential_or_account_header_partitions(header: str) -> None:
+ assert compute_cache_partition({header: "one"}) != compute_cache_partition({header: "two"})
+
+
+def test_non_credential_headers_do_not_fragment_the_cache() -> None:
+ base = compute_cache_partition({"x-api-key": "k"})
+ noisy = {
+ "x-api-key": "k",
+ "idempotency-key": "per-request-nonce",
+ "x-headroom-proxy-token": "proxy-secret",
+ "user-agent": "sdk/1.0",
+ "anthropic-version": "2023-06-01",
+ "content-type": "application/json",
+ }
+ assert compute_cache_partition(noisy) == base
+
+
+def test_no_credential_and_no_principal_is_the_shared_operator_partition() -> None:
+ assert compute_cache_partition({"user-agent": "x"}) == ANONYMOUS_PARTITION
+
+
+def test_partition_is_keyed_not_a_plain_hash_of_the_credential() -> None:
+ part = compute_cache_partition({"x-api-key": "sk-ant-A"})
+ for plain in (
+ hashlib.sha256(b"sk-ant-A").hexdigest()[:32],
+ hashlib.sha256(b"x-api-key:sk-ant-A").hexdigest()[:32],
+ ):
+ assert plain not in part
+
+
+def test_authenticated_principal_partitions_callers_sharing_a_credential() -> None:
+ same_key = {"authorization": "Bearer operator-key"}
+ assert compute_cache_partition(same_key, principal="alice") != compute_cache_partition(
+ same_key, principal="bob"
+ )
+
+
+def test_semantic_cache_refuses_calls_without_a_partition() -> None:
+ cache = SemanticCache()
+ with pytest.raises(TypeError):
+ cache._compute_key([{"role": "user", "content": "x"}], "m") # type: ignore[call-arg]
+
+
+async def test_semantic_cache_isolates_partitions() -> None:
+ cache = SemanticCache()
+ msgs = [{"role": "user", "content": "Say hi."}]
+ await cache.set(msgs, "m", b"A-body", {}, partition="p_a")
+ assert await cache.get(msgs, "m", partition="p_b") is None
+ hit = await cache.get(msgs, "m", partition="p_a")
+ assert hit is not None and hit.response_body == b"A-body"
+
+
+# ---------------------------------------------------------------------------
+# End to end through the real handlers
+# ---------------------------------------------------------------------------
+
+
+def _client(*, cache_enabled: bool = True) -> TestClient:
+ return TestClient(
+ create_app(
+ ProxyConfig(
+ optimize=False,
+ cache_enabled=cache_enabled,
+ rate_limit_enabled=False,
+ cost_tracking_enabled=False,
+ log_requests=False,
+ ccr_inject_tool=False,
+ ccr_handle_responses=False,
+ ccr_context_tracking=False,
+ image_optimize=False,
+ )
+ )
+ )
+
+
+def _anthropic_reply(text: str) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ json={
+ "id": "msg_1",
+ "type": "message",
+ "role": "assistant",
+ "content": [{"type": "text", "text": text}],
+ "usage": {
+ "input_tokens": 10,
+ "output_tokens": 3,
+ "cache_read_input_tokens": 0,
+ "cache_creation_input_tokens": 0,
+ },
+ },
+ )
+
+
+def _openai_reply(text: str) -> httpx.Response:
+ return httpx.Response(
+ 200,
+ json={
+ "id": "chatcmpl_1",
+ "object": "chat.completion",
+ "choices": [
+ {
+ "index": 0,
+ "message": {"role": "assistant", "content": text},
+ "finish_reason": "stop",
+ }
+ ],
+ "usage": {"prompt_tokens": 10, "completion_tokens": 3, "total_tokens": 13},
+ },
+ )
+
+
+ANTHROPIC_BODY = {
+ "model": "claude-haiku-4-5",
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "What is my account's secret plan?"}],
+ "stream": False,
+}
+OPENAI_BODY = {
+ "model": "gpt-4o-mini",
+ "max_tokens": 64,
+ "messages": [{"role": "user", "content": "What is my account's secret plan?"}],
+ "stream": False,
+}
+
+
+def _drive(path: str, body: dict, reply, header: str, fmt: str): # noqa: ANN001
+ """Send A, B (different credential), A again; return bodies and upstream owners."""
+ seen: list[str] = []
+ out: list[str] = []
+ with _client() as client:
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ lowered = {k.lower(): v for k, v in headers.items()}
+ owner = "A" if "key-A" in lowered.get(header, "") else "B"
+ seen.append(owner)
+ return reply(f"private answer for {owner}")
+
+ proxy._retry_request = _fake_retry
+ extra = {"anthropic-version": "2023-06-01"} if path == "/v1/messages" else {}
+ for who in ("A", "B", "A"):
+ r = client.post(path, headers={header: fmt.format(who), **extra}, json=body)
+ assert r.status_code == 200, r.text
+ out.append(r.text)
+ return out, seen
+
+
+ "path,body,reply,header,fmt",
+ [
+ ("/v1/messages", ANTHROPIC_BODY, _anthropic_reply, "x-api-key", "key-{}"),
+ ("/v1/chat/completions", OPENAI_BODY, _openai_reply, "authorization", "Bearer key-{}"),
+ # Cookie and Proxy-Authorization are forwarded upstream, so a
+ # cookie-authenticated gateway must not let two sessions share a reply.
+ ("/v1/messages", ANTHROPIC_BODY, _anthropic_reply, "cookie", "session=key-{}"),
+ ("/v1/chat/completions", OPENAI_BODY, _openai_reply, "cookie", "session=key-{}"),
+ (
+ "/v1/messages",
+ ANTHROPIC_BODY,
+ _anthropic_reply,
+ "proxy-authorization",
+ "Basic key-{}",
+ ),
+ (
+ "/v1/chat/completions",
+ OPENAI_BODY,
+ _openai_reply,
+ "proxy-authorization",
+ "Basic key-{}",
+ ),
+ ],
+ ids=[
+ "anthropic",
+ "openai",
+ "anthropic-cookie",
+ "openai-cookie",
+ "anthropic-proxy-authorization",
+ "openai-proxy-authorization",
+ ],
+)
+def test_second_caller_is_never_served_first_callers_cached_response(
+ path, body, reply, header, fmt
+) -> None:
+ (a, b, a_again), upstream = _drive(path, body, reply, header, fmt)
+ # B must reach the upstream under its own key and never see A's answer.
+ assert "private answer for A" not in b
+ assert "private answer for B" in b
+ # A's repeat is still a cache hit: single-credential hit rate is unchanged.
+ assert "private answer for A" in a_again
+ assert upstream == ["A", "B"]
+
+
+def test_installed_identity_resolver_partitions_callers_on_one_operator_key() -> None:
+ identity.set_identity_resolver(
+ lambda request, *, default: request.headers.get("x-test-user", "")
+ )
+ try:
+ with _client() as client:
+ proxy = client.app.state.proxy
+ calls: list[str] = []
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ calls.append("x")
+ return _anthropic_reply(f"answer {len(calls)}")
+
+ proxy._retry_request = _fake_retry
+ common = {"x-api-key": "shared-operator-key", "anthropic-version": "2023-06-01"}
+ r1 = client.post(
+ "/v1/messages", headers={**common, "x-test-user": "alice"}, json=ANTHROPIC_BODY
+ )
+ r2 = client.post(
+ "/v1/messages", headers={**common, "x-test-user": "bob"}, json=ANTHROPIC_BODY
+ )
+ assert "answer 1" in r1.text
+ assert "answer 2" in r2.text
+ assert len(calls) == 2
+ finally:
+ identity.set_identity_resolver(None)
+
+
+def _failing_resolver(request, *, default): # noqa: ANN001
+ raise RuntimeError("identity backend unavailable")
+
+
+def _empty_resolver(request, *, default): # noqa: ANN001
+ # Installed, but cannot establish a principal for this caller.
+ return ""
+
+
+UNRESOLVED_RESOLVERS = pytest.mark.parametrize(
+ "resolver", [_failing_resolver, _empty_resolver], ids=["raises", "empty"]
+)
+
+
+@UNRESOLVED_RESOLVERS
+def test_failing_identity_resolver_yields_no_partition(resolver) -> None: # noqa: ANN001
+ identity.set_identity_resolver(resolver)
+ try:
+ request = SimpleNamespace(headers={"x-api-key": "shared-operator-key"})
+ assert compute_request_cache_partition(request) is None
+ finally:
+ identity.set_identity_resolver(None)
+
+
+def test_no_installed_resolver_keeps_the_credential_partition() -> None:
+ request = SimpleNamespace(headers={"x-api-key": "shared-operator-key"})
+ assert compute_request_cache_partition(request) == compute_cache_partition(request.headers)
+
+
+ "path,body,reply,headers",
+ [
+ (
+ "/v1/messages",
+ ANTHROPIC_BODY,
+ _anthropic_reply,
+ {"x-api-key": "shared-operator-key", "anthropic-version": "2023-06-01"},
+ ),
+ (
+ "/v1/chat/completions",
+ OPENAI_BODY,
+ _openai_reply,
+ {"authorization": "Bearer shared-operator-key"},
+ ),
+ ],
+ ids=["anthropic", "openai"],
+)
+@UNRESOLVED_RESOLVERS
+def test_failing_identity_resolver_bypasses_the_response_cache(
+ resolver, # noqa: ANN001
+ path, # noqa: ANN001
+ body, # noqa: ANN001
+ reply, # noqa: ANN001
+ headers, # noqa: ANN001
+) -> None:
+ # A resolver that raises or returns no principal must not collapse tenants
+ # on one operator key into the credential-only partition: the cache is
+ # neither read nor written.
+ identity.set_identity_resolver(resolver)
+ try:
+ with _client() as client:
+ proxy = client.app.state.proxy
+ calls: list[str] = []
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ calls.append("x")
+ return reply(f"answer {len(calls)}")
+
+ proxy._retry_request = _fake_retry
+ texts = [
+ client.post(path, headers={**headers, "x-test-user": who}, json=body).text
+ for who in ("alice", "bob", "alice")
+ ]
+ assert "answer 1" in texts[0]
+ assert "answer 1" not in texts[1] and "answer 2" in texts[1]
+ assert "answer 3" in texts[2]
+ assert len(calls) == 3
+ finally:
+ identity.set_identity_resolver(None)
+
+
+ "path,body,reply,headers",
+ [
+ ("/v1/messages", ANTHROPIC_BODY, _anthropic_reply, {"x-api-key": "k"}),
+ ("/v1/chat/completions", OPENAI_BODY, _openai_reply, {"authorization": "Bearer k"}),
+ ],
+ ids=["anthropic", "openai"],
+)
+def test_cache_disabled_requests_skip_identity_resolution(path, body, reply, headers) -> None:
+ # No response cache means no partition is needed, so the (possibly
+ # remote) identity resolver must not be consulted for it.
+ resolved: list[str] = []
+
+ def _counting_resolver(request, *, default): # noqa: ANN001
+ resolved.append("x")
+ return "alice"
+
+ identity.set_identity_resolver(_counting_resolver)
+ try:
+ with _client(cache_enabled=False) as client:
+ proxy = client.app.state.proxy
+
+ async def _fake_retry(method, url, headers, body, stream=False, **kwargs): # noqa: ANN001
+ return reply("ok")
+
+ proxy._retry_request = _fake_retry
+ extra = {"anthropic-version": "2023-06-01"} if path == "/v1/messages" else {}
+ assert client.post(path, headers={**headers, **extra}, json=body).status_code == 200
+ assert resolved == []
+ finally:
+ identity.set_identity_resolver(None)
diff --git a/tests/test_proxy_semantic_cache_key.py b/tests/test_proxy_semantic_cache_key.py
index 7d1c9c628..67e1c35c6 100644
--- a/tests/test_proxy_semantic_cache_key.py
+++ b/tests/test_proxy_semantic_cache_key.py
@@ -22,7 +22,7 @@ MODEL = "claude-haiku-4-5"
def _key(cache: SemanticCache, **kw) -> str:
- return cache._compute_key(MESSAGES, MODEL, **kw)
+ return cache._compute_key(MESSAGES, MODEL, partition="p_test", **kw)
# --- too loose: different inputs must NOT collide (the bug) --------------------
@@ -145,8 +145,8 @@ def test_message_cache_control_breakpoint_move_same_key():
}
]
messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]
- assert cache._compute_key(messages_with_cc, MODEL) == cache._compute_key(
- messages_without_cc, MODEL
+ assert cache._compute_key(messages_with_cc, MODEL, partition="p_test") == cache._compute_key(
+ messages_without_cc, MODEL, partition="p_test"
)
@@ -155,7 +155,9 @@ def test_message_content_change_still_distinct_key():
cache = SemanticCache()
a = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]
b = [{"role": "user", "content": [{"type": "text", "text": "goodbye"}]}]
- assert cache._compute_key(a, MODEL) != cache._compute_key(b, MODEL)
+ assert cache._compute_key(a, MODEL, partition="p_test") != cache._compute_key(
+ b, MODEL, partition="p_test"
+ )
# --- behavioral get/set: collision prevented end to end -----------------------
@@ -165,11 +167,13 @@ async def test_get_set_collision_prevented():
"""Store under system A; fetching with system B is a MISS (no contamination),
fetching with system A is a HIT."""
cache = SemanticCache()
- await cache.set(MESSAGES, MODEL, b"french-body", {}, system="Answer only in French.")
+ await cache.set(
+ MESSAGES, MODEL, b"french-body", {}, partition="p_test", system="Answer only in French."
+ )
- miss = await cache.get(MESSAGES, MODEL, system="Answer only in English.")
+ miss = await cache.get(MESSAGES, MODEL, partition="p_test", system="Answer only in English.")
assert miss is None
- hit = await cache.get(MESSAGES, MODEL, system="Answer only in French.")
+ hit = await cache.get(MESSAGES, MODEL, partition="p_test", system="Answer only in French.")
assert hit is not None
assert hit.response_body == b"french-body"
diff --git a/tests/test_proxy_semantic_cache_key_policy.py b/tests/test_proxy_semantic_cache_key_policy.py
index 09a817e90..07e8a2a27 100644
--- a/tests/test_proxy_semantic_cache_key_policy.py
+++ b/tests/test_proxy_semantic_cache_key_policy.py
@@ -12,11 +12,11 @@ MODEL = "claude-haiku-4-5"
def test_semantic_cache_key_distinguishes_response_shaping_fields() -> None:
- assert compute_semantic_cache_key(MESSAGES, MODEL, system="French") != (
- compute_semantic_cache_key(MESSAGES, MODEL, system="English")
+ assert compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", system="French") != (
+ compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", system="English")
)
- assert compute_semantic_cache_key(MESSAGES, MODEL, temperature=0.0) != (
- compute_semantic_cache_key(MESSAGES, MODEL, temperature=1.0)
+ assert compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", temperature=0.0) != (
+ compute_semantic_cache_key(MESSAGES, MODEL, partition="p_test", temperature=1.0)
)
@@ -24,8 +24,12 @@ def test_semantic_cache_key_ignores_moved_cache_control() -> None:
with_cache_control = [{"type": "text", "text": "sys", "cache_control": {"type": "ephemeral"}}]
without_cache_control = [{"type": "text", "text": "sys"}]
- assert compute_semantic_cache_key(MESSAGES, MODEL, system=with_cache_control) == (
- compute_semantic_cache_key(MESSAGES, MODEL, system=without_cache_control)
+ assert compute_semantic_cache_key(
+ MESSAGES, MODEL, partition="p_test", system=with_cache_control
+ ) == (
+ compute_semantic_cache_key(
+ MESSAGES, MODEL, partition="p_test", system=without_cache_control
+ )
)
@@ -38,8 +42,8 @@ def test_semantic_cache_key_ignores_moved_message_cache_control() -> None:
}
]
messages_without_cc = [{"role": "user", "content": [{"type": "text", "text": "hello"}]}]
- assert compute_semantic_cache_key(messages_with_cc, MODEL) == (
- compute_semantic_cache_key(messages_without_cc, MODEL)
+ assert compute_semantic_cache_key(messages_with_cc, MODEL, partition="p_test") == (
+ compute_semantic_cache_key(messages_without_cc, MODEL, partition="p_test")
)
diff --git a/tests/test_semantic_cache_key_policy.py b/tests/test_semantic_cache_key_policy.py
index 44797b857..330989764 100644
--- a/tests/test_semantic_cache_key_policy.py
+++ b/tests/test_semantic_cache_key_policy.py
@@ -30,19 +30,23 @@ def test_strip_cache_control_recurses_through_dicts_and_lists() -> None:
def test_compute_semantic_cache_key_is_stable_for_identical_inputs() -> None:
kwargs = {"system": "sys", "tools": [{"name": "read"}], "temperature": 0.2}
- assert compute_semantic_cache_key(MESSAGES, MODEL, **kwargs) == compute_semantic_cache_key(
+ assert compute_semantic_cache_key(
+ MESSAGES, MODEL, partition="p_test", **kwargs
+ ) == compute_semantic_cache_key(
MESSAGES,
MODEL,
+ partition="p_test",
**kwargs,
)
def test_compute_semantic_cache_key_distinguishes_response_shaping_fields() -> None:
assert compute_semantic_cache_key(
- MESSAGES, MODEL, temperature=0.0
+ MESSAGES, MODEL, partition="p_test", temperature=0.0
) != compute_semantic_cache_key(
MESSAGES,
MODEL,
+ partition="p_test",
temperature=1.0,
)
@@ -53,10 +57,12 @@ def test_compute_semantic_cache_key_ignores_moved_cache_control_breakpoints() ->
assert compute_semantic_cache_key(
MESSAGES,
MODEL,
+ partition="p_test",
system=with_breakpoint,
) == compute_semantic_cache_key(
MESSAGES,
MODEL,
+ partition="p_test",
system=without_breakpoint,
)
@@ -64,8 +70,11 @@ def test_compute_semantic_cache_key_ignores_moved_cache_control_breakpoints() ->
def test_semantic_cache_private_key_wrapper_delegates_to_policy() -> None:
cache = SemanticCache()
kwargs = {"system": "sys", "tools": [{"name": "read"}], "temperature": 0.2}
- assert cache._compute_key(MESSAGES, MODEL, **kwargs) == compute_semantic_cache_key(
+ assert cache._compute_key(
+ MESSAGES, MODEL, partition="p_test", **kwargs
+ ) == compute_semantic_cache_key(
MESSAGES,
MODEL,
+ partition="p_test",
**kwargs,
)