Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache): namespace SemanticCacheLayer by provider, model and metadata (#4473)

Parideboy99cdaaf1c7de
diff --git a/headroom/cache/semantic.py b/headroom/cache/semantic.py
index 946fada32..8e14f9359 100644
--- a/headroom/cache/semantic.py
+++ b/headroom/cache/semantic.py
@@ -36,6 +36,7 @@ Usage:
from __future__ import annotations
import hashlib
+import json
import time
from collections import OrderedDict
from collections.abc import Callable
@@ -74,6 +75,10 @@ class CacheEntry:
# Hash of the full messages for exact matching
messages_hash: str = ""
+ # Cache namespace (provider, model, metadata) so entries never replay
+ # across different provider/model/metadata contexts
+ namespace: str = ""
+
@dataclass
class SemanticCacheConfig:
@@ -122,7 +127,7 @@ class SemanticCache:
# LRU cache: key -> CacheEntry
self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
- # Exact hash index: messages_hash -> key
+ # Exact hash index: namespace-scoped hash -> key
self._hash_index: dict[str, str] = {}
# Statistics
@@ -134,6 +139,7 @@ class SemanticCache:
self,
query: str,
messages_hash: str | None = None,
+ namespace: str = "",
) -> CacheEntry | None:
"""
Look up a cached entry.
@@ -141,6 +147,8 @@ class SemanticCache:
Args:
query: Query text to search for
messages_hash: Optional exact hash for fast lookup
+ namespace: Cache namespace (provider, model, metadata); entries are
+ never returned outside their own namespace
Returns:
CacheEntry if found, None otherwise
@@ -149,13 +157,16 @@ class SemanticCache:
# Try exact hash match first
if messages_hash and self.config.use_exact_matching:
- key = self._hash_index.get(messages_hash)
+ index_key = self._combined_key(namespace, messages_hash)
+ key = self._hash_index.get(index_key)
if key and key in self._cache:
entry = self._cache[key]
- # Verify the stored entry really belongs to this request. Guards
- # against a stale index mapping ever pointing at an entry that was
- # overwritten by a different conversation sharing the same key.
- if entry.messages_hash == messages_hash:
+ # Verify the stored entry really belongs to this request and
+ # namespace. Guards against a stale index mapping ever pointing
+ # at an entry that was overwritten by a different conversation
+ # sharing the same key, and against replaying a cached response
+ # across provider/model/metadata contexts.
+ if entry.messages_hash == messages_hash and entry.namespace == namespace:
self._touch(key)
self._hits += 1
return entry
@@ -173,7 +184,7 @@ class SemanticCache:
# messages_hash above, which is context-complete and safe.
if self._embedding_fn and query.strip():
query_embedding = self._embedding_fn(query)
- best_match, best_similarity = self._find_similar(query_embedding)
+ best_match, best_similarity = self._find_similar(query_embedding, namespace)
if best_similarity >= self.config.similarity_threshold:
self._touch(best_match)
@@ -188,6 +199,7 @@ class SemanticCache:
query: str,
response: Any,
messages_hash: str | None = None,
+ namespace: str = "",
) -> str:
"""
Store a response in the cache.
@@ -196,6 +208,7 @@ class SemanticCache:
query: Query text
response: Response to cache
messages_hash: Optional exact hash for fast lookup
+ namespace: Cache namespace (provider, model, metadata)
Returns:
Cache key for the entry
@@ -206,8 +219,11 @@ class SemanticCache:
# a trailing user message ("continue", "yes", "run the tests") but differ
# in earlier context must NOT collide on one query-derived slot and
# overwrite each other. Fall back to the query hash only when no
- # messages_hash is supplied (e.g. embedding-only usage).
- key = messages_hash or self._generate_key(query)
+ # messages_hash is supplied (e.g. embedding-only usage). The key always
+ # includes the namespace so entries from different provider/model/
+ # metadata contexts never share a slot.
+ base = messages_hash or self._generate_key(query)
+ key = self._combined_key(namespace, base)
# Evict if adding a NEW key would exceed capacity. Overwriting a key that
# is already present is an in-place update that does not grow the map, so
@@ -235,13 +251,14 @@ class SemanticCache:
created_at=now,
last_accessed=now,
messages_hash=messages_hash or "",
+ namespace=namespace,
)
self._cache[key] = entry
# Index by hash for fast exact matching
if messages_hash:
- self._hash_index[messages_hash] = key
+ self._hash_index[self._combined_key(namespace, messages_hash)] = key
return key
@@ -250,7 +267,7 @@ class SemanticCache:
if key in self._cache:
entry = self._cache.pop(key)
if entry.messages_hash:
- self._hash_index.pop(entry.messages_hash, None)
+ self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
return True
return False
@@ -276,12 +293,15 @@ class SemanticCache:
def _find_similar(
self,
query_embedding: list[float],
+ namespace: str = "",
) -> tuple[str, float]:
- """Find the most similar cached entry."""
+ """Find the most similar cached entry within the same namespace."""
best_key = ""
best_similarity = -1.0
for key, entry in self._cache.items():
+ if entry.namespace != namespace:
+ continue
if not entry.embedding:
continue
@@ -325,7 +345,7 @@ class SemanticCache:
if self._cache:
key, entry = self._cache.popitem(last=False)
if entry.messages_hash:
- self._hash_index.pop(entry.messages_hash, None)
+ self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
self._evictions += 1
def _cleanup_expired(self) -> None:
@@ -343,7 +363,15 @@ class SemanticCache:
for key in expired:
entry = self._cache.pop(key)
if entry.messages_hash:
- self._hash_index.pop(entry.messages_hash, None)
+ self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
+
+ def _combined_key(self, namespace: str, base: str) -> str:
+ """Hash a namespace with an exact messages_hash or query-derived key.
+
+ Combining namespace + base (separated by a NUL so a namespace can't be
+ confused with a hash prefix) truncates like the existing hashes.
+ """
+ return hashlib.sha256((namespace + "\0" + base).encode()).hexdigest()[:24]
def _generate_key(self, query: str) -> str:
"""Generate a cache key for a query."""
@@ -405,9 +433,15 @@ class SemanticCacheLayer:
# Extract query for semantic matching
query = context.query or self._extract_query(messages)
messages_hash = self._compute_messages_hash(messages)
+ namespace = self._compute_namespace(context)
+
+ # Unserializable context: never share cached responses across contexts,
+ # so skip the cache and just delegate to the provider optimizer.
+ if namespace is None:
+ return self.provider_optimizer.optimize(messages, context, config)
# Check semantic cache
- cached = self.cache.get(query, messages_hash)
+ cached = self.cache.get(query, messages_hash, namespace)
if cached:
return CacheResult(
messages=messages,
@@ -440,15 +474,24 @@ class SemanticCacheLayer:
Args:
messages: Original messages
response: Response from LLM
- context: Optional context with query
+ context: Optional context with query. Pass the same OptimizationContext
+ used for `process`; omitting it stores under the unscoped
+ namespace, which only matches lookups whose context also has
+ empty provider, model and metadata.
Returns:
Cache key
"""
query = (context.query if context else None) or self._extract_query(messages)
messages_hash = self._compute_messages_hash(messages)
+ namespace = self._compute_namespace(context)
+
+ # Unserializable context: don't store anything that could be replayed
+ # for a different provider/model/metadata.
+ if namespace is None:
+ return ""
- return self.cache.put(query, response, messages_hash)
+ return self.cache.put(query, response, messages_hash, namespace)
def get_stats(self) -> dict[str, Any]:
"""Get combined statistics."""
@@ -457,6 +500,28 @@ class SemanticCacheLayer:
"provider_optimizer": self.provider_optimizer.name,
}
+ def _compute_namespace(self, context: OptimizationContext | None) -> str | None:
+ """Compute the cache namespace from provider, model, and metadata.
+
+ request_id and timestamp are deliberately excluded so they never affect
+ matching. Returns None when the context isn't JSON-serializable (e.g.
+ unserializable metadata), so callers skip lookup/storage rather than
+ replaying entries across contexts.
+ """
+ if context is None:
+ context = OptimizationContext()
+ try:
+ return json.dumps(
+ {
+ "provider": context.provider,
+ "model": context.model,
+ "metadata": context.metadata,
+ },
+ sort_keys=True,
+ )
+ except (TypeError, ValueError):
+ return None
+
def _extract_query(self, messages: list[dict[str, Any]]) -> str:
"""Extract the last user query from messages."""
for msg in reversed(messages):
@@ -473,8 +538,6 @@ class SemanticCacheLayer:
def _compute_messages_hash(self, messages: list[dict[str, Any]]) -> str:
"""Compute a hash of all messages."""
- import json
-
try:
content = json.dumps(messages, sort_keys=True)
return hashlib.sha256(content.encode()).hexdigest()[:24]
diff --git a/tests/test_cache/test_semantic.py b/tests/test_cache/test_semantic.py
index f5923f1cb..4143762e0 100644
--- a/tests/test_cache/test_semantic.py
+++ b/tests/test_cache/test_semantic.py
@@ -1,6 +1,8 @@
"""Tests for SemanticCache and SemanticCacheLayer."""
+import hashlib
import time
+from datetime import datetime
import pytest
@@ -238,6 +240,63 @@ class TestSemanticCache:
cache.put(" \n\t", "response-C", messages_hash="ctxC")
assert cache.get(" ", messages_hash="ctxD") is None
+ def test_namespace_isolation_exact_match(self, cache):
+ """Same messages_hash in different namespaces must not collide."""
+ cache.put("continue", "response A", messages_hash="shared", namespace="ns-a")
+ cache.put("continue", "response B", messages_hash="shared", namespace="ns-b")
+
+ assert (
+ cache.get("continue", messages_hash="shared", namespace="ns-a").response == "response A"
+ )
+ assert (
+ cache.get("continue", messages_hash="shared", namespace="ns-b").response == "response B"
+ )
+ # A third namespace that never stored the hash is a miss.
+ assert cache.get("continue", messages_hash="shared", namespace="ns-c") is None
+
+ def test_namespace_isolation_semantic(self):
+ """Semantic matches never cross namespace boundaries."""
+
+ def embedding(text: str) -> list[float]:
+ return [1.0, 0.0, 0.0] if "weather" in text.lower() else [0.0, 1.0, 0.0]
+
+ config = SemanticCacheConfig(similarity_threshold=0.9)
+ cache = SemanticCache(config, embedding_fn=embedding)
+
+ cache.put("What is the weather?", "sunny", namespace="ns-a")
+
+ # Same query, different namespace: no hit.
+ assert cache.get("How is the weather?", namespace="ns-b") is None
+ # Same namespace: still hits.
+ assert cache.get("How is the weather?", namespace="ns-a").response == "sunny"
+
+ def test_query_derived_key_includes_namespace(self, cache):
+ """Without messages_hash the query-derived key is still namespace-scoped."""
+ key_a = cache.put("run the tests", "response A", namespace="ns-a")
+ key_b = cache.put("run the tests", "response B", namespace="ns-b")
+
+ # Different namespaces produce different keys for the same query.
+ assert key_a != key_b
+ # Re-storing in the same namespace reuses the key (in-place update).
+ key_a_again = cache.put("run the tests", "response A2", namespace="ns-a")
+ assert key_a_again == key_a
+
+ def test_max_entries_enforced_globally_across_namespaces(self):
+ """Eviction counts entries across all namespaces, not per namespace."""
+ config = SemanticCacheConfig(max_entries=2)
+ cache = SemanticCache(config)
+
+ cache.put("query1", "response1", messages_hash="h1", namespace="ns-a")
+ cache.put("query2", "response2", messages_hash="h2", namespace="ns-b")
+
+ # Cache is full: a new entry in a third namespace evicts the LRU entry
+ # even though that entry belongs to a different namespace.
+ cache.put("query3", "response3", messages_hash="h3", namespace="ns-c")
+
+ assert cache.get("query1", messages_hash="h1", namespace="ns-a") is None
+ assert cache.get("query2", messages_hash="h2", namespace="ns-b") is not None
+ assert cache.get("query3", messages_hash="h3", namespace="ns-c") is not None
+
class TestSemanticCacheLayer:
"""Test SemanticCacheLayer functionality."""
@@ -373,3 +432,149 @@ class TestSemanticCacheLayer:
result = layer.process(messages, context)
assert result.semantic_cache_hit is True
+
+ def test_different_model_no_hit(self, layer):
+ """Cached responses must not replay across models."""
+ messages = [{"role": "user", "content": "Find alpha."}]
+ context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")
+ context_b = OptimizationContext(provider="anthropic", model="claude-3-sonnet")
+
+ layer.store_response(messages, {"answer": "opus response"}, context_a)
+
+ result = layer.process(messages, context_b)
+ assert result.semantic_cache_hit is False
+
+ def test_different_provider_no_hit(self, layer):
+ """Cached responses must not replay across providers."""
+ messages = [{"role": "user", "content": "Find alpha."}]
+ context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")
+ context_b = OptimizationContext(provider="bedrock", model="claude-3-opus")
+
+ layer.store_response(messages, {"answer": "opus response"}, context_a)
+
+ result = layer.process(messages, context_b)
+ assert result.semantic_cache_hit is False
+
+ def test_different_metadata_no_hit(self, layer):
+ """Cached responses must not replay across different metadata."""
+ messages = [{"role": "user", "content": "Find alpha."}]
+ context_a = OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ metadata={"temperature": 0.7},
+ )
+ context_b = OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ metadata={"temperature": 0.9},
+ )
+
+ layer.store_response(messages, {"answer": "opus response"}, context_a)
+
+ result = layer.process(messages, context_b)
+ assert result.semantic_cache_hit is False
+
+ def test_embedding_different_model_no_hit_same_context_hit(self):
+ """Semantic embedding lookup is namespace-scoped too."""
+ messages = [{"role": "user", "content": "What is 2+2?"}]
+
+ def deterministic_embedding(text: str) -> list[float]:
+ digest = hashlib.sha256(text.encode()).digest()
+ return [float(b) / 255.0 for b in digest[:4]]
+
+ layer = SemanticCacheLayer(
+ AnthropicCacheOptimizer(),
+ similarity_threshold=0.9,
+ max_entries=100,
+ ttl_seconds=60,
+ embedding_fn=deterministic_embedding,
+ )
+
+ context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")
+ context_b = OptimizationContext(provider="anthropic", model="claude-3-sonnet")
+
+ layer.store_response(messages, {"text": "4"}, context_a)
+
+ # Same context: exact messages_hash hit via the embedding path's store.
+ assert layer.process(messages, context_a).semantic_cache_hit is True
+ # Different model: namespace differs, so no hit even though the query
+ # embedding is identical.
+ assert layer.process(messages, context_b).semantic_cache_hit is False
+
+ def test_request_id_and_timestamp_do_not_affect_matching(self, layer):
+ """request_id and timestamp are not part of the namespace."""
+ messages = [{"role": "user", "content": "Hello!"}]
+ context_a = OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ request_id="req-1",
+ timestamp=datetime(2024, 1, 1),
+ )
+ context_b = OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ request_id="req-2",
+ timestamp=datetime(2025, 6, 15),
+ )
+
+ layer.store_response(messages, {"text": "hi"}, context_a)
+
+ result = layer.process(messages, context_b)
+ assert result.semantic_cache_hit is True
+ assert result.cached_response == {"text": "hi"}
+
+ def test_unserializable_metadata_store_returns_empty_and_no_hit(self, layer):
+ """Unserializable metadata yields a None namespace: store refuses and
+ lookup skips the cache entirely."""
+ messages = [{"role": "user", "content": "Hello!"}]
+ context = OptimizationContext(
+ provider="anthropic",
+ model="claude-3-opus",
+ metadata={"x": object()},
+ )
+
+ key = layer.store_response(messages, {"text": "hi"}, context)
+ assert key == ""
+
+ result = layer.process(messages, context)
+ assert result.semantic_cache_hit is False
+
+
+class TestSemanticCoverage:
+ def test_process_namespace_none_delegates(self):
+ from headroom.cache import AnthropicCacheOptimizer, OptimizationContext, SemanticCacheLayer
+
+ layer = SemanticCacheLayer(AnthropicCacheOptimizer())
+ messages = [{"role": "user", "content": "hi"}]
+ context = OptimizationContext(provider="anthropic", model="m", metadata={"x": object()})
+ result = layer.process(messages, context)
+ assert result.semantic_cache_hit is False
+
+ def test_invalidate_namespaced_entry(self):
+ from headroom.cache.semantic import SemanticCache, SemanticCacheConfig
+
+ cache = SemanticCache(SemanticCacheConfig())
+ key = cache.put("q", "r", messages_hash="h1", namespace="ns-a")
+ assert cache.get("q", messages_hash="h1", namespace="ns-a") is not None
+ assert cache.invalidate(key) is True
+ assert cache.get("q", messages_hash="h1", namespace="ns-a") is None
+ cache.put("q", "r2", messages_hash="h1", namespace="ns-a")
+ assert cache.get("q", messages_hash="h1", namespace="ns-a").response == "r2"
+
+ def test_evict_oldest_removes_namespaced_index(self):
+ from headroom.cache.semantic import SemanticCache, SemanticCacheConfig
+
+ cache = SemanticCache(SemanticCacheConfig(max_entries=1))
+ cache.put("q1", "r1", messages_hash="h1", namespace="ns-a")
+ cache.put("q2", "r2", messages_hash="h2", namespace="ns-b")
+ assert cache.get("q1", messages_hash="h1", namespace="ns-a") is None
+
+ def test_context_free_store_policy(self):
+ from headroom.cache import AnthropicCacheOptimizer, OptimizationContext, SemanticCacheLayer
+
+ layer = SemanticCacheLayer(AnthropicCacheOptimizer())
+ messages = [{"role": "user", "content": "Hello!"}]
+ layer.store_response(messages, {"text": "hi"})
+ assert layer.process(messages, OptimizationContext()).semantic_cache_hit is True
+ ctx = OptimizationContext(provider="anthropic", model="claude-3-opus")
+ assert layer.process(messages, ctx).semantic_cache_hit is False