Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache): namespace SemanticCacheLayer by provider, model and metadata (#4473)
Parideboy99cdaaf1c7de
diff --git a/headroom/cache/semantic.py b/headroom/cache/semantic.pyindex 946fada32..8e14f9359 100644--- a/headroom/cache/semantic.py+++ b/headroom/cache/semantic.py@@ -36,6 +36,7 @@ Usage:from __future__ import annotationsimport hashlib+import jsonimport timefrom collections import OrderedDictfrom collections.abc import Callable@@ -74,6 +75,10 @@ class CacheEntry:# Hash of the full messages for exact matchingmessages_hash: str = ""+ # Cache namespace (provider, model, metadata) so entries never replay+ # across different provider/model/metadata contexts+ namespace: str = ""+@dataclassclass SemanticCacheConfig:@@ -122,7 +127,7 @@ class SemanticCache:# LRU cache: key -> CacheEntryself._cache: OrderedDict[str, CacheEntry] = OrderedDict()- # Exact hash index: messages_hash -> key+ # Exact hash index: namespace-scoped hash -> keyself._hash_index: dict[str, str] = {}# Statistics@@ -134,6 +139,7 @@ class SemanticCache:self,query: str,messages_hash: str | None = None,+ namespace: str = "",) -> CacheEntry | None:"""Look up a cached entry.@@ -141,6 +147,8 @@ class SemanticCache:Args:query: Query text to search formessages_hash: Optional exact hash for fast lookup+ namespace: Cache namespace (provider, model, metadata); entries are+ never returned outside their own namespaceReturns:CacheEntry if found, None otherwise@@ -149,13 +157,16 @@ class SemanticCache:# Try exact hash match firstif messages_hash and self.config.use_exact_matching:- key = self._hash_index.get(messages_hash)+ index_key = self._combined_key(namespace, messages_hash)+ key = self._hash_index.get(index_key)if key and key in self._cache:entry = self._cache[key]- # Verify the stored entry really belongs to this request. Guards- # against a stale index mapping ever pointing at an entry that was- # overwritten by a different conversation sharing the same key.- if entry.messages_hash == messages_hash:+ # Verify the stored entry really belongs to this request and+ # namespace. Guards against a stale index mapping ever pointing+ # at an entry that was overwritten by a different conversation+ # sharing the same key, and against replaying a cached response+ # across provider/model/metadata contexts.+ if entry.messages_hash == messages_hash and entry.namespace == namespace:self._touch(key)self._hits += 1return entry@@ -173,7 +184,7 @@ class SemanticCache:# messages_hash above, which is context-complete and safe.if self._embedding_fn and query.strip():query_embedding = self._embedding_fn(query)- best_match, best_similarity = self._find_similar(query_embedding)+ best_match, best_similarity = self._find_similar(query_embedding, namespace)if best_similarity >= self.config.similarity_threshold:self._touch(best_match)@@ -188,6 +199,7 @@ class SemanticCache:query: str,response: Any,messages_hash: str | None = None,+ namespace: str = "",) -> str:"""Store a response in the cache.@@ -196,6 +208,7 @@ class SemanticCache:query: Query textresponse: Response to cachemessages_hash: Optional exact hash for fast lookup+ namespace: Cache namespace (provider, model, metadata)Returns:Cache key for the entry@@ -206,8 +219,11 @@ class SemanticCache:# a trailing user message ("continue", "yes", "run the tests") but differ# in earlier context must NOT collide on one query-derived slot and# overwrite each other. Fall back to the query hash only when no- # messages_hash is supplied (e.g. embedding-only usage).- key = messages_hash or self._generate_key(query)+ # messages_hash is supplied (e.g. embedding-only usage). The key always+ # includes the namespace so entries from different provider/model/+ # metadata contexts never share a slot.+ base = messages_hash or self._generate_key(query)+ key = self._combined_key(namespace, base)# Evict if adding a NEW key would exceed capacity. Overwriting a key that# is already present is an in-place update that does not grow the map, so@@ -235,13 +251,14 @@ class SemanticCache:created_at=now,last_accessed=now,messages_hash=messages_hash or "",+ namespace=namespace,)self._cache[key] = entry# Index by hash for fast exact matchingif messages_hash:- self._hash_index[messages_hash] = key+ self._hash_index[self._combined_key(namespace, messages_hash)] = keyreturn key@@ -250,7 +267,7 @@ class SemanticCache:if key in self._cache:entry = self._cache.pop(key)if entry.messages_hash:- self._hash_index.pop(entry.messages_hash, None)+ self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)return Truereturn False@@ -276,12 +293,15 @@ class SemanticCache:def _find_similar(self,query_embedding: list[float],+ namespace: str = "",) -> tuple[str, float]:- """Find the most similar cached entry."""+ """Find the most similar cached entry within the same namespace."""best_key = ""best_similarity = -1.0for key, entry in self._cache.items():+ if entry.namespace != namespace:+ continueif not entry.embedding:continue@@ -325,7 +345,7 @@ class SemanticCache:if self._cache:key, entry = self._cache.popitem(last=False)if entry.messages_hash:- self._hash_index.pop(entry.messages_hash, None)+ self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)self._evictions += 1def _cleanup_expired(self) -> None:@@ -343,7 +363,15 @@ class SemanticCache:for key in expired:entry = self._cache.pop(key)if entry.messages_hash:- self._hash_index.pop(entry.messages_hash, None)+ self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)++ def _combined_key(self, namespace: str, base: str) -> str:+ """Hash a namespace with an exact messages_hash or query-derived key.++ Combining namespace + base (separated by a NUL so a namespace can't be+ confused with a hash prefix) truncates like the existing hashes.+ """+ return hashlib.sha256((namespace + "\0" + base).encode()).hexdigest()[:24]def _generate_key(self, query: str) -> str:"""Generate a cache key for a query."""@@ -405,9 +433,15 @@ class SemanticCacheLayer:# Extract query for semantic matchingquery = context.query or self._extract_query(messages)messages_hash = self._compute_messages_hash(messages)+ namespace = self._compute_namespace(context)++ # Unserializable context: never share cached responses across contexts,+ # so skip the cache and just delegate to the provider optimizer.+ if namespace is None:+ return self.provider_optimizer.optimize(messages, context, config)# Check semantic cache- cached = self.cache.get(query, messages_hash)+ cached = self.cache.get(query, messages_hash, namespace)if cached:return CacheResult(messages=messages,@@ -440,15 +474,24 @@ class SemanticCacheLayer:Args:messages: Original messagesresponse: Response from LLM- context: Optional context with query+ context: Optional context with query. Pass the same OptimizationContext+ used for `process`; omitting it stores under the unscoped+ namespace, which only matches lookups whose context also has+ empty provider, model and metadata.Returns:Cache key"""query = (context.query if context else None) or self._extract_query(messages)messages_hash = self._compute_messages_hash(messages)+ namespace = self._compute_namespace(context)++ # Unserializable context: don't store anything that could be replayed+ # for a different provider/model/metadata.+ if namespace is None:+ return ""- return self.cache.put(query, response, messages_hash)+ return self.cache.put(query, response, messages_hash, namespace)def get_stats(self) -> dict[str, Any]:"""Get combined statistics."""@@ -457,6 +500,28 @@ class SemanticCacheLayer:"provider_optimizer": self.provider_optimizer.name,}+ def _compute_namespace(self, context: OptimizationContext | None) -> str | None:+ """Compute the cache namespace from provider, model, and metadata.++ request_id and timestamp are deliberately excluded so they never affect+ matching. Returns None when the context isn't JSON-serializable (e.g.+ unserializable metadata), so callers skip lookup/storage rather than+ replaying entries across contexts.+ """+ if context is None:+ context = OptimizationContext()+ try:+ return json.dumps(+ {+ "provider": context.provider,+ "model": context.model,+ "metadata": context.metadata,+ },+ sort_keys=True,+ )+ except (TypeError, ValueError):+ return None+def _extract_query(self, messages: list[dict[str, Any]]) -> str:"""Extract the last user query from messages."""for msg in reversed(messages):@@ -473,8 +538,6 @@ class SemanticCacheLayer:def _compute_messages_hash(self, messages: list[dict[str, Any]]) -> str:"""Compute a hash of all messages."""- import json-try:content = json.dumps(messages, sort_keys=True)return hashlib.sha256(content.encode()).hexdigest()[:24]diff --git a/tests/test_cache/test_semantic.py b/tests/test_cache/test_semantic.pyindex f5923f1cb..4143762e0 100644--- a/tests/test_cache/test_semantic.py+++ b/tests/test_cache/test_semantic.py@@ -1,6 +1,8 @@"""Tests for SemanticCache and SemanticCacheLayer."""+import hashlibimport time+from datetime import datetimeimport pytest@@ -238,6 +240,63 @@ class TestSemanticCache:cache.put(" \n\t", "response-C", messages_hash="ctxC")assert cache.get(" ", messages_hash="ctxD") is None+ def test_namespace_isolation_exact_match(self, cache):+ """Same messages_hash in different namespaces must not collide."""+ cache.put("continue", "response A", messages_hash="shared", namespace="ns-a")+ cache.put("continue", "response B", messages_hash="shared", namespace="ns-b")++ assert (+ cache.get("continue", messages_hash="shared", namespace="ns-a").response == "response A"+ )+ assert (+ cache.get("continue", messages_hash="shared", namespace="ns-b").response == "response B"+ )+ # A third namespace that never stored the hash is a miss.+ assert cache.get("continue", messages_hash="shared", namespace="ns-c") is None++ def test_namespace_isolation_semantic(self):+ """Semantic matches never cross namespace boundaries."""++ def embedding(text: str) -> list[float]:+ return [1.0, 0.0, 0.0] if "weather" in text.lower() else [0.0, 1.0, 0.0]++ config = SemanticCacheConfig(similarity_threshold=0.9)+ cache = SemanticCache(config, embedding_fn=embedding)++ cache.put("What is the weather?", "sunny", namespace="ns-a")++ # Same query, different namespace: no hit.+ assert cache.get("How is the weather?", namespace="ns-b") is None+ # Same namespace: still hits.+ assert cache.get("How is the weather?", namespace="ns-a").response == "sunny"++ def test_query_derived_key_includes_namespace(self, cache):+ """Without messages_hash the query-derived key is still namespace-scoped."""+ key_a = cache.put("run the tests", "response A", namespace="ns-a")+ key_b = cache.put("run the tests", "response B", namespace="ns-b")++ # Different namespaces produce different keys for the same query.+ assert key_a != key_b+ # Re-storing in the same namespace reuses the key (in-place update).+ key_a_again = cache.put("run the tests", "response A2", namespace="ns-a")+ assert key_a_again == key_a++ def test_max_entries_enforced_globally_across_namespaces(self):+ """Eviction counts entries across all namespaces, not per namespace."""+ config = SemanticCacheConfig(max_entries=2)+ cache = SemanticCache(config)++ cache.put("query1", "response1", messages_hash="h1", namespace="ns-a")+ cache.put("query2", "response2", messages_hash="h2", namespace="ns-b")++ # Cache is full: a new entry in a third namespace evicts the LRU entry+ # even though that entry belongs to a different namespace.+ cache.put("query3", "response3", messages_hash="h3", namespace="ns-c")++ assert cache.get("query1", messages_hash="h1", namespace="ns-a") is None+ assert cache.get("query2", messages_hash="h2", namespace="ns-b") is not None+ assert cache.get("query3", messages_hash="h3", namespace="ns-c") is not None+class TestSemanticCacheLayer:"""Test SemanticCacheLayer functionality."""@@ -373,3 +432,149 @@ class TestSemanticCacheLayer:result = layer.process(messages, context)assert result.semantic_cache_hit is True++ def test_different_model_no_hit(self, layer):+ """Cached responses must not replay across models."""+ messages = [{"role": "user", "content": "Find alpha."}]+ context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")+ context_b = OptimizationContext(provider="anthropic", model="claude-3-sonnet")++ layer.store_response(messages, {"answer": "opus response"}, context_a)++ result = layer.process(messages, context_b)+ assert result.semantic_cache_hit is False++ def test_different_provider_no_hit(self, layer):+ """Cached responses must not replay across providers."""+ messages = [{"role": "user", "content": "Find alpha."}]+ context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")+ context_b = OptimizationContext(provider="bedrock", model="claude-3-opus")++ layer.store_response(messages, {"answer": "opus response"}, context_a)++ result = layer.process(messages, context_b)+ assert result.semantic_cache_hit is False++ def test_different_metadata_no_hit(self, layer):+ """Cached responses must not replay across different metadata."""+ messages = [{"role": "user", "content": "Find alpha."}]+ context_a = OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ metadata={"temperature": 0.7},+ )+ context_b = OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ metadata={"temperature": 0.9},+ )++ layer.store_response(messages, {"answer": "opus response"}, context_a)++ result = layer.process(messages, context_b)+ assert result.semantic_cache_hit is False++ def test_embedding_different_model_no_hit_same_context_hit(self):+ """Semantic embedding lookup is namespace-scoped too."""+ messages = [{"role": "user", "content": "What is 2+2?"}]++ def deterministic_embedding(text: str) -> list[float]:+ digest = hashlib.sha256(text.encode()).digest()+ return [float(b) / 255.0 for b in digest[:4]]++ layer = SemanticCacheLayer(+ AnthropicCacheOptimizer(),+ similarity_threshold=0.9,+ max_entries=100,+ ttl_seconds=60,+ embedding_fn=deterministic_embedding,+ )++ context_a = OptimizationContext(provider="anthropic", model="claude-3-opus")+ context_b = OptimizationContext(provider="anthropic", model="claude-3-sonnet")++ layer.store_response(messages, {"text": "4"}, context_a)++ # Same context: exact messages_hash hit via the embedding path's store.+ assert layer.process(messages, context_a).semantic_cache_hit is True+ # Different model: namespace differs, so no hit even though the query+ # embedding is identical.+ assert layer.process(messages, context_b).semantic_cache_hit is False++ def test_request_id_and_timestamp_do_not_affect_matching(self, layer):+ """request_id and timestamp are not part of the namespace."""+ messages = [{"role": "user", "content": "Hello!"}]+ context_a = OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ request_id="req-1",+ timestamp=datetime(2024, 1, 1),+ )+ context_b = OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ request_id="req-2",+ timestamp=datetime(2025, 6, 15),+ )++ layer.store_response(messages, {"text": "hi"}, context_a)++ result = layer.process(messages, context_b)+ assert result.semantic_cache_hit is True+ assert result.cached_response == {"text": "hi"}++ def test_unserializable_metadata_store_returns_empty_and_no_hit(self, layer):+ """Unserializable metadata yields a None namespace: store refuses and+ lookup skips the cache entirely."""+ messages = [{"role": "user", "content": "Hello!"}]+ context = OptimizationContext(+ provider="anthropic",+ model="claude-3-opus",+ metadata={"x": object()},+ )++ key = layer.store_response(messages, {"text": "hi"}, context)+ assert key == ""++ result = layer.process(messages, context)+ assert result.semantic_cache_hit is False+++class TestSemanticCoverage:+ def test_process_namespace_none_delegates(self):+ from headroom.cache import AnthropicCacheOptimizer, OptimizationContext, SemanticCacheLayer++ layer = SemanticCacheLayer(AnthropicCacheOptimizer())+ messages = [{"role": "user", "content": "hi"}]+ context = OptimizationContext(provider="anthropic", model="m", metadata={"x": object()})+ result = layer.process(messages, context)+ assert result.semantic_cache_hit is False++ def test_invalidate_namespaced_entry(self):+ from headroom.cache.semantic import SemanticCache, SemanticCacheConfig++ cache = SemanticCache(SemanticCacheConfig())+ key = cache.put("q", "r", messages_hash="h1", namespace="ns-a")+ assert cache.get("q", messages_hash="h1", namespace="ns-a") is not None+ assert cache.invalidate(key) is True+ assert cache.get("q", messages_hash="h1", namespace="ns-a") is None+ cache.put("q", "r2", messages_hash="h1", namespace="ns-a")+ assert cache.get("q", messages_hash="h1", namespace="ns-a").response == "r2"++ def test_evict_oldest_removes_namespaced_index(self):+ from headroom.cache.semantic import SemanticCache, SemanticCacheConfig++ cache = SemanticCache(SemanticCacheConfig(max_entries=1))+ cache.put("q1", "r1", messages_hash="h1", namespace="ns-a")+ cache.put("q2", "r2", messages_hash="h2", namespace="ns-b")+ assert cache.get("q1", messages_hash="h1", namespace="ns-a") is None++ def test_context_free_store_policy(self):+ from headroom.cache import AnthropicCacheOptimizer, OptimizationContext, SemanticCacheLayer++ layer = SemanticCacheLayer(AnthropicCacheOptimizer())+ messages = [{"role": "user", "content": "Hello!"}]+ layer.store_response(messages, {"text": "hi"})+ assert layer.process(messages, OptimizationContext()).semantic_cache_hit is True+ ctx = OptimizationContext(provider="anthropic", model="claude-3-opus")+ assert layer.process(messages, ctx).semantic_cache_hit is False