Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(cache/semantic): don't semantic-match an empty query across contexts (#3226)
Abhay Singh455f4f263ce2
diff --git a/headroom/cache/semantic.py b/headroom/cache/semantic.pyindex 0ea2effba..946fada32 100644--- a/headroom/cache/semantic.py+++ b/headroom/cache/semantic.py@@ -160,8 +160,18 @@ class SemanticCache:self._hits += 1return entry- # Try semantic similarity if we have embedding function- if self._embedding_fn:+ # Try semantic similarity if we have embedding function.+ #+ # Only for a NON-EMPTY query: the query is the last user message, and in+ # agent/tool traffic the overwhelming majority of turns are tool_result+ # continuations whose extracted query is "" (no text block). Embedding+ # matching on "" makes every such turn ~identical to every other (a real+ # sentence embedder maps "" to a fixed non-zero vector), so an empty+ # query would false-hit and serve one conversation's response to an+ # unrelated one — precisely the cross-context collision the messages_hash+ # key is chosen to avoid. An empty query may still hit via the exact+ # messages_hash above, which is context-complete and safe.+ if self._embedding_fn and query.strip():query_embedding = self._embedding_fn(query)best_match, best_similarity = self._find_similar(query_embedding)@@ -208,9 +218,13 @@ class SemanticCache:while key not in self._cache and len(self._cache) >= self.config.max_entries:self._evict_oldest()- # Generate embedding if available+ # Generate embedding if available — but never for an empty/blank query.+ # A stored empty-query entry with an embedding would be a false-match+ # target for the semantic get() path; leaving its embedding empty makes+ # _find_similar skip it (it ignores entries with no embedding), so an+ # empty-query entry is reachable only by its exact messages_hash.embedding: list[float] = []- if self._embedding_fn:+ if self._embedding_fn and query.strip():embedding = self._embedding_fn(query)now = time.time()diff --git a/tests/test_cache/test_semantic.py b/tests/test_cache/test_semantic.pyindex c24d261f9..f5923f1cb 100644--- a/tests/test_cache/test_semantic.py+++ b/tests/test_cache/test_semantic.py@@ -208,6 +208,36 @@ class TestSemanticCache:entry = cache.get("What time is it?")assert entry is None+ def test_empty_query_never_semantic_matches(self):+ """An empty extracted query must not trigger cross-context false hits.++ The query is the last user message; in agent/tool traffic most turns are+ tool_result continuations whose extracted query is "". A real embedder+ maps "" to a fixed non-zero vector, so without a guard every empty-query+ turn would be ~identical to every other and serve one conversation's+ response to an unrelated one. An empty query may only ever hit via the+ exact messages_hash (which is context-complete).+ """++ def const_embedding(text: str) -> list[float]:+ # Realistic: a non-zero, identical vector for every input (incl. "").+ return [0.5, 0.5, 0.5]++ config = SemanticCacheConfig(similarity_threshold=0.9)+ cache = SemanticCache(config, embedding_fn=const_embedding)++ # Conversation A: an empty-query turn (unique full-context hash).+ cache.put("", "response-A", messages_hash="ctxA")++ # Conversation B: a different empty-query turn — must NOT get A's answer.+ assert cache.get("", messages_hash="ctxB") is None+ # Its own exact hash still works.+ assert cache.get("", messages_hash="ctxA").response == "response-A"++ # A whitespace-only query is treated the same as empty.+ cache.put(" \n\t", "response-C", messages_hash="ctxC")+ assert cache.get(" ", messages_hash="ctxD") is None+class TestSemanticCacheLayer:"""Test SemanticCacheLayer functionality."""