Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(cache/semantic): don't semantic-match an empty query across contexts (#3226)

Abhay Singh455f4f263ce2
diff --git a/headroom/cache/semantic.py b/headroom/cache/semantic.py
index 0ea2effba..946fada32 100644
--- a/headroom/cache/semantic.py
+++ b/headroom/cache/semantic.py
@@ -160,8 +160,18 @@ class SemanticCache:
self._hits += 1
return entry
- # Try semantic similarity if we have embedding function
- if self._embedding_fn:
+ # Try semantic similarity if we have embedding function.
+ #
+ # Only for a NON-EMPTY query: the query is the last user message, and in
+ # agent/tool traffic the overwhelming majority of turns are tool_result
+ # continuations whose extracted query is "" (no text block). Embedding
+ # matching on "" makes every such turn ~identical to every other (a real
+ # sentence embedder maps "" to a fixed non-zero vector), so an empty
+ # query would false-hit and serve one conversation's response to an
+ # unrelated one — precisely the cross-context collision the messages_hash
+ # key is chosen to avoid. An empty query may still hit via the exact
+ # messages_hash above, which is context-complete and safe.
+ if self._embedding_fn and query.strip():
query_embedding = self._embedding_fn(query)
best_match, best_similarity = self._find_similar(query_embedding)
@@ -208,9 +218,13 @@ class SemanticCache:
while key not in self._cache and len(self._cache) >= self.config.max_entries:
self._evict_oldest()
- # Generate embedding if available
+ # Generate embedding if available — but never for an empty/blank query.
+ # A stored empty-query entry with an embedding would be a false-match
+ # target for the semantic get() path; leaving its embedding empty makes
+ # _find_similar skip it (it ignores entries with no embedding), so an
+ # empty-query entry is reachable only by its exact messages_hash.
embedding: list[float] = []
- if self._embedding_fn:
+ if self._embedding_fn and query.strip():
embedding = self._embedding_fn(query)
now = time.time()
diff --git a/tests/test_cache/test_semantic.py b/tests/test_cache/test_semantic.py
index c24d261f9..f5923f1cb 100644
--- a/tests/test_cache/test_semantic.py
+++ b/tests/test_cache/test_semantic.py
@@ -208,6 +208,36 @@ class TestSemanticCache:
entry = cache.get("What time is it?")
assert entry is None
+ def test_empty_query_never_semantic_matches(self):
+ """An empty extracted query must not trigger cross-context false hits.
+
+ The query is the last user message; in agent/tool traffic most turns are
+ tool_result continuations whose extracted query is "". A real embedder
+ maps "" to a fixed non-zero vector, so without a guard every empty-query
+ turn would be ~identical to every other and serve one conversation's
+ response to an unrelated one. An empty query may only ever hit via the
+ exact messages_hash (which is context-complete).
+ """
+
+ def const_embedding(text: str) -> list[float]:
+ # Realistic: a non-zero, identical vector for every input (incl. "").
+ return [0.5, 0.5, 0.5]
+
+ config = SemanticCacheConfig(similarity_threshold=0.9)
+ cache = SemanticCache(config, embedding_fn=const_embedding)
+
+ # Conversation A: an empty-query turn (unique full-context hash).
+ cache.put("", "response-A", messages_hash="ctxA")
+
+ # Conversation B: a different empty-query turn — must NOT get A's answer.
+ assert cache.get("", messages_hash="ctxB") is None
+ # Its own exact hash still works.
+ assert cache.get("", messages_hash="ctxA").response == "response-A"
+
+ # A whitespace-only query is treated the same as empty.
+ cache.put(" \n\t", "response-C", messages_hash="ctxC")
+ assert cache.get(" ", messages_hash="ctxD") is None
+
class TestSemanticCacheLayer:
"""Test SemanticCacheLayer functionality."""