[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
feat(search): cache query embeddings per searcher (#436)
Aakash Suresh115254ebe89b
diff --git a/packages/leann-core/src/leann/searcher_base.py b/packages/leann-core/src/leann/searcher_base.pyindex be1ee4c..77cf593 100644--- a/packages/leann-core/src/leann/searcher_base.py+++ b/packages/leann-core/src/leann/searcher_base.py@@ -1,5 +1,7 @@+import hashlibimport jsonfrom abc import ABC, abstractmethod+from collections import OrderedDictfrom pathlib import Pathfrom typing import Any, Literal, Optional@@ -9,20 +11,65 @@ from .embedding_server_manager import EmbeddingServerManagerfrom .interface import LeannBackendSearcherInterface-class ReusableZMQConnection:- """Persistent ZMQ REQ connection to the embedding server.+class QueryEmbeddingCache:+ """LRU cache for query embeddings to avoid recomputation."""- The socket is created once and reused across searches instead of being- rebuilt per query. It lives on the searcher for the searcher's lifetime,- and `LeannChat` and the LlamaIndex retrievers hold a searcher for as long- as they live.+ def __init__(self, max_size: int = 1000, embedding_model: Optional[str] = None):+ self.cache: OrderedDict[str, np.ndarray] = OrderedDict()+ self.max_size = max_size+ self.embedding_model = embedding_model or ""- **A `zmq.Socket` is not thread-safe.** A searcher sharing this connection- must not have `search` called concurrently from multiple threads; give- each thread its own searcher. The previous per-query socket did not carry- this constraint.+ def _hash_query(self, query: str, query_template: Optional[str] = None) -> str:+ """Create hash key for query."""+ key_data = {+ "query": query,+ "template": query_template or "",+ "model": self.embedding_model,+ }+ key_str = json.dumps(key_data, sort_keys=True)+ return hashlib.sha256(key_str.encode()).hexdigest()- """+ def get(self, query: str, query_template: Optional[str] = None) -> Optional[np.ndarray]:+ """Get cached embedding if exists.++ Returns a copy of the stored vector with shape (D,).+ Callers that need batch shape should reshape.+ """+ key = self._hash_query(query, query_template)+ cached = self.cache.get(key)+ if cached is None:+ return None+ # Mark as most-recently-used; without this, eviction below is FIFO and+ # a hot query can be dropped while a never-reused one survives.+ self.cache.move_to_end(key)+ return cached.copy()++ def put(self, query: str, embedding: np.ndarray, query_template: Optional[str] = None):+ """Cache embedding (stores a 1-D vector of shape (D,))."""+ key = self._hash_query(query, query_template)++ # Normalize to 1-D so cache hits always return a consistent shape+ vec = np.asarray(embedding, dtype=np.float32).reshape(-1)++ # max_size <= 0 disables the cache; storing nothing keeps `get` a miss+ # and avoids evicting from an empty mapping.+ if self.max_size <= 0:+ return++ if key in self.cache:+ self.cache.move_to_end(key)+ elif len(self.cache) >= self.max_size:+ self.cache.popitem(last=False) # evict least-recently-used++ self.cache[key] = vec.copy()++ def clear(self):+ """Clear cache."""+ self.cache.clear()+++class ReusableZMQConnection:+ """Reusable ZMQ connection to avoid creating new context/socket per request."""def __init__(self):self.context = None@@ -136,6 +183,10 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):)# Optimization: Query embedding cache+ cache_size = kwargs.get("query_cache_size", 1000)+ self.query_cache = QueryEmbeddingCache(+ max_size=cache_size, embedding_model=self.embedding_model+ )# Optimization: Reusable ZMQ connectionself.zmq_connection = ReusableZMQConnection()@@ -215,6 +266,15 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):Returns:Query embedding as numpy array with shape (1, D)"""+ # Store original query for caching (before template is applied)+ original_query = query++ # Check cache first (before applying template). Cache stores (D,);+ # always return (1, D) to match uncached paths.+ cached = self.query_cache.get(original_query, query_template)+ if cached is not None:+ return np.asarray(cached, dtype=np.float32).reshape(1, -1)+# Apply query template BEFORE any computation path# This ensures template is applied consistently for both server and fallback pathsif query_template:@@ -238,6 +298,9 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):0:1] # Return (1, D) shape+ # Cache the result (use original query before template)+ self.query_cache.put(original_query, embedding[0], query_template)+return embeddingexcept Exception as e:print(f"⚠️ Embedding server failed: {e}")@@ -254,6 +317,9 @@ class BaseSearcher(LeannBackendSearcherInterface, ABC):provider_options=self.embedding_options,)+ # Cache the result (use original query before template)+ self.query_cache.put(original_query, embedding[0], query_template)+return embeddingdef _compute_embedding_via_server(self, chunks: list, zmq_port: int) -> np.ndarray:diff --git a/tests/test_query_embedding_cache.py b/tests/test_query_embedding_cache.pynew file mode 100644index 0000000..3db1df9--- /dev/null+++ b/tests/test_query_embedding_cache.py@@ -0,0 +1,86 @@+"""Unit tests for QueryEmbeddingCache (no native backends required)."""++import numpy as np+from leann.searcher_base import QueryEmbeddingCache+++class TestQueryEmbeddingCache:+ def test_put_get_roundtrip(self):+ cache = QueryEmbeddingCache(max_size=8)+ emb = np.array([0.1, 0.2, 0.3], dtype=np.float32)+ cache.put("hello", emb)+ got = cache.get("hello")+ assert got is not None+ assert got.shape == (3,)+ assert np.allclose(got, emb)++ def test_template_is_part_of_key(self):+ cache = QueryEmbeddingCache(max_size=8)+ cache.put("q", np.ones(4, dtype=np.float32), query_template="A: ")+ assert cache.get("q", query_template="A: ") is not None+ assert cache.get("q", query_template="B: ") is None+ assert cache.get("q") is None++ def test_cache_miss_returns_none(self):+ cache = QueryEmbeddingCache(max_size=8)+ assert cache.get("missing") is None++ def test_lru_eviction(self):+ cache = QueryEmbeddingCache(max_size=2)+ cache.put("a", np.array([1.0], dtype=np.float32))+ cache.put("b", np.array([2.0], dtype=np.float32))+ cache.put("c", np.array([3.0], dtype=np.float32)) # evicts "a"+ assert cache.get("a") is None+ assert cache.get("b") is not None+ assert cache.get("c") is not None++ def test_put_normalizes_2d_to_1d(self):+ cache = QueryEmbeddingCache(max_size=4)+ cache.put("q", np.array([[9.0, 8.0, 7.0]], dtype=np.float32))+ got = cache.get("q")+ assert got is not None+ assert got.shape == (3,)+ assert np.allclose(got, [9.0, 8.0, 7.0])++ def test_clear(self):+ cache = QueryEmbeddingCache(max_size=4)+ cache.put("q", np.ones(2, dtype=np.float32))+ cache.clear()+ assert cache.get("q") is None+++def test_eviction_is_lru_not_fifo():+ """A recently-read entry must outlive an older untouched one.++ `get` previously did not reorder, so eviction fell back to insertion+ order: a hot query could be dropped while a never-reused one survived.+ """+ cache = QueryEmbeddingCache(max_size=2)+ cache.put("a", np.ones(4, dtype=np.float32))+ cache.put("b", np.full(4, 2.0, dtype=np.float32))++ assert cache.get("a") is not None # 'a' becomes most-recently-used+ cache.put("c", np.full(4, 3.0, dtype=np.float32))++ assert cache.get("a") is not None, "LRU must keep the recently-read entry"+ assert cache.get("b") is None, "least-recently-used entry should be evicted"+ assert cache.get("c") is not None+++def test_key_is_scoped_to_the_embedding_model():+ """Two caches on different models must not collide on the same query."""+ a = QueryEmbeddingCache(max_size=4, embedding_model="model-A")+ b = QueryEmbeddingCache(max_size=4, embedding_model="model-B")+ assert a._hash_query("same query") != b._hash_query("same query")+++def test_max_size_zero_disables_the_cache():+ """`query_cache_size=0` is the natural way to turn the cache off.++ Eviction previously ran on an empty mapping and raised+ `KeyError: 'dictionary is empty'` on the first put.+ """+ cache = QueryEmbeddingCache(max_size=0)+ cache.put("a", np.ones(4, dtype=np.float32)) # must not raise+ assert cache.get("a") is None+ assert len(cache.cache) == 0