[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
Fix MPS memory pathologies on Apple Silicon (#340)
ww26b99910a6f2e5
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 2c8f243..e22633a 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -12,7 +12,6 @@ import subprocessimport timeimport warningsfrom abc import ABC, abstractmethod-from collections import Counter, defaultdictfrom dataclasses import dataclass, fieldfrom pathlib import Pathfrom typing import Any, Literal, Optional, Union@@ -279,12 +278,7 @@ class PassageManager:class BM25Index(ABC):- """Minimal contract for a BM25-style sparse index over LEANN passages.-- Concrete implementations today: `BM25Scorer` (in-memory, fit-on-search).- Planned: an FTS5-backed implementation that builds at index-build time- and queries memory-bounded. See the design issue for the broader plan.- """+ """Minimal contract for a BM25-style sparse index over LEANN passages."""@abstractmethoddef fit(self, documents: list[dict[str, Any]]) -> None:@@ -304,88 +298,11 @@ class BM25Index(ABC):"""-class BM25Scorer(BM25Index):- def __init__(self, k1: float = 1.2, b: float = 0.75):- self.k1 = k1- self.b = b- self.doc_freqs = None # How many docs contain each term (DF)- self.doc_lengths = {} # How long each doc is (in words)- self.word_counts = {} # How many times each word appears in each doc (TF)- self.avg_doc_length = None- self.corpus_size = None- self.idlist = set() # List of all document IDs for easier searching-- def _tokenize(self, text: str) -> list[str]:- return re.sub(r"[^\w\s]", "", text).lower().split()-- def fit(self, documents: list[dict[str, Any]]):- """- Build BM25 statistics from a document corpus.- Must be called before scoring.- """- self.corpus_size = len(documents)- self.doc_lengths = {}- self.word_counts = {}- self.idlist = set()- doc_freqs = defaultdict(int)-- for doc_data in documents:- doc_id = doc_data["id"]- words = self._tokenize(doc_data["text"])- doc_length = len(words)- self.doc_lengths[doc_id] = doc_length-- unique_words = set(words)- for word in unique_words:- doc_freqs[word] += 1- self.word_counts[doc_id] = dict(Counter(words))- self.idlist.add(doc_id)-- self.doc_freqs = dict(doc_freqs)- self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)-- def score(self, query_words: list[str], document_id: str) -> float:- if (- self.doc_freqs is None- or self.doc_lengths == {}- or self.word_counts == {}- or self.avg_doc_length is None- or self.corpus_size is None- ):- raise ValueError("BM25 model not fitted. Call fit() before scoring.")-- passage_words = self.word_counts[document_id]- passage_length = sum(passage_words.values())- score = 0.0- for word in query_words:- if word not in self.doc_freqs:- continue- word_freq = passage_words[word] if word in passage_words else 0- idf = np.log(- (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1- )- tf = (word_freq * (self.k1 + 1)) / (- word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))- )- score += idf * tf- return score-- def search(self, query: str, top_k: int = 5) -> list[SearchResult]:- query_words = self._tokenize(query)- scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}- sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)- return [- SearchResult(id=doc_id, score=score, text="", metadata={})- for doc_id, score in sorted_scores[:top_k]- ]--class Fts5BM25Index(BM25Index):"""BM25 over a SQLite FTS5 virtual table, persisted on disk.Built once at `leann build` time, queried memory-bounded at search time.- Avoids the in-memory term-frequency table that BM25Scorer keeps in RAM and- that gets re-fit on every cold start. See #327 for the broader plan.+ SQLite owns the on-disk term/posting data; queries hit `bm25()` directly."""# SQLite's FTS5 bm25() returns lower-is-better. We negate so the rest of@@ -425,9 +342,8 @@ class Fts5BM25Index(BM25Index):conn.close()def search(self, query: str, top_k: int = 5) -> list["SearchResult"]:- # Match the BM25Scorer tokenization for query consistency: strip- # punctuation, lowercase, OR the terms together. Avoids FTS5 query- # syntax surprises (`:`, `*`, etc.).+ # Strip punctuation, lowercase, OR the terms together. Avoids FTS5+ # query syntax surprises (`:`, `*`, etc.) for natural-language queries.terms = re.sub(r"[^\w\s]", "", query).lower().split()if not terms:return []@@ -459,16 +375,13 @@ class LeannBuilder:embedding_mode: str = "sentence-transformers",embedding_options: Optional[dict[str, Any]] = None,prebuild_bm25: bool = False,- bm25_backend: str = "memory",+ bm25_backend: str = "fts5",**backend_kwargs,):- if bm25_backend not in ("memory", "fts5"):- raise ValueError(- f"Unknown bm25_backend: {bm25_backend!r}. Expected 'memory' or 'fts5'."- )+ if bm25_backend != "fts5":+ logger.warning(f"bm25_backend={bm25_backend!r} is deprecated; using 'fts5'.")+ bm25_backend = "fts5"self.bm25_backend = bm25_backend- # If user picked fts5 explicitly, treat that as opting into prebuild —- # FTS5 only makes sense as a build-time artifact.self.prebuild_bm25 = prebuild_bm25 or bm25_backend == "fts5"self.backend_name = backend_name# Normalize incompatible combinations early (for consistent metadata)@@ -687,14 +600,9 @@ class LeannBuilder:meta_data["is_pruned"] = bool(is_recompute)if self.prebuild_bm25:- if self.bm25_backend == "fts5":- self._build_bm25_fts5(index_dir, index_name)- meta_data["bm25_backend"] = "fts5"- meta_data["bm25_db"] = f"{index_name}.bm25.sqlite"- else:- self._build_bm25_snapshot(index_dir, index_name)- meta_data["bm25_snapshot"] = f"{index_name}.bm25.pkl"- meta_data["bm25_backend"] = "memory"+ self._build_bm25_fts5(index_dir, index_name)+ meta_data["bm25_backend"] = "fts5"+ meta_data["bm25_db"] = f"{index_name}.bm25.sqlite"with open(leann_meta_path, "w", encoding="utf-8") as f:json.dump(meta_data, f, indent=2)@@ -712,21 +620,6 @@ class LeannBuilder:index.close()logger.info(f"Wrote BM25 FTS5 index to {db_path}")- def _build_bm25_snapshot(self, index_dir: Path, index_name: str) -> None:- """Fit BM25Scorer on self.chunks and pickle alongside the index.-- Lets LeannSearcher._init_bm25 load the fitted scorer on first BM25 query- instead of re-fitting against the passage JSONL — which scans every- passage and builds the full TF table in RAM, dominating first-search- latency on larger corpora.- """- bm25_path = index_dir / f"{index_name}.bm25.pkl"- scorer = BM25Scorer()- scorer.fit(self.chunks)- with open(bm25_path, "wb") as f:- pickle.dump(scorer, f, protocol=pickle.HIGHEST_PROTOCOL)- logger.info(f"Wrote BM25 snapshot to {bm25_path}")-def build_index_from_arrays(self, index_path: str, ids: list, embeddings: np.ndarray):"""Build an index from pre-computed embedding arrays.@@ -1559,31 +1452,41 @@ class LeannSearcher:f"falling back to fit-on-search.")- snapshot_name = self.meta_data.get("bm25_snapshot")- if snapshot_name:- snapshot_path = meta_dir / snapshot_name- if snapshot_path.exists():- try:- with open(snapshot_path, "rb") as f:- self.bm25_scorer = pickle.load(f)- logger.info(f"Loaded BM25 snapshot from {snapshot_path}")- return- except Exception as exc:- logger.warning(- f"Failed to load BM25 snapshot at {snapshot_path}, "- f"falling back to fit-on-search: {exc}"- )-- # No artifact (older indexes) or load failed: fit on the fly.- self.bm25_scorer = BM25Scorer()+ # No FTS5 artifact: build one on the fly from passages.+ db_path = meta_dir / (Path(self.meta_path_str).stem.replace(".meta", "") + ".bm25.sqlite")+ index = Fts5BM25Index(str(db_path))passages = []for passage_file in self.passage_manager.passage_files.values():- with open(passage_file, encoding="utf-8") as f:- for line in f:- if line.strip():- data = json.loads(line)- passages.append(data)- self.bm25_scorer.fit(passages)+ try:+ with open(passage_file, encoding="utf-8") as f:+ for line in f:+ if line.strip():+ try:+ passages.append(json.loads(line))+ except json.JSONDecodeError as exc:+ logger.warning(f"Skipping malformed JSONL in {passage_file}: {exc}")+ except FileNotFoundError:+ logger.warning(f"Passage file missing: {passage_file}")++ if not passages:+ logger.error(+ "No passages found for on-demand BM25 index. "+ "BM25/hybrid search will return empty results. "+ "Re-run 'leann build' to regenerate passage files."+ )+ return++ try:+ index.fit(passages)+ except (PermissionError, OSError) as exc:+ logger.error(+ f"Cannot write BM25 index to {db_path}: {exc}. "+ f"Ensure the index directory is writable, or rebuild with prebuild_bm25=True."+ )+ return++ self.bm25_scorer = index+ logger.info(f"Built FTS5 BM25 index on-demand at {db_path}")def _bm25_search(self, query: str, top_k: int = 5) -> list[SearchResult]:"""Perform BM25 search on raw passages"""diff --git a/packages/leann-core/src/leann/embedding_compute.py b/packages/leann-core/src/leann/embedding_compute.pyindex 00f7c84..34c02e5 100644--- a/packages/leann-core/src/leann/embedding_compute.py+++ b/packages/leann-core/src/leann/embedding_compute.py@@ -481,11 +481,10 @@ def compute_embeddings_sentence_transformers(torch.backends.cudnn.deterministic = Falsetorch.cuda.set_per_process_memory_fraction(0.9)elif device == "mps":- try:- if hasattr(torch.mps, "set_per_process_memory_fraction"):- torch.mps.set_per_process_memory_fraction(0.9)- except AttributeError:- logger.warning("Some MPS optimizations not available in this PyTorch version")+ # No device-level init for MPS. set_per_process_memory_fraction causes+ # greedy allocation; torch.compile causes graph buffer bloat. Cache+ # clearing is handled per-batch in the compute loop below.+ passelif device == "cpu":# TODO: Haven't tested this yettorch.set_num_threads(min(8, os.cpu_count() or 4))@@ -586,7 +585,7 @@ def compute_embeddings_sentence_transformers(logger.warning(f"FP16 optimization failed: {e}")# Apply torch.compile optimization- if device in ["cuda", "mps"]:+ if device == "cuda":try:model = torch.compile(model, mode="reduce-overhead", dynamic=True)logger.info(f"Applied torch.compile optimization: {model_name}")@@ -659,7 +658,7 @@ def compute_embeddings_sentence_transformers(hf_model.eval()# Optional compile on supported devices- if device in ["cuda", "mps"]:+ if device == "cuda":try:hf_model = torch.compile(hf_model, mode="reduce-overhead", dynamic=True)logger.info(@@ -714,6 +713,11 @@ def compute_embeddings_sentence_transformers(pooled = masked.sum(dim=1) / lengthsbatch_embeddings = pooled.detach().to("cpu").float().numpy()all_embeddings.append(batch_embeddings)+ if device == "mps":+ try:+ torch.mps.empty_cache()+ except (RuntimeError, AttributeError):+ passembeddings = np.vstack(all_embeddings).astype(np.float32, copy=False)try: