[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
[bm25 2/5] feat: opt-in build-time BM25 snapshot to skip fit-on-first-search (#332)
Abia1b759f1dd8b
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 0f996e0..41ac788 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -388,8 +388,10 @@ class LeannBuilder:dimensions: Optional[int] = None,embedding_mode: str = "sentence-transformers",embedding_options: Optional[dict[str, Any]] = None,+ prebuild_bm25: bool = False,**backend_kwargs,):+ self.prebuild_bm25 = prebuild_bm25self.backend_name = backend_name# Normalize incompatible combinations early (for consistent metadata)if backend_name == "hnsw":@@ -605,9 +607,29 @@ class LeannBuilder:is_recompute = self.backend_kwargs.get("is_recompute", True)meta_data["is_compact"] = is_compactmeta_data["is_pruned"] = bool(is_recompute)++ if self.prebuild_bm25:+ self._build_bm25_snapshot(index_dir, index_name)+ meta_data["bm25_snapshot"] = f"{index_name}.bm25.pkl"+with open(leann_meta_path, "w", encoding="utf-8") as f:json.dump(meta_data, f, indent=2)+ def _build_bm25_snapshot(self, index_dir: Path, index_name: str) -> None:+ """Fit BM25Scorer on self.chunks and pickle alongside the index.++ Lets LeannSearcher._init_bm25 load the fitted scorer on first BM25 query+ instead of re-fitting against the passage JSONL — which scans every+ passage and builds the full TF table in RAM, dominating first-search+ latency on larger corpora.+ """+ bm25_path = index_dir / f"{index_name}.bm25.pkl"+ scorer = BM25Scorer()+ scorer.fit(self.chunks)+ with open(bm25_path, "wb") as f:+ pickle.dump(scorer, f, protocol=pickle.HIGHEST_PROTOCOL)+ logger.info(f"Wrote BM25 snapshot to {bm25_path}")+def build_index_from_arrays(self, index_path: str, ids: list, embeddings: np.ndarray):"""Build an index from pre-computed embedding arrays.@@ -1423,9 +1445,24 @@ class LeannSearcher:return enriched_resultsdef _init_bm25(self) -> None:- """Initialize BM25 scorer"""+ """Initialize BM25 scorer, preferring a build-time snapshot when present."""+ snapshot_name = self.meta_data.get("bm25_snapshot")+ if snapshot_name:+ snapshot_path = Path(self.meta_path_str).parent / snapshot_name+ if snapshot_path.exists():+ try:+ with open(snapshot_path, "rb") as f:+ self.bm25_scorer = pickle.load(f)+ logger.info(f"Loaded BM25 snapshot from {snapshot_path}")+ return+ except Exception as exc:+ logger.warning(+ f"Failed to load BM25 snapshot at {snapshot_path}, "+ f"falling back to fit-on-search: {exc}"+ )++ # No snapshot (older indexes) or snapshot load failed: fit on the fly.self.bm25_scorer = BM25Scorer()- # Load all the files directlypassages = []for passage_file in self.passage_manager.passage_files.values():with open(passage_file, encoding="utf-8") as f: