[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
[bm25 5/5] refactor: drop in-memory BM25Scorer; FTS5 is the only implementation (#335)
Abi84d9a0a998f5
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex 2c8f243..1dfda90 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -315,70 +315,96 @@ class BM25Scorer(BM25Index):self.corpus_size = Noneself.idlist = set() # List of all document IDs for easier searching- def _tokenize(self, text: str) -> list[str]:- return re.sub(r"[^\w\s]", "", text).lower().split()+ Today's only implementation is `Fts5BM25Index` (SQLite FTS5, built at+ `leann build` time, queried memory-bounded). The historical in-memory+ `BM25Scorer` was removed once the FTS5 path was stable.+ """++ @abstractmethod+ def fit(self, documents: list[dict[str, Any]]) -> None:+ """Build the index from a corpus.- def fit(self, documents: list[dict[str, Any]]):+ `documents` is a list of `{"id": str, "text": str, ...}` entries. Extra+ fields are ignored by BM25 implementations but preserved by the caller+ for use elsewhere."""- Build BM25 statistics from a document corpus.- Must be called before scoring.++ @abstractmethod+ def search(self, query: str, top_k: int = 5) -> list["SearchResult"]:+ """Return up to `top_k` SearchResult entries ranked by descending score.++ Returned SearchResults have `id` and `score` populated; `text` and+ `metadata` are filled in by `LeannSearcher` from the passage store."""- self.corpus_size = len(documents)- self.doc_lengths = {}- self.word_counts = {}- self.idlist = set()- doc_freqs = defaultdict(int)-- for doc_data in documents:- doc_id = doc_data["id"]- words = self._tokenize(doc_data["text"])- doc_length = len(words)- self.doc_lengths[doc_id] = doc_length-- unique_words = set(words)- for word in unique_words:- doc_freqs[word] += 1- self.word_counts[doc_id] = dict(Counter(words))- self.idlist.add(doc_id)-- self.doc_freqs = dict(doc_freqs)- self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)-- def score(self, query_words: list[str], document_id: str) -> float:- if (- self.doc_freqs is None- or self.doc_lengths == {}- or self.word_counts == {}- or self.avg_doc_length is None- or self.corpus_size is None- ):- raise ValueError("BM25 model not fitted. Call fit() before scoring.")-- passage_words = self.word_counts[document_id]- passage_length = sum(passage_words.values())- score = 0.0- for word in query_words:- if word not in self.doc_freqs:- continue- word_freq = passage_words[word] if word in passage_words else 0- idf = np.log(- (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1- )- tf = (word_freq * (self.k1 + 1)) / (- word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))+++class Fts5BM25Index(BM25Index):+ """BM25 over a SQLite FTS5 virtual table, persisted on disk.++ Built once at `leann build` time, queried memory-bounded at search time.+ SQLite owns the on-disk term/posting data; queries hit `bm25()` directly.+ """++ # SQLite's FTS5 bm25() returns lower-is-better. We negate so the rest of+ # LeannSearcher (and the hybrid fusion math) can keep higher-is-better.+ _SCHEMA = (+ "CREATE VIRTUAL TABLE bm25_passages USING fts5("+ "id UNINDEXED, text, tokenize='unicode61 remove_diacritics 2'"+ ")"+ )++ def __init__(self, db_path: str):+ self._db_path = db_path+ self._conn: Optional[Any] = None++ def _connect(self):+ import sqlite3++ if self._conn is None:+ self._conn = sqlite3.connect(self._db_path)+ return self._conn++ def fit(self, documents: list[dict[str, Any]]) -> None:+ import sqlite3++ # Fresh DB every fit — fit() is a one-shot bulk-load.+ if os.path.exists(self._db_path):+ os.unlink(self._db_path)+ conn = sqlite3.connect(self._db_path)+ try:+ conn.execute(self._SCHEMA)+ conn.executemany(+ "INSERT INTO bm25_passages(id, text) VALUES (?, ?)",+ ((d["id"], d.get("text", "")) for d in documents),)- score += idf * tf- return score+ conn.commit()+ finally:+ conn.close()- def search(self, query: str, top_k: int = 5) -> list[SearchResult]:- query_words = self._tokenize(query)- scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}- sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)+ def search(self, query: str, top_k: int = 5) -> list["SearchResult"]:+ # Strip punctuation, lowercase, OR the terms together. Avoids FTS5+ # query syntax surprises (`:`, `*`, etc.) for natural-language queries.+ terms = re.sub(r"[^\w\s]", "", query).lower().split()+ if not terms:+ return []+ fts5_query = " OR ".join(terms)+ conn = self._connect()+ rows = conn.execute(+ "SELECT id, -bm25(bm25_passages) AS score "+ "FROM bm25_passages WHERE bm25_passages MATCH ? "+ "ORDER BY score DESC LIMIT ?",+ (fts5_query, top_k),+ ).fetchall()return [- SearchResult(id=doc_id, score=score, text="", metadata={})- for doc_id, score in sorted_scores[:top_k]+ SearchResult(id=doc_id, score=float(score), text="", metadata={})+ for doc_id, score in rows]+ def close(self) -> None:+ if self._conn is not None:+ self._conn.close()+ self._conn = None+class Fts5BM25Index(BM25Index):"""BM25 over a SQLite FTS5 virtual table, persisted on disk.