Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(api): restore BM25Scorer + dedupe Fts5BM25Index after botched merge (#341)

Abia996b3512283
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index 1dfda90..2c8f243 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -315,96 +315,70 @@ class BM25Scorer(BM25Index):
self.corpus_size = None
self.idlist = set() # List of all document IDs for easier searching
- Today's only implementation is `Fts5BM25Index` (SQLite FTS5, built at
- `leann build` time, queried memory-bounded). The historical in-memory
- `BM25Scorer` was removed once the FTS5 path was stable.
- """
-
- @abstractmethod
- def fit(self, documents: list[dict[str, Any]]) -> None:
- """Build the index from a corpus.
+ def _tokenize(self, text: str) -> list[str]:
+ return re.sub(r"[^\w\s]", "", text).lower().split()
- `documents` is a list of `{"id": str, "text": str, ...}` entries. Extra
- fields are ignored by BM25 implementations but preserved by the caller
- for use elsewhere.
+ def fit(self, documents: list[dict[str, Any]]):
"""
-
- @abstractmethod
- def search(self, query: str, top_k: int = 5) -> list["SearchResult"]:
- """Return up to `top_k` SearchResult entries ranked by descending score.
-
- Returned SearchResults have `id` and `score` populated; `text` and
- `metadata` are filled in by `LeannSearcher` from the passage store.
+ Build BM25 statistics from a document corpus.
+ Must be called before scoring.
"""
-
-
-class Fts5BM25Index(BM25Index):
- """BM25 over a SQLite FTS5 virtual table, persisted on disk.
-
- Built once at `leann build` time, queried memory-bounded at search time.
- SQLite owns the on-disk term/posting data; queries hit `bm25()` directly.
- """
-
- # SQLite's FTS5 bm25() returns lower-is-better. We negate so the rest of
- # LeannSearcher (and the hybrid fusion math) can keep higher-is-better.
- _SCHEMA = (
- "CREATE VIRTUAL TABLE bm25_passages USING fts5("
- "id UNINDEXED, text, tokenize='unicode61 remove_diacritics 2'"
- ")"
- )
-
- def __init__(self, db_path: str):
- self._db_path = db_path
- self._conn: Optional[Any] = None
-
- def _connect(self):
- import sqlite3
-
- if self._conn is None:
- self._conn = sqlite3.connect(self._db_path)
- return self._conn
-
- def fit(self, documents: list[dict[str, Any]]) -> None:
- import sqlite3
-
- # Fresh DB every fit — fit() is a one-shot bulk-load.
- if os.path.exists(self._db_path):
- os.unlink(self._db_path)
- conn = sqlite3.connect(self._db_path)
- try:
- conn.execute(self._SCHEMA)
- conn.executemany(
- "INSERT INTO bm25_passages(id, text) VALUES (?, ?)",
- ((d["id"], d.get("text", "")) for d in documents),
+ self.corpus_size = len(documents)
+ self.doc_lengths = {}
+ self.word_counts = {}
+ self.idlist = set()
+ doc_freqs = defaultdict(int)
+
+ for doc_data in documents:
+ doc_id = doc_data["id"]
+ words = self._tokenize(doc_data["text"])
+ doc_length = len(words)
+ self.doc_lengths[doc_id] = doc_length
+
+ unique_words = set(words)
+ for word in unique_words:
+ doc_freqs[word] += 1
+ self.word_counts[doc_id] = dict(Counter(words))
+ self.idlist.add(doc_id)
+
+ self.doc_freqs = dict(doc_freqs)
+ self.avg_doc_length = sum(self.doc_lengths.values()) / len(self.doc_lengths)
+
+ def score(self, query_words: list[str], document_id: str) -> float:
+ if (
+ self.doc_freqs is None
+ or self.doc_lengths == {}
+ or self.word_counts == {}
+ or self.avg_doc_length is None
+ or self.corpus_size is None
+ ):
+ raise ValueError("BM25 model not fitted. Call fit() before scoring.")
+
+ passage_words = self.word_counts[document_id]
+ passage_length = sum(passage_words.values())
+ score = 0.0
+ for word in query_words:
+ if word not in self.doc_freqs:
+ continue
+ word_freq = passage_words[word] if word in passage_words else 0
+ idf = np.log(
+ (self.corpus_size - self.doc_freqs[word] + 0.5) / (self.doc_freqs[word] + 0.5) + 1
)
- conn.commit()
- finally:
- conn.close()
+ tf = (word_freq * (self.k1 + 1)) / (
+ word_freq + self.k1 * (1 - self.b + self.b * (passage_length / self.avg_doc_length))
+ )
+ score += idf * tf
+ return score
- def search(self, query: str, top_k: int = 5) -> list["SearchResult"]:
- # Strip punctuation, lowercase, OR the terms together. Avoids FTS5
- # query syntax surprises (`:`, `*`, etc.) for natural-language queries.
- terms = re.sub(r"[^\w\s]", "", query).lower().split()
- if not terms:
- return []
- fts5_query = " OR ".join(terms)
- conn = self._connect()
- rows = conn.execute(
- "SELECT id, -bm25(bm25_passages) AS score "
- "FROM bm25_passages WHERE bm25_passages MATCH ? "
- "ORDER BY score DESC LIMIT ?",
- (fts5_query, top_k),
- ).fetchall()
+ def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+ query_words = self._tokenize(query)
+ scores = {doc_id: self.score(query_words, doc_id) for doc_id in self.idlist}
+ sorted_scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)
return [
- SearchResult(id=doc_id, score=float(score), text="", metadata={})
- for doc_id, score in rows
+ SearchResult(id=doc_id, score=score, text="", metadata={})
+ for doc_id, score in sorted_scores[:top_k]
]
- def close(self) -> None:
- if self._conn is not None:
- self._conn.close()
- self._conn = None
-
class Fts5BM25Index(BM25Index):
"""BM25 over a SQLite FTS5 virtual table, persisted on disk.