[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(search): preserve mixed-script boundaries in FTS5 BM25 (#415)
Yifan Chen0c20c50b4d7a
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex 75df727..57da7a0 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -71,3 +71,8 @@ fixes). Newest entries at the bottom.- Remove the seven duplicate index command parsers and handlers introduced by #269 after #285 had already supplied them. This fixes `argparse.ArgumentError: conflicting subparser: index-browser`, which prevented every CLI command, including help, from running.- Retain the #285 command interface (`--max-count`, `--index-name`, `--no-recompute`), application readers, and shared builder; native indexing and standalone wheel reader packaging are unchanged.++## 2026-09-06: Preserve mixed-script words in BM25 search++- Keep Latin words and numbers separate from adjacent Chinese, Japanese, and Korean n-grams when building SQLite FTS5 indexes and parsing keyword queries. For example, `Python数据库SQL` can be retrieved by `Python`, `SQL`, or a mixed-script query.+- Existing BM25 artifacts remain readable. Rebuild indexes containing mixed-script text to regenerate tokens that were previously joined at script boundaries.diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex e374a46..0ff88e6 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -45,12 +45,15 @@ _CJK_RUN = re.compile(f"[{_CJK_CHARACTERS}]+")def _fts5_cjk_ngrams(match: re.Match[str]) -> str:"""Expand a CJK run into unigram and bigram tokens for SQLite FTS5."""text = match.group()- return " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))])+ tokens = " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))])+ # Keep adjacent non-CJK words separate from the first and last n-grams.+ return f" {tokens} "def _fts5_cjk_query(query: str) -> str:"""Build a safe FTS5 query that requires every CJK bigram in each term."""- tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|\w+", query.lower())+ # Unicode \w includes CJK, so exclude it from the non-CJK alternative.+ tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|[^\W{_CJK_CHARACTERS}]+", query.lower())terms = []for token in tokens:if _CJK_RUN.fullmatch(token):diff --git a/tests/test_fts5_bm25.py b/tests/test_fts5_bm25.pyindex 3561a8b..0f00aa6 100644--- a/tests/test_fts5_bm25.py+++ b/tests/test_fts5_bm25.py@@ -1,3 +1,4 @@+import pytestfrom leann.api import Fts5BM25Index@@ -30,3 +31,45 @@ def test_fts5_bm25_keeps_legacy_database_query_format(tmp_path):assert [result.id for result in reopened.search("database")] == ["database"]finally:reopened.close()++[email protected]("cjk_text", ["数据库", "データベース", "데이터베이스"])[email protected]("query", ["Python", "SQL"])+def test_fts5_bm25_preserves_words_adjacent_to_cjk(tmp_path, cjk_text, query):+ db_path = tmp_path / "mixed.sqlite"+ index = Fts5BM25Index(str(db_path))+ index.fit(+ [+ {"id": "mixed", "text": f"Python{cjk_text}SQL"},+ {"id": "unrelated", "text": "unrelated document"},+ ]+ )+ index.close()++ reopened = Fts5BM25Index(str(db_path))+ try:+ assert [result.id for result in reopened.search(query)] == ["mixed"]+ finally:+ reopened.close()++[email protected]("query", ["Python数据库", "数据库Python", "Python数据库SQL", "2026数据库"])+def test_fts5_bm25_splits_mixed_script_query_terms(tmp_path, query):+ db_path = tmp_path / "queries.sqlite"+ index = Fts5BM25Index(str(db_path))+ index.fit(+ [+ {"id": "database", "text": "数据库检索系统"},+ {"id": "partial", "text": "数据分析"},+ {"id": "unrelated", "text": "image classification"},+ ]+ )+ index.close()++ reopened = Fts5BM25Index(str(db_path))+ try:+ # Match the CJK term independently of the Latin/number terms, while+ # still requiring all its bigrams (the partial match must be excluded).+ assert [result.id for result in reopened.search(query)] == ["database"]+ finally:+ reopened.close()