Skip to content

armgit / LEANN

Public Imported from GitHub
0

[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.

fix(search): preserve mixed-script boundaries in FTS5 BM25 (#415)

Yifan Chen0c20c50b4d7a
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index 75df727..57da7a0 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -71,3 +71,8 @@ fixes). Newest entries at the bottom.
- Remove the seven duplicate index command parsers and handlers introduced by #269 after #285 had already supplied them. This fixes `argparse.ArgumentError: conflicting subparser: index-browser`, which prevented every CLI command, including help, from running.
- Retain the #285 command interface (`--max-count`, `--index-name`, `--no-recompute`), application readers, and shared builder; native indexing and standalone wheel reader packaging are unchanged.
+
+## 2026-09-06: Preserve mixed-script words in BM25 search
+
+- Keep Latin words and numbers separate from adjacent Chinese, Japanese, and Korean n-grams when building SQLite FTS5 indexes and parsing keyword queries. For example, `Python数据库SQL` can be retrieved by `Python`, `SQL`, or a mixed-script query.
+- Existing BM25 artifacts remain readable. Rebuild indexes containing mixed-script text to regenerate tokens that were previously joined at script boundaries.
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.py
index e374a46..0ff88e6 100644
--- a/packages/leann-core/src/leann/api.py
+++ b/packages/leann-core/src/leann/api.py
@@ -45,12 +45,15 @@ _CJK_RUN = re.compile(f"[{_CJK_CHARACTERS}]+")
def _fts5_cjk_ngrams(match: re.Match[str]) -> str:
"""Expand a CJK run into unigram and bigram tokens for SQLite FTS5."""
text = match.group()
- return " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))])
+ tokens = " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))])
+ # Keep adjacent non-CJK words separate from the first and last n-grams.
+ return f" {tokens} "
def _fts5_cjk_query(query: str) -> str:
"""Build a safe FTS5 query that requires every CJK bigram in each term."""
- tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|\w+", query.lower())
+ # Unicode \w includes CJK, so exclude it from the non-CJK alternative.
+ tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|[^\W{_CJK_CHARACTERS}]+", query.lower())
terms = []
for token in tokens:
if _CJK_RUN.fullmatch(token):
diff --git a/tests/test_fts5_bm25.py b/tests/test_fts5_bm25.py
index 3561a8b..0f00aa6 100644
--- a/tests/test_fts5_bm25.py
+++ b/tests/test_fts5_bm25.py
@@ -1,3 +1,4 @@
+import pytest
from leann.api import Fts5BM25Index
@@ -30,3 +31,45 @@ def test_fts5_bm25_keeps_legacy_database_query_format(tmp_path):
assert [result.id for result in reopened.search("database")] == ["database"]
finally:
reopened.close()
+
+
[email protected]("cjk_text", ["数据库", "データベース", "데이터베이스"])
[email protected]("query", ["Python", "SQL"])
+def test_fts5_bm25_preserves_words_adjacent_to_cjk(tmp_path, cjk_text, query):
+ db_path = tmp_path / "mixed.sqlite"
+ index = Fts5BM25Index(str(db_path))
+ index.fit(
+ [
+ {"id": "mixed", "text": f"Python{cjk_text}SQL"},
+ {"id": "unrelated", "text": "unrelated document"},
+ ]
+ )
+ index.close()
+
+ reopened = Fts5BM25Index(str(db_path))
+ try:
+ assert [result.id for result in reopened.search(query)] == ["mixed"]
+ finally:
+ reopened.close()
+
+
[email protected]("query", ["Python数据库", "数据库Python", "Python数据库SQL", "2026数据库"])
+def test_fts5_bm25_splits_mixed_script_query_terms(tmp_path, query):
+ db_path = tmp_path / "queries.sqlite"
+ index = Fts5BM25Index(str(db_path))
+ index.fit(
+ [
+ {"id": "database", "text": "数据库检索系统"},
+ {"id": "partial", "text": "数据分析"},
+ {"id": "unrelated", "text": "image classification"},
+ ]
+ )
+ index.close()
+
+ reopened = Fts5BM25Index(str(db_path))
+ try:
+ # Match the CJK term independently of the Latin/number terms, while
+ # still requiring all its bigrams (the partial match must be excluded).
+ assert [result.id for result in reopened.search(query)] == ["database"]
+ finally:
+ reopened.close()