[MLsys2026 Best Paper]: https://arxiv.org/abs/2506.08276. RAG on Everything with LEANN. Enjoy 97% storage savings while running a fast, accurate, and 100% private RAG application on your personal device.
Python aifaissgpt-osslangchainllama-indexllmlocalstorageoffline-firstollamaprivacypythonragretrieval-augmented-generationvector-databasevector-searchvectors
fix(search): support CJK terms in FTS5 BM25 (#401)
kv864794092ff4da6
diff --git a/packages/leann-core/src/leann/api.py b/packages/leann-core/src/leann/api.pyindex f5f8f4d..99c4432 100644--- a/packages/leann-core/src/leann/api.py+++ b/packages/leann-core/src/leann/api.py@@ -39,6 +39,30 @@ logger = logging.getLogger(__name__)PASSAGE_ID_SCHEME_SEQUENTIAL = "sequential"PASSAGE_ID_SCHEME_CONTENT_HASH = "content-hash"+_CJK_CHARACTERS = "\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\u3040-\u30ff\uac00-\ud7af"+_CJK_RUN = re.compile(f"[{_CJK_CHARACTERS}]+")+++def _fts5_cjk_ngrams(match: re.Match[str]) -> str:+ """Expand a CJK run into unigram and bigram tokens for SQLite FTS5."""+ text = match.group()+ return " ".join([*text, *(text[i : i + 2] for i in range(len(text) - 1))])+++def _fts5_cjk_query(query: str) -> str:+ """Build a safe FTS5 query that requires every CJK bigram in each term."""+ tokens = re.findall(rf"[{_CJK_CHARACTERS}]+|\w+", query.lower())+ terms = []+ for token in tokens:+ if _CJK_RUN.fullmatch(token):+ ngrams = (+ [token] if len(token) == 1 else [token[i : i + 2] for i in range(len(token) - 1)]+ )+ terms.append("(" + " AND ".join(f'"{ngram}"' for ngram in ngrams) + ")")+ else:+ terms.append(f'"{token}"')+ return " OR ".join(terms)+def get_registered_backends() -> list[str]:"""Get list of registered backend names."""@@ -322,15 +346,18 @@ class Fts5BM25Index(BM25Index):")")- def __init__(self, db_path: str):+ def __init__(self, db_path: str, cjk_ngrams: Optional[bool] = None):self._db_path = db_pathself._conn: Optional[Any] = None+ self._cjk_ngrams = cjk_ngramsdef _connect(self):import sqlite3if self._conn is None:self._conn = sqlite3.connect(self._db_path)+ if self._cjk_ngrams is None:+ self._cjk_ngrams = bool(self._conn.execute("PRAGMA user_version").fetchone()[0])return self._conndef fit(self, documents: list[dict[str, Any]]) -> None:@@ -339,25 +366,39 @@ class Fts5BM25Index(BM25Index):# Fresh DB every fit — fit() is a one-shot bulk-load.if os.path.exists(self._db_path):os.unlink(self._db_path)+ if self._cjk_ngrams is None:+ self._cjk_ngrams = Trueconn = sqlite3.connect(self._db_path)try:conn.execute(self._SCHEMA)+ conn.execute(f"PRAGMA user_version = {int(self._cjk_ngrams)}")conn.executemany("INSERT INTO bm25_passages(id, text) VALUES (?, ?)",- ((d["id"], d.get("text", "")) for d in documents),+ (+ (+ d["id"],+ _CJK_RUN.sub(_fts5_cjk_ngrams, d.get("text", ""))+ if self._cjk_ngrams+ else d.get("text", ""),+ )+ for d in documents+ ),)conn.commit()finally:conn.close()def search(self, query: str, top_k: int = 5) -> list["SearchResult"]:- # Strip punctuation, lowercase, OR the terms together. Avoids FTS5- # query syntax surprises (`:`, `*`, etc.) for natural-language queries.- terms = re.sub(r"[^\w\s]", "", query).lower().split()- if not terms:- return []- fts5_query = " OR ".join(terms)+ # Expand CJK terms to the same n-grams used at index time. Older+ # databases retain their legacy query behaviour via PRAGMA user_version.conn = self._connect()+ if self._cjk_ngrams:+ fts5_query = _fts5_cjk_query(query)+ else:+ terms = re.sub(r"[^\w\s]", "", query).lower().split()+ fts5_query = " OR ".join(terms)+ if not fts5_query:+ return []rows = conn.execute("SELECT id, -bm25(bm25_passages) AS score ""FROM bm25_passages WHERE bm25_passages MATCH ? "diff --git a/tests/test_fts5_bm25.py b/tests/test_fts5_bm25.pynew file mode 100644index 0000000..3561a8b--- /dev/null+++ b/tests/test_fts5_bm25.py@@ -0,0 +1,32 @@+from leann.api import Fts5BM25Index+++def test_fts5_bm25_matches_chinese_substrings_after_reopen(tmp_path):+ db_path = tmp_path / "passages.sqlite"+ index = Fts5BM25Index(str(db_path))+ index.fit(+ [+ {"id": "database", "text": "数据库检索系统"},+ {"id": "image", "text": "图像分类系统"},+ ]+ )+ index.close()++ reopened = Fts5BM25Index(str(db_path))+ try:+ assert [result.id for result in reopened.search("数据库")] == ["database"]+ finally:+ reopened.close()+++def test_fts5_bm25_keeps_legacy_database_query_format(tmp_path):+ db_path = tmp_path / "legacy.sqlite"+ index = Fts5BM25Index(str(db_path), cjk_ngrams=False)+ index.fit([{"id": "database", "text": "database retrieval"}])+ index.close()++ reopened = Fts5BM25Index(str(db_path))+ try:+ assert [result.id for result in reopened.search("database")] == ["database"]+ finally:+ reopened.close()