Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: make keyword search find Chinese, Japanese and Korean words (#17261)

Nilay22d801e80984
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index c56a19d4222..7bda5da8f10 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -12908,12 +12908,15 @@ def _mcp_search_index(specs: list[dict]) -> sqlite3.Connection:
conn.execute(
"CREATE VIRTUAL TABLE tool_text USING fts5(description, params, tokenize='porter unicode61')"
)
+ from storage.rag_db import fts_text # noqa: PLC0415
+
conn.executemany(
- "INSERT INTO tool_names(rowid, name) VALUES (?, ?)", [(i, r[0]) for i, r in enumerate(rows)]
+ "INSERT INTO tool_names(rowid, name) VALUES (?, ?)",
+ [(i, fts_text(r[0])) for i, r in enumerate(rows)],
)
conn.executemany(
"INSERT INTO tool_text(rowid, description, params) VALUES (?, ?, ?)",
- [(i, r[1], r[2]) for i, r in enumerate(rows)],
+ [(i, fts_text(r[1]), fts_text(r[2])) for i, r in enumerate(rows)],
)
_MCP_SEARCH_INDEXES[digest] = conn
while len(_MCP_SEARCH_INDEXES) > 4:
diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.py
index 72e1a8abae4..59d57692065 100644
--- a/studio/backend/core/rag/store.py
+++ b/studio/backend/core/rag/store.py
@@ -69,15 +69,36 @@ def _now() -> str:
_TOKEN = re.compile(r"\w+", re.UNICODE)
+_CJK_RUN = re.compile(f"([{rag_db.CJK}]+)")
# Quotes mark a word being named rather than used; non-greedy and single-line so an unclosed quote spans nothing.
_QUOTED = re.compile(r"\"([^\"\n]+)\"|\u201c([^\u201d\n]+)\u201d|'([^'\n]+)'|`([^`\n]+)`")
+def _terms(token: str) -> list[str]:
+ terms = []
+ for i, part in enumerate(_CJK_RUN.split(token)):
+ if i % 2:
+ terms.extend(" ".join(part[j : j + 2]) for j in range(max(1, len(part) - 1)))
+ elif part:
+ terms.append(part)
+ return terms
+
+
+def _or_terms(tokens) -> str:
+ # Dedupe CJK phrases only: a repeated plain word keeps its bm25 weight.
+ seen = set()
+ return " OR ".join(
+ f'"{t}"'
+ for token in tokens
+ for t in _terms(token)
+ if " " not in t or not (t in seen or seen.add(t))
+ )
+
+
def _match_query(query: str) -> str:
"""User text -> safe FTS5 OR-of-quoted-terms query; quoting defuses FTS5
operators. "" (no tokens) means no lexical results."""
- toks = _TOKEN.findall(query.lower())
- return " OR ".join(f'"{t}"' for t in toks)
+ return _or_terms(_TOKEN.findall(query.lower()))
# Fixed list for reproducibility; 'no'/'not' deliberately excluded (they carry meaning).
@@ -120,10 +141,10 @@ def conversation_match_queries(query: str) -> list[str]:
for token in _TOKEN.findall("".join(match))
)
content = [t for t in tokens if t not in _ARCHIVE_STOPWORDS or t in quoted] or tokens
- permissive = " OR ".join(f'"{t}"' for t in content)
+ permissive = _or_terms(content)
if not identifiers:
return [permissive]
- focused = " OR ".join(f'"{t}"' for t in identifiers)
+ focused = " OR ".join('"' + " ".join(rag_db.fts_text(t).split()) + '"' for t in identifiers)
return [focused] if focused == permissive else [focused, permissive]
@@ -421,7 +442,7 @@ def _insert_chunk(
)
conn.execute(
"INSERT INTO chunks_fts(text, chunk_id, scope) VALUES(?,?,?)",
- (chunk.text, chunk_id, scope),
+ (rag_db.fts_text(chunk.text), chunk_id, scope),
)
return cur.lastrowid
@@ -607,10 +628,14 @@ def _copy_chunk_rows(
"FROM chunks WHERE document_id=?",
(target_id, target_id, scope, source_id),
)
- conn.execute(
- "INSERT INTO chunks_fts(text, chunk_id, scope) "
- "SELECT text, id, scope FROM chunks WHERE document_id=?",
- (target_id,),
+ conn.executemany(
+ "INSERT INTO chunks_fts(text, chunk_id, scope) VALUES(?,?,?)",
+ [
+ (rag_db.fts_text(r["text"]), r["id"], r["scope"])
+ for r in conn.execute(
+ "SELECT text, id, scope FROM chunks WHERE document_id=?", (target_id,)
+ ).fetchall()
+ ],
)
return chunk_ids
diff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.py
index ad0ac007f11..7766dba704a 100644
--- a/studio/backend/storage/rag_db.py
+++ b/studio/backend/storage/rag_db.py
@@ -28,6 +28,14 @@ except Exception as exc: # noqa: BLE001 - any import failure disables RAG
_RAG_UNAVAILABLE_MSG = "RAG unavailable: sqlite-vec extension could not be loaded"
+# Han, Kana and Hangul. unicode61 keeps an unspaced run as one token, so chunks_fts holds them one per token.
+CJK = "\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff\uff66-\uff9f\U00020000-\U0002fa1f"
+_CJK_CHAR = re.compile(f"[{CJK}]")
+
+
+def fts_text(text: str) -> str:
+ return _CJK_CHAR.sub(r" \g<0> ", text)
+
class RagExtensionUnavailable(RuntimeError):
"""sqlite-vec is installed but its native library will not load (a missing vec0 binary in the venv is the
@@ -282,6 +290,16 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:
"ON documents(linked_folder_id) WHERE linked_folder_id IS NOT NULL"
)
ensure_linked_folder_columns(conn)
+ if conn.execute("PRAGMA user_version").fetchone()[0] < 1:
+ conn.executemany(
+ "UPDATE chunks_fts SET text=? WHERE rowid=?",
+ [
+ (fts_text(text), rowid)
+ for rowid, text in conn.execute("SELECT rowid, text FROM chunks_fts")
+ if text and _CJK_CHAR.search(text)
+ ],
+ )
+ conn.execute("PRAGMA user_version = 1")
conn.commit()
diff --git a/studio/backend/tests/test_mcp_servers.py b/studio/backend/tests/test_mcp_servers.py
index 61ea255efe0..bf117a78c31 100644
--- a/studio/backend/tests/test_mcp_servers.py
+++ b/studio/backend/tests/test_mcp_servers.py
@@ -2608,6 +2608,25 @@ def test_mcp_tool_search_never_finds_an_app_only_tool(tmp_path, monkeypatch, sea
assert tools_mod.mcp_tool_spec("mcp__srv1__widget_refresh") is None
+def test_mcp_tool_search_finds_cjk_descriptions(tmp_path, monkeypatch):
+ from core.inference import tools as tools_mod
+
+ _cache_server_tools(
+ tmp_path,
+ monkeypatch,
+ [
+ _PING,
+ {"name": "refund", "description": "处理退款申请"},
+ {"name": "henkin", "description": "返金を処理します"},
+ {"name": "hwanbul", "description": "환불 처리"},
+ ],
+ )
+ search = tools_mod.mcp_tool_search_specs
+ assert _names(search({"query": "退款"})) == ["mcp__srv1__refund"]
+ assert _names(search({"query": "返金"})) == ["mcp__srv1__henkin"]
+ assert _names(search({"query": "환불"})) == ["mcp__srv1__hwanbul"]
+
+
def test_mcp_tool_search_finds_an_alias_by_its_raw_name(tmp_path, monkeypatch):
from core.inference import tools as tools_mod
diff --git a/studio/backend/tests/test_rag_store.py b/studio/backend/tests/test_rag_store.py
index 407e242957b..096453bce8e 100644
--- a/studio/backend/tests/test_rag_store.py
+++ b/studio/backend/tests/test_rag_store.py
@@ -62,6 +62,60 @@ def test_match_query_sanitizes_special_chars():
assert store._match_query('AND OR "quote" (paren) -dash') != ""
+def test_lexical_finds_words_in_unspaced_cjk_text(rag_conn):
+ _add_doc(rag_conn, "kb_a", "zh", "zh.txt", "h1", ["我们的退款政策:购买后30天内可以退款。"])
+ _add_doc(rag_conn, "kb_a", "ja", "ja.txt", "h2", ["返金ポリシー:30日以内に返金します。"])
+ _add_doc(rag_conn, "kb_a", "ko", "ko.txt", "h3", ["환불 정책은 30일 이내입니다."])
+ _add_doc(rag_conn, "kb_a", "en", "en.txt", "h4", ["Our refund policy: refunds within 30 days."])
+
+ def hits(query):
+ return [cid for cid, _ in store.search_lexical(rag_conn, "kb_a", query, 10)]
+
+ assert hits("退款政策是什么?")[0] == "zh:0"
+ assert hits("退款") == ["zh:0"]
+ assert hits("返金") == ["ja:0"]
+ assert hits("ポリシー") == ["ja:0"]
+ assert hits("환불은 어떻게 되나요") == ["ko:0"]
+ assert hits("refund policy") == ["en:0"]
+ assert hits("退") == ["zh:0"]
+
+
+def test_focused_query_keeps_a_cjk_identifier_whole(rag_conn):
+ _add_doc(rag_conn, "kb_a", "a", "a.txt", "h1", ["案件123的状态"])
+ _add_doc(rag_conn, "kb_a", "b", "b.txt", "h2", ["案件124的状态"])
+ focused = store.conversation_match_queries("current status of 案件123")[0]
+ assert store.lexical_matching_ids(rag_conn, ["a:0", "b:0"], focused) == {"a:0"}
+
+
+def test_repeated_cjk_text_builds_each_phrase_once():
+ terms = store._match_query("订单:苹果手机一台。" * 200).split(" OR ")
+ assert len(terms) == len(set(terms)) == 6
+ assert store._match_query("refund refund policy") == '"refund" OR "refund" OR "policy"'
+
+
+def test_existing_index_is_rebuilt_for_cjk_search(rag_conn):
+ from storage import rag_db
+
+ text = "我们的退款政策:购买后30天内可以退款。"
+ _add_doc(rag_conn, "kb_a", "zh", "zh.txt", "h1", [text])
+ _add_doc(rag_conn, "kb_a", "en", "en.txt", "h2", ["alpha bravo"])
+ rag_conn.execute("UPDATE chunks_fts SET text=? WHERE chunk_id='zh:0'", (text,))
+ rag_conn.execute("PRAGMA user_version = 0")
+ rag_conn.commit()
+ assert store.search_lexical(rag_conn, "kb_a", "退款", 10) == []
+
+ rag_db.reset_schema_state_for_tests()
+ for _ in range(2):
+ conn = rag_db.get_connection()
+ try:
+ assert [cid for cid, _ in store.search_lexical(conn, "kb_a", "退款", 10)] == ["zh:0"]
+ assert [cid for cid, _ in store.search_lexical(conn, "kb_a", "alpha", 10)] == ["en:0"]
+ finally:
+ conn.close()
+ rag_db.reset_schema_state_for_tests()
+ assert store.chunks_by_id(rag_conn, ["zh:0"])["zh:0"]["text"] == text
+
+
def test_lexical_does_not_crash_on_punctuation(rag_conn):
_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha bravo"])
store.search_lexical(rag_conn, "kb_a", 'NEAR("x" AND', 5)