Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: make keyword search find Chinese, Japanese and Korean words (#17261)
Nilay22d801e80984
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex c56a19d4222..7bda5da8f10 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -12908,12 +12908,15 @@ def _mcp_search_index(specs: list[dict]) -> sqlite3.Connection:conn.execute("CREATE VIRTUAL TABLE tool_text USING fts5(description, params, tokenize='porter unicode61')")+ from storage.rag_db import fts_text # noqa: PLC0415+conn.executemany(- "INSERT INTO tool_names(rowid, name) VALUES (?, ?)", [(i, r[0]) for i, r in enumerate(rows)]+ "INSERT INTO tool_names(rowid, name) VALUES (?, ?)",+ [(i, fts_text(r[0])) for i, r in enumerate(rows)],)conn.executemany("INSERT INTO tool_text(rowid, description, params) VALUES (?, ?, ?)",- [(i, r[1], r[2]) for i, r in enumerate(rows)],+ [(i, fts_text(r[1]), fts_text(r[2])) for i, r in enumerate(rows)],)_MCP_SEARCH_INDEXES[digest] = connwhile len(_MCP_SEARCH_INDEXES) > 4:diff --git a/studio/backend/core/rag/store.py b/studio/backend/core/rag/store.pyindex 72e1a8abae4..59d57692065 100644--- a/studio/backend/core/rag/store.py+++ b/studio/backend/core/rag/store.py@@ -69,15 +69,36 @@ def _now() -> str:_TOKEN = re.compile(r"\w+", re.UNICODE)+_CJK_RUN = re.compile(f"([{rag_db.CJK}]+)")# Quotes mark a word being named rather than used; non-greedy and single-line so an unclosed quote spans nothing._QUOTED = re.compile(r"\"([^\"\n]+)\"|\u201c([^\u201d\n]+)\u201d|'([^'\n]+)'|`([^`\n]+)`")+def _terms(token: str) -> list[str]:+ terms = []+ for i, part in enumerate(_CJK_RUN.split(token)):+ if i % 2:+ terms.extend(" ".join(part[j : j + 2]) for j in range(max(1, len(part) - 1)))+ elif part:+ terms.append(part)+ return terms+++def _or_terms(tokens) -> str:+ # Dedupe CJK phrases only: a repeated plain word keeps its bm25 weight.+ seen = set()+ return " OR ".join(+ f'"{t}"'+ for token in tokens+ for t in _terms(token)+ if " " not in t or not (t in seen or seen.add(t))+ )++def _match_query(query: str) -> str:"""User text -> safe FTS5 OR-of-quoted-terms query; quoting defuses FTS5operators. "" (no tokens) means no lexical results."""- toks = _TOKEN.findall(query.lower())- return " OR ".join(f'"{t}"' for t in toks)+ return _or_terms(_TOKEN.findall(query.lower()))# Fixed list for reproducibility; 'no'/'not' deliberately excluded (they carry meaning).@@ -120,10 +141,10 @@ def conversation_match_queries(query: str) -> list[str]:for token in _TOKEN.findall("".join(match)))content = [t for t in tokens if t not in _ARCHIVE_STOPWORDS or t in quoted] or tokens- permissive = " OR ".join(f'"{t}"' for t in content)+ permissive = _or_terms(content)if not identifiers:return [permissive]- focused = " OR ".join(f'"{t}"' for t in identifiers)+ focused = " OR ".join('"' + " ".join(rag_db.fts_text(t).split()) + '"' for t in identifiers)return [focused] if focused == permissive else [focused, permissive]@@ -421,7 +442,7 @@ def _insert_chunk()conn.execute("INSERT INTO chunks_fts(text, chunk_id, scope) VALUES(?,?,?)",- (chunk.text, chunk_id, scope),+ (rag_db.fts_text(chunk.text), chunk_id, scope),)return cur.lastrowid@@ -607,10 +628,14 @@ def _copy_chunk_rows("FROM chunks WHERE document_id=?",(target_id, target_id, scope, source_id),)- conn.execute(- "INSERT INTO chunks_fts(text, chunk_id, scope) "- "SELECT text, id, scope FROM chunks WHERE document_id=?",- (target_id,),+ conn.executemany(+ "INSERT INTO chunks_fts(text, chunk_id, scope) VALUES(?,?,?)",+ [+ (rag_db.fts_text(r["text"]), r["id"], r["scope"])+ for r in conn.execute(+ "SELECT text, id, scope FROM chunks WHERE document_id=?", (target_id,)+ ).fetchall()+ ],)return chunk_idsdiff --git a/studio/backend/storage/rag_db.py b/studio/backend/storage/rag_db.pyindex ad0ac007f11..7766dba704a 100644--- a/studio/backend/storage/rag_db.py+++ b/studio/backend/storage/rag_db.py@@ -28,6 +28,14 @@ except Exception as exc: # noqa: BLE001 - any import failure disables RAG_RAG_UNAVAILABLE_MSG = "RAG unavailable: sqlite-vec extension could not be loaded"+# Han, Kana and Hangul. unicode61 keeps an unspaced run as one token, so chunks_fts holds them one per token.+CJK = "\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uf900-\ufaff\uff66-\uff9f\U00020000-\U0002fa1f"+_CJK_CHAR = re.compile(f"[{CJK}]")+++def fts_text(text: str) -> str:+ return _CJK_CHAR.sub(r" \g<0> ", text)+class RagExtensionUnavailable(RuntimeError):"""sqlite-vec is installed but its native library will not load (a missing vec0 binary in the venv is the@@ -282,6 +290,16 @@ def _ensure_schema(conn: sqlite3.Connection) -> None:"ON documents(linked_folder_id) WHERE linked_folder_id IS NOT NULL")ensure_linked_folder_columns(conn)+ if conn.execute("PRAGMA user_version").fetchone()[0] < 1:+ conn.executemany(+ "UPDATE chunks_fts SET text=? WHERE rowid=?",+ [+ (fts_text(text), rowid)+ for rowid, text in conn.execute("SELECT rowid, text FROM chunks_fts")+ if text and _CJK_CHAR.search(text)+ ],+ )+ conn.execute("PRAGMA user_version = 1")conn.commit()diff --git a/studio/backend/tests/test_mcp_servers.py b/studio/backend/tests/test_mcp_servers.pyindex 61ea255efe0..bf117a78c31 100644--- a/studio/backend/tests/test_mcp_servers.py+++ b/studio/backend/tests/test_mcp_servers.py@@ -2608,6 +2608,25 @@ def test_mcp_tool_search_never_finds_an_app_only_tool(tmp_path, monkeypatch, seaassert tools_mod.mcp_tool_spec("mcp__srv1__widget_refresh") is None+def test_mcp_tool_search_finds_cjk_descriptions(tmp_path, monkeypatch):+ from core.inference import tools as tools_mod++ _cache_server_tools(+ tmp_path,+ monkeypatch,+ [+ _PING,+ {"name": "refund", "description": "处理退款申请"},+ {"name": "henkin", "description": "返金を処理します"},+ {"name": "hwanbul", "description": "환불 처리"},+ ],+ )+ search = tools_mod.mcp_tool_search_specs+ assert _names(search({"query": "退款"})) == ["mcp__srv1__refund"]+ assert _names(search({"query": "返金"})) == ["mcp__srv1__henkin"]+ assert _names(search({"query": "환불"})) == ["mcp__srv1__hwanbul"]++def test_mcp_tool_search_finds_an_alias_by_its_raw_name(tmp_path, monkeypatch):from core.inference import tools as tools_moddiff --git a/studio/backend/tests/test_rag_store.py b/studio/backend/tests/test_rag_store.pyindex 407e242957b..096453bce8e 100644--- a/studio/backend/tests/test_rag_store.py+++ b/studio/backend/tests/test_rag_store.py@@ -62,6 +62,60 @@ def test_match_query_sanitizes_special_chars():assert store._match_query('AND OR "quote" (paren) -dash') != ""+def test_lexical_finds_words_in_unspaced_cjk_text(rag_conn):+ _add_doc(rag_conn, "kb_a", "zh", "zh.txt", "h1", ["我们的退款政策:购买后30天内可以退款。"])+ _add_doc(rag_conn, "kb_a", "ja", "ja.txt", "h2", ["返金ポリシー:30日以内に返金します。"])+ _add_doc(rag_conn, "kb_a", "ko", "ko.txt", "h3", ["환불 정책은 30일 이내입니다."])+ _add_doc(rag_conn, "kb_a", "en", "en.txt", "h4", ["Our refund policy: refunds within 30 days."])++ def hits(query):+ return [cid for cid, _ in store.search_lexical(rag_conn, "kb_a", query, 10)]++ assert hits("退款政策是什么?")[0] == "zh:0"+ assert hits("退款") == ["zh:0"]+ assert hits("返金") == ["ja:0"]+ assert hits("ポリシー") == ["ja:0"]+ assert hits("환불은 어떻게 되나요") == ["ko:0"]+ assert hits("refund policy") == ["en:0"]+ assert hits("退") == ["zh:0"]+++def test_focused_query_keeps_a_cjk_identifier_whole(rag_conn):+ _add_doc(rag_conn, "kb_a", "a", "a.txt", "h1", ["案件123的状态"])+ _add_doc(rag_conn, "kb_a", "b", "b.txt", "h2", ["案件124的状态"])+ focused = store.conversation_match_queries("current status of 案件123")[0]+ assert store.lexical_matching_ids(rag_conn, ["a:0", "b:0"], focused) == {"a:0"}+++def test_repeated_cjk_text_builds_each_phrase_once():+ terms = store._match_query("订单:苹果手机一台。" * 200).split(" OR ")+ assert len(terms) == len(set(terms)) == 6+ assert store._match_query("refund refund policy") == '"refund" OR "refund" OR "policy"'+++def test_existing_index_is_rebuilt_for_cjk_search(rag_conn):+ from storage import rag_db++ text = "我们的退款政策:购买后30天内可以退款。"+ _add_doc(rag_conn, "kb_a", "zh", "zh.txt", "h1", [text])+ _add_doc(rag_conn, "kb_a", "en", "en.txt", "h2", ["alpha bravo"])+ rag_conn.execute("UPDATE chunks_fts SET text=? WHERE chunk_id='zh:0'", (text,))+ rag_conn.execute("PRAGMA user_version = 0")+ rag_conn.commit()+ assert store.search_lexical(rag_conn, "kb_a", "退款", 10) == []++ rag_db.reset_schema_state_for_tests()+ for _ in range(2):+ conn = rag_db.get_connection()+ try:+ assert [cid for cid, _ in store.search_lexical(conn, "kb_a", "退款", 10)] == ["zh:0"]+ assert [cid for cid, _ in store.search_lexical(conn, "kb_a", "alpha", 10)] == ["en:0"]+ finally:+ conn.close()+ rag_db.reset_schema_state_for_tests()+ assert store.chunks_by_id(rag_conn, ["zh:0"])["zh:0"]["text"] == text++def test_lexical_does_not_crash_on_punctuation(rag_conn):_add_doc(rag_conn, "kb_a", "d1", "f", "h1", ["alpha bravo"])store.search_lexical(rag_conn, "kb_a", 'NEAR("x" AND', 5)