🚀 The fast, Pythonic way to build MCP servers and clients.
fix: support Unicode tokens in BM25 search (#4889)
nate nowack258554d93c57
diff --git a/fastmcp_slim/fastmcp/server/transforms/search/bm25.py b/fastmcp_slim/fastmcp/server/transforms/search/bm25.pyindex 447db8cac..b08a5a963 100644--- a/fastmcp_slim/fastmcp/server/transforms/search/bm25.py+++ b/fastmcp_slim/fastmcp/server/transforms/search/bm25.py@@ -3,6 +3,7 @@import hashlibimport mathimport re+import unicodedatafrom collections.abc import Sequencefrom typing import Annotated, Any@@ -16,8 +17,9 @@ from fastmcp.tools.base import Tooldef _tokenize(text: str) -> list[str]:- """Lowercase, split on non-alphanumeric, filter short tokens."""- return [t for t in re.split(r"[^a-z0-9]+", text.lower()) if len(t) > 1]+ """Normalize and extract Unicode alphanumeric tokens."""+ normalized = unicodedata.normalize("NFKC", text).casefold()+ return re.findall(r"[^\W_]{2,}", normalized)class _BM25Index:diff --git a/tests/server/transforms/test_search.py b/tests/server/transforms/test_search.pyindex 810a05bdd..09a822ce6 100644--- a/tests/server/transforms/test_search.py+++ b/tests/server/transforms/test_search.py@@ -436,6 +436,18 @@ class TestBM25Index:index.build(["alpha beta gamma"])assert index.query("zzz", 5) == []+ def test_unicode_tokens(self):+ index = _BM25Index()+ index.build(["show portfolio", "показать портфель"])+ assert index.query("портфель", 5) == [1]++ def test_unicode_normalization(self):+ index = _BM25Index()+ index.build(["Straße PORTFOLIO cafe\u0301"])+ assert index.query("STRASSE", 5) == [0]+ assert index.query("portfolio", 5) == [0]+ assert index.query("café", 5) == [0]+# ---------------------------------------------------------------------------# call_tool self-reference guard