Skip to content

🚀 The fast, Pythonic way to build MCP servers and clients.

fix: support Unicode tokens in BM25 search (#4889)

nate nowack258554d93c57
diff --git a/fastmcp_slim/fastmcp/server/transforms/search/bm25.py b/fastmcp_slim/fastmcp/server/transforms/search/bm25.py
index 447db8cac..b08a5a963 100644
--- a/fastmcp_slim/fastmcp/server/transforms/search/bm25.py
+++ b/fastmcp_slim/fastmcp/server/transforms/search/bm25.py
@@ -3,6 +3,7 @@
import hashlib
import math
import re
+import unicodedata
from collections.abc import Sequence
from typing import Annotated, Any
@@ -16,8 +17,9 @@ from fastmcp.tools.base import Tool
def _tokenize(text: str) -> list[str]:
- """Lowercase, split on non-alphanumeric, filter short tokens."""
- return [t for t in re.split(r"[^a-z0-9]+", text.lower()) if len(t) > 1]
+ """Normalize and extract Unicode alphanumeric tokens."""
+ normalized = unicodedata.normalize("NFKC", text).casefold()
+ return re.findall(r"[^\W_]{2,}", normalized)
class _BM25Index:
diff --git a/tests/server/transforms/test_search.py b/tests/server/transforms/test_search.py
index 810a05bdd..09a822ce6 100644
--- a/tests/server/transforms/test_search.py
+++ b/tests/server/transforms/test_search.py
@@ -436,6 +436,18 @@ class TestBM25Index:
index.build(["alpha beta gamma"])
assert index.query("zzz", 5) == []
+ def test_unicode_tokens(self):
+ index = _BM25Index()
+ index.build(["show portfolio", "показать портфель"])
+ assert index.query("портфель", 5) == [1]
+
+ def test_unicode_normalization(self):
+ index = _BM25Index()
+ index.build(["Straße PORTFOLIO cafe\u0301"])
+ assert index.query("STRASSE", 5) == [0]
+ assert index.query("portfolio", 5) == [0]
+ assert index.query("café", 5) == [0]
+
# ---------------------------------------------------------------------------
# call_tool self-reference guard