Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix: repair KWS symbol cleanup regex

游雁c26848f6480f
diff --git a/funasr/utils/kws_utils.py b/funasr/utils/kws_utils.py
index 1086050e4..11de3238b 100644
--- a/funasr/utils/kws_utils.py
+++ b/funasr/utils/kws_utils.py
@@ -7,7 +7,7 @@ from collections import defaultdict
from typing import List, Optional, Tuple
-symbol_str = '[’!"#$%&\'()*+,-./:;<>=?@,。?★、…【】《》?“”‘’![\\]^_`{|}~\s]+'
+symbol_str = r'''[’!"#$%&'()*+,\-./:;<>=?@,。?★、…【】《》?“”‘’!\[\]\\^_`{|}~\s]+'''
def split_mixed_label(input_str):
diff --git a/tests/run_test.py b/tests/run_test.py
index 9ff648bc1..e32ff12b2 100644
--- a/tests/run_test.py
+++ b/tests/run_test.py
@@ -7,6 +7,11 @@ import unittest
from fnmatch import fnmatch
+PROJECT_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+if PROJECT_ROOT not in sys.path:
+ sys.path.insert(0, PROJECT_ROOT)
+
+
def gather_test_cases(test_dir, pattern, list_tests):
case_list = []
for dirpath, dirnames, filenames in os.walk(test_dir):
diff --git a/tests/test_kws_utils.py b/tests/test_kws_utils.py
new file mode 100644
index 000000000..79fe4cdc8
--- /dev/null
+++ b/tests/test_kws_utils.py
@@ -0,0 +1,23 @@
+import re
+import unittest
+
+from funasr.utils.kws_utils import query_token_set, split_mixed_label, symbol_str
+
+
+class KwsUtilsTest(unittest.TestCase):
+ def test_symbol_regex_strips_punctuation_and_whitespace(self):
+ self.assertEqual(re.sub(symbol_str, "", "abc!@# 中文\t★"), "abc中文")
+
+ def test_split_mixed_label_lowercases_ascii_words(self):
+ self.assertEqual(split_mixed_label("Hello中文"), ["hello", "中", "文"])
+
+ def test_query_token_set_strips_symbols_before_character_lookup(self):
+ symbol_table = {"a": 1, "中": 2, "<unk>": 0}
+ tokens, token_ids = query_token_set("A!中", symbol_table, {})
+
+ self.assertEqual(tokens, ("a", "中"))
+ self.assertEqual(token_ids, (1, 2))
+
+
+if __name__ == "__main__":
+ unittest.main()