Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(tokenizer): ignore empty non-linguistic symbols to avoid a hang

Mohammad Hijjawi5156f09ee35c
diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index d25c98926..69e8cd3a4 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -44,6 +44,9 @@ class CharTokenizer(BaseTokenizer):
self.non_linguistic_symbols = set()
else:
self.non_linguistic_symbols = set(non_linguistic_symbols)
+ # An empty symbol (e.g. a blank line in the file) matches every position
+ # and would make text2tokens loop forever.
+ self.non_linguistic_symbols.discard("")
self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
self.split_with_space = split_with_space
self.seg_dict = None
diff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.py
index ab479242a..0005483e3 100644
--- a/funasr/tokenizer/phoneme_tokenizer.py
+++ b/funasr/tokenizer/phoneme_tokenizer.py
@@ -599,6 +599,9 @@ class PhonemeTokenizer(AbsTokenizer):
self.non_linguistic_symbols = set()
else:
self.non_linguistic_symbols = set(non_linguistic_symbols)
+ # An empty symbol (e.g. a blank line in the file) matches every position
+ # and would make text2tokens loop forever.
+ self.non_linguistic_symbols.discard("")
self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
def __repr__(self):
diff --git a/tests/test_char_tokenizer_non_linguistic_symbols.py b/tests/test_char_tokenizer_non_linguistic_symbols.py
new file mode 100644
index 000000000..7644ba1ea
--- /dev/null
+++ b/tests/test_char_tokenizer_non_linguistic_symbols.py
@@ -0,0 +1,44 @@
+"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever."""
+
+import os
+import tempfile
+import unittest
+
+from funasr.tokenizer.char_tokenizer import CharTokenizer
+from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer
+
+
+class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase):
+ def test_blank_line_in_symbol_file_is_ignored(self):
+ with tempfile.TemporaryDirectory() as tmpdir:
+ path = os.path.join(tmpdir, "nlsyms.txt")
+ with open(path, "w", encoding="utf-8") as f:
+ f.write("<noise>\n\n")
+ for remove in (False, True):
+ with self.subTest(remove_non_linguistic_symbols=remove):
+ tokenizer = CharTokenizer(
+ non_linguistic_symbols=path, remove_non_linguistic_symbols=remove
+ )
+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})
+ expected = ["你", "好"] if remove else ["你", "<noise>", "好"]
+ self.assertEqual(tokenizer.text2tokens("你<noise>好"), expected)
+
+ def test_empty_symbol_in_iterable_is_ignored(self):
+ tokenizer = CharTokenizer(non_linguistic_symbols=["", "<noise>"])
+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})
+ self.assertEqual(tokenizer.text2tokens("a <noise>b"), ["a", "<noise>", "b"])
+
+
+class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase):
+ def test_empty_symbol_is_ignored(self):
+ tokenizer = PhonemeTokenizer(
+ g2p_type=None,
+ non_linguistic_symbols=["", "<noise>"],
+ remove_non_linguistic_symbols=True,
+ )
+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})
+ self.assertEqual(tokenizer.text2tokens("a<noise>b"), ["ab"])
+
+
+if __name__ == "__main__":
+ unittest.main()