Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(tokenizer): ignore empty non-linguistic symbols to avoid a hang
Mohammad Hijjawi5156f09ee35c
diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.pyindex d25c98926..69e8cd3a4 100644--- a/funasr/tokenizer/char_tokenizer.py+++ b/funasr/tokenizer/char_tokenizer.py@@ -44,6 +44,9 @@ class CharTokenizer(BaseTokenizer):self.non_linguistic_symbols = set()else:self.non_linguistic_symbols = set(non_linguistic_symbols)+ # An empty symbol (e.g. a blank line in the file) matches every position+ # and would make text2tokens loop forever.+ self.non_linguistic_symbols.discard("")self.remove_non_linguistic_symbols = remove_non_linguistic_symbolsself.split_with_space = split_with_spaceself.seg_dict = Nonediff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.pyindex ab479242a..0005483e3 100644--- a/funasr/tokenizer/phoneme_tokenizer.py+++ b/funasr/tokenizer/phoneme_tokenizer.py@@ -599,6 +599,9 @@ class PhonemeTokenizer(AbsTokenizer):self.non_linguistic_symbols = set()else:self.non_linguistic_symbols = set(non_linguistic_symbols)+ # An empty symbol (e.g. a blank line in the file) matches every position+ # and would make text2tokens loop forever.+ self.non_linguistic_symbols.discard("")self.remove_non_linguistic_symbols = remove_non_linguistic_symbolsdef __repr__(self):diff --git a/tests/test_char_tokenizer_non_linguistic_symbols.py b/tests/test_char_tokenizer_non_linguistic_symbols.pynew file mode 100644index 000000000..7644ba1ea--- /dev/null+++ b/tests/test_char_tokenizer_non_linguistic_symbols.py@@ -0,0 +1,44 @@+"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever."""++import os+import tempfile+import unittest++from funasr.tokenizer.char_tokenizer import CharTokenizer+from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer+++class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase):+ def test_blank_line_in_symbol_file_is_ignored(self):+ with tempfile.TemporaryDirectory() as tmpdir:+ path = os.path.join(tmpdir, "nlsyms.txt")+ with open(path, "w", encoding="utf-8") as f:+ f.write("<noise>\n\n")+ for remove in (False, True):+ with self.subTest(remove_non_linguistic_symbols=remove):+ tokenizer = CharTokenizer(+ non_linguistic_symbols=path, remove_non_linguistic_symbols=remove+ )+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})+ expected = ["你", "好"] if remove else ["你", "<noise>", "好"]+ self.assertEqual(tokenizer.text2tokens("你<noise>好"), expected)++ def test_empty_symbol_in_iterable_is_ignored(self):+ tokenizer = CharTokenizer(non_linguistic_symbols=["", "<noise>"])+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})+ self.assertEqual(tokenizer.text2tokens("a <noise>b"), ["a", "<noise>", "b"])+++class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase):+ def test_empty_symbol_is_ignored(self):+ tokenizer = PhonemeTokenizer(+ g2p_type=None,+ non_linguistic_symbols=["", "<noise>"],+ remove_non_linguistic_symbols=True,+ )+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})+ self.assertEqual(tokenizer.text2tokens("a<noise>b"), ["ab"])+++if __name__ == "__main__":+ unittest.main()