Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
test: retain tokenizer regressions alongside normalization fixes
zhifu gaoa3050a3ea7e5
diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.ymlindex a6f2b474c..370553668 100644--- a/.github/workflows/test-numpy-compatibility.yml+++ b/.github/workflows/test-numpy-compatibility.yml@@ -9,6 +9,9 @@ on:- "tests/test_cli.py"- "funasr/auto/auto_model.py"- "funasr/tokenizer/sentencepiece_tokenizer.py"+ - "funasr/tokenizer/char_tokenizer.py"+ - "funasr/tokenizer/phoneme_tokenizer.py"+ - "tests/test_char_tokenizer_non_linguistic_symbols.py"- "tests/test_sentencepiece_tokenizer.py"- "tests/test_speaker_optional_dependency.py"- "funasr/utils/load_utils.py"@@ -50,6 +53,9 @@ on:- "tests/test_cli.py"- "funasr/auto/auto_model.py"- "funasr/tokenizer/sentencepiece_tokenizer.py"+ - "funasr/tokenizer/char_tokenizer.py"+ - "funasr/tokenizer/phoneme_tokenizer.py"+ - "tests/test_char_tokenizer_non_linguistic_symbols.py"- "tests/test_sentencepiece_tokenizer.py"- "tests/test_speaker_optional_dependency.py"- "funasr/utils/load_utils.py"@@ -135,6 +141,7 @@ jobs:tests/test_pypi_metadata.py \tests/test_sensevoice_word_timestamps.py \tests/test_sentencepiece_tokenizer.py \+ tests/test_char_tokenizer_non_linguistic_symbols.py \tests/test_pcm_input_format.py \tests/test_realtime_ws_service.py \tests/test_realtime_ws_hf_overrides.py \diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.pyindex d25c98926..69e8cd3a4 100644--- a/funasr/tokenizer/char_tokenizer.py+++ b/funasr/tokenizer/char_tokenizer.py@@ -44,6 +44,9 @@ class CharTokenizer(BaseTokenizer):self.non_linguistic_symbols = set()else:self.non_linguistic_symbols = set(non_linguistic_symbols)+ # An empty symbol (e.g. a blank line in the file) matches every position+ # and would make text2tokens loop forever.+ self.non_linguistic_symbols.discard("")self.remove_non_linguistic_symbols = remove_non_linguistic_symbolsself.split_with_space = split_with_spaceself.seg_dict = Nonediff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.pyindex ab479242a..0005483e3 100644--- a/funasr/tokenizer/phoneme_tokenizer.py+++ b/funasr/tokenizer/phoneme_tokenizer.py@@ -599,6 +599,9 @@ class PhonemeTokenizer(AbsTokenizer):self.non_linguistic_symbols = set()else:self.non_linguistic_symbols = set(non_linguistic_symbols)+ # An empty symbol (e.g. a blank line in the file) matches every position+ # and would make text2tokens loop forever.+ self.non_linguistic_symbols.discard("")self.remove_non_linguistic_symbols = remove_non_linguistic_symbolsdef __repr__(self):diff --git a/tests/test_char_tokenizer_non_linguistic_symbols.py b/tests/test_char_tokenizer_non_linguistic_symbols.pynew file mode 100644index 000000000..7644ba1ea--- /dev/null+++ b/tests/test_char_tokenizer_non_linguistic_symbols.py@@ -0,0 +1,44 @@+"""Tokenizers must ignore empty non-linguistic symbols instead of looping forever."""++import os+import tempfile+import unittest++from funasr.tokenizer.char_tokenizer import CharTokenizer+from funasr.tokenizer.phoneme_tokenizer import PhonemeTokenizer+++class CharTokenizerNonLinguisticSymbolsTest(unittest.TestCase):+ def test_blank_line_in_symbol_file_is_ignored(self):+ with tempfile.TemporaryDirectory() as tmpdir:+ path = os.path.join(tmpdir, "nlsyms.txt")+ with open(path, "w", encoding="utf-8") as f:+ f.write("<noise>\n\n")+ for remove in (False, True):+ with self.subTest(remove_non_linguistic_symbols=remove):+ tokenizer = CharTokenizer(+ non_linguistic_symbols=path, remove_non_linguistic_symbols=remove+ )+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})+ expected = ["你", "好"] if remove else ["你", "<noise>", "好"]+ self.assertEqual(tokenizer.text2tokens("你<noise>好"), expected)++ def test_empty_symbol_in_iterable_is_ignored(self):+ tokenizer = CharTokenizer(non_linguistic_symbols=["", "<noise>"])+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})+ self.assertEqual(tokenizer.text2tokens("a <noise>b"), ["a", "<noise>", "b"])+++class PhonemeTokenizerNonLinguisticSymbolsTest(unittest.TestCase):+ def test_empty_symbol_is_ignored(self):+ tokenizer = PhonemeTokenizer(+ g2p_type=None,+ non_linguistic_symbols=["", "<noise>"],+ remove_non_linguistic_symbols=True,+ )+ self.assertEqual(tokenizer.non_linguistic_symbols, {"<noise>"})+ self.assertEqual(tokenizer.text2tokens("a<noise>b"), ["ab"])+++if __name__ == "__main__":+ unittest.main()