Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(tokenizer): treat special-token strings as text to prevent batch crashes (#3110)
liuweiqing57e2f48c2c02
diff --git a/funasr/models/sense_voice/whisper_lib/tokenizer.py b/funasr/models/sense_voice/whisper_lib/tokenizer.pyindex 4a7520519..4722b8f18 100644--- a/funasr/models/sense_voice/whisper_lib/tokenizer.py+++ b/funasr/models/sense_voice/whisper_lib/tokenizer.py@@ -194,11 +194,24 @@ class Tokenizer:def encode(self, text, **kwargs):"""Encode.-+Args:text: Text tensor or string input.**kwargs: Additional keyword arguments.- """++ ``tiktoken`` rejects any text containing a special-token string+ (e.g. ``<|no|>``, ``<|zh|>``, ``<|nospeech|>``) by default+ (``disallowed_special="all"``). ASR models such as Fun-ASR-Nano+ occasionally emit such special-token strings as part of their+ transcription text; re-encoding that text for downstream tasks+ (forced alignment, loss computation, ...) then crashes the whole+ batch with a single bad sample (see issue #3110).++ Treat special-token strings as ordinary text unless the caller+ explicitly opts into a stricter policy. Callers that already pass+ ``allowed_special`` / ``disallowed_special`` are left untouched.+ """+ kwargs.setdefault("disallowed_special", ())return self.encoding.encode(text, **kwargs)def decode(self, token_ids: List[int], **kwargs) -> str:diff --git a/tests/test_sensevoice_tokenizer_special_tokens.py b/tests/test_sensevoice_tokenizer_special_tokens.pynew file mode 100644index 000000000..73c3ab4fe--- /dev/null+++ b/tests/test_sensevoice_tokenizer_special_tokens.py@@ -0,0 +1,110 @@+"""Unit tests for SenseVoice Tokenizer handling of special-token strings.++Regression guard for issue #3110: ASR models such as Fun-ASR-Nano occasionally+emit special-token strings (e.g. ``<|no|>``, a language tag) as part of the+transcription text. Re-encoding that text via ``tiktoken.Encoding.encode``+crashes by default (``disallowed_special="all"``), which takes down the whole+batch on a single bad sample during forced alignment / loss computation.+``Tokenizer.encode`` now treats special-token strings as ordinary text unless+the caller explicitly opts into a stricter policy.++These tests build a minimal in-memory tiktoken encoding, so they run without+the Fun-ASR-Nano ``multilingual.tiktoken`` vocab (which ships with the model,+not this repo) and without a GPU or model download.+"""++import unittest++import tiktoken++from funasr.models.sense_voice.whisper_lib.tokenizer import Tokenizer+++def _build_test_encoding() -> tiktoken.Encoding:+ """A minimal byte-level BPE encoding with the specials Tokenizer needs.++ Every single byte is its own token, so arbitrary text can be encoded+ without the model's multilingual.tiktoken vocab file.+ """+ mergeable_ranks = {bytes([b]): b for b in range(256)}+ n_vocab = len(mergeable_ranks)++ # Specials required by Tokenizer.__post_init__ (startoftranscript/translate/+ # transcribe) plus a few language tags. "<|no|>" is the Norwegian tag that+ # triggered #3110.+ required_specials = [+ "<|startoftranscript|>",+ "<|no|>",+ "<|zh|>",+ "<|en|>",+ "<|translate|>",+ "<|transcribe|>",+ "<|startoflm|>",+ "<|startofprev|>",+ "<|nospeech|>",+ "<|notimestamps|>",+ "<|0.00|>",+ ]+ special_tokens = {}+ for tok in required_specials:+ special_tokens[tok] = n_vocab+ n_vocab += 1++ return tiktoken.Encoding(+ name="test-encoding",+ pat_str=(+ r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| """+ r"""?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""+ ),+ mergeable_ranks=mergeable_ranks,+ special_tokens=special_tokens,+ )+++def _build_test_tokenizer() -> Tokenizer:+ return Tokenizer(encoding=_build_test_encoding(), num_languages=1)+++class TestTokenizerEncodeSpecialTokens(unittest.TestCase):+ def setUp(self):+ self.tokenizer = _build_test_tokenizer()+ self.no_id = self.tokenizer.special_tokens["<|no|>"]++ def test_plain_text_encodes(self):+ ids = self.tokenizer.encode("hello world")+ self.assertIsInstance(ids, list)+ self.assertTrue(ids)++ def test_language_tag_in_text_does_not_raise(self):+ # The exact failure from issue #3110: ASR output containing "<|no|>"+ # used to crash with "disallowed special token '<|no|>'".+ ids = self.tokenizer.encode("hello <|no|> world")+ self.assertIsInstance(ids, list)+ self.assertTrue(ids)++ def test_nospeech_tag_in_text_does_not_raise(self):+ ids = self.tokenizer.encode("<|nospeech|> something")+ self.assertIsInstance(ids, list)+ self.assertTrue(ids)++ def test_language_tag_alone_does_not_raise(self):+ ids = self.tokenizer.encode("<|no|>")+ self.assertIsInstance(ids, list)+ self.assertTrue(ids)++ def test_caller_disallowed_special_is_respected(self):+ # A caller that explicitly wants the strict behaviour must still get+ # it (setdefault must not override an explicit kwargs).+ with self.assertRaises(ValueError):+ self.tokenizer.encode("<|no|>", disallowed_special="all")++ def test_allowed_special_all_keeps_special_token_ids(self):+ # funasr/models/sense_voice/whisper_lib/decoding.py calls+ # tokenizer.encode(prompt, allowed_special="all"); the special token+ # must still be encoded as its token id, not as raw bytes.+ ids = self.tokenizer.encode("<|no|>", allowed_special="all")+ self.assertEqual(ids, [self.no_id])+++if __name__ == "__main__":+ unittest.main()