Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(auto-model): align split punctuation tokens (#3469)
zhifu gao6829ba784505
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.pyindex f24506b6f..dd94a3fd2 100644--- a/funasr/auto/auto_model.py+++ b/funasr/auto/auto_model.py@@ -195,29 +195,51 @@ def _merge_timestamp_units(text, words, timestamps, punc_array, punc_model):def normalize(value):return "".join(value.split()).casefold()+ if len(words) != len(timestamps):+ return None++ aligned_text = ""+ character_timestamps = []+ for word, timestamp in zip(words, timestamps):+ word_text = normalize(word)+ if (+ not word_text+ or not isinstance(timestamp, (list, tuple))+ or len(timestamp) < 2+ or timestamp[1] < timestamp[0]+ ):+ return None+ start, end = timestamp[:2]+ duration = end - start+ aligned_text += word_text+ for index in range(len(word_text)):+ character_timestamps.append(+ [+ start + duration * index // len(word_text),+ start + duration * (index + 1) // len(word_text),+ ]+ )+merged_timestamps = []- word_index = 0+ character_index = 0for token in expanded_tokens:token_text = normalize(token)- if not token_text:- return None- start_index = word_index- word_text = ""- while word_index < len(words) and len(word_text) < len(token_text):- word_text += normalize(words[word_index])- word_index += 1- if word_text != token_text or start_index == word_index:- return None- if not all(- isinstance(item, (list, tuple)) and len(item) >= 2- for item in timestamps[start_index:word_index]+ token_end = character_index + len(token_text)+ if (+ not token_text+ or aligned_text[character_index:token_end] != token_text+ or token_end > len(character_timestamps)):return Nonemerged_timestamps.append(- [timestamps[start_index][0], timestamps[word_index - 1][1]]+ [+ character_timestamps[character_index][0],+ character_timestamps[token_end - 1][1],+ ])+ character_index = token_end- if word_index != len(words):+ if character_index != len(character_timestamps):return Nonereturn " ".join(expanded_tokens), merged_timestampsdiff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.pyindex 8211156dd..db0823382 100644--- a/tests/test_punc_model_none.py+++ b/tests/test_punc_model_none.py@@ -328,6 +328,68 @@ class TestPuncModelNone(unittest.TestCase):],)+ @patch(+ "funasr.auto.auto_model._get_punc_tokens",+ return_value=["aon", "storyi", "ca", "说", "差", "距"],+ )+ @patch("funasr.auto.auto_model.slice_padding_audio_samples")+ @patch("funasr.auto.auto_model.load_audio_text_image_video")+ @patch("funasr.auto.auto_model.prepare_data_iterator")+ def test_sentence_timestamp_splits_one_asr_word_across_punctuation_tokens(+ self, mock_prep, mock_load, mock_slice, _mock_get_punc_tokens+ ):+ """Punctuation token boundaries may occur inside one timestamped ASR word."""+ punc_model = MagicMock()+ punc_model.punc_list = None+ am = self._make_auto_model(punc_model=punc_model)+ tag = "<|zh|><|NEUTRAL|><|Speech|><|woitn|>"+ results_seq = [+ [{"key": "test_utt", "value": [[0, 1200]]}],+ [+ {+ "text": f"{tag}aon storyica说差距",+ "timestamp": [+ [0, 100],+ [100, 900],+ [900, 1000],+ [1000, 1100],+ [1100, 1200],+ ],+ "words": ["aon", "storyica", "说", "差", "距"],+ }+ ],+ [+ {+ "text": "aon storyica.说差距。",+ "punc_array": [1, 1, 3, 1, 1, 3],+ }+ ],+ ]+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))+ mock_prep.return_value = (["test_utt"], [np.zeros(19200, dtype=np.float32)])+ mock_load.return_value = np.zeros(19200, dtype=np.float32)+ mock_slice.return_value = ([np.zeros(19200, dtype=np.float32)], [19200])++ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)++ self.assertEqual(+ results[0]["sentence_info"],+ [+ {+ "text": "aon storyica.",+ "start": 0,+ "end": 900,+ "timestamp": [[0, 100], [100, 700], [700, 900]],+ },+ {+ "text": "说差距。",+ "start": 900,+ "end": 1200,+ "timestamp": [[900, 1000], [1000, 1100], [1100, 1200]],+ },+ ],+ )+@patch("funasr.auto.auto_model.distribute_spk")@patch("funasr.auto.auto_model.postprocess")@patch("funasr.auto.auto_model.sv_chunk")