Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(auto-model): preserve VAD segments on punctuation mismatch (#3434)
zhifu gao0a9cc006424a
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.pyindex 2d3ca92b6..2581ee0b7 100644--- a/funasr/auto/auto_model.py+++ b/funasr/auto/auto_model.py@@ -68,6 +68,42 @@ def _join_vad_texts(texts):return joined+def _vad_segment_sentences(restored_data, vadsegments):+ """Build readable sentence records directly from VAD-aligned ASR chunks."""+ sentences = []+ for result, vadsegment in zip(restored_data, vadsegments):+ text = re.sub(r"<\|[^|]*\|>", "", str(result.get("text", ""))).strip()+ if not text:+ continue++ timestamps = []+ raw_timestamps = result.get("timestamp")+ if raw_timestamps is None:+ raw_timestamps = result.get("timestamps", [])+ for item in raw_timestamps or []:+ if isinstance(item, dict):+ start = item.get("start_time")+ end = item.get("end_time")+ if start is None or end is None:+ continue+ timestamps.append([int(float(start) * 1000), int(float(end) * 1000)])+ elif isinstance(item, (list, tuple)) and len(item) >= 2:+ timestamps.append([int(item[0]), int(item[1])])++ start = timestamps[0][0] if timestamps else vadsegment[0]+ end = timestamps[-1][1] if timestamps else vadsegment[1]+ sentences.append(+ {+ "start": start,+ "end": end,+ "text": text,+ "sentence": text,+ "timestamp": timestamps,+ }+ )+ return sentences++def _get_punc_tokens(text, punc_array, punc_model):"""Return the surface tokens represented by a CT-Transformer punctuation array."""try:@@ -1018,6 +1054,7 @@ class AutoModel:timestamp_text = punc_input_textsentence_timestamps = result.get("timestamp", [])+ punc_alignment_failed = Falseif punc_res is not None:try:punc_length = len(punc_array)@@ -1038,6 +1075,7 @@ class AutoModel:timestamp_text, sentence_timestamps = merged_unitsif punc_array is not None and punc_length != len(sentence_timestamps):punc_array = None+ punc_alignment_failed = Truesurface_sentence_list = Noneif aligned_word_text is not None and punc_array is not None:surface_sentence_list = _timestamp_sentences_from_surface(@@ -1131,25 +1169,17 @@ class AutoModel:if not len(result["text"].strip()):sentence_list = []elif self.punc_model is None and punc_res is None and not sentence_timestamps:- sentence_list = []- for rest, vadsegment in zip(restored_data, vadsegments):- text = str(rest.get("text", "")).strip()- if not text:- continue- sentence_list.append(- {- "start": vadsegment[0],- "end": vadsegment[1],- "text": text,- "sentence": text,- "timestamp": [],- }- )+ sentence_list = _vad_segment_sentences(restored_data, vadsegments)elif punc_res is None:logging.warning("punc_model is required for sentence_timestamp, skipping sentence segmentation.")sentence_list = []+ elif punc_alignment_failed:+ logging.warning(+ "punctuation timestamps could not be aligned, falling back to VAD segments."+ )+ sentence_list = _vad_segment_sentences(restored_data, vadsegments)elif surface_sentence_list is not None:sentence_list = surface_sentence_listelse:diff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.pyindex 97303b104..8211156dd 100644--- a/tests/test_punc_model_none.py+++ b/tests/test_punc_model_none.py@@ -274,6 +274,60 @@ class TestPuncModelNone(unittest.TestCase):],)+ @patch("funasr.auto.auto_model.slice_padding_audio_samples")+ @patch("funasr.auto.auto_model.load_audio_text_image_video")+ @patch("funasr.auto.auto_model.prepare_data_iterator")+ def test_sentence_timestamp_falls_back_to_vad_when_punctuation_alignment_fails(+ self, mock_prep, mock_load, mock_slice+ ):+ """A malformed long-audio punctuation array must not collapse all VAD segments."""+ am = self._make_auto_model(punc_model=MagicMock())+ tag = "<|zh|><|NEUTRAL|><|Speech|><|woitn|>"+ results_seq = [+ [{"key": "test_utt", "value": [[100, 1100], [1300, 2300]]}],+ [+ {+ "text": f"{tag}第一句",+ "timestamp": [[0, 300], [300, 600], [600, 900]],+ "words": ["第", "一", "句"],+ }+ ],+ [+ {+ "text": f"{tag}第二句",+ "timestamp": [[0, 300], [300, 600], [600, 900]],+ "words": ["第", "二", "句"],+ }+ ],+ [{"text": "第一句。第二句。", "punc_array": [3]}],+ ]+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))+ mock_prep.return_value = (["test_utt"], [np.zeros(40000, dtype=np.float32)])+ mock_load.return_value = np.zeros(40000, dtype=np.float32)+ mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000])++ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)++ self.assertEqual(+ results[0]["sentence_info"],+ [+ {+ "start": 100,+ "end": 1000,+ "text": "第一句",+ "sentence": "第一句",+ "timestamp": [[100, 400], [400, 700], [700, 1000]],+ },+ {+ "start": 1300,+ "end": 2200,+ "text": "第二句",+ "sentence": "第二句",+ "timestamp": [[1300, 1600], [1600, 1900], [1900, 2200]],+ },+ ],+ )+@patch("funasr.auto.auto_model.distribute_spk")@patch("funasr.auto.auto_model.postprocess")@patch("funasr.auto.auto_model.sv_chunk")