Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
Return VAD sentence timestamps for SenseVoice (#3414)
zhifu gao99b703009a61
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.pyindex 6546f7c80..2d3ca92b6 100644--- a/funasr/auto/auto_model.py+++ b/funasr/auto/auto_model.py@@ -1130,6 +1130,21 @@ class AutoModel:elif kwargs.get("sentence_timestamp", False):if not len(result["text"].strip()):sentence_list = []+ elif self.punc_model is None and punc_res is None and not sentence_timestamps:+ sentence_list = []+ for rest, vadsegment in zip(restored_data, vadsegments):+ text = str(rest.get("text", "")).strip()+ if not text:+ continue+ sentence_list.append(+ {+ "start": vadsegment[0],+ "end": vadsegment[1],+ "text": text,+ "sentence": text,+ "timestamp": [],+ }+ )elif punc_res is None:logging.warning("punc_model is required for sentence_timestamp, skipping sentence segmentation."diff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.pyindex 26e354c72..97303b104 100644--- a/tests/test_punc_model_none.py+++ b/tests/test_punc_model_none.py@@ -87,6 +87,66 @@ class TestPuncModelNone(unittest.TestCase):# sentence_info should be empty list since punc_res is unavailableself.assertEqual(results[0].get("sentence_info"), [])+ @patch("funasr.auto.auto_model.slice_padding_audio_samples")+ @patch("funasr.auto.auto_model.load_audio_text_image_video")+ @patch("funasr.auto.auto_model.prepare_data_iterator")+ def test_sentence_timestamp_uses_vad_segments_when_asr_has_no_timestamps(+ self, mock_prep, mock_load, mock_slice+ ):+ """SenseVoice should expose its per-VAD text as timestamped sentences."""+ am = self._make_auto_model(punc_model=None)+ results_seq = [+ [{"key": "test_utt", "value": [[120, 800], [1050, 1900]]}],+ [{"text": "first phrase"}],+ [{"text": "second phrase"}],+ ]+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))+ mock_prep.return_value = (["test_utt"], [np.zeros(32000, dtype=np.float32)])+ mock_load.return_value = np.zeros(32000, dtype=np.float32)+ mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000])++ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)++ self.assertEqual(+ results[0]["sentence_info"],+ [+ {+ "start": 120,+ "end": 800,+ "text": "first phrase",+ "sentence": "first phrase",+ "timestamp": [],+ },+ {+ "start": 1050,+ "end": 1900,+ "text": "second phrase",+ "sentence": "second phrase",+ "timestamp": [],+ },+ ],+ )++ @patch("funasr.auto.auto_model.slice_padding_audio_samples")+ @patch("funasr.auto.auto_model.load_audio_text_image_video")+ @patch("funasr.auto.auto_model.prepare_data_iterator")+ def test_vad_fallback_requires_punc_model_to_be_absent(+ self, mock_prep, mock_load, mock_slice+ ):+ am = self._make_auto_model(punc_model=MagicMock())+ results_seq = [+ [{"key": "test_utt", "value": [[120, 800]]}],+ [{"text": "phrase", "timestamps": []}],+ ]+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))+ mock_prep.return_value = (["test_utt"], [np.zeros(16000, dtype=np.float32)])+ mock_load.return_value = np.zeros(16000, dtype=np.float32)+ mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000])++ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)++ self.assertEqual(results[0]["sentence_info"], [])+@patch("funasr.auto.auto_model.slice_padding_audio_samples")@patch("funasr.auto.auto_model.load_audio_text_image_video")@patch("funasr.auto.auto_model.prepare_data_iterator")