Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

Return VAD sentence timestamps for SenseVoice (#3414)

zhifu gao99b703009a61
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index 6546f7c80..2d3ca92b6 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -1130,6 +1130,21 @@ class AutoModel:
elif kwargs.get("sentence_timestamp", False):
if not len(result["text"].strip()):
sentence_list = []
+ elif self.punc_model is None and punc_res is None and not sentence_timestamps:
+ sentence_list = []
+ for rest, vadsegment in zip(restored_data, vadsegments):
+ text = str(rest.get("text", "")).strip()
+ if not text:
+ continue
+ sentence_list.append(
+ {
+ "start": vadsegment[0],
+ "end": vadsegment[1],
+ "text": text,
+ "sentence": text,
+ "timestamp": [],
+ }
+ )
elif punc_res is None:
logging.warning(
"punc_model is required for sentence_timestamp, skipping sentence segmentation."
diff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.py
index 26e354c72..97303b104 100644
--- a/tests/test_punc_model_none.py
+++ b/tests/test_punc_model_none.py
@@ -87,6 +87,66 @@ class TestPuncModelNone(unittest.TestCase):
# sentence_info should be empty list since punc_res is unavailable
self.assertEqual(results[0].get("sentence_info"), [])
+ @patch("funasr.auto.auto_model.slice_padding_audio_samples")
+ @patch("funasr.auto.auto_model.load_audio_text_image_video")
+ @patch("funasr.auto.auto_model.prepare_data_iterator")
+ def test_sentence_timestamp_uses_vad_segments_when_asr_has_no_timestamps(
+ self, mock_prep, mock_load, mock_slice
+ ):
+ """SenseVoice should expose its per-VAD text as timestamped sentences."""
+ am = self._make_auto_model(punc_model=None)
+ results_seq = [
+ [{"key": "test_utt", "value": [[120, 800], [1050, 1900]]}],
+ [{"text": "first phrase"}],
+ [{"text": "second phrase"}],
+ ]
+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))
+ mock_prep.return_value = (["test_utt"], [np.zeros(32000, dtype=np.float32)])
+ mock_load.return_value = np.zeros(32000, dtype=np.float32)
+ mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000])
+
+ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)
+
+ self.assertEqual(
+ results[0]["sentence_info"],
+ [
+ {
+ "start": 120,
+ "end": 800,
+ "text": "first phrase",
+ "sentence": "first phrase",
+ "timestamp": [],
+ },
+ {
+ "start": 1050,
+ "end": 1900,
+ "text": "second phrase",
+ "sentence": "second phrase",
+ "timestamp": [],
+ },
+ ],
+ )
+
+ @patch("funasr.auto.auto_model.slice_padding_audio_samples")
+ @patch("funasr.auto.auto_model.load_audio_text_image_video")
+ @patch("funasr.auto.auto_model.prepare_data_iterator")
+ def test_vad_fallback_requires_punc_model_to_be_absent(
+ self, mock_prep, mock_load, mock_slice
+ ):
+ am = self._make_auto_model(punc_model=MagicMock())
+ results_seq = [
+ [{"key": "test_utt", "value": [[120, 800]]}],
+ [{"text": "phrase", "timestamps": []}],
+ ]
+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))
+ mock_prep.return_value = (["test_utt"], [np.zeros(16000, dtype=np.float32)])
+ mock_load.return_value = np.zeros(16000, dtype=np.float32)
+ mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000])
+
+ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)
+
+ self.assertEqual(results[0]["sentence_info"], [])
+
@patch("funasr.auto.auto_model.slice_padding_audio_samples")
@patch("funasr.auto.auto_model.load_audio_text_image_video")
@patch("funasr.auto.auto_model.prepare_data_iterator")