Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(auto-model): preserve VAD segments on punctuation mismatch (#3434)

zhifu gao0a9cc006424a
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index 2d3ca92b6..2581ee0b7 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -68,6 +68,42 @@ def _join_vad_texts(texts):
return joined
+def _vad_segment_sentences(restored_data, vadsegments):
+ """Build readable sentence records directly from VAD-aligned ASR chunks."""
+ sentences = []
+ for result, vadsegment in zip(restored_data, vadsegments):
+ text = re.sub(r"<\|[^|]*\|>", "", str(result.get("text", ""))).strip()
+ if not text:
+ continue
+
+ timestamps = []
+ raw_timestamps = result.get("timestamp")
+ if raw_timestamps is None:
+ raw_timestamps = result.get("timestamps", [])
+ for item in raw_timestamps or []:
+ if isinstance(item, dict):
+ start = item.get("start_time")
+ end = item.get("end_time")
+ if start is None or end is None:
+ continue
+ timestamps.append([int(float(start) * 1000), int(float(end) * 1000)])
+ elif isinstance(item, (list, tuple)) and len(item) >= 2:
+ timestamps.append([int(item[0]), int(item[1])])
+
+ start = timestamps[0][0] if timestamps else vadsegment[0]
+ end = timestamps[-1][1] if timestamps else vadsegment[1]
+ sentences.append(
+ {
+ "start": start,
+ "end": end,
+ "text": text,
+ "sentence": text,
+ "timestamp": timestamps,
+ }
+ )
+ return sentences
+
+
def _get_punc_tokens(text, punc_array, punc_model):
"""Return the surface tokens represented by a CT-Transformer punctuation array."""
try:
@@ -1018,6 +1054,7 @@ class AutoModel:
timestamp_text = punc_input_text
sentence_timestamps = result.get("timestamp", [])
+ punc_alignment_failed = False
if punc_res is not None:
try:
punc_length = len(punc_array)
@@ -1038,6 +1075,7 @@ class AutoModel:
timestamp_text, sentence_timestamps = merged_units
if punc_array is not None and punc_length != len(sentence_timestamps):
punc_array = None
+ punc_alignment_failed = True
surface_sentence_list = None
if aligned_word_text is not None and punc_array is not None:
surface_sentence_list = _timestamp_sentences_from_surface(
@@ -1131,25 +1169,17 @@ class AutoModel:
if not len(result["text"].strip()):
sentence_list = []
elif self.punc_model is None and punc_res is None and not sentence_timestamps:
- sentence_list = []
- for rest, vadsegment in zip(restored_data, vadsegments):
- text = str(rest.get("text", "")).strip()
- if not text:
- continue
- sentence_list.append(
- {
- "start": vadsegment[0],
- "end": vadsegment[1],
- "text": text,
- "sentence": text,
- "timestamp": [],
- }
- )
+ sentence_list = _vad_segment_sentences(restored_data, vadsegments)
elif punc_res is None:
logging.warning(
"punc_model is required for sentence_timestamp, skipping sentence segmentation."
)
sentence_list = []
+ elif punc_alignment_failed:
+ logging.warning(
+ "punctuation timestamps could not be aligned, falling back to VAD segments."
+ )
+ sentence_list = _vad_segment_sentences(restored_data, vadsegments)
elif surface_sentence_list is not None:
sentence_list = surface_sentence_list
else:
diff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.py
index 97303b104..8211156dd 100644
--- a/tests/test_punc_model_none.py
+++ b/tests/test_punc_model_none.py
@@ -274,6 +274,60 @@ class TestPuncModelNone(unittest.TestCase):
],
)
+ @patch("funasr.auto.auto_model.slice_padding_audio_samples")
+ @patch("funasr.auto.auto_model.load_audio_text_image_video")
+ @patch("funasr.auto.auto_model.prepare_data_iterator")
+ def test_sentence_timestamp_falls_back_to_vad_when_punctuation_alignment_fails(
+ self, mock_prep, mock_load, mock_slice
+ ):
+ """A malformed long-audio punctuation array must not collapse all VAD segments."""
+ am = self._make_auto_model(punc_model=MagicMock())
+ tag = "<|zh|><|NEUTRAL|><|Speech|><|woitn|>"
+ results_seq = [
+ [{"key": "test_utt", "value": [[100, 1100], [1300, 2300]]}],
+ [
+ {
+ "text": f"{tag}第一句",
+ "timestamp": [[0, 300], [300, 600], [600, 900]],
+ "words": ["第", "一", "句"],
+ }
+ ],
+ [
+ {
+ "text": f"{tag}第二句",
+ "timestamp": [[0, 300], [300, 600], [600, 900]],
+ "words": ["第", "二", "句"],
+ }
+ ],
+ [{"text": "第一句。第二句。", "punc_array": [3]}],
+ ]
+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))
+ mock_prep.return_value = (["test_utt"], [np.zeros(40000, dtype=np.float32)])
+ mock_load.return_value = np.zeros(40000, dtype=np.float32)
+ mock_slice.return_value = ([np.zeros(16000, dtype=np.float32)], [16000])
+
+ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)
+
+ self.assertEqual(
+ results[0]["sentence_info"],
+ [
+ {
+ "start": 100,
+ "end": 1000,
+ "text": "第一句",
+ "sentence": "第一句",
+ "timestamp": [[100, 400], [400, 700], [700, 1000]],
+ },
+ {
+ "start": 1300,
+ "end": 2200,
+ "text": "第二句",
+ "sentence": "第二句",
+ "timestamp": [[1300, 1600], [1600, 1900], [1900, 2200]],
+ },
+ ],
+ )
+
@patch("funasr.auto.auto_model.distribute_spk")
@patch("funasr.auto.auto_model.postprocess")
@patch("funasr.auto.auto_model.sv_chunk")