Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(auto-model): align split punctuation tokens (#3469)

zhifu gao6829ba784505
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index f24506b6f..dd94a3fd2 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -195,29 +195,51 @@ def _merge_timestamp_units(text, words, timestamps, punc_array, punc_model):
def normalize(value):
return "".join(value.split()).casefold()
+ if len(words) != len(timestamps):
+ return None
+
+ aligned_text = ""
+ character_timestamps = []
+ for word, timestamp in zip(words, timestamps):
+ word_text = normalize(word)
+ if (
+ not word_text
+ or not isinstance(timestamp, (list, tuple))
+ or len(timestamp) < 2
+ or timestamp[1] < timestamp[0]
+ ):
+ return None
+ start, end = timestamp[:2]
+ duration = end - start
+ aligned_text += word_text
+ for index in range(len(word_text)):
+ character_timestamps.append(
+ [
+ start + duration * index // len(word_text),
+ start + duration * (index + 1) // len(word_text),
+ ]
+ )
+
merged_timestamps = []
- word_index = 0
+ character_index = 0
for token in expanded_tokens:
token_text = normalize(token)
- if not token_text:
- return None
- start_index = word_index
- word_text = ""
- while word_index < len(words) and len(word_text) < len(token_text):
- word_text += normalize(words[word_index])
- word_index += 1
- if word_text != token_text or start_index == word_index:
- return None
- if not all(
- isinstance(item, (list, tuple)) and len(item) >= 2
- for item in timestamps[start_index:word_index]
+ token_end = character_index + len(token_text)
+ if (
+ not token_text
+ or aligned_text[character_index:token_end] != token_text
+ or token_end > len(character_timestamps)
):
return None
merged_timestamps.append(
- [timestamps[start_index][0], timestamps[word_index - 1][1]]
+ [
+ character_timestamps[character_index][0],
+ character_timestamps[token_end - 1][1],
+ ]
)
+ character_index = token_end
- if word_index != len(words):
+ if character_index != len(character_timestamps):
return None
return " ".join(expanded_tokens), merged_timestamps
diff --git a/tests/test_punc_model_none.py b/tests/test_punc_model_none.py
index 8211156dd..db0823382 100644
--- a/tests/test_punc_model_none.py
+++ b/tests/test_punc_model_none.py
@@ -328,6 +328,68 @@ class TestPuncModelNone(unittest.TestCase):
],
)
+ @patch(
+ "funasr.auto.auto_model._get_punc_tokens",
+ return_value=["aon", "storyi", "ca", "说", "差", "距"],
+ )
+ @patch("funasr.auto.auto_model.slice_padding_audio_samples")
+ @patch("funasr.auto.auto_model.load_audio_text_image_video")
+ @patch("funasr.auto.auto_model.prepare_data_iterator")
+ def test_sentence_timestamp_splits_one_asr_word_across_punctuation_tokens(
+ self, mock_prep, mock_load, mock_slice, _mock_get_punc_tokens
+ ):
+ """Punctuation token boundaries may occur inside one timestamped ASR word."""
+ punc_model = MagicMock()
+ punc_model.punc_list = None
+ am = self._make_auto_model(punc_model=punc_model)
+ tag = "<|zh|><|NEUTRAL|><|Speech|><|woitn|>"
+ results_seq = [
+ [{"key": "test_utt", "value": [[0, 1200]]}],
+ [
+ {
+ "text": f"{tag}aon storyica说差距",
+ "timestamp": [
+ [0, 100],
+ [100, 900],
+ [900, 1000],
+ [1000, 1100],
+ [1100, 1200],
+ ],
+ "words": ["aon", "storyica", "说", "差", "距"],
+ }
+ ],
+ [
+ {
+ "text": "aon storyica.说差距。",
+ "punc_array": [1, 1, 3, 1, 1, 3],
+ }
+ ],
+ ]
+ am.inference = MagicMock(side_effect=lambda *args, **kwargs: results_seq.pop(0))
+ mock_prep.return_value = (["test_utt"], [np.zeros(19200, dtype=np.float32)])
+ mock_load.return_value = np.zeros(19200, dtype=np.float32)
+ mock_slice.return_value = ([np.zeros(19200, dtype=np.float32)], [19200])
+
+ results = am.inference_with_vad("dummy_input", sentence_timestamp=True)
+
+ self.assertEqual(
+ results[0]["sentence_info"],
+ [
+ {
+ "text": "aon storyica.",
+ "start": 0,
+ "end": 900,
+ "timestamp": [[0, 100], [100, 700], [700, 900]],
+ },
+ {
+ "text": "说差距。",
+ "start": 900,
+ "end": 1200,
+ "timestamp": [[900, 1000], [1000, 1100], [1100, 1200]],
+ },
+ ],
+ )
+
@patch("funasr.auto.auto_model.distribute_spk")
@patch("funasr.auto.auto_model.postprocess")
@patch("funasr.auto.auto_model.sv_chunk")