Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(sensevoice): normalize first word before joining subpieces

LauraGPT2ddabb2c910e
diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml
index bdaaf26ec..d23e0c8ff 100644
--- a/.github/workflows/test-numpy-compatibility.yml
+++ b/.github/workflows/test-numpy-compatibility.yml
@@ -8,10 +8,12 @@ on:
- "funasr/frontends/**"
- "funasr/models/emotion2vec/**"
- "funasr/models/campplus/**"
+ - "funasr/models/sense_voice/model.py"
- "funasr/models/eend/utils/feature.py"
- "tests/test_numpy_compatibility.py"
- "tests/test_numpy_cluster_compatibility.py"
- "tests/test_eend_feature_compatibility.py"
+ - "tests/test_sensevoice_word_timestamps.py"
- "tests/test_pypi_metadata.py"
- ".github/workflows/test-numpy-compatibility.yml"
push:
@@ -22,10 +24,12 @@ on:
- "funasr/frontends/**"
- "funasr/models/emotion2vec/**"
- "funasr/models/campplus/**"
+ - "funasr/models/sense_voice/model.py"
- "funasr/models/eend/utils/feature.py"
- "tests/test_numpy_compatibility.py"
- "tests/test_numpy_cluster_compatibility.py"
- "tests/test_eend_feature_compatibility.py"
+ - "tests/test_sensevoice_word_timestamps.py"
- "tests/test_pypi_metadata.py"
- ".github/workflows/test-numpy-compatibility.yml"
@@ -71,6 +75,7 @@ jobs:
tests/test_numpy_cluster_compatibility.py \
tests/test_eend_feature_compatibility.py \
tests/test_pypi_metadata.py \
+ tests/test_sensevoice_word_timestamps.py \
--junitxml=numpy-results.xml
- uses: actions/upload-artifact@v4
if: always()
diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.py
index c10c663d9..c2f89eefd 100644
--- a/funasr/models/sense_voice/model.py
+++ b/funasr/models/sense_voice/model.py
@@ -1095,6 +1095,7 @@ class SenseVoiceSmall(nn.Module):
if word == "▁":
continue
if i == 0:
+ word = word[1:] if word.startswith("▁") else word
# timestamp_new.append([word, start, end])
timestamp_new.append([start, end])
words_new.append(word)
diff --git a/tests/test_sensevoice_word_timestamps.py b/tests/test_sensevoice_word_timestamps.py
new file mode 100644
index 000000000..5f1a4e9be
--- /dev/null
+++ b/tests/test_sensevoice_word_timestamps.py
@@ -0,0 +1,81 @@
+"""Word timestamps use the same SentencePiece boundaries for the first word."""
+
+from copy import deepcopy
+
+import pytest
+
+from funasr.models.sense_voice.model import SenseVoiceSmall
+
+
+ ("tokens", "words", "timestamps"),
+ [
+ pytest.param(
+ ["\u2581Hello", "\u2581world", "."],
+ ["Hello", "world", "."],
+ [[0, 125], [125, 250], [250, 375]],
+ id="first-word-marker",
+ ),
+ pytest.param(
+ ["\u2581Hel", "lo", "\u2581world", "."],
+ ["Hello", "world", "."],
+ [[0, 250], [250, 375], [375, 500]],
+ id="first-word-subpieces",
+ ),
+ pytest.param(
+ ["Hel", "lo", "\u2581wor", "ld", "."],
+ ["Hello", "world", "."],
+ [[0, 250], [250, 500], [500, 625]],
+ id="unmarked-first-word-and-later-boundary",
+ ),
+ pytest.param(
+ ["\u2581", "\u2581Hel", "lo"],
+ ["Hello"],
+ [[125, 375]],
+ id="standalone-leading-marker",
+ ),
+ pytest.param(
+ ["\u2581", "Hel", "lo"],
+ ["Hello"],
+ [[125, 375]],
+ id="standalone-marker-before-unmarked-word",
+ ),
+ pytest.param(
+ ["\u2581\u4f60", "\u597d", "\u3002"],
+ ["\u4f60", "\u597d", "\u3002"],
+ [[0, 125], [125, 250], [250, 375]],
+ id="chinese-first-word",
+ ),
+ pytest.param(
+ ["\u2581Hello", "\u4e16", "\u754c"],
+ ["Hello", "\u4e16", "\u754c"],
+ [[0, 125], [125, 250], [250, 375]],
+ id="mixed-script-boundaries",
+ ),
+ pytest.param(["\u2581", "\u2581"], [], [], id="only-markers"),
+ pytest.param([], [], [], id="empty"),
+ pytest.param(
+ ["under\u2581score"], ["under\u2581score"], [[0, 125]], id="nonleading-marker"
+ ),
+ pytest.param(
+ ["\u2581I'm", "\u2581here", "."],
+ ["I'm", "here", "."],
+ [[0, 125], [125, 250], [250, 375]],
+ id="preserve-punctuation",
+ ),
+ pytest.param(
+ ["A", "\u2581B", "C"], ["A", "BC"], [[0, 125], [125, 375]], id="word-boundary"
+ ),
+ ],
+)
+def test_post_preserves_word_boundaries_and_timestamp_alignment(tokens, words, timestamps):
+ token_timestamps = [[token, index / 8, (index + 1) / 8] for index, token in enumerate(tokens)]
+ original = deepcopy(token_timestamps)
+
+ # post uses no model state; importing the real class requires no checkpoint.
+ result_timestamps, result_words = SenseVoiceSmall.post(None, token_timestamps)
+
+ assert result_words == words
+ assert result_timestamps == timestamps
+ assert len(result_words) == len(result_timestamps)
+ assert token_timestamps == original