Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix: keep token separators in timestamp_sentence raw_text (#3746)

Mohammad Hijjawib0fa25b12c10
diff --git a/funasr/utils/timestamp_tools.py b/funasr/utils/timestamp_tools.py
index 85cc42eba..fb77702eb 100644
--- a/funasr/utils/timestamp_tools.py
+++ b/funasr/utils/timestamp_tools.py
@@ -189,11 +189,10 @@ def timestamp_sentence(
punc_id = int(punc_id) if punc_id is not None else 1
sentence_end = timestamp[1] if timestamp is not None else sentence_end
- sentence_text_seg = (
- sentence_text_seg[:-1] if sentence_text_seg and sentence_text_seg[-1] == " " else sentence_text_seg
- )
if punc_id > 1:
sentence_text += punc_list[punc_id - 2]
+ if sentence_text_seg.endswith(" "):
+ sentence_text_seg = sentence_text_seg[:-1]
if return_raw_text:
res.append(
{
diff --git a/tests/test_timestamp_tools.py b/tests/test_timestamp_tools.py
index 6b058d130..d5af9408e 100644
--- a/tests/test_timestamp_tools.py
+++ b/tests/test_timestamp_tools.py
@@ -53,3 +53,14 @@ def test_timestamp_sentence_en_missing_trailing_timestamp_matches_zh_fallback():
assert en_result[1]["start"] is None
assert en_result[1]["start"] == zh_result[1]["start"]
+
+
+def test_timestamp_sentence_raw_text_keeps_token_separators():
+ result = timestamp_sentence(
+ punc_id_list=[1, 2, 1, 1, 3],
+ timestamp_postprocessed=[[0, 100], [100, 200], [200, 300], [300, 400], [400, 500]],
+ text_postprocessed="我 用 iphone pro max",
+ return_raw_text=True,
+ )
+
+ assert [sentence["raw_text"] for sentence in result] == ["我 用", "iphone pro max"]