Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(nano): correct punctuation timestamps across VAD merges (#3703)

Bruce-Yii41778c45de50
diff --git a/funasr/models/fun_asr_nano/inference_vllm.py b/funasr/models/fun_asr_nano/inference_vllm.py
index 78f567c2d..db9828a83 100644
--- a/funasr/models/fun_asr_nano/inference_vllm.py
+++ b/funasr/models/fun_asr_nano/inference_vllm.py
@@ -694,7 +694,10 @@ class FunASRNanoVLLM:
@torch.no_grad()
def _compute_timestamps(self, encoder_out, encoder_out_lens, text):
"""CTC forced alignment for character-level timestamps."""
- from funasr.models.fun_asr_nano.tools.utils import forced_align
+ from funasr.models.fun_asr_nano.tools.utils import (
+ anchor_punctuation_timestamps,
+ forced_align,
+ )
decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)
ctc_logits = self.ctc.log_softmax(decoder_out)
@@ -709,6 +712,9 @@ class FunASRNanoVLLM:
ts["token"] = self.ctc_tokenizer.decode([ts["token"]])
ts["start_time"] = ts["start_time"] * 6 * 10 / 1000
ts["end_time"] = ts["end_time"] * 6 * 10 / 1000
+ # See model.py: anchor punctuation at the preceding spoken token's end
+ # (issue #3702); punctuation has no acoustic extent to align.
+ anchor_punctuation_timestamps(timestamps)
return timestamps
@classmethod
diff --git a/funasr/models/fun_asr_nano/inference_vllm_pipeline.py b/funasr/models/fun_asr_nano/inference_vllm_pipeline.py
index 49241bc5b..468e10e7d 100644
--- a/funasr/models/fun_asr_nano/inference_vllm_pipeline.py
+++ b/funasr/models/fun_asr_nano/inference_vllm_pipeline.py
@@ -322,7 +322,10 @@ class FunASRNanoVLLMPipeline:
def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):
"""Compute CTC timestamps for all segments with VAD offsets."""
- from funasr.models.fun_asr_nano.tools.utils import forced_align
+ from funasr.models.fun_asr_nano.tools.utils import (
+ anchor_punctuation_timestamps,
+ forced_align,
+ )
all_timestamps = []
for seg_audio, vad_seg, text in zip(segment_audios, vad_segments, asr_results):
@@ -356,6 +359,13 @@ class FunASRNanoVLLMPipeline:
ts["token"] = self.asr_engine.ctc_tokenizer.decode([ts["token"]])
ts["start_time"] = ts["start_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
ts["end_time"] = ts["end_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000
+ # Same invariant as the offline/vLLM paths (issue #3702), applied
+ # per independently aligned VAD segment: punctuation carries no
+ # acoustic extent, so pin sandwiched punctuation at the preceding
+ # spoken token's end. Segment-local on purpose — speech in a
+ # later VAD segment must not rewrite this segment's trailing
+ # punctuation.
+ anchor_punctuation_timestamps(timestamps)
all_timestamps.extend(timestamps)
except Exception as e:
logger.debug(f"Timestamp failed for segment: {e}")
diff --git a/funasr/models/fun_asr_nano/model.py b/funasr/models/fun_asr_nano/model.py
index 01a1f88d8..049ba6a06 100644
--- a/funasr/models/fun_asr_nano/model.py
+++ b/funasr/models/fun_asr_nano/model.py
@@ -25,7 +25,7 @@ from .ctc import CTC
from .checkpoint_utils import disable_incomplete_ctc, normalize_checkpoint_state
from .device_utils import resolve_autocast_device_type
from .llm_forward_opts import configure_llm_forward
-from .tools.utils import forced_align
+from .tools.utils import anchor_punctuation_timestamps, forced_align
dtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}
@@ -1075,6 +1075,12 @@ class FunASRNano(nn.Module):
timestamp["token"] = self.ctc_tokenizer.decode([timestamp["token"]])
timestamp["start_time"] = timestamp["start_time"] * 6 * 10 / 1000
timestamp["end_time"] = timestamp["end_time"] * 6 * 10 / 1000
+ # Punctuation tokens have no acoustic realization; without anchoring,
+ # forced alignment places them at the next sentence's onset inside
+ # merged multi-sentence windows (issue #3702). Applied after decoding
+ # (tokens are still integer ids above). ctc_timestamps is left
+ # untouched: greedy CTC text carries no punctuation to anchor.
+ anchor_punctuation_timestamps(result["timestamps"])
if ibest_writer is not None:
ibest_writer["text"][key[0]] = response.replace("\n", " ")
diff --git a/funasr/models/fun_asr_nano/tools/utils.py b/funasr/models/fun_asr_nano/tools/utils.py
index 63ec6cb8f..a1389c4ad 100644
--- a/funasr/models/fun_asr_nano/tools/utils.py
+++ b/funasr/models/fun_asr_nano/tools/utils.py
@@ -1,4 +1,5 @@
from itertools import groupby
+import re
import librosa
import soundfile as sf
@@ -77,4 +78,97 @@ def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0)
)
except Exception:
pass
- return items
\ No newline at end of file
+ return items
+
+
+_SPECIAL_TOKEN_RE = re.compile(r"^<[^>]*>$")
+# tiktoken decodes a partial UTF-8 byte sequence (one id of a multi-id
+# character, e.g. SenseVoice ids [10958, 245] for 郗) as U+FFFD. All three
+# Nano timestamp paths decode each token id independently, so such fragments
+# reach classification as "�". They are spoken content, never punctuation.
+_UNDECODABLE_FRAGMENT_MARK = "�"
+# Supported sentence punctuation: the closed set of terminators and clause
+# separators that ASR emits as structural markup with no acoustic
+# realization — the only tokens the anchoring policy is entitled to collapse.
+# Grounded in the official SenseVoice multilingual.tiktoken vocabulary
+# (FunAudioLLM/Fun-ASR-Nano-2512 @272c57b, sha256
+# 74797963e813193436aabcff7c1c235d37de8097b71c563ec8b63b7a515c718), where
+# every character below is a single-id Po token. Symbol tokens are
+# deliberately excluded: they carry real acoustic extent as spoken semantic
+# content, e.g. "++" (id 24754, Unicode Sm) in 我用C++。你好。 and "$" (id 3,
+# Sc) in 价格是$10。你好。 — anchoring them would delete spoken timing (PR
+# #3703 review). Anything outside this set defaults to spoken, so unknown
+# tokens fail safe rather than lose their span.
+_SENTENCE_PUNCTUATION_CHARS = frozenset("。!?,、;:.,!?;:")
+
+
+def _is_sentence_punctuation(token):
+ """True iff every character of ``token`` is supported sentence punctuation."""
+ return bool(token) and all(char in _SENTENCE_PUNCTUATION_CHARS for char in token)
+
+
+def _classify_timestamp_token(token):
+ """Classify one decoded timestamp token into spoken/punctuation/special.
+
+ Special tokens (``<sil>``-shaped) are their own class: they are neither
+ spoken content nor punctuation, so they neither trigger anchoring nor
+ serve as anchors. Undecodable byte fragments (containing U+FFFD) are
+ spoken: they are slices of a multi-id character with real acoustic
+ extent, and collapsing their spans would delete spoken timing (issue
+ #3703). Punctuation is only the supported sentence-punctuation set
+ (see ``_SENTENCE_PUNCTUATION_CHARS``); every other non-word token —
+ semantic symbols such as ``++`` or ``$`` — is spoken and keeps its span.
+ Non-string tokens (integer ids, if called pre-decode) fall back to
+ spoken to preserve pass-through behavior.
+ """
+ if not isinstance(token, str) or not token:
+ return "spoken"
+ if _SPECIAL_TOKEN_RE.match(token):
+ return "special"
+ if _UNDECODABLE_FRAGMENT_MARK in token:
+ return "spoken"
+ if _is_sentence_punctuation(token):
+ return "punctuation"
+ return "spoken"
+
+
+def anchor_punctuation_timestamps(timestamps):
+ """Anchor punctuation-only tokens at the preceding spoken token's end.
+
+ CTC forced alignment must place every target token — including punctuation
+ tokens with no acoustic realization — on at least one frame. Inside a
+ merged multi-sentence window (see FunASR issue #3702) such a token lands
+ at the next sentence's onset frame, so the current sentence's punctuation
+ timestamp reads as the next sentence start. Punctuation carries no acoustic
+ extent, so a punctuation span that is followed by more speech is pinned
+ (zero-width) at the end of the speech it terminates.
+
+ Leading punctuation (no predecessor) and trailing punctuation (no
+ successor, e.g. single-segment output) are left untouched, as are special
+ tokens such as ``<sil>``. This preserves existing single-segment behavior.
+
+ Args:
+ timestamps: List of ``{"token": str, "start_time": float,
+ "end_time": float}`` dicts in time order. Mutated in place.
+
+ Returns:
+ The same list, for convenient chaining.
+ """
+ last_spoken_end = None
+ total = len(timestamps)
+ for index, ts in enumerate(timestamps):
+ kind = _classify_timestamp_token(ts.get("token", ""))
+ if kind != "punctuation":
+ if kind == "spoken":
+ end = ts.get("end_time")
+ if isinstance(end, (int, float)):
+ last_spoken_end = end
+ continue
+ following_spoken = any(
+ _classify_timestamp_token(later.get("token", "")) == "spoken"
+ for later in timestamps[index + 1 : total]
+ )
+ if last_spoken_end is not None and following_spoken:
+ ts["start_time"] = last_spoken_end
+ ts["end_time"] = last_spoken_end
+ return timestamps
diff --git a/tests/test_fun_asr_nano_byte_fragment_timestamps.py b/tests/test_fun_asr_nano_byte_fragment_timestamps.py
new file mode 100644
index 000000000..3045c88bd
--- /dev/null
+++ b/tests/test_fun_asr_nano_byte_fragment_timestamps.py
@@ -0,0 +1,205 @@
+"""Byte-fragment spoken tokens must never be treated as punctuation.
+
+Review regression for FunASR PR #3703 (issue #3702 follow-up): Nano's CTC
+tokenizer is byte-based, so one spoken character can span several token ids
+(official SenseVoiceTokenizer: 郗 -> [10958, 245]). All three Nano timestamp
+paths decode each token id independently, and each fragment decodes to U+FFFD
+while the id sequence decodes to real speech. The punctuation classifier must
+not collapse such fragments' valid acoustic spans.
+
+These tests instantiate the repository's real ``SenseVoiceTokenizer``
+production path (``SenseVoiceTokenizer`` factory -> ``get_tokenizer`` ->
+``get_encoding`` -> production ``Tokenizer``) against a deterministic
+temporary offline ``.tiktoken`` vocab, so they run without the Fun-ASR-Nano
+``multilingual.tiktoken`` model artifact (which ships with the model, not
+this repo) and without weights/GPU. The fixture vocab mirrors the official
+vocab shape: ordinary CJK chars and punctuation are single ids, while 郗
+(deliberately given no merged rank) splits into single-byte ids whose
+independent decodes are U+FFFD — the exact byte-fragment mechanism
+(official: 郗 -> [10958, 245]). Official-artifact cross-check
+(SenseVoiceTokenizer from FunAudioLLM/Fun-ASR-Nano-2512
+``multilingual.tiktoken`` @272c57b, sha256 74797963...): 郗 -> [10958, 245],
+per-id decodes U+FFFD, roundtrip intact; helper/vLLM/pipeline probes preserve
+[0.60, 0.72]/[0.84, 0.96]-style spans post-fix (see PR review evidence).
+"""
+
+import base64
+import types
+
+import numpy as np
+import pytest
+import torch
+
+from funasr.models.fun_asr_nano.tools.utils import (
+ _classify_timestamp_token,
+ anchor_punctuation_timestamps,
+ forced_align,
+)
+from funasr.tokenizer.whisper_tokenizer import SenseVoiceTokenizer
+
+TEXT = "我叫郗明。你好。"
+FRAMES_PER_TOKEN = 10
+FRAME_TO_SEC = 6 * 10 / 1000
+
+
+def _write_fragment_vocab(path):
+ """Deterministic offline vocab: byte-level base plus single-id merged
+ ranks for every fixture char except 郗, which stays byte-split. Format
+ matches production ``.tiktoken`` files (``base64(token) rank`` lines)."""
+ lines = []
+ for byte in range(256):
+ lines.append(f"{base64.b64encode(bytes([byte])).decode()} {byte}")
+ next_rank = 256
+ # Prefix pair first so BPE completes each 3-byte merge (tiktoken merges
+ # lowest-rank pairs first); 郗 gets no ranks and stays byte-split.
+ for char in "我叫明你好。,":
+ raw = char.encode("utf-8")
+ assert len(raw) == 3, char
+ lines.append(f"{base64.b64encode(raw[:2]).decode()} {next_rank}")
+ next_rank += 1
+ lines.append(f"{base64.b64encode(raw).decode()} {next_rank}")
+ next_rank += 1
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def sensevoice_tokenizer(tmp_path):
+ """The real production Tokenizer via the SenseVoiceTokenizer factory."""
+ vocab = tmp_path / "byte-fragment.tiktoken"
+ _write_fragment_vocab(vocab)
+ tokenizer = SenseVoiceTokenizer(vocab_path=str(vocab))
+ target_ids = tokenizer.encode(TEXT)
+ assert tokenizer.decode(target_ids) == TEXT
+ return tokenizer
+
+
+def _peaky_log_probs(target_ids, vocab_size, frames_per_token=FRAMES_PER_TOKEN):
+ n_frames = len(target_ids) * frames_per_token
+ log_probs = torch.full((n_frames, vocab_size), -30.0)
+ for i, tid in enumerate(target_ids):
+ log_probs[i * frames_per_token : (i + 1) * frames_per_token, tid] = 0.0
+ return log_probs
+
+
+def _blank_id(tokenizer):
+ return tokenizer.get_vocab_size() - 1
+
+
+def _production_timestamps(tokenizer, target_ids):
+ """Mirror the production decode-per-id + scale path shared by all three
+ Nano timestamp methods, over the repo's real forced_align."""
+ items = forced_align(
+ _peaky_log_probs(target_ids, tokenizer.get_vocab_size()),
+ torch.tensor(target_ids, dtype=torch.int64),
+ _blank_id(tokenizer),
+ )
+ assert [it["token"] for it in items] == list(target_ids)
+ return [
+ {
+ "token": tokenizer.decode([it["token"]]),
+ "start_time": it["start_time"] * FRAME_TO_SEC,
+ "end_time": it["end_time"] * FRAME_TO_SEC,
+ }
+ for it in items
+ ]
+
+
+def test_undecodable_fragment_is_spoken_not_punctuation():
+ assert _classify_timestamp_token("�") == "spoken"
+ assert _classify_timestamp_token("��") == "spoken"
+ # Genuine punctuation still classifies as punctuation.
+ assert _classify_timestamp_token("。") == "punctuation"
+ assert _classify_timestamp_token(",") == "punctuation"
+ # Special tokens keep their own class.
+ assert _classify_timestamp_token("<sil>") == "special"
+
+
+def test_byte_fragment_spans_survive_anchoring(sensevoice_tokenizer):
+ tokenizer = sensevoice_tokenizer
+ target_ids = tokenizer.encode(TEXT)
+ assert len(target_ids) > len(TEXT), "郗 must stay byte-split"
+ assert tokenizer.decode(target_ids) == TEXT
+ assert any(tokenizer.decode([i]) == "�" for i in target_ids)
+
+ timestamps = _production_timestamps(tokenizer, target_ids)
+ before = [(t["token"], t["start_time"], t["end_time"]) for t in timestamps]
+ anchor_punctuation_timestamps(timestamps)
+
+ for (token, start, end), ts in zip(before, timestamps):
+ if token == "�":
+ assert (ts["start_time"], ts["end_time"]) == (start, end)
+ assert ts["end_time"] > ts["start_time"], "fragment keeps acoustic extent"
+ # Genuine 。 between speech still anchors (control).
+ punct = [t for t in timestamps if t["token"] == "。"]
+ assert punct, "expected genuine punctuation in fixtures"
+ assert punct[0]["start_time"] == punct[0]["end_time"]
+
+
+def test_vllm_method_preserves_byte_fragment_spans(sensevoice_tokenizer):
+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+ tokenizer = sensevoice_tokenizer
+ target_ids = tokenizer.encode(TEXT)
+ n_frames = len(target_ids) * FRAMES_PER_TOKEN
+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+ engine = FunASRNanoVLLM.__new__(FunASRNanoVLLM)
+ engine.ctc_decoder = lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames]))
+ engine.ctc = types.SimpleNamespace(log_softmax=lambda d: d)
+ engine.ctc_tokenizer = tokenizer
+ engine.blank_id = _blank_id(tokenizer)
+
+ result = engine._compute_timestamps(
+ torch.zeros(1, n_frames, 8), torch.tensor([n_frames]), TEXT
+ )
+ fragments = [t for t in result if t["token"] == "�"]
+ assert len(fragments) >= 2
+ for t in fragments:
+ assert t["end_time"] > t["start_time"]
+ assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)
+
+
+def test_pipeline_method_preserves_byte_fragment_spans(
+ sensevoice_tokenizer, monkeypatch
+):
+ from funasr.models.fun_asr_nano.inference_vllm_pipeline import (
+ FunASRNanoVLLMPipeline,
+ )
+
+ tokenizer = sensevoice_tokenizer
+ target_ids = tokenizer.encode(TEXT)
+ n_frames = len(target_ids) * FRAMES_PER_TOKEN
+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+ monkeypatch.setattr(
+ "funasr.utils.load_utils.extract_fbank",
+ lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
+ )
+ pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
+ pipeline.device = "cpu"
+ pipeline.asr_engine = types.SimpleNamespace(
+ frontend=object(),
+ audio_encoder=lambda s, sl: (
+ torch.zeros(1, n_frames, 8),
+ torch.tensor([n_frames]),
+ ),
+ ctc_decoder=lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames])),
+ ctc=types.SimpleNamespace(log_softmax=lambda d: d),
+ ctc_tokenizer=tokenizer,
+ blank_id=_blank_id(tokenizer),
+ )
+ result = pipeline._compute_all_timestamps(
+ [np.zeros(32000, dtype=np.float32)], [[1000, 9000]], [TEXT]
+ )
+ fragments = [t for t in result if t["token"] == "�"]
+ assert len(fragments) >= 2
+ for t in fragments:
+ assert t["end_time"] > t["start_time"]
+ # +1s VAD offset applied on top of preserved spans: compare against the
+ # un-offset helper path for the identical target sequence.
+ expected = _production_timestamps(tokenizer, target_ids)
+ expected_frags = [t for t in expected if t["token"] == "�"]
+ assert len(fragments) == len(expected_frags)
+ for got, want in zip(fragments, expected_frags):
+ assert got["start_time"] == pytest.approx(want["start_time"] + 1.0)
+ assert got["end_time"] == pytest.approx(want["end_time"] + 1.0)
diff --git a/tests/test_fun_asr_nano_punctuation_timestamps.py b/tests/test_fun_asr_nano_punctuation_timestamps.py
new file mode 100644
index 000000000..4f3e19141
--- /dev/null
+++ b/tests/test_fun_asr_nano_punctuation_timestamps.py
@@ -0,0 +1,164 @@
+"""Punctuation-timestamp anchoring for Fun-ASR-Nano forced alignment.
+
+Issue #3702: with merge_vad, a sentence-final punctuation token (which has no
+acoustic realization) is placed by CTC forced alignment at the next sentence's
+onset frame, so the current sentence's punctuation timestamp lands on the next
+sentence start. Proven with synthetic peaky emissions through the repo's exact
+``forced_align`` (see run evidence): in a merged [sent1 + silence + sent2]
+window the punctuation lands at the silence end, while the single-sentence
+control keeps it at the window end.
+
+These tests pin the post-processing rule: a punctuation-only token that is
+followed by more speech is anchored (zero-width) at the preceding spoken
+token's end. Punctuation carries no acoustic extent, so anchoring invents no
+duration. Leading punctuation (no predecessor) and trailing punctuation (no
+successor, e.g. single-segment output) are left untouched, preserving
+existing single-segment behavior.
+"""
+
+import copy
+
+import pytest
+
+from funasr.models.fun_asr_nano.tools.utils import anchor_punctuation_timestamps
+
+
+def _ts(token, start, end):
+ return {"token": token, "start_time": start, "end_time": end}
+
+
+def test_punctuation_anchored_at_previous_spoken_end():
+ # Mirrors the proven repro: 好 ends at 19, 。misplaced at [72, 73).
+ timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("他", 78, 79)]
+ result = anchor_punctuation_timestamps(timestamps)
+ assert result[1] == {"token": "。", "start_time": 19, "end_time": 19}
+ assert result[0] == {"token": "好", "start_time": 18, "end_time": 19}
+ assert result[2] == {"token": "他", "start_time": 78, "end_time": 79}
+
+
+def test_leading_punctuation_without_predecessor_is_kept():
+ timestamps = [_ts("。", 5, 6), _ts("好", 18, 19)]
+ before = copy.deepcopy(timestamps)
+ assert anchor_punctuation_timestamps(timestamps) == before
+
+
+def test_consecutive_punctuation_share_one_anchor():
+ timestamps = [_ts("好", 18, 19), _ts(",", 50, 51), _ts("。", 72, 73), _ts("他", 78, 79)]
+ result = anchor_punctuation_timestamps(timestamps)
+ assert (result[1]["start_time"], result[1]["end_time"]) == (19, 19)
+ assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)
+ assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)
+
+
+def test_spoken_tokens_are_never_rewritten():
+ timestamps = [_ts("你", 12, 13), _ts("好", 18, 19), _ts("。", 39, 40)]
+ before = copy.deepcopy(timestamps)
+ result = anchor_punctuation_timestamps(timestamps)
+ assert result[0] == before[0]
+ # Single-sentence control shape: punctuation already at window end is kept.
+ assert result[2] == before[2]
+
+
+def test_special_tokens_are_not_treated_as_punctuation():
+ timestamps = [_ts("<sil>", 0, 5), _ts("好", 18, 19)]
+ before = copy.deepcopy(timestamps)
+ assert anchor_punctuation_timestamps(timestamps) == before
+
+
+def test_punctuation_followed_only_by_special_is_kept():
+ # No later real speech: the trailing <sil> must not make 。 eligible.
+ timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("<sil>", 80, 85)]
+ before = copy.deepcopy(timestamps)
+ assert anchor_punctuation_timestamps(timestamps) == before
+
+
+def test_special_token_is_not_used_as_predecessor_anchor():
+ # The <sil> between speech and punctuation must not become the anchor.
+ timestamps = [_ts("好", 18, 19), _ts("<sil>", 30, 35), _ts("。", 72, 73), _ts("他", 78, 79)]
+ result = anchor_punctuation_timestamps(timestamps)
+ assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)
+ assert (result[1]["start_time"], result[1]["end_time"]) == (30, 35)
+ assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)
+
+
+def test_pipeline_segment_local_anchor_boundary(monkeypatch):
+ """Pipeline segment boundary: anchoring applies per VAD segment, never
+ across segments. segA shows intra-segment anchoring (misplaced 。 before
+ later speech in the SAME aligned segment anchors); segB shows a
+ segment-final 。 staying put even though segC contains later speech;
+ VAD offsets and spoken spans pass through verbatim."""
+ import types
+
+ import numpy as np
+ import torch
+
+ from funasr.models.fun_asr_nano.inference_vllm_pipeline import FunASRNanoVLLMPipeline
+
+ id2char = {2: "好", 3: "。", 4: "他", 5: "在", 6: "是"}
+ spans = iter(
+ [
+ [
+ {"token": 2, "start_time": 18, "end_time": 19},
+ {"token": 3, "start_time": 72, "end_time": 73},
+ {"token": 4, "start_time": 78, "end_time": 79},
+ ],
+ [
+ {"token": 6, "start_time": 10, "end_time": 12},
+ {"token": 3, "start_time": 40, "end_time": 41},
+ ],
+ [{"token": 5, "start_time": 50, "end_time": 56}],
+ ]
+ )
+
+ class StubTok:
+ def encode(self, text):
+ return [1] * max(1, len(text))
+
+ def decode(self, ids):
+ return "".join(id2char.get(i, "?") for i in ids)
+
+ engine = types.SimpleNamespace(
+ frontend=object(),
+ device="cpu",
+ blank_id=0,
+ audio_encoder=lambda s, sl: (torch.zeros(1, 10, 8), torch.tensor([10])),
+ ctc_decoder=lambda e, l: (torch.zeros(1, 10, 8), torch.tensor([10])),
+ ctc=types.SimpleNamespace(log_softmax=lambda d: d),
+ ctc_tokenizer=StubTok(),
+ )
+ monkeypatch.setattr(
+ "funasr.utils.load_utils.extract_fbank",
+ lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
+ )
+ monkeypatch.setattr(
+ "funasr.models.fun_asr_nano.tools.utils.forced_align",
+ lambda *a, **k: next(spans),
+ )
+
+ pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
+ pipeline.asr_engine = engine
+ pipeline.device = "cpu"
+ result = pipeline._compute_all_timestamps(
+ [
+ np.zeros(32000, dtype=np.float32),
+ np.zeros(16000, dtype=np.float32),
+ np.zeros(16000, dtype=np.float32),
+ ],
+ [[1000, 3000], [5000, 6000], [9000, 10000]],
+ ["segAtext", "segBtext", "segCtext"],
+ )
+
+ assert [t["token"] for t in result] == ["好", "。", "他", "是", "。", "在"]
+ # segA (offset 1.0s): intra-segment 。 anchors at 好.end bit-identically.
+ assert result[1]["start_time"] == result[0]["end_time"]
+ assert result[1]["end_time"] == result[0]["end_time"]
+ assert result[0]["end_time"] == pytest.approx(2.14)
+ assert result[2]["start_time"] == pytest.approx(5.68)
+ # segB (offset 5.0s): segment-final 。 keeps its aligned span even though
+ # segC contains later speech — cross-segment speech must not trigger it.
+ assert result[4]["start_time"] == pytest.approx(7.4)
+ assert result[4]["end_time"] == pytest.approx(7.46)
+ assert result[3]["end_time"] == pytest.approx(5.72)
+ # segC (offset 9.0s) passes through untouched.
+ assert result[5]["start_time"] == pytest.approx(12.0)
+ assert result[5]["end_time"] == pytest.approx(12.36)
diff --git a/tests/test_fun_asr_nano_semantic_symbol_timestamps.py b/tests/test_fun_asr_nano_semantic_symbol_timestamps.py
new file mode 100644
index 000000000..1240b1876
--- /dev/null
+++ b/tests/test_fun_asr_nano_semantic_symbol_timestamps.py
@@ -0,0 +1,309 @@
+"""Semantic-symbol tokens must keep their acoustic spans (PR #3703 rework).
+
+LauraGPT's blocking review on PR #3703 (comment 3996970511): the accepted
+U+FFFD byte-fragment guard fixed undecodable fragments, but the classifier
+still treated every non-word character as punctuation. With the official
+SenseVoice vocabulary, ``我用C++。你好。`` encodes ``++`` as one token (id
+24754, Unicode Sm) and ``价格是$10。你好。`` encodes ``$`` as one token (id 3,
+Sc). Both decode without replacement characters, are spoken semantic
+content, and had their aligned spans [1.8, 2.4] collapsed to [1.8, 1.8] by
+the actual ``FunASRNanoVLLM._compute_timestamps`` (the pipeline path after
+its +1s VAD offset: [2.8, 3.4] -> [2.8, 2.8]).
+
+The classifier now anchors only the supported sentence-punctuation set
+(``_SENTENCE_PUNCTUATION_CHARS``); everything else defaults to spoken.
+
+These tests instantiate the repository's real ``SenseVoiceTokenizer``
+production path (``SenseVoiceTokenizer`` factory -> ``get_tokenizer`` ->
+``get_encoding`` -> production ``Tokenizer``) against a deterministic
+temporary offline ``.tiktoken`` vocab, so they run without the Fun-ASR-Nano
+``multilingual.tiktoken`` model artifact (which ships with the model, not
+this repo) and without weights/GPU. The fixture vocab mirrors the official
+vocab shape for both reviewer cases: CJK chars and ``。`` are single ids,
+``++`` is one merged id (official: id 24754), ``$`` decodes standalone
+(official: id 3, a single-byte rank), ``C`` stays a single-byte id
+(official: id 34), and ``10`` is one merged id (official: id 3254).
+Official-artifact cross-check (SenseVoiceTokenizer from
+FunAudioLLM/Fun-ASR-Nano-2512 ``multilingual.tiktoken`` @272c57b, sha256
+74797963e813193436aabcff7c1c235d37de8097b71c563ec8b63b7a515c718):
+``我用C++。你好。`` -> [51405, 53719, 34, 24754, 1542, 48934, 50371, 1542]
+and ``价格是$10。你好。`` -> [48869, 52258, 51971, 3, 3254, 1542, 48934,
+50371, 1542]; the symbol ids decode standalone to ``++`` / ``$`` while
+``。`` is id 1542.
+"""
+
+import base64
+import types
+
+import numpy as np
+import pytest
+import torch
+
+from funasr.models.fun_asr_nano.tools.utils import (
+ _classify_timestamp_token,
+ _is_sentence_punctuation,
+ anchor_punctuation_timestamps,
+ forced_align,
+)
+from funasr.tokenizer.whisper_tokenizer import SenseVoiceTokenizer
+
+TEXT_PLUS = "我用C++。你好。"
+TEXT_CURRENCY = "价格是$10。你好。"
+FRAMES_PER_TOKEN = 10
+FRAME_TO_SEC = 6 * 10 / 1000
+
+# Representative official-vocab symbol tokens (FunAudioLLM/Fun-ASR-Nano-2512
+# @272c57b): all are single-id spoken semantic content, never sentence
+# punctuation. Categories are Unicode general categories.
+OFFICIAL_SYMBOL_TOKENS = [
+ ("++", "Sm"),
+ ("+", "Sm"),
+ ("=", "Sm"),
+ ("<", "Sm"),
+ (">", "Sm"),
+ ("^", "Sk"),
+ ("~", "Sm"),
+ ("$", "Sc"),
+ ("€", "Sc"),
+ ("℃", "So"),
+ ("°", "So"),
+ ("%", "Po"),
+ ("#", "Po"),
+ ("@", "Po"),
+ ("&", "Po"),
+ ("*", "Po"),
+ ("/", "Po"),
+ ("-", "Pd"),
+ ("—", "Pd"),
+ ("…", "Po"),
+ ('"', "Pe"),
+ ("(", "Ps"),
+ ("「", "Ps"),
+ ("》", "Pe"),
+]
+
+# Sentence punctuation the anchoring policy supports: terminators and clause
+# separators that ASR emits as structural markup with no acoustic extent.
+SUPPORTED_SENTENCE_PUNCTUATION = [
+ "。",
+ ",",
+ "、",
+ ";",
+ ":",
+ "!",
+ "?",
+ ".",
+ ",",
+ "!",
+ "?",
+ ";",
+ ":",
+]
+
+
+def _write_semantic_symbol_vocab(path):
+ """Deterministic offline vocab: byte-level base plus merged ranks that
+ mirror the official reviewer-case tokenization. Format matches production
+ ``.tiktoken`` files (``base64(token) rank`` lines)."""
+ lines = []
+ for byte in range(256):
+ lines.append(f"{base64.b64encode(bytes([byte])).decode()} {byte}")
+ next_rank = 256
+ # CJK chars and 。/,: prefix pair first so BPE completes each 3-byte
+ # merge (tiktoken merges lowest-rank pairs first).
+ for char in "我用你好价格是。,":
+ raw = char.encode("utf-8")
+ assert len(raw) == 3, char
+ lines.append(f"{base64.b64encode(raw[:2]).decode()} {next_rank}")
+ next_rank += 1
+ lines.append(f"{base64.b64encode(raw).decode()} {next_rank}")
+ next_rank += 1
+ # Multi-char symbol tokens, mirroring official ids 24754 ("++") and 3254
+ # ("10"): single merged ranks, so each decodes standalone as one token.
+ for token in ("++", "10"):
+ lines.append(f"{base64.b64encode(token.encode('utf-8')).decode()} {next_rank}")
+ next_rank += 1
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def sensevoice_tokenizer(tmp_path):
+ """The real production Tokenizer via the SenseVoiceTokenizer factory."""
+ vocab = tmp_path / "semantic-symbol.tiktoken"
+ _write_semantic_symbol_vocab(vocab)
+ tokenizer = SenseVoiceTokenizer(vocab_path=str(vocab))
+ for text in (TEXT_PLUS, TEXT_CURRENCY):
+ assert tokenizer.decode(tokenizer.encode(text)) == text
+ return tokenizer
+
+
+def _peaky_log_probs(target_ids, vocab_size, frames_per_token=FRAMES_PER_TOKEN):
+ n_frames = len(target_ids) * frames_per_token
+ log_probs = torch.full((n_frames, vocab_size), -30.0)
+ for i, tid in enumerate(target_ids):
+ log_probs[i * frames_per_token : (i + 1) * frames_per_token, tid] = 0.0
+ return log_probs
+
+
+def _blank_id(tokenizer):
+ return tokenizer.get_vocab_size() - 1
+
+
+def _production_timestamps(tokenizer, target_ids):
+ """Mirror the production decode-per-id + scale path shared by all three
+ Nano timestamp methods, over the repo's real forced_align."""
+ items = forced_align(
+ _peaky_log_probs(target_ids, tokenizer.get_vocab_size()),
+ torch.tensor(target_ids, dtype=torch.int64),
+ _blank_id(tokenizer),
+ )
+ assert [it["token"] for it in items] == list(target_ids)
+ return [
+ {
+ "token": tokenizer.decode([it["token"]]),
+ "start_time": it["start_time"] * FRAME_TO_SEC,
+ "end_time": it["end_time"] * FRAME_TO_SEC,
+ }
+ for it in items
+ ]
+
+
+def test_supported_sentence_punctuation_anchors():
+ for token in SUPPORTED_SENTENCE_PUNCTUATION:
+ assert _is_sentence_punctuation(token), token
+ assert _classify_timestamp_token(token) == "punctuation", token
+ # Combinations of supported punctuation stay punctuation.
+ assert _is_sentence_punctuation("!?")
+ assert _is_sentence_punctuation("!!")
+ assert _classify_timestamp_token("!?") == "punctuation"
+
+
+def test_semantic_symbols_are_spoken_not_punctuation():
+ for token, _category in OFFICIAL_SYMBOL_TOKENS:
+ assert _classify_timestamp_token(token) == "spoken", token
+ assert not _is_sentence_punctuation(token), token
+ # A symbol glued to sentence punctuation must not anchor either.
+ assert _classify_timestamp_token('。"') == "spoken"
+ assert _classify_timestamp_token("%,") == "spoken"
+ # Undecodable byte fragments stay spoken; special tokens keep their class.
+ assert _classify_timestamp_token("�") == "spoken"
+ assert _classify_timestamp_token("<sil>") == "special"
+
+
+def test_symbol_token_ids_decode_standalone(sensevoice_tokenizer):
+ tokenizer = sensevoice_tokenizer
+ plus_ids = tokenizer.encode(TEXT_PLUS)
+ currency_ids = tokenizer.encode(TEXT_CURRENCY)
+ assert tokenizer.decode(plus_ids) == TEXT_PLUS
+ assert tokenizer.decode(currency_ids) == TEXT_CURRENCY
+ # ++ and $ must be single ids that decode standalone (official: one id
+ # each, 24754 and 3), not per-character or byte-split.
+ plus_tokens = [tokenizer.decode([i]) for i in plus_ids]
+ currency_tokens = [tokenizer.decode([i]) for i in currency_ids]
+ assert "++" in plus_tokens, plus_tokens
+ assert "$" in currency_tokens, currency_tokens
+ assert _classify_timestamp_token("++") == "spoken"
+ assert _classify_timestamp_token("$") == "spoken"
+
+
+def test_semantic_symbol_spans_survive_anchoring(sensevoice_tokenizer):
+ tokenizer = sensevoice_tokenizer
+ for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):
+ target_ids = tokenizer.encode(text)
+ timestamps = _production_timestamps(tokenizer, target_ids)
+ before = [(t["token"], t["start_time"], t["end_time"]) for t in timestamps]
+ anchor_punctuation_timestamps(timestamps)
+
+ symbols = [t for t in timestamps if t["token"] == symbol]
+ assert symbols, f"expected {symbol!r} in {text!r}"
+ for (token, start, end), ts in zip(before, timestamps):
+ if token == symbol:
+ # The acoustic span is preserved verbatim: no collapse.
+ assert (ts["start_time"], ts["end_time"]) == (start, end)
+ assert ts["end_time"] > ts["start_time"], f"{symbol} keeps acoustic extent"
+ # Genuine 。 between speech still anchors (zero-width control): it
+ # pins at the preceding spoken token's end, which for these texts is
+ # the symbol or the token right after it.
+ punct = [t for t in timestamps if t["token"] == "。"]
+ assert punct
+ assert punct[0]["start_time"] == punct[0]["end_time"]
+ punct_index = next(i for i, t in enumerate(before) if t[0] == "。")
+ assert before[punct_index - 1][2] == punct[0]["start_time"]
+
+
+def test_vllm_method_preserves_semantic_symbol_spans(sensevoice_tokenizer):
+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM
+
+ tokenizer = sensevoice_tokenizer
+ for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):
+ target_ids = tokenizer.encode(text)
+ n_frames = len(target_ids) * FRAMES_PER_TOKEN
+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+ engine = FunASRNanoVLLM.__new__(FunASRNanoVLLM)
+ engine.ctc_decoder = lambda e, l, lp=log_probs, nf=n_frames: (
+ lp.unsqueeze(0),
+ torch.tensor([nf]),
+ )
+ engine.ctc = types.SimpleNamespace(log_softmax=lambda d: d)
+ engine.ctc_tokenizer = tokenizer
+ engine.blank_id = _blank_id(tokenizer)
+
+ result = engine._compute_timestamps(
+ torch.zeros(1, n_frames, 8), torch.tensor([n_frames]), text
+ )
+ symbols = [t for t in result if t["token"] == symbol]
+ assert symbols
+ for t in symbols:
+ assert t["end_time"] > t["start_time"], f"{symbol} span preserved"
+ # Genuine punctuation still anchors between speech.
+ assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)
+
+
+def test_pipeline_method_preserves_semantic_symbol_spans(sensevoice_tokenizer, monkeypatch):
+ from funasr.models.fun_asr_nano.inference_vllm_pipeline import (
+ FunASRNanoVLLMPipeline,
+ )
+
+ tokenizer = sensevoice_tokenizer
+ for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):
+ target_ids = tokenizer.encode(text)
+ n_frames = len(target_ids) * FRAMES_PER_TOKEN
+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())
+
+ monkeypatch.setattr(
+ "funasr.utils.load_utils.extract_fbank",
+ lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),
+ )
+ pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)
+ pipeline.device = "cpu"
+ pipeline.asr_engine = types.SimpleNamespace(
+ frontend=object(),
+ audio_encoder=lambda s, sl, nf=n_frames: (
+ torch.zeros(1, nf, 8),
+ torch.tensor([nf]),
+ ),
+ ctc_decoder=lambda e, l, lp=log_probs, nf=n_frames: (
+ lp.unsqueeze(0),
+ torch.tensor([nf]),
+ ),
+ ctc=types.SimpleNamespace(log_softmax=lambda d: d),
+ ctc_tokenizer=tokenizer,
+ blank_id=_blank_id(tokenizer),
+ )
+ result = pipeline._compute_all_timestamps(
+ [np.zeros(32000, dtype=np.float32)], [[1000, 9000]], [text]
+ )
+ symbols = [t for t in result if t["token"] == symbol]
+ assert symbols
+ # +1s VAD offset applied on top of preserved spans: compare against
+ # the un-offset helper path for the identical target sequence.
+ expected = _production_timestamps(tokenizer, target_ids)
+ expected_symbols = [t for t in expected if t["token"] == symbol]
+ assert len(symbols) == len(expected_symbols)
+ for got, want in zip(symbols, expected_symbols):
+ assert got["start_time"] == pytest.approx(want["start_time"] + 1.0)
+ assert got["end_time"] == pytest.approx(want["end_time"] + 1.0)
+ assert got["end_time"] > got["start_time"], f"{symbol} span preserved"
+ # Genuine punctuation still anchors between speech.
+ assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)