Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(nano): correct punctuation timestamps across VAD merges (#3703)
Bruce-Yii41778c45de50
diff --git a/funasr/models/fun_asr_nano/inference_vllm.py b/funasr/models/fun_asr_nano/inference_vllm.pyindex 78f567c2d..db9828a83 100644--- a/funasr/models/fun_asr_nano/inference_vllm.py+++ b/funasr/models/fun_asr_nano/inference_vllm.py@@ -694,7 +694,10 @@ class FunASRNanoVLLM:@torch.no_grad()def _compute_timestamps(self, encoder_out, encoder_out_lens, text):"""CTC forced alignment for character-level timestamps."""- from funasr.models.fun_asr_nano.tools.utils import forced_align+ from funasr.models.fun_asr_nano.tools.utils import (+ anchor_punctuation_timestamps,+ forced_align,+ )decoder_out, decoder_out_lens = self.ctc_decoder(encoder_out, encoder_out_lens)ctc_logits = self.ctc.log_softmax(decoder_out)@@ -709,6 +712,9 @@ class FunASRNanoVLLM:ts["token"] = self.ctc_tokenizer.decode([ts["token"]])ts["start_time"] = ts["start_time"] * 6 * 10 / 1000ts["end_time"] = ts["end_time"] * 6 * 10 / 1000+ # See model.py: anchor punctuation at the preceding spoken token's end+ # (issue #3702); punctuation has no acoustic extent to align.+ anchor_punctuation_timestamps(timestamps)return timestamps@classmethoddiff --git a/funasr/models/fun_asr_nano/inference_vllm_pipeline.py b/funasr/models/fun_asr_nano/inference_vllm_pipeline.pyindex 49241bc5b..468e10e7d 100644--- a/funasr/models/fun_asr_nano/inference_vllm_pipeline.py+++ b/funasr/models/fun_asr_nano/inference_vllm_pipeline.py@@ -322,7 +322,10 @@ class FunASRNanoVLLMPipeline:def _compute_all_timestamps(self, segment_audios, vad_segments, asr_results):"""Compute CTC timestamps for all segments with VAD offsets."""- from funasr.models.fun_asr_nano.tools.utils import forced_align+ from funasr.models.fun_asr_nano.tools.utils import (+ anchor_punctuation_timestamps,+ forced_align,+ )all_timestamps = []for seg_audio, vad_seg, text in zip(segment_audios, vad_segments, asr_results):@@ -356,6 +359,13 @@ class FunASRNanoVLLMPipeline:ts["token"] = self.asr_engine.ctc_tokenizer.decode([ts["token"]])ts["start_time"] = ts["start_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000ts["end_time"] = ts["end_time"] * 6 * 10 / 1000 + vad_offset_ms / 1000+ # Same invariant as the offline/vLLM paths (issue #3702), applied+ # per independently aligned VAD segment: punctuation carries no+ # acoustic extent, so pin sandwiched punctuation at the preceding+ # spoken token's end. Segment-local on purpose — speech in a+ # later VAD segment must not rewrite this segment's trailing+ # punctuation.+ anchor_punctuation_timestamps(timestamps)all_timestamps.extend(timestamps)except Exception as e:logger.debug(f"Timestamp failed for segment: {e}")diff --git a/funasr/models/fun_asr_nano/model.py b/funasr/models/fun_asr_nano/model.pyindex 01a1f88d8..049ba6a06 100644--- a/funasr/models/fun_asr_nano/model.py+++ b/funasr/models/fun_asr_nano/model.py@@ -25,7 +25,7 @@ from .ctc import CTCfrom .checkpoint_utils import disable_incomplete_ctc, normalize_checkpoint_statefrom .device_utils import resolve_autocast_device_typefrom .llm_forward_opts import configure_llm_forward-from .tools.utils import forced_align+from .tools.utils import anchor_punctuation_timestamps, forced_aligndtype_map = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}@@ -1075,6 +1075,12 @@ class FunASRNano(nn.Module):timestamp["token"] = self.ctc_tokenizer.decode([timestamp["token"]])timestamp["start_time"] = timestamp["start_time"] * 6 * 10 / 1000timestamp["end_time"] = timestamp["end_time"] * 6 * 10 / 1000+ # Punctuation tokens have no acoustic realization; without anchoring,+ # forced alignment places them at the next sentence's onset inside+ # merged multi-sentence windows (issue #3702). Applied after decoding+ # (tokens are still integer ids above). ctc_timestamps is left+ # untouched: greedy CTC text carries no punctuation to anchor.+ anchor_punctuation_timestamps(result["timestamps"])if ibest_writer is not None:ibest_writer["text"][key[0]] = response.replace("\n", " ")diff --git a/funasr/models/fun_asr_nano/tools/utils.py b/funasr/models/fun_asr_nano/tools/utils.pyindex 63ec6cb8f..a1389c4ad 100644--- a/funasr/models/fun_asr_nano/tools/utils.py+++ b/funasr/models/fun_asr_nano/tools/utils.py@@ -1,4 +1,5 @@from itertools import groupby+import reimport librosaimport soundfile as sf@@ -77,4 +78,97 @@ def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0))except Exception:pass- return items\ No newline at end of file+ return items+++_SPECIAL_TOKEN_RE = re.compile(r"^<[^>]*>$")+# tiktoken decodes a partial UTF-8 byte sequence (one id of a multi-id+# character, e.g. SenseVoice ids [10958, 245] for 郗) as U+FFFD. All three+# Nano timestamp paths decode each token id independently, so such fragments+# reach classification as "�". They are spoken content, never punctuation.+_UNDECODABLE_FRAGMENT_MARK = "�"+# Supported sentence punctuation: the closed set of terminators and clause+# separators that ASR emits as structural markup with no acoustic+# realization — the only tokens the anchoring policy is entitled to collapse.+# Grounded in the official SenseVoice multilingual.tiktoken vocabulary+# (FunAudioLLM/Fun-ASR-Nano-2512 @272c57b, sha256+# 74797963e813193436aabcff7c1c235d37de8097b71c563ec8b63b7a515c718), where+# every character below is a single-id Po token. Symbol tokens are+# deliberately excluded: they carry real acoustic extent as spoken semantic+# content, e.g. "++" (id 24754, Unicode Sm) in 我用C++。你好。 and "$" (id 3,+# Sc) in 价格是$10。你好。 — anchoring them would delete spoken timing (PR+# #3703 review). Anything outside this set defaults to spoken, so unknown+# tokens fail safe rather than lose their span.+_SENTENCE_PUNCTUATION_CHARS = frozenset("。!?,、;:.,!?;:")+++def _is_sentence_punctuation(token):+ """True iff every character of ``token`` is supported sentence punctuation."""+ return bool(token) and all(char in _SENTENCE_PUNCTUATION_CHARS for char in token)+++def _classify_timestamp_token(token):+ """Classify one decoded timestamp token into spoken/punctuation/special.++ Special tokens (``<sil>``-shaped) are their own class: they are neither+ spoken content nor punctuation, so they neither trigger anchoring nor+ serve as anchors. Undecodable byte fragments (containing U+FFFD) are+ spoken: they are slices of a multi-id character with real acoustic+ extent, and collapsing their spans would delete spoken timing (issue+ #3703). Punctuation is only the supported sentence-punctuation set+ (see ``_SENTENCE_PUNCTUATION_CHARS``); every other non-word token —+ semantic symbols such as ``++`` or ``$`` — is spoken and keeps its span.+ Non-string tokens (integer ids, if called pre-decode) fall back to+ spoken to preserve pass-through behavior.+ """+ if not isinstance(token, str) or not token:+ return "spoken"+ if _SPECIAL_TOKEN_RE.match(token):+ return "special"+ if _UNDECODABLE_FRAGMENT_MARK in token:+ return "spoken"+ if _is_sentence_punctuation(token):+ return "punctuation"+ return "spoken"+++def anchor_punctuation_timestamps(timestamps):+ """Anchor punctuation-only tokens at the preceding spoken token's end.++ CTC forced alignment must place every target token — including punctuation+ tokens with no acoustic realization — on at least one frame. Inside a+ merged multi-sentence window (see FunASR issue #3702) such a token lands+ at the next sentence's onset frame, so the current sentence's punctuation+ timestamp reads as the next sentence start. Punctuation carries no acoustic+ extent, so a punctuation span that is followed by more speech is pinned+ (zero-width) at the end of the speech it terminates.++ Leading punctuation (no predecessor) and trailing punctuation (no+ successor, e.g. single-segment output) are left untouched, as are special+ tokens such as ``<sil>``. This preserves existing single-segment behavior.++ Args:+ timestamps: List of ``{"token": str, "start_time": float,+ "end_time": float}`` dicts in time order. Mutated in place.++ Returns:+ The same list, for convenient chaining.+ """+ last_spoken_end = None+ total = len(timestamps)+ for index, ts in enumerate(timestamps):+ kind = _classify_timestamp_token(ts.get("token", ""))+ if kind != "punctuation":+ if kind == "spoken":+ end = ts.get("end_time")+ if isinstance(end, (int, float)):+ last_spoken_end = end+ continue+ following_spoken = any(+ _classify_timestamp_token(later.get("token", "")) == "spoken"+ for later in timestamps[index + 1 : total]+ )+ if last_spoken_end is not None and following_spoken:+ ts["start_time"] = last_spoken_end+ ts["end_time"] = last_spoken_end+ return timestampsdiff --git a/tests/test_fun_asr_nano_byte_fragment_timestamps.py b/tests/test_fun_asr_nano_byte_fragment_timestamps.pynew file mode 100644index 000000000..3045c88bd--- /dev/null+++ b/tests/test_fun_asr_nano_byte_fragment_timestamps.py@@ -0,0 +1,205 @@+"""Byte-fragment spoken tokens must never be treated as punctuation.++Review regression for FunASR PR #3703 (issue #3702 follow-up): Nano's CTC+tokenizer is byte-based, so one spoken character can span several token ids+(official SenseVoiceTokenizer: 郗 -> [10958, 245]). All three Nano timestamp+paths decode each token id independently, and each fragment decodes to U+FFFD+while the id sequence decodes to real speech. The punctuation classifier must+not collapse such fragments' valid acoustic spans.++These tests instantiate the repository's real ``SenseVoiceTokenizer``+production path (``SenseVoiceTokenizer`` factory -> ``get_tokenizer`` ->+``get_encoding`` -> production ``Tokenizer``) against a deterministic+temporary offline ``.tiktoken`` vocab, so they run without the Fun-ASR-Nano+``multilingual.tiktoken`` model artifact (which ships with the model, not+this repo) and without weights/GPU. The fixture vocab mirrors the official+vocab shape: ordinary CJK chars and punctuation are single ids, while 郗+(deliberately given no merged rank) splits into single-byte ids whose+independent decodes are U+FFFD — the exact byte-fragment mechanism+(official: 郗 -> [10958, 245]). Official-artifact cross-check+(SenseVoiceTokenizer from FunAudioLLM/Fun-ASR-Nano-2512+``multilingual.tiktoken`` @272c57b, sha256 74797963...): 郗 -> [10958, 245],+per-id decodes U+FFFD, roundtrip intact; helper/vLLM/pipeline probes preserve+[0.60, 0.72]/[0.84, 0.96]-style spans post-fix (see PR review evidence).+"""++import base64+import types++import numpy as np+import pytest+import torch++from funasr.models.fun_asr_nano.tools.utils import (+ _classify_timestamp_token,+ anchor_punctuation_timestamps,+ forced_align,+)+from funasr.tokenizer.whisper_tokenizer import SenseVoiceTokenizer++TEXT = "我叫郗明。你好。"+FRAMES_PER_TOKEN = 10+FRAME_TO_SEC = 6 * 10 / 1000+++def _write_fragment_vocab(path):+ """Deterministic offline vocab: byte-level base plus single-id merged+ ranks for every fixture char except 郗, which stays byte-split. Format+ matches production ``.tiktoken`` files (``base64(token) rank`` lines)."""+ lines = []+ for byte in range(256):+ lines.append(f"{base64.b64encode(bytes([byte])).decode()} {byte}")+ next_rank = 256+ # Prefix pair first so BPE completes each 3-byte merge (tiktoken merges+ # lowest-rank pairs first); 郗 gets no ranks and stays byte-split.+ for char in "我叫明你好。,":+ raw = char.encode("utf-8")+ assert len(raw) == 3, char+ lines.append(f"{base64.b64encode(raw[:2]).decode()} {next_rank}")+ next_rank += 1+ lines.append(f"{base64.b64encode(raw).decode()} {next_rank}")+ next_rank += 1+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")+++def sensevoice_tokenizer(tmp_path):+ """The real production Tokenizer via the SenseVoiceTokenizer factory."""+ vocab = tmp_path / "byte-fragment.tiktoken"+ _write_fragment_vocab(vocab)+ tokenizer = SenseVoiceTokenizer(vocab_path=str(vocab))+ target_ids = tokenizer.encode(TEXT)+ assert tokenizer.decode(target_ids) == TEXT+ return tokenizer+++def _peaky_log_probs(target_ids, vocab_size, frames_per_token=FRAMES_PER_TOKEN):+ n_frames = len(target_ids) * frames_per_token+ log_probs = torch.full((n_frames, vocab_size), -30.0)+ for i, tid in enumerate(target_ids):+ log_probs[i * frames_per_token : (i + 1) * frames_per_token, tid] = 0.0+ return log_probs+++def _blank_id(tokenizer):+ return tokenizer.get_vocab_size() - 1+++def _production_timestamps(tokenizer, target_ids):+ """Mirror the production decode-per-id + scale path shared by all three+ Nano timestamp methods, over the repo's real forced_align."""+ items = forced_align(+ _peaky_log_probs(target_ids, tokenizer.get_vocab_size()),+ torch.tensor(target_ids, dtype=torch.int64),+ _blank_id(tokenizer),+ )+ assert [it["token"] for it in items] == list(target_ids)+ return [+ {+ "token": tokenizer.decode([it["token"]]),+ "start_time": it["start_time"] * FRAME_TO_SEC,+ "end_time": it["end_time"] * FRAME_TO_SEC,+ }+ for it in items+ ]+++def test_undecodable_fragment_is_spoken_not_punctuation():+ assert _classify_timestamp_token("�") == "spoken"+ assert _classify_timestamp_token("��") == "spoken"+ # Genuine punctuation still classifies as punctuation.+ assert _classify_timestamp_token("。") == "punctuation"+ assert _classify_timestamp_token(",") == "punctuation"+ # Special tokens keep their own class.+ assert _classify_timestamp_token("<sil>") == "special"+++def test_byte_fragment_spans_survive_anchoring(sensevoice_tokenizer):+ tokenizer = sensevoice_tokenizer+ target_ids = tokenizer.encode(TEXT)+ assert len(target_ids) > len(TEXT), "郗 must stay byte-split"+ assert tokenizer.decode(target_ids) == TEXT+ assert any(tokenizer.decode([i]) == "�" for i in target_ids)++ timestamps = _production_timestamps(tokenizer, target_ids)+ before = [(t["token"], t["start_time"], t["end_time"]) for t in timestamps]+ anchor_punctuation_timestamps(timestamps)++ for (token, start, end), ts in zip(before, timestamps):+ if token == "�":+ assert (ts["start_time"], ts["end_time"]) == (start, end)+ assert ts["end_time"] > ts["start_time"], "fragment keeps acoustic extent"+ # Genuine 。 between speech still anchors (control).+ punct = [t for t in timestamps if t["token"] == "。"]+ assert punct, "expected genuine punctuation in fixtures"+ assert punct[0]["start_time"] == punct[0]["end_time"]+++def test_vllm_method_preserves_byte_fragment_spans(sensevoice_tokenizer):+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM++ tokenizer = sensevoice_tokenizer+ target_ids = tokenizer.encode(TEXT)+ n_frames = len(target_ids) * FRAMES_PER_TOKEN+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())++ engine = FunASRNanoVLLM.__new__(FunASRNanoVLLM)+ engine.ctc_decoder = lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames]))+ engine.ctc = types.SimpleNamespace(log_softmax=lambda d: d)+ engine.ctc_tokenizer = tokenizer+ engine.blank_id = _blank_id(tokenizer)++ result = engine._compute_timestamps(+ torch.zeros(1, n_frames, 8), torch.tensor([n_frames]), TEXT+ )+ fragments = [t for t in result if t["token"] == "�"]+ assert len(fragments) >= 2+ for t in fragments:+ assert t["end_time"] > t["start_time"]+ assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)+++def test_pipeline_method_preserves_byte_fragment_spans(+ sensevoice_tokenizer, monkeypatch+):+ from funasr.models.fun_asr_nano.inference_vllm_pipeline import (+ FunASRNanoVLLMPipeline,+ )++ tokenizer = sensevoice_tokenizer+ target_ids = tokenizer.encode(TEXT)+ n_frames = len(target_ids) * FRAMES_PER_TOKEN+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())++ monkeypatch.setattr(+ "funasr.utils.load_utils.extract_fbank",+ lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),+ )+ pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)+ pipeline.device = "cpu"+ pipeline.asr_engine = types.SimpleNamespace(+ frontend=object(),+ audio_encoder=lambda s, sl: (+ torch.zeros(1, n_frames, 8),+ torch.tensor([n_frames]),+ ),+ ctc_decoder=lambda e, l: (log_probs.unsqueeze(0), torch.tensor([n_frames])),+ ctc=types.SimpleNamespace(log_softmax=lambda d: d),+ ctc_tokenizer=tokenizer,+ blank_id=_blank_id(tokenizer),+ )+ result = pipeline._compute_all_timestamps(+ [np.zeros(32000, dtype=np.float32)], [[1000, 9000]], [TEXT]+ )+ fragments = [t for t in result if t["token"] == "�"]+ assert len(fragments) >= 2+ for t in fragments:+ assert t["end_time"] > t["start_time"]+ # +1s VAD offset applied on top of preserved spans: compare against the+ # un-offset helper path for the identical target sequence.+ expected = _production_timestamps(tokenizer, target_ids)+ expected_frags = [t for t in expected if t["token"] == "�"]+ assert len(fragments) == len(expected_frags)+ for got, want in zip(fragments, expected_frags):+ assert got["start_time"] == pytest.approx(want["start_time"] + 1.0)+ assert got["end_time"] == pytest.approx(want["end_time"] + 1.0)diff --git a/tests/test_fun_asr_nano_punctuation_timestamps.py b/tests/test_fun_asr_nano_punctuation_timestamps.pynew file mode 100644index 000000000..4f3e19141--- /dev/null+++ b/tests/test_fun_asr_nano_punctuation_timestamps.py@@ -0,0 +1,164 @@+"""Punctuation-timestamp anchoring for Fun-ASR-Nano forced alignment.++Issue #3702: with merge_vad, a sentence-final punctuation token (which has no+acoustic realization) is placed by CTC forced alignment at the next sentence's+onset frame, so the current sentence's punctuation timestamp lands on the next+sentence start. Proven with synthetic peaky emissions through the repo's exact+``forced_align`` (see run evidence): in a merged [sent1 + silence + sent2]+window the punctuation lands at the silence end, while the single-sentence+control keeps it at the window end.++These tests pin the post-processing rule: a punctuation-only token that is+followed by more speech is anchored (zero-width) at the preceding spoken+token's end. Punctuation carries no acoustic extent, so anchoring invents no+duration. Leading punctuation (no predecessor) and trailing punctuation (no+successor, e.g. single-segment output) are left untouched, preserving+existing single-segment behavior.+"""++import copy++import pytest++from funasr.models.fun_asr_nano.tools.utils import anchor_punctuation_timestamps+++def _ts(token, start, end):+ return {"token": token, "start_time": start, "end_time": end}+++def test_punctuation_anchored_at_previous_spoken_end():+ # Mirrors the proven repro: 好 ends at 19, 。misplaced at [72, 73).+ timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("他", 78, 79)]+ result = anchor_punctuation_timestamps(timestamps)+ assert result[1] == {"token": "。", "start_time": 19, "end_time": 19}+ assert result[0] == {"token": "好", "start_time": 18, "end_time": 19}+ assert result[2] == {"token": "他", "start_time": 78, "end_time": 79}+++def test_leading_punctuation_without_predecessor_is_kept():+ timestamps = [_ts("。", 5, 6), _ts("好", 18, 19)]+ before = copy.deepcopy(timestamps)+ assert anchor_punctuation_timestamps(timestamps) == before+++def test_consecutive_punctuation_share_one_anchor():+ timestamps = [_ts("好", 18, 19), _ts(",", 50, 51), _ts("。", 72, 73), _ts("他", 78, 79)]+ result = anchor_punctuation_timestamps(timestamps)+ assert (result[1]["start_time"], result[1]["end_time"]) == (19, 19)+ assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)+ assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)+++def test_spoken_tokens_are_never_rewritten():+ timestamps = [_ts("你", 12, 13), _ts("好", 18, 19), _ts("。", 39, 40)]+ before = copy.deepcopy(timestamps)+ result = anchor_punctuation_timestamps(timestamps)+ assert result[0] == before[0]+ # Single-sentence control shape: punctuation already at window end is kept.+ assert result[2] == before[2]+++def test_special_tokens_are_not_treated_as_punctuation():+ timestamps = [_ts("<sil>", 0, 5), _ts("好", 18, 19)]+ before = copy.deepcopy(timestamps)+ assert anchor_punctuation_timestamps(timestamps) == before+++def test_punctuation_followed_only_by_special_is_kept():+ # No later real speech: the trailing <sil> must not make 。 eligible.+ timestamps = [_ts("好", 18, 19), _ts("。", 72, 73), _ts("<sil>", 80, 85)]+ before = copy.deepcopy(timestamps)+ assert anchor_punctuation_timestamps(timestamps) == before+++def test_special_token_is_not_used_as_predecessor_anchor():+ # The <sil> between speech and punctuation must not become the anchor.+ timestamps = [_ts("好", 18, 19), _ts("<sil>", 30, 35), _ts("。", 72, 73), _ts("他", 78, 79)]+ result = anchor_punctuation_timestamps(timestamps)+ assert (result[2]["start_time"], result[2]["end_time"]) == (19, 19)+ assert (result[1]["start_time"], result[1]["end_time"]) == (30, 35)+ assert (result[3]["start_time"], result[3]["end_time"]) == (78, 79)+++def test_pipeline_segment_local_anchor_boundary(monkeypatch):+ """Pipeline segment boundary: anchoring applies per VAD segment, never+ across segments. segA shows intra-segment anchoring (misplaced 。 before+ later speech in the SAME aligned segment anchors); segB shows a+ segment-final 。 staying put even though segC contains later speech;+ VAD offsets and spoken spans pass through verbatim."""+ import types++ import numpy as np+ import torch++ from funasr.models.fun_asr_nano.inference_vllm_pipeline import FunASRNanoVLLMPipeline++ id2char = {2: "好", 3: "。", 4: "他", 5: "在", 6: "是"}+ spans = iter(+ [+ [+ {"token": 2, "start_time": 18, "end_time": 19},+ {"token": 3, "start_time": 72, "end_time": 73},+ {"token": 4, "start_time": 78, "end_time": 79},+ ],+ [+ {"token": 6, "start_time": 10, "end_time": 12},+ {"token": 3, "start_time": 40, "end_time": 41},+ ],+ [{"token": 5, "start_time": 50, "end_time": 56}],+ ]+ )++ class StubTok:+ def encode(self, text):+ return [1] * max(1, len(text))++ def decode(self, ids):+ return "".join(id2char.get(i, "?") for i in ids)++ engine = types.SimpleNamespace(+ frontend=object(),+ device="cpu",+ blank_id=0,+ audio_encoder=lambda s, sl: (torch.zeros(1, 10, 8), torch.tensor([10])),+ ctc_decoder=lambda e, l: (torch.zeros(1, 10, 8), torch.tensor([10])),+ ctc=types.SimpleNamespace(log_softmax=lambda d: d),+ ctc_tokenizer=StubTok(),+ )+ monkeypatch.setattr(+ "funasr.utils.load_utils.extract_fbank",+ lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),+ )+ monkeypatch.setattr(+ "funasr.models.fun_asr_nano.tools.utils.forced_align",+ lambda *a, **k: next(spans),+ )++ pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)+ pipeline.asr_engine = engine+ pipeline.device = "cpu"+ result = pipeline._compute_all_timestamps(+ [+ np.zeros(32000, dtype=np.float32),+ np.zeros(16000, dtype=np.float32),+ np.zeros(16000, dtype=np.float32),+ ],+ [[1000, 3000], [5000, 6000], [9000, 10000]],+ ["segAtext", "segBtext", "segCtext"],+ )++ assert [t["token"] for t in result] == ["好", "。", "他", "是", "。", "在"]+ # segA (offset 1.0s): intra-segment 。 anchors at 好.end bit-identically.+ assert result[1]["start_time"] == result[0]["end_time"]+ assert result[1]["end_time"] == result[0]["end_time"]+ assert result[0]["end_time"] == pytest.approx(2.14)+ assert result[2]["start_time"] == pytest.approx(5.68)+ # segB (offset 5.0s): segment-final 。 keeps its aligned span even though+ # segC contains later speech — cross-segment speech must not trigger it.+ assert result[4]["start_time"] == pytest.approx(7.4)+ assert result[4]["end_time"] == pytest.approx(7.46)+ assert result[3]["end_time"] == pytest.approx(5.72)+ # segC (offset 9.0s) passes through untouched.+ assert result[5]["start_time"] == pytest.approx(12.0)+ assert result[5]["end_time"] == pytest.approx(12.36)diff --git a/tests/test_fun_asr_nano_semantic_symbol_timestamps.py b/tests/test_fun_asr_nano_semantic_symbol_timestamps.pynew file mode 100644index 000000000..1240b1876--- /dev/null+++ b/tests/test_fun_asr_nano_semantic_symbol_timestamps.py@@ -0,0 +1,309 @@+"""Semantic-symbol tokens must keep their acoustic spans (PR #3703 rework).++LauraGPT's blocking review on PR #3703 (comment 3996970511): the accepted+U+FFFD byte-fragment guard fixed undecodable fragments, but the classifier+still treated every non-word character as punctuation. With the official+SenseVoice vocabulary, ``我用C++。你好。`` encodes ``++`` as one token (id+24754, Unicode Sm) and ``价格是$10。你好。`` encodes ``$`` as one token (id 3,+Sc). Both decode without replacement characters, are spoken semantic+content, and had their aligned spans [1.8, 2.4] collapsed to [1.8, 1.8] by+the actual ``FunASRNanoVLLM._compute_timestamps`` (the pipeline path after+its +1s VAD offset: [2.8, 3.4] -> [2.8, 2.8]).++The classifier now anchors only the supported sentence-punctuation set+(``_SENTENCE_PUNCTUATION_CHARS``); everything else defaults to spoken.++These tests instantiate the repository's real ``SenseVoiceTokenizer``+production path (``SenseVoiceTokenizer`` factory -> ``get_tokenizer`` ->+``get_encoding`` -> production ``Tokenizer``) against a deterministic+temporary offline ``.tiktoken`` vocab, so they run without the Fun-ASR-Nano+``multilingual.tiktoken`` model artifact (which ships with the model, not+this repo) and without weights/GPU. The fixture vocab mirrors the official+vocab shape for both reviewer cases: CJK chars and ``。`` are single ids,+``++`` is one merged id (official: id 24754), ``$`` decodes standalone+(official: id 3, a single-byte rank), ``C`` stays a single-byte id+(official: id 34), and ``10`` is one merged id (official: id 3254).+Official-artifact cross-check (SenseVoiceTokenizer from+FunAudioLLM/Fun-ASR-Nano-2512 ``multilingual.tiktoken`` @272c57b, sha256+74797963e813193436aabcff7c1c235d37de8097b71c563ec8b63b7a515c718):+``我用C++。你好。`` -> [51405, 53719, 34, 24754, 1542, 48934, 50371, 1542]+and ``价格是$10。你好。`` -> [48869, 52258, 51971, 3, 3254, 1542, 48934,+50371, 1542]; the symbol ids decode standalone to ``++`` / ``$`` while+``。`` is id 1542.+"""++import base64+import types++import numpy as np+import pytest+import torch++from funasr.models.fun_asr_nano.tools.utils import (+ _classify_timestamp_token,+ _is_sentence_punctuation,+ anchor_punctuation_timestamps,+ forced_align,+)+from funasr.tokenizer.whisper_tokenizer import SenseVoiceTokenizer++TEXT_PLUS = "我用C++。你好。"+TEXT_CURRENCY = "价格是$10。你好。"+FRAMES_PER_TOKEN = 10+FRAME_TO_SEC = 6 * 10 / 1000++# Representative official-vocab symbol tokens (FunAudioLLM/Fun-ASR-Nano-2512+# @272c57b): all are single-id spoken semantic content, never sentence+# punctuation. Categories are Unicode general categories.+OFFICIAL_SYMBOL_TOKENS = [+ ("++", "Sm"),+ ("+", "Sm"),+ ("=", "Sm"),+ ("<", "Sm"),+ (">", "Sm"),+ ("^", "Sk"),+ ("~", "Sm"),+ ("$", "Sc"),+ ("€", "Sc"),+ ("℃", "So"),+ ("°", "So"),+ ("%", "Po"),+ ("#", "Po"),+ ("@", "Po"),+ ("&", "Po"),+ ("*", "Po"),+ ("/", "Po"),+ ("-", "Pd"),+ ("—", "Pd"),+ ("…", "Po"),+ ('"', "Pe"),+ ("(", "Ps"),+ ("「", "Ps"),+ ("》", "Pe"),+]++# Sentence punctuation the anchoring policy supports: terminators and clause+# separators that ASR emits as structural markup with no acoustic extent.+SUPPORTED_SENTENCE_PUNCTUATION = [+ "。",+ ",",+ "、",+ ";",+ ":",+ "!",+ "?",+ ".",+ ",",+ "!",+ "?",+ ";",+ ":",+]+++def _write_semantic_symbol_vocab(path):+ """Deterministic offline vocab: byte-level base plus merged ranks that+ mirror the official reviewer-case tokenization. Format matches production+ ``.tiktoken`` files (``base64(token) rank`` lines)."""+ lines = []+ for byte in range(256):+ lines.append(f"{base64.b64encode(bytes([byte])).decode()} {byte}")+ next_rank = 256+ # CJK chars and 。/,: prefix pair first so BPE completes each 3-byte+ # merge (tiktoken merges lowest-rank pairs first).+ for char in "我用你好价格是。,":+ raw = char.encode("utf-8")+ assert len(raw) == 3, char+ lines.append(f"{base64.b64encode(raw[:2]).decode()} {next_rank}")+ next_rank += 1+ lines.append(f"{base64.b64encode(raw).decode()} {next_rank}")+ next_rank += 1+ # Multi-char symbol tokens, mirroring official ids 24754 ("++") and 3254+ # ("10"): single merged ranks, so each decodes standalone as one token.+ for token in ("++", "10"):+ lines.append(f"{base64.b64encode(token.encode('utf-8')).decode()} {next_rank}")+ next_rank += 1+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")+++def sensevoice_tokenizer(tmp_path):+ """The real production Tokenizer via the SenseVoiceTokenizer factory."""+ vocab = tmp_path / "semantic-symbol.tiktoken"+ _write_semantic_symbol_vocab(vocab)+ tokenizer = SenseVoiceTokenizer(vocab_path=str(vocab))+ for text in (TEXT_PLUS, TEXT_CURRENCY):+ assert tokenizer.decode(tokenizer.encode(text)) == text+ return tokenizer+++def _peaky_log_probs(target_ids, vocab_size, frames_per_token=FRAMES_PER_TOKEN):+ n_frames = len(target_ids) * frames_per_token+ log_probs = torch.full((n_frames, vocab_size), -30.0)+ for i, tid in enumerate(target_ids):+ log_probs[i * frames_per_token : (i + 1) * frames_per_token, tid] = 0.0+ return log_probs+++def _blank_id(tokenizer):+ return tokenizer.get_vocab_size() - 1+++def _production_timestamps(tokenizer, target_ids):+ """Mirror the production decode-per-id + scale path shared by all three+ Nano timestamp methods, over the repo's real forced_align."""+ items = forced_align(+ _peaky_log_probs(target_ids, tokenizer.get_vocab_size()),+ torch.tensor(target_ids, dtype=torch.int64),+ _blank_id(tokenizer),+ )+ assert [it["token"] for it in items] == list(target_ids)+ return [+ {+ "token": tokenizer.decode([it["token"]]),+ "start_time": it["start_time"] * FRAME_TO_SEC,+ "end_time": it["end_time"] * FRAME_TO_SEC,+ }+ for it in items+ ]+++def test_supported_sentence_punctuation_anchors():+ for token in SUPPORTED_SENTENCE_PUNCTUATION:+ assert _is_sentence_punctuation(token), token+ assert _classify_timestamp_token(token) == "punctuation", token+ # Combinations of supported punctuation stay punctuation.+ assert _is_sentence_punctuation("!?")+ assert _is_sentence_punctuation("!!")+ assert _classify_timestamp_token("!?") == "punctuation"+++def test_semantic_symbols_are_spoken_not_punctuation():+ for token, _category in OFFICIAL_SYMBOL_TOKENS:+ assert _classify_timestamp_token(token) == "spoken", token+ assert not _is_sentence_punctuation(token), token+ # A symbol glued to sentence punctuation must not anchor either.+ assert _classify_timestamp_token('。"') == "spoken"+ assert _classify_timestamp_token("%,") == "spoken"+ # Undecodable byte fragments stay spoken; special tokens keep their class.+ assert _classify_timestamp_token("�") == "spoken"+ assert _classify_timestamp_token("<sil>") == "special"+++def test_symbol_token_ids_decode_standalone(sensevoice_tokenizer):+ tokenizer = sensevoice_tokenizer+ plus_ids = tokenizer.encode(TEXT_PLUS)+ currency_ids = tokenizer.encode(TEXT_CURRENCY)+ assert tokenizer.decode(plus_ids) == TEXT_PLUS+ assert tokenizer.decode(currency_ids) == TEXT_CURRENCY+ # ++ and $ must be single ids that decode standalone (official: one id+ # each, 24754 and 3), not per-character or byte-split.+ plus_tokens = [tokenizer.decode([i]) for i in plus_ids]+ currency_tokens = [tokenizer.decode([i]) for i in currency_ids]+ assert "++" in plus_tokens, plus_tokens+ assert "$" in currency_tokens, currency_tokens+ assert _classify_timestamp_token("++") == "spoken"+ assert _classify_timestamp_token("$") == "spoken"+++def test_semantic_symbol_spans_survive_anchoring(sensevoice_tokenizer):+ tokenizer = sensevoice_tokenizer+ for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):+ target_ids = tokenizer.encode(text)+ timestamps = _production_timestamps(tokenizer, target_ids)+ before = [(t["token"], t["start_time"], t["end_time"]) for t in timestamps]+ anchor_punctuation_timestamps(timestamps)++ symbols = [t for t in timestamps if t["token"] == symbol]+ assert symbols, f"expected {symbol!r} in {text!r}"+ for (token, start, end), ts in zip(before, timestamps):+ if token == symbol:+ # The acoustic span is preserved verbatim: no collapse.+ assert (ts["start_time"], ts["end_time"]) == (start, end)+ assert ts["end_time"] > ts["start_time"], f"{symbol} keeps acoustic extent"+ # Genuine 。 between speech still anchors (zero-width control): it+ # pins at the preceding spoken token's end, which for these texts is+ # the symbol or the token right after it.+ punct = [t for t in timestamps if t["token"] == "。"]+ assert punct+ assert punct[0]["start_time"] == punct[0]["end_time"]+ punct_index = next(i for i, t in enumerate(before) if t[0] == "。")+ assert before[punct_index - 1][2] == punct[0]["start_time"]+++def test_vllm_method_preserves_semantic_symbol_spans(sensevoice_tokenizer):+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM++ tokenizer = sensevoice_tokenizer+ for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):+ target_ids = tokenizer.encode(text)+ n_frames = len(target_ids) * FRAMES_PER_TOKEN+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())++ engine = FunASRNanoVLLM.__new__(FunASRNanoVLLM)+ engine.ctc_decoder = lambda e, l, lp=log_probs, nf=n_frames: (+ lp.unsqueeze(0),+ torch.tensor([nf]),+ )+ engine.ctc = types.SimpleNamespace(log_softmax=lambda d: d)+ engine.ctc_tokenizer = tokenizer+ engine.blank_id = _blank_id(tokenizer)++ result = engine._compute_timestamps(+ torch.zeros(1, n_frames, 8), torch.tensor([n_frames]), text+ )+ symbols = [t for t in result if t["token"] == symbol]+ assert symbols+ for t in symbols:+ assert t["end_time"] > t["start_time"], f"{symbol} span preserved"+ # Genuine punctuation still anchors between speech.+ assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)+++def test_pipeline_method_preserves_semantic_symbol_spans(sensevoice_tokenizer, monkeypatch):+ from funasr.models.fun_asr_nano.inference_vllm_pipeline import (+ FunASRNanoVLLMPipeline,+ )++ tokenizer = sensevoice_tokenizer+ for text, symbol in ((TEXT_PLUS, "++"), (TEXT_CURRENCY, "$")):+ target_ids = tokenizer.encode(text)+ n_frames = len(target_ids) * FRAMES_PER_TOKEN+ log_probs = _peaky_log_probs(target_ids, tokenizer.get_vocab_size())++ monkeypatch.setattr(+ "funasr.utils.load_utils.extract_fbank",+ lambda *a, **k: (torch.zeros(1, 4, 80), torch.tensor([[4]])),+ )+ pipeline = FunASRNanoVLLMPipeline.__new__(FunASRNanoVLLMPipeline)+ pipeline.device = "cpu"+ pipeline.asr_engine = types.SimpleNamespace(+ frontend=object(),+ audio_encoder=lambda s, sl, nf=n_frames: (+ torch.zeros(1, nf, 8),+ torch.tensor([nf]),+ ),+ ctc_decoder=lambda e, l, lp=log_probs, nf=n_frames: (+ lp.unsqueeze(0),+ torch.tensor([nf]),+ ),+ ctc=types.SimpleNamespace(log_softmax=lambda d: d),+ ctc_tokenizer=tokenizer,+ blank_id=_blank_id(tokenizer),+ )+ result = pipeline._compute_all_timestamps(+ [np.zeros(32000, dtype=np.float32)], [[1000, 9000]], [text]+ )+ symbols = [t for t in result if t["token"] == symbol]+ assert symbols+ # +1s VAD offset applied on top of preserved spans: compare against+ # the un-offset helper path for the identical target sequence.+ expected = _production_timestamps(tokenizer, target_ids)+ expected_symbols = [t for t in expected if t["token"] == symbol]+ assert len(symbols) == len(expected_symbols)+ for got, want in zip(symbols, expected_symbols):+ assert got["start_time"] == pytest.approx(want["start_time"] + 1.0)+ assert got["end_time"] == pytest.approx(want["end_time"] + 1.0)+ assert got["end_time"] > got["start_time"], f"{symbol} span preserved"+ # Genuine punctuation still anchors between speech.+ assert any(t["token"] == "。" and t["start_time"] == t["end_time"] for t in result)