Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix: use converted token_ids for alignment for sensevoice model with timestamp output (#2429)
Han Zhang3c349ac0531b
diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.pyindex 945a30181..7971cc1e2 100644--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@ -919,14 +919,20 @@ class SenseVoiceSmall(nn.Module):timestamp = []tokens = tokenizer.text2tokens(text)[4:]+ token_back_to_id = tokenizer.tokens2ids(tokens)+ token_ids = []+ for tok_ls in token_back_to_id:+ if tok_ls: token_ids.extend(tok_ls)+ else: token_ids.append(124)+logits_speech = self.ctc.softmax(encoder_out)[i, 4 : encoder_out_lens[i].item(), :]pred = logits_speech.argmax(-1).cpu()logits_speech[pred == self.blank_id, self.blank_id] = 0align = ctc_forced_align(logits_speech.unsqueeze(0).float(),- torch.Tensor(token_int[4:]).unsqueeze(0).long().to(logits_speech.device),+ torch.Tensor(token_ids).unsqueeze(0).long().to(logits_speech.device),(encoder_out_lens[i] - 4).long(),- torch.tensor(len(token_int) - 4).unsqueeze(0).long().to(logits_speech.device),+ torch.tensor(len(token_ids)).unsqueeze(0).long().to(logits_speech.device),ignore_id=self.ignore_id,)pred = groupby(align[0, : encoder_out_lens[i]])diff --git a/funasr/utils/timestamp_tools.py b/funasr/utils/timestamp_tools.pyindex 995e179c4..37ce88683 100644--- a/funasr/utils/timestamp_tools.py+++ b/funasr/utils/timestamp_tools.py@@ -160,7 +160,7 @@ def timestamp_sentence(punc_id = int(punc_id) if punc_id is not None else 1sentence_end = timestamp[1] if timestamp is not None else sentence_endsentence_text_seg = (- sentence_text_seg[:-1] if sentence_text_seg[-1] == " " else sentence_text_seg+ sentence_text_seg[:-1] if sentence_text_seg and sentence_text_seg[-1] == " " else sentence_text_seg)if punc_id > 1:sentence_text += punc_list[punc_id - 2]