Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
decoding
游雁67329a74a529
diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.pyindex 22272eefc..9be5abe54 100644--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@ -16,6 +16,7 @@ from funasr.train_utils.device_funcs import force_gatherablefrom . import whisper_lib as whisperfrom funasr.utils.load_utils import load_audio_text_image_video, extract_fbankfrom funasr.utils.datadir_writer import DatadirWriter+from funasr.models.ctc.ctc import CTCfrom funasr.register import tables@@ -1264,7 +1265,7 @@ class SenseVoiceSANM(nn.Module):if isinstance(task, str):task = [task]task = "".join([f"<|{x}|>" for x in task])-+sos = kwargs.get("model_conf").get("sos")if isinstance(sos, str):initial_prompt = kwargs.get("initial_prompt", f"<|startoftranscript|>{task}")@@ -1278,7 +1279,9 @@ class SenseVoiceSANM(nn.Module):language = DecodingOptions.get("language", None)language = None if language == "auto" else languageinitial_prompt = kwargs.get("initial_prompt", f"{task}")- initial_prompt_lid = f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt+ initial_prompt_lid = (+ f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt+ )initial_prompt_lid_int = tokenizer.encode(initial_prompt_lid, allowed_special="all")sos_int = [sos] + initial_prompt_lid_inteos = kwargs.get("model_conf").get("eos")@@ -1311,9 +1314,7 @@ class SenseVoiceSANM(nn.Module):)self.beam_search.event_score_ga = DecodingOptions.get("gain_tokens_score", [1, 1, 1, 1])- encoder_out, encoder_out_lens = self.encode(- speech[None, :, :], speech_lengths- )+ encoder_out, encoder_out_lens = self.encode(speech[None, :, :], speech_lengths)if text_token_int is not None:i = 0@@ -1392,3 +1393,250 @@ class SenseVoiceSANM(nn.Module):ibest_writer["text"][key[i]] = textreturn results, meta_data+++from funasr.models.paraformer.search import Hypothesis+from funasr.utils import postprocess_utils++[email protected]("model_classes", "SenseVoiceSANMCTC")+class SenseVoiceSANMCTC(nn.Module):+ """CTC-attention hybrid Encoder-Decoder model"""++ def __init__(+ self,+ specaug: str = None,+ specaug_conf: dict = None,+ normalize: str = None,+ normalize_conf: dict = None,+ encoder: str = None,+ encoder_conf: dict = None,+ ctc_conf: dict = None,+ input_size: int = 80,+ vocab_size: int = -1,+ ignore_id: int = -1,+ blank_id: int = 0,+ sos: int = 1,+ eos: int = 2,+ length_normalized_loss: bool = False,+ **kwargs,+ ):++ super().__init__()++ if specaug is not None:+ specaug_class = tables.specaug_classes.get(specaug)+ specaug = specaug_class(**specaug_conf)+ if normalize is not None:+ normalize_class = tables.normalize_classes.get(normalize)+ normalize = normalize_class(**normalize_conf)+ encoder_class = tables.encoder_classes.get(encoder)+ encoder = encoder_class(input_size=input_size, **encoder_conf)+ encoder_output_size = encoder.output_size()++ if ctc_conf is None:+ ctc_conf = {}+ ctc = CTC(odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf)++ self.blank_id = blank_id+ self.sos = sos if sos is not None else vocab_size - 1+ self.eos = eos if eos is not None else vocab_size - 1+ self.vocab_size = vocab_size+ self.ignore_id = ignore_id+ self.specaug = specaug+ self.normalize = normalize+ self.encoder = encoder+ self.error_calculator = None++ self.ctc = ctc++ self.length_normalized_loss = length_normalized_loss++ def forward(+ self,+ speech: torch.Tensor,+ speech_lengths: torch.Tensor,+ text: torch.Tensor,+ text_lengths: torch.Tensor,+ **kwargs,+ ):+ """Encoder + Decoder + Calc loss+ Args:+ speech: (Batch, Length, ...)+ speech_lengths: (Batch, )+ text: (Batch, Length)+ text_lengths: (Batch,)+ """+ # import pdb;+ # pdb.set_trace()+ if len(text_lengths.size()) > 1:+ text_lengths = text_lengths[:, 0]+ if len(speech_lengths.size()) > 1:+ speech_lengths = speech_lengths[:, 0]++ batch_size = speech.shape[0]++ # 1. Encoder+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)++ loss_ctc, cer_ctc = None, None+ stats = dict()++ loss_ctc, cer_ctc = self._calc_ctc_loss(encoder_out, encoder_out_lens, text, text_lengths)++ loss = loss_ctc++ # Collect total loss stats+ stats["loss"] = torch.clone(loss.detach())++ # force_gatherable: to-device and to-tensor if scalar for DataParallel+ if self.length_normalized_loss:+ batch_size = int((text_lengths + 1).sum())+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)+ return loss, stats, weight++ def encode(+ self,+ speech: torch.Tensor,+ speech_lengths: torch.Tensor,+ **kwargs,+ ):+ """Frontend + Encoder. Note that this method is used by asr_inference.py+ Args:+ speech: (Batch, Length, ...)+ speech_lengths: (Batch, )+ ind: int+ """++ # Data augmentation+ if self.specaug is not None and self.training:+ speech, speech_lengths = self.specaug(speech, speech_lengths)++ # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN+ if self.normalize is not None:+ speech, speech_lengths = self.normalize(speech, speech_lengths)++ # Forward encoder+ # feats: (Batch, Length, Dim)+ # -> encoder_out: (Batch, Length2, Dim2)+ encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)++ return encoder_out, encoder_out_lens++ def _calc_ctc_loss(+ self,+ encoder_out: torch.Tensor,+ encoder_out_lens: torch.Tensor,+ ys_pad: torch.Tensor,+ ys_pad_lens: torch.Tensor,+ ):+ # Calc CTC loss+ loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)++ # Calc CER using CTC+ cer_ctc = None+ if not self.training and self.error_calculator is not None:+ ys_hat = self.ctc.argmax(encoder_out).data+ cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)+ return loss_ctc, cer_ctc++ def inference(+ self,+ data_in,+ data_lengths=None,+ key: list = None,+ tokenizer=None,+ frontend=None,+ **kwargs,+ ):++ if kwargs.get("batch_size", 1) > 1:+ raise NotImplementedError("batch decoding is not implemented")++ meta_data = {}+ if (+ isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"+ ): # fbank+ speech, speech_lengths = data_in, data_lengths+ if len(speech.shape) < 3:+ speech = speech[None, :, :]+ if speech_lengths is None:+ speech_lengths = speech.shape[1]+ else:+ # extract fbank feats+ time1 = time.perf_counter()+ audio_sample_list = load_audio_text_image_video(+ data_in,+ fs=frontend.fs,+ audio_fs=kwargs.get("fs", 16000),+ data_type=kwargs.get("data_type", "sound"),+ tokenizer=tokenizer,+ )+ time2 = time.perf_counter()+ meta_data["load_data"] = f"{time2 - time1:0.3f}"+ speech, speech_lengths = extract_fbank(+ audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend+ )+ time3 = time.perf_counter()+ meta_data["extract_feat"] = f"{time3 - time2:0.3f}"+ meta_data["batch_data_time"] = (+ speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000+ )++ speech = speech.to(device=kwargs["device"])+ speech_lengths = speech_lengths.to(device=kwargs["device"])+ # Encoder+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)+ if isinstance(encoder_out, tuple):+ encoder_out = encoder_out[0]++ # c. Passed the encoder result and the beam search+ ctc_logits = self.ctc.log_softmax(encoder_out)++ results = []+ b, n, d = encoder_out.size()+ if isinstance(key[0], (list, tuple)):+ key = key[0]+ if len(key) < b:+ key = key * b+ for i in range(b):+ x = ctc_logits[i, : encoder_out_lens[i], :]+ yseq = x.argmax(dim=-1)+ yseq = torch.unique_consecutive(yseq, dim=-1)+ yseq = torch.tensor([self.sos] + yseq.tolist() + [self.eos], device=yseq.device)+ nbest_hyps = [Hypothesis(yseq=yseq)]++ for nbest_idx, hyp in enumerate(nbest_hyps):+ ibest_writer = None+ if kwargs.get("output_dir") is not None:+ if not hasattr(self, "writer"):+ self.writer = DatadirWriter(kwargs.get("output_dir"))+ ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"]++ # remove sos/eos and get results+ last_pos = -1+ if isinstance(hyp.yseq, list):+ token_int = hyp.yseq[1:last_pos]+ else:+ token_int = hyp.yseq[1:last_pos].tolist()++ # remove blank symbol id, which is assumed to be 0+ token_int = list(+ filter(+ lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int+ )+ )++ # Change integer-ids to tokens+ token = tokenizer.ids2tokens(token_int)+ text = tokenizer.tokens2text(token)++ text_postprocessed, _ = postprocess_utils.sentence_postprocess(token)+ result_i = {"key": key[i], "token": token, "text": text_postprocessed}+ results.append(result_i)++ if ibest_writer is not None:+ ibest_writer["token"][key[i]] = " ".join(token)+ ibest_writer["text"][key[i]] = text_postprocessed++ return results, meta_data