Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
sensevoice sanm
游雁7d06f581dbe6
diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.pyindex 56e61e7ab..a633a8d13 100644--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@ -966,3 +966,419 @@ class SenseVoiceFSMN(nn.Module):ibest_writer["text"][key[i]] = textreturn results, meta_data++[email protected]("model_classes", "SenseVoiceSANM")+class SenseVoiceSANM(nn.Module):++ def __init__(+ self,+ specaug: str = None,+ specaug_conf: dict = None,+ normalize: str = None,+ normalize_conf: dict = None,+ encoder: str = None,+ encoder_conf: dict = None,+ decoder: str = None,+ decoder_conf: dict = None,+ input_size: int = 80,+ vocab_size: int = -1,+ ignore_id: int = -1,+ blank_id: int = 0,+ sos: int = 1,+ eos: int = 2,+ lsm_weight: float = 0.0,+ length_normalized_loss: bool = False,+ report_cer: bool = True,+ report_wer: bool = True,+ sym_space: str = "<space>",+ sym_blank: str = "<blank>",+ # extract_feats_in_collect_stats: bool = True,+ share_embedding: bool = False,+ # preencoder: Optional[AbsPreEncoder] = None,+ # postencoder: Optional[AbsPostEncoder] = None,+ **kwargs,+ ):++ super().__init__()++ if specaug is not None:+ specaug_class = tables.specaug_classes.get(specaug)+ specaug = specaug_class(**specaug_conf)+ if normalize is not None:+ normalize_class = tables.normalize_classes.get(normalize)+ normalize = normalize_class(**normalize_conf)+ encoder_class = tables.encoder_classes.get(encoder)+ encoder = encoder_class(input_size=input_size, **encoder_conf)+ encoder_output_size = encoder.output_size()++ decoder_class = tables.decoder_classes.get(decoder)+ decoder = decoder_class(+ vocab_size=vocab_size,+ encoder_output_size=encoder_output_size,+ **decoder_conf,+ )++ self.blank_id = blank_id+ self.sos = sos if sos is not None else vocab_size - 1+ self.eos = eos if eos is not None else vocab_size - 1+ self.vocab_size = vocab_size+ self.ignore_id = ignore_id++ self.specaug = specaug+ self.normalize = normalize+ self.encoder = encoder++ self.decoder = decoder++ self.criterion_att = LabelSmoothingLoss(+ size=vocab_size,+ padding_idx=ignore_id,+ smoothing=lsm_weight,+ normalize_length=length_normalized_loss,+ )++ self.error_calculator = None++ self.share_embedding = share_embedding+ if self.share_embedding:+ self.decoder.embed = None++ self.length_normalized_loss = length_normalized_loss+ self.beam_search = None++ def forward(+ self,+ speech: torch.Tensor,+ speech_lengths: torch.Tensor,+ text: torch.Tensor,+ text_lengths: torch.Tensor,+ **kwargs,+ ):+ target_mask = kwargs.get("target_mask", None)++ # import pdb;+ # pdb.set_trace()+ if len(text_lengths.size()) > 1:+ text_lengths = text_lengths[:, 0]+ if len(speech_lengths.size()) > 1:+ speech_lengths = speech_lengths[:, 0]++ batch_size, frames, _ = speech.shape+ _, text_tokens = text.shape++ if self.activation_checkpoint:+ from torch.utils.checkpoint import checkpoint++ encoder_out, encoder_out_lens = checkpoint(+ self.encode, speech, speech_lengths, use_reentrant=False+ )+ else:+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)++ loss_att, acc_att, cer_att, wer_att = self._calc_att_loss(+ encoder_out, encoder_out_lens, text, text_lengths, target_mask=target_mask+ )++ loss = loss_att+ stats = {}+ stats["acc"] = acc_att+ stats["loss"] = torch.clone(loss.detach())+ stats["batch_size"] = batch_size+ stats["batch_size_x_frames"] = frames * batch_size+ stats["batch_size_real_frames"] = speech_lengths.sum().item()+ stats["padding_frames"] = stats["batch_size_x_frames"] - stats["batch_size_real_frames"]+ stats["batch_size_x_tokens"] = text_tokens * batch_size+ stats["batch_size_real_tokens"] = text_lengths.sum().item()+ stats["padding_tokens"] = stats["batch_size_x_tokens"] - stats["batch_size_real_tokens"]+ stats["batch_size_x_frames_plus_tokens"] = (text_tokens + frames) * batch_size++ # force_gatherable: to-device and to-tensor if scalar for DataParallel+ if self.length_normalized_loss:+ batch_size = int((text_lengths + 1).sum())+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)+ return loss, stats, weight++ def encode(+ self,+ speech: torch.Tensor,+ speech_lengths: torch.Tensor,+ **kwargs,+ ):+ """Frontend + Encoder. Note that this method is used by asr_inference.py+ Args:+ speech: (Batch, Length, ...)+ speech_lengths: (Batch, )+ ind: int+ """+ with autocast(False):++ # Data augmentation+ if self.specaug is not None and self.training:+ speech, speech_lengths = self.specaug(speech, speech_lengths)++ # Forward encoder+ # feats: (Batch, Length, Dim)+ # -> encoder_out: (Batch, Length2, Dim2)++ encoder_out, encoder_out_lens, _ = self.encoder(speech, speech_lengths)+ if isinstance(encoder_out, (tuple, list)):+ encoder_out = encoder_out[0]++ return encoder_out, encoder_out_lens++ def _calc_att_loss(+ self,+ encoder_out: torch.Tensor,+ encoder_out_lens: torch.Tensor,+ ys_pad: torch.Tensor,+ ys_pad_lens: torch.Tensor,+ **kwargs,+ ):+ target_mask = kwargs.get("target_mask", None)+ stats = {}++ # 1. Forward decoder+ decoder_out = self.decoder(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)+ if isinstance(decoder_out, (list, tuple)):+ decoder_out = decoder_out[0]++ # 2. Compute attention loss+ mask = torch.ones_like(ys_pad) * (-1)+ ys_pad_mask = (ys_pad * target_mask + mask * (1 - target_mask)).to(torch.int64)+ ys_pad_mask[ys_pad_mask == 0] = -1+ loss_att = self.criterion_att(decoder_out[:, :-1, :], ys_pad_mask[:, 1:])++ with torch.no_grad():+ preds = torch.argmax(decoder_out, -1)+ acc_att = compute_accuracy(+ preds[:, :-1], ys_pad_mask[:, 1:], ignore_label=self.ignore_id+ )++ return loss_att, acc_att, None, None++ def init_beam_search(+ self,+ **kwargs,+ ):+ from .search import BeamSearch++ from funasr.models.transformer.scorers.length_bonus import LengthBonus++ # 1. Build ASR model+ scorers = {}++ scorers.update(+ decoder=self.decoder,+ length_bonus=LengthBonus(self.vocab_size),+ )++ weights = dict(+ decoder=1.0,+ ctc=0.0,+ lm=0.0,+ ngram=0.0,+ length_bonus=kwargs.get("penalty", 0.0),+ )+ beam_search = BeamSearch(+ beam_size=kwargs.get("beam_size", 5),+ weights=weights,+ scorers=scorers,+ sos=None,+ eos=None,+ vocab_size=self.vocab_size,+ token_list=None,+ pre_beam_score_key="full",+ )++ self.beam_search = beam_search++ def inference(+ self,+ data_in,+ data_lengths=None,+ key: list = None,+ tokenizer=None,+ frontend=None,+ **kwargs,+ ):+ if kwargs.get("batch_size", 1) > 1:+ raise NotImplementedError("batch decoding is not implemented")++ # init beamsearch+ if not hasattr(self, "beam_search") or self.beam_search is None:+ logging.info("enable beam_search")+ self.init_beam_search(**kwargs)+ self.nbest = kwargs.get("nbest", 1)++ if frontend is None and not hasattr(self, "frontend"):+ frontend_class = tables.frontend_classes.get("WhisperFrontend")+ frontend = frontend_class(+ n_mels=self.model.dims.n_mels, do_pad_trim=kwargs.get("do_pad_trim", True)+ )+ self.frontend = frontend+ else:+ frontend = frontend if frontend is not None else self.frontend++ meta_data = {}+ if (+ isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"+ ): # fbank+ speech, speech_lengths = data_in, data_lengths+ if len(speech.shape) < 3:+ speech = speech[None, :, :]+ if speech_lengths is None:+ speech_lengths = speech.shape[1]+ else:+ # extract fbank feats+ time1 = time.perf_counter()+ audio_sample_list = load_audio_text_image_video(+ data_in,+ fs=frontend.fs if hasattr(frontend, "fs") else 16000,+ audio_fs=kwargs.get("fs", 16000),+ data_type=kwargs.get("data_type", "sound"),+ tokenizer=tokenizer,+ )++ if (+ isinstance(kwargs.get("data_type", None), (list, tuple))+ and len(kwargs.get("data_type", [])) > 1+ ):+ audio_sample_list, text_token_int_list = audio_sample_list+ text_token_int = text_token_int_list[0]+ else:+ text_token_int = None++ time2 = time.perf_counter()+ meta_data["load_data"] = f"{time2 - time1:0.3f}"+ speech, speech_lengths = extract_fbank(+ audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend+ )+ time3 = time.perf_counter()+ meta_data["extract_feat"] = f"{time3 - time2:0.3f}"+ frame_shift = frontend.frame_shift if hasattr(frontend, "frame_shift") else 10+ lfr_n = frontend.lfr_n if hasattr(frontend, "lfr_n") else 1+ meta_data["batch_data_time"] = speech_lengths.sum().item() * frame_shift * lfr_n / 1000++ speech = speech.to(device=kwargs["device"])[0, :, :]+ speech_lengths = speech_lengths.to(device=kwargs["device"])++ DecodingOptions = kwargs.get("DecodingOptions", {})+ task = DecodingOptions.get("task", "ASR")+ if isinstance(task, str):+ task = [task]+ task = "".join([f"<|{x}|>" for x in task])+ initial_prompt = kwargs.get("initial_prompt", f"<|startoftranscript|>{task}")++ language = DecodingOptions.get("language", None)+ language = None if language == "auto" else language++ sos = f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt+ sos_int = tokenizer.encode(sos, allowed_special="all")+ eos = kwargs.get("model_conf").get("eos")+ eos_int = tokenizer.encode(eos, allowed_special="all")+ self.beam_search.sos = sos_int+ self.beam_search.eos = eos_int[0]++ # Paramterts for rich decoding+ self.beam_search.emo_unk = tokenizer.encode(+ DecodingOptions.get("emo_unk_token", "<|SPECIAL_TOKEN_1|>"), allowed_special="all"+ )[0]+ self.beam_search.emo_unk_score = 1+ self.beam_search.emo_tokens = tokenizer.encode(+ DecodingOptions.get("emo_target_tokens", "<|HAPPY|><|SAD|><|ANGRY|>"),+ allowed_special="all",+ )+ self.beam_search.emo_scores = DecodingOptions.get("emo_target_threshold", [0.1, 0.1, 0.1])++ self.beam_search.event_bg_token = tokenizer.encode(+ DecodingOptions.get("gain_tokens_bg", "<|Speech|><|BGM|><|Applause|><|Laughter|>"),+ allowed_special="all",+ )+ self.beam_search.event_ed_token = tokenizer.encode(+ DecodingOptions.get("gain_tokens_ed", "<|/Speech|><|/BGM|><|/Applause|><|/Laughter|>"),+ allowed_special="all",+ )+ self.beam_search.event_score_ga = DecodingOptions.get("gain_tokens_score", [1, 1, 1, 1])++ encoder_out, encoder_out_lens = self.encode(+ speech[None, :, :].permute(0, 2, 1), speech_lengths+ )++ if text_token_int is not None:+ i = 0+ results = []+ ibest_writer = None+ if kwargs.get("output_dir") is not None:+ if not hasattr(self, "writer"):+ self.writer = DatadirWriter(kwargs.get("output_dir"))+ ibest_writer = self.writer[f"1best_recog"]++ # 1. Forward decoder+ ys_pad = torch.tensor(sos_int + text_token_int, dtype=torch.int64).to(kwargs["device"])[+ None, :+ ]+ ys_pad_lens = torch.tensor([len(sos_int + text_token_int)], dtype=torch.int64).to(+ kwargs["device"]+ )[None, :]+ decoder_out = self.model.decoder(+ x=ys_pad, xa=encoder_out, hlens=encoder_out_lens, ys_in_lens=ys_pad_lens+ )++ token_int = decoder_out.argmax(-1)[0, :].tolist()+ text = tokenizer.decode(token_int)++ result_i = {"key": key[i], "text": text}+ results.append(result_i)++ if ibest_writer is not None:+ # ibest_writer["token"][key[i]] = " ".join(token)+ ibest_writer["text"][key[i]] = text+ return results, meta_data++ # c. Passed the encoder result and the beam search+ nbest_hyps = self.beam_search(+ x=encoder_out[0],+ maxlenratio=kwargs.get("maxlenratio", 0.0),+ minlenratio=kwargs.get("minlenratio", 0.0),+ )++ nbest_hyps = nbest_hyps[: self.nbest]++ results = []+ b, n, d = encoder_out.size()+ for i in range(b):++ for nbest_idx, hyp in enumerate(nbest_hyps):+ ibest_writer = None+ if kwargs.get("output_dir") is not None:+ if not hasattr(self, "writer"):+ self.writer = DatadirWriter(kwargs.get("output_dir"))+ ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"]++ # remove sos/eos and get results+ last_pos = -1+ if isinstance(hyp.yseq, list):+ token_int = hyp.yseq[1:last_pos]+ else:+ token_int = hyp.yseq[1:last_pos].tolist()++ # # remove blank symbol id, which is assumed to be 0+ # token_int = list(+ # filter(+ # lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int+ # )+ # )++ # Change integer-ids to tokens+ # token = tokenizer.ids2tokens(token_int)+ text = tokenizer.decode(token_int)++ result_i = {"key": key[i], "text": text}+ results.append(result_i)++ if ibest_writer is not None:+ # ibest_writer["token"][key[i]] = " ".join(token)+ ibest_writer["text"][key[i]] = text++ return results, meta_data