Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
decoding
游雁45d7aa900476
diff --cc docs/images/wechat.pngindex 705ff75de,491a1c3df..8d37700c9Binary files differdiff --cc examples/industrial_data_pretraining/ctc/demo.pyindex 85a748a65,1afb4b54f..b9d1647b7--- a/examples/industrial_data_pretraining/ctc/demo.py+++ b/examples/industrial_data_pretraining/ctc/demo.py@@@ -6,8 -6,10 +6,11 @@@import sysfrom funasr import AutoModel- model_dir=sys.argv[1]- input_file=sys.argv[2]+++ model_dir = "/Users/zhifu/Downloads/modelscope_models/ctc_model"+ input_file = (+ "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav"+ )model = AutoModel(model=model_dir,diff --cc funasr/auto/auto_model.pyindex 91e80d8be,e30eb0976..a6cd3a652--- a/funasr/auto/auto_model.py+++ b/funasr/auto/auto_model.py@@@ -324,7 -324,7 +324,7 @@@ class AutoModelinput, input_len=input_len, model=self.vad_model, kwargs=self.vad_kwargs, **cfg)end_vad = time.time()--++# FIX(gcf): concat the vad clips for sense vocie model for better aedif kwargs.get("merge_vad", False):for i in range(len(res)):@@@ -466,22 -466,25 +466,22 @@@result[k] = restored_data[j][k]else:result[k] += restored_data[j][k]-++ if not len(result["text"].strip()):+ continuereturn_raw_text = kwargs.get("return_raw_text", False)# step.3 compute punc model+ raw_text = Noneif self.punc_model is not None:- if not len(result["text"].strip()):- if return_raw_text:- result["raw_text"] = ""- else:- deep_update(self.punc_kwargs, cfg)- punc_res = self.inference(- result["text"], model=self.punc_model, kwargs=self.punc_kwargs, **cfg- )- raw_text = copy.copy(result["text"])- if return_raw_text:- result["raw_text"] = raw_text- result["text"] = punc_res[0]["text"]- else:- raw_text = None+ deep_update(self.punc_kwargs, cfg)+ punc_res = self.inference(+ result["text"], model=self.punc_model, kwargs=self.punc_kwargs, **cfg+ )+ raw_text = copy.copy(result["text"])+ if return_raw_text:+ result["raw_text"] = raw_text+ result["text"] = punc_res[0]["text"]-+# speaker embedding cluster after resortedif self.spk_model is not None and kwargs.get("return_spk_res", True):if raw_text is None:@@@ -602,6 -605,12 +602,6 @@@)with torch.no_grad():- export_dir = export_utils.export(model=model, data_in=data_list, **kwargs)-- if type == "onnx":- export_dir = export_utils.export_onnx(model=model, data_in=data_list, **kwargs)- else:- export_dir = export_utils.export_torchscripts(- model=model, data_in=data_list, **kwargs- )++ export_dir = export_utils.export(model=model, data_in=data_list, **kwargs)return export_dirdiff --cc funasr/models/paraformer/cif_predictor.pyindex 05e283a24,7490310c9..0856eede2--- a/funasr/models/paraformer/cif_predictor.py+++ b/funasr/models/paraformer/cif_predictor.py@@@ -494,6 -494,6 +494,8 @@@ class CifPredictorV2Export(torch.nn.Modtoken_num_floor = torch.floor(token_num)return hidden, alphas, token_num_floor++++@torch.jit.scriptdef cif_v1_export(hidden, alphas, threshold: float):device = hidden.device@@@ -516,9 -516,9 +518,7 @@@fires[fire_idxs] = 1fires = fires + prefix_sum - prefix_sum_floor-- prefix_sum_hidden = torch.cumsum(-- alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1-- )++ prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)frames = prefix_sum_hidden[fire_idxs]shift_frames = torch.roll(frames, 1, dims=0)@@@ -530,26 -530,25 +530,22 @@@shift_frames[shift_batch_idxs] = 0remains = fires - torch.floor(fires)-- remain_frames = (-- remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]-- )++ remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]shift_remain_frames = torch.roll(remain_frames, 1, dims=0)shift_remain_frames[shift_batch_idxs] = 0frames = frames - shift_frames + shift_remain_frames - remain_frames- max_label_len = alphas.sum(dim=-1)- max_label_len = torch.floor(max_label_len).max().to(dtype=torch.int64)+ max_label_len = batch_len.max()-- frame_fires = torch.zeros(-- batch_size, max_label_len, hidden_size, dtype=dtype, device=device-- )++ frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)frame_fires_idxs = indices < batch_len.unsqueeze(1)frame_fires[frame_fires_idxs] = framesreturn frame_fires, fires++@torch.jit.scriptdef cif_export(hidden, alphas, threshold: float):batch_size, len_time, hidden_size = hidden.size()@@@ -693,11 -692,11 +689,8 @@@ def cif_v1(hidden, alphas, threshold)device = hidden.devicedtype = hidden.dtypebatch_size, len_time, hidden_size = hidden.size()-- frames = torch.zeros(batch_size, len_time, hidden_size,-- dtype=dtype, device=device)-- prefix_sum_hidden = torch.cumsum(-- alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1-- )++ frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)++ prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)frames = prefix_sum_hidden[fire_idxs]shift_frames = torch.roll(frames, 1, dims=0)@@@ -709,21 -708,21 +702,16 @@@shift_frames[shift_batch_idxs] = 0remains = fires - torch.floor(fires)-- remain_frames = (-- remains[fire_idxs].unsqueeze(-1).tile((1,-- hidden_size)) * hidden[fire_idxs]-- )++ remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]shift_remain_frames = torch.roll(remain_frames, 1, dims=0)shift_remain_frames[shift_batch_idxs] = 0frames = frames - shift_frames + shift_remain_frames - remain_frames- max_label_len = torch.round(alphas.sum(-1)).int().max() # torch.round to calculate the max length+ max_label_len = batch_len.max()-- frame_fires = torch.zeros(-- batch_size, max_label_len, hidden_size, dtype=dtype, device=device-- )++ frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)frame_fires_idxs = indices < batch_len.unsqueeze(1)frame_fires[frame_fires_idxs] = framesdiff --cc funasr/models/sense_voice/model.pyindex 97f1b1913,697f50c00..a9b2149af--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@@ -1384,3 -1394,269 +1386,279 @@@ class SenseVoiceSANM(nn.Module)ibest_writer["text"][key[i]] = textreturn results, meta_data+++ from funasr.models.paraformer.search import Hypothesis+ from funasr.utils import postprocess_utils+++ @tables.register("model_classes", "SenseVoiceSANMCTC")+ class SenseVoiceSANMCTC(nn.Module):+ """CTC-attention hybrid Encoder-Decoder model"""++ def __init__(+ self,+ specaug: str = None,+ specaug_conf: dict = None,+ normalize: str = None,+ normalize_conf: dict = None,+ encoder: str = None,+ encoder_conf: dict = None,+ ctc_conf: dict = None,+ input_size: int = 80,+ vocab_size: int = -1,+ ignore_id: int = -1,+ blank_id: int = 0,+ sos: int = 1,+ eos: int = 2,+ length_normalized_loss: bool = False,+ **kwargs,+ ):++ super().__init__()++ if specaug is not None:+ specaug_class = tables.specaug_classes.get(specaug)+ specaug = specaug_class(**specaug_conf)+ if normalize is not None:+ normalize_class = tables.normalize_classes.get(normalize)+ normalize = normalize_class(**normalize_conf)+ encoder_class = tables.encoder_classes.get(encoder)+ encoder = encoder_class(input_size=input_size, **encoder_conf)+ encoder_output_size = encoder.output_size()++ if ctc_conf is None:+ ctc_conf = {}+ ctc = CTC(odim=vocab_size, encoder_output_size=encoder_output_size, **ctc_conf)++ self.blank_id = blank_id+ self.sos = sos if sos is not None else vocab_size - 1+ self.eos = eos if eos is not None else vocab_size - 1+ self.vocab_size = vocab_size+ self.ignore_id = ignore_id+ self.specaug = specaug+ self.normalize = normalize+ self.encoder = encoder+ self.error_calculator = None++ self.ctc = ctc++ self.length_normalized_loss = length_normalized_loss+ self.encoder_output_size = encoder_output_size++ self.lid_dict = {"zh": 3, "en": 4, "yue": 7, "ja": 11, "ko": 12, "nospeech": 13}+ self.textnorm_dict = {"withtextnorm": 14, "wotextnorm": 15}+ self.embed = torch.nn.Embedding(8 + len(self.lid_dict) + len(self.textnorm_dict), 560)++ def forward(+ self,+ speech: torch.Tensor,+ speech_lengths: torch.Tensor,+ text: torch.Tensor,+ text_lengths: torch.Tensor,+ **kwargs,+ ):+ """Encoder + Decoder + Calc loss+ Args:+ speech: (Batch, Length, ...)+ speech_lengths: (Batch, )+ text: (Batch, Length)+ text_lengths: (Batch,)+ """+ # import pdb;+ # pdb.set_trace()+ if len(text_lengths.size()) > 1:+ text_lengths = text_lengths[:, 0]+ if len(speech_lengths.size()) > 1:+ speech_lengths = speech_lengths[:, 0]++ batch_size = speech.shape[0]++ # 1. Encoder+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)++ loss_ctc, cer_ctc = None, None+ stats = dict()++ loss_ctc, cer_ctc = self._calc_ctc_loss(encoder_out, encoder_out_lens, text, text_lengths)++ loss = loss_ctc++ # Collect total loss stats+ stats["loss"] = torch.clone(loss.detach())++ # force_gatherable: to-device and to-tensor if scalar for DataParallel+ if self.length_normalized_loss:+ batch_size = int((text_lengths + 1).sum())+ loss, stats, weight = force_gatherable((loss, stats, batch_size), loss.device)+ return loss, stats, weight++ def encode(+ self,+ speech: torch.Tensor,+ speech_lengths: torch.Tensor,+ **kwargs,+ ):+ """Frontend + Encoder. Note that this method is used by asr_inference.py+ Args:+ speech: (Batch, Length, ...)+ speech_lengths: (Batch, )+ ind: int+ """++ # Data augmentation+ if self.specaug is not None and self.training:+ speech, speech_lengths = self.specaug(speech, speech_lengths)++ # Normalization for feature: e.g. Global-CMVN, Utterance-CMVN+ if self.normalize is not None:+ speech, speech_lengths = self.normalize(speech, speech_lengths)++ # Forward encoder+ # feats: (Batch, Length, Dim)+ # -> encoder_out: (Batch, Length2, Dim2)+ encoder_out, encoder_out_lens = self.encoder(speech, speech_lengths)++ return encoder_out, encoder_out_lens++ def _calc_ctc_loss(+ self,+ encoder_out: torch.Tensor,+ encoder_out_lens: torch.Tensor,+ ys_pad: torch.Tensor,+ ys_pad_lens: torch.Tensor,+ ):+ # Calc CTC loss+ loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)++ # Calc CER using CTC+ cer_ctc = None+ if not self.training and self.error_calculator is not None:+ ys_hat = self.ctc.argmax(encoder_out).data+ cer_ctc = self.error_calculator(ys_hat.cpu(), ys_pad.cpu(), is_ctc=True)+ return loss_ctc, cer_ctc++ def inference(+ self,+ data_in,+ data_lengths=None,+ key: list = None,+ tokenizer=None,+ frontend=None,+ **kwargs,+ ):++ if kwargs.get("batch_size", 1) > 1:+ raise NotImplementedError("batch decoding is not implemented")++ meta_data = {}+ if (+ isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"+ ): # fbank+ speech, speech_lengths = data_in, data_lengths+ if len(speech.shape) < 3:+ speech = speech[None, :, :]+ if speech_lengths is None:+ speech_lengths = speech.shape[1]+ else:+ # extract fbank feats+ time1 = time.perf_counter()+ audio_sample_list = load_audio_text_image_video(+ data_in,+ fs=frontend.fs,+ audio_fs=kwargs.get("fs", 16000),+ data_type=kwargs.get("data_type", "sound"),+ tokenizer=tokenizer,+ )+ time2 = time.perf_counter()+ meta_data["load_data"] = f"{time2 - time1:0.3f}"+ speech, speech_lengths = extract_fbank(+ audio_sample_list, data_type=kwargs.get("data_type", "sound"), frontend=frontend+ )+ time3 = time.perf_counter()+ meta_data["extract_feat"] = f"{time3 - time2:0.3f}"+ meta_data["batch_data_time"] = (+ speech_lengths.sum().item() * frontend.frame_shift * frontend.lfr_n / 1000+ )++ speech = speech.to(device=kwargs["device"])+ speech_lengths = speech_lengths.to(device=kwargs["device"])++ language = kwargs.get("language", None)+ if language is not None:- language_query = self.embed(torch.LongTensor([[self.lid_dict[language] if language in self.lid_dict else 0]]).to(speech.device)).repeat(speech.size(0), 1, 1)++ language_query = self.embed(++ torch.LongTensor(++ [[self.lid_dict[language] if language in self.lid_dict else 0]]++ ).to(speech.device)++ ).repeat(speech.size(0), 1, 1)+ else:- language_query = self.embed(torch.LongTensor([[0]]).to(speech.device)).repeat(speech.size(0), 1, 1)++ language_query = self.embed(torch.LongTensor([[0]]).to(speech.device)).repeat(++ speech.size(0), 1, 1++ )+ textnorm = kwargs.get("text_norm", "wotextnorm")- textnorm_query = self.embed(torch.LongTensor([[self.textnorm_dict[textnorm]]]).to(speech.device)).repeat(speech.size(0), 1, 1)++ textnorm_query = self.embed(++ torch.LongTensor([[self.textnorm_dict[textnorm]]]).to(speech.device)++ ).repeat(speech.size(0), 1, 1)+ speech = torch.cat((textnorm_query, speech), dim=1)+ speech_lengths += 1+- event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(speech.size(0), 1, 1)++ event_emo_query = self.embed(torch.LongTensor([[1, 2]]).to(speech.device)).repeat(++ speech.size(0), 1, 1++ )+ input_query = torch.cat((language_query, event_emo_query), dim=1)+ speech = torch.cat((input_query, speech), dim=1)+ speech_lengths += 3++ # Encoder+ encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)+ if isinstance(encoder_out, tuple):+ encoder_out = encoder_out[0]++ # c. Passed the encoder result and the beam search+ ctc_logits = self.ctc.log_softmax(encoder_out)++ results = []+ b, n, d = encoder_out.size()+ if isinstance(key[0], (list, tuple)):+ key = key[0]+ if len(key) < b:+ key = key * b+ for i in range(b):+ x = ctc_logits[i, : encoder_out_lens[i], :]+ yseq = x.argmax(dim=-1)+ yseq = torch.unique_consecutive(yseq, dim=-1)+ yseq = torch.tensor([self.sos] + yseq.tolist() + [self.eos], device=yseq.device)+ nbest_hyps = [Hypothesis(yseq=yseq)]++ for nbest_idx, hyp in enumerate(nbest_hyps):+ ibest_writer = None+ if kwargs.get("output_dir") is not None:+ if not hasattr(self, "writer"):+ self.writer = DatadirWriter(kwargs.get("output_dir"))+ ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"]++ # remove sos/eos and get results+ last_pos = -1+ if isinstance(hyp.yseq, list):+ token_int = hyp.yseq[1:last_pos]+ else:+ token_int = hyp.yseq[1:last_pos].tolist()++ # remove blank symbol id, which is assumed to be 0+ token_int = list(+ filter(+ lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int+ )+ )++ # Change integer-ids to tokens+ text = tokenizer.decode(token_int)++ result_i = {"key": key[i], "text": text}+ results.append(result_i)++ if ibest_writer is not None:+ ibest_writer["token"][key[i]] = " ".join(token)+ ibest_writer["text"][key[i]] = text_postprocessed++ return results, meta_data