Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
decoding
游雁15bb198d4af5
diff --git a/examples/industrial_data_pretraining/sense_voice/demo_fsmn.py b/examples/industrial_data_pretraining/sense_voice/demo_fsmn.pyindex b94cba7f4..e063e1fde 100644--- a/examples/industrial_data_pretraining/sense_voice/demo_fsmn.py+++ b/examples/industrial_data_pretraining/sense_voice/demo_fsmn.py@@ -23,5 +23,5 @@ DecodingOptions = {"gain_event": True,}-res = model.generate(input=input_wav, batch_size_s=0, DecodingOptions=DecodingOptions)+res = model.generate(input=input_wav, batch_size_s=0, DecodingOptions=DecodingOptions, beam_size=5)print(res)diff --git a/funasr/models/sense_voice/decoder.py b/funasr/models/sense_voice/decoder.pyindex 8a4a2ce6a..19d9c1680 100644--- a/funasr/models/sense_voice/decoder.py+++ b/funasr/models/sense_voice/decoder.py@@ -337,6 +337,29 @@ class SenseVoiceDecoder(nn.Module):return x+ def init_state(self, x):+ state = {}++ return state++ def final_score(self, state) -> float:+ """Score eos (optional).++ Args:+ state: Scorer state for prefix tokens++ Returns:+ float: final score++ """+ return 0.0++ def score(self, ys, state, x):+ """Score."""+ ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0)+ logp = self.forward(ys.unsqueeze(0), x.unsqueeze(0), cache=state)+ return logp.squeeze(0)[-1, :], state+class MultiHeadedAttentionSANMDecoder(nn.Module):"""Multi-Head Attention layer.diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.pyindex 82ccc55e1..d5e4130db 100644--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@ -396,6 +396,42 @@ class SenseVoiceRWKV(nn.Module):return loss_att, acc_att, None, None+ def init_beam_search(+ self,+ **kwargs,+ ):+ from .search import BeamSearch++ from funasr.models.transformer.scorers.length_bonus import LengthBonus++ # 1. Build ASR model+ scorers = {}++ scorers.update(+ decoder=self.model.decoder,+ length_bonus=LengthBonus(self.vocab_size),+ )++ weights = dict(+ decoder=1.0,+ ctc=0.0,+ lm=0.0,+ ngram=0.0,+ length_bonus=kwargs.get("penalty", 0.0),+ )+ beam_search = BeamSearch(+ beam_size=kwargs.get("beam_size", 5),+ weights=weights,+ scorers=scorers,+ sos=None,+ eos=None,+ vocab_size=self.vocab_size,+ token_list=None,+ pre_beam_score_key="full",+ )++ self.beam_search = beam_search+def inference(self,data_in,@@ -408,6 +444,12 @@ class SenseVoiceRWKV(nn.Module):if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")+ # init beamsearch+ if not hasattr(self, "beam_search") or self.beam_search is None:+ logging.info("enable beam_search")+ self.init_beam_search(**kwargs)+ self.nbest = kwargs.get("nbest", 1)+if frontend is None and not hasattr(self, "frontend"):frontend_class = tables.frontend_classes.get("WhisperFrontend")frontend = frontend_class(@@ -456,25 +498,65 @@ class SenseVoiceRWKV(nn.Module):task = [task]task = "".join([f"<|{x}|>" for x in task])initial_prompt = kwargs.get("initial_prompt", f"<|startoftranscript|>{task}")- DecodingOptions["initial_prompt"] = initial_promptlanguage = DecodingOptions.get("language", None)language = None if language == "auto" else language- DecodingOptions["language"] = language- DecodingOptions["vocab_path"] = kwargs["tokenizer_conf"].get("vocab_path", None)+ sos = f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt+ sos_int = tokenizer.encode(sos, allowed_special="all")+ eos = kwargs.get("model_conf").get("eos")+ eos_int = tokenizer.encode(eos, allowed_special="all")+ self.beam_search.sos = sos_int+ self.beam_search.eos = eos_int[0]- if "without_timestamps" not in DecodingOptions:- DecodingOptions["without_timestamps"] = True+ encoder_out, encoder_out_lens = self.encode(+ speech[None, :, :].permute(0, 2, 1), speech_lengths+ )- options = whisper.DecodingOptions(**DecodingOptions)+ # c. Passed the encoder result and the beam search+ nbest_hyps = self.beam_search(+ x=encoder_out[0],+ maxlenratio=kwargs.get("maxlenratio", 0.0),+ minlenratio=kwargs.get("minlenratio", 0.0),+ )++ nbest_hyps = nbest_hyps[: self.nbest]- result = whisper.decode(self.model, speech, options)- text = f"{result.text}"results = []- result_i = {"key": key[0], "text": text}+ b, n, d = encoder_out.size()+ for i in range(b):- results.append(result_i)+ for nbest_idx, hyp in enumerate(nbest_hyps):+ ibest_writer = None+ if kwargs.get("output_dir") is not None:+ if not hasattr(self, "writer"):+ self.writer = DatadirWriter(kwargs.get("output_dir"))+ ibest_writer = self.writer[f"{nbest_idx + 1}best_recog"]++ # remove sos/eos and get results+ last_pos = -1+ if isinstance(hyp.yseq, list):+ token_int = hyp.yseq[1:last_pos]+ else:+ token_int = hyp.yseq[1:last_pos].tolist()++ # # remove blank symbol id, which is assumed to be 0+ # token_int = list(+ # filter(+ # lambda x: x != self.eos and x != self.sos and x != self.blank_id, token_int+ # )+ # )++ # Change integer-ids to tokens+ # token = tokenizer.ids2tokens(token_int)+ text = tokenizer.decode(token_int)++ result_i = {"key": key[i], "text": text}+ results.append(result_i)++ if ibest_writer is not None:+ # ibest_writer["token"][key[i]] = " ".join(token)+ ibest_writer["text"][key[i]] = textreturn results, meta_data