Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
update with main (#1786)
zhifu gao32e783664534
diff --git a/funasr/auto/auto_frontend.py b/funasr/auto/auto_frontend.pyindex 696a51e08..501d1ab71 100644--- a/funasr/auto/auto_frontend.py+++ b/funasr/auto/auto_frontend.py@@ -60,7 +60,7 @@ class AutoFrontend:result_list = []num_samples = len(data_list)- pbar = tqdm(colour="blue", total=num_samples + 1, dynamic_ncols=True)+ # pbar = tqdm(colour="blue", total=num_samples + 1, dynamic_ncols=True)time0 = time.perf_counter()for beg_idx in range(0, num_samples, batch_size):@@ -95,15 +95,15 @@ class AutoFrontend:"input": speech,"input_len": speech_lengths,"key": key_batch,- data_type: "fbank",+ "data_type": "fbank",}result_list.append(batch)- pbar.update(1)- description = f"{meta_data}, "- pbar.set_description(description)+ # pbar.update(1)+ # description = f"{meta_data}, "+ # pbar.set_description(description)time_end = time.perf_counter()- pbar.set_description(f"time escaped total: {time_end - time0:0.3f}")+ # pbar.set_description(f"time escaped total: {time_end - time0:0.3f}")return result_listdiff --git a/funasr/datasets/audio_datasets/espnet_samplers.py b/funasr/datasets/audio_datasets/espnet_samplers.pyindex b358fa379..004201e53 100644--- a/funasr/datasets/audio_datasets/espnet_samplers.py+++ b/funasr/datasets/audio_datasets/espnet_samplers.py@@ -147,7 +147,9 @@ class EspnetStyleBatchSampler(DistributedSampler):start_idx = self.rank * batches_per_rankend_idx = start_idx + batches_per_rankrank_batches = buffer_batches[start_idx + self.start_step : end_idx]+self.batch_num = len(rank_batches)+logging.info(f"rank: {self.rank}, dataloader start from step: {self.start_step}, batch_num: {end_idx-start_idx}, batch_num_after_step: {len(rank_batches)}")diff --git a/funasr/models/llm_asr/adaptor.py b/funasr/models/llm_asr/adaptor.pyindex 8c2a80434..9b79ed28a 100644--- a/funasr/models/llm_asr/adaptor.py+++ b/funasr/models/llm_asr/adaptor.py@@ -1,5 +1,7 @@import torchimport torch.nn as nn+import torch.nn.functional as F+from funasr.models.transformer.utils.nets_utils import make_pad_maskfrom funasr.register import tables@@ -63,3 +65,64 @@ class EncoderProjectorQFormer(nn.Module):query_proj = self.norm(self.linear(query_output.last_hidden_state))return query_proj++[email protected]("adaptor_classes", "Transformer")+class Transformer(nn.Module):+ def __init__(+ self, downsample_rate=2, encoder_dim=1280, llm_dim=4096, ffn_dim: int = 2048, **kwargs+ ):+ super().__init__()+ self.k = downsample_rate+ self.encoder_dim = encoder_dim+ self.llm_dim = llm_dim+ self.linear1 = nn.Linear(self.encoder_dim * self.k, ffn_dim)+ self.relu = nn.ReLU()+ self.linear2 = nn.Linear(ffn_dim, self.llm_dim)+ from funasr.models.transformer.encoder import EncoderLayer+ from funasr.models.transformer.attention import MultiHeadedAttention+ from funasr.models.transformer.positionwise_feed_forward import PositionwiseFeedForward++ self.blocks = nn.ModuleList(+ [+ EncoderLayer(+ llm_dim,+ MultiHeadedAttention(+ kwargs.get("attention_heads", 8),+ llm_dim,+ kwargs.get("attention_dropout_rate", 0.0),+ ),+ PositionwiseFeedForward(+ llm_dim,+ llm_dim // 4,+ kwargs.get("dropout_rate", 0.0),+ ),+ kwargs.get("dropout_rate", 0.0),+ )+ for i in range(kwargs.get("n_layer", 2))+ ]+ )++ def forward(self, x, ilens=None):++ batch_size, seq_len, dim = x.size()+ # num_frames_to_discard = seq_len % self.k+ chunk_num = (seq_len - 1) // self.k + 1+ pad_num = chunk_num * self.k - seq_len+ x = F.pad(x, (0, 0, 0, pad_num, 0, 0), value=0.0)+ # if num_frames_to_discard > 0:+ # x = x[:, :-num_frames_to_discard, :]+ seq_len = x.size(1)++ x = x.contiguous()+ x = x.view(batch_size, chunk_num, dim * self.k)+ x = self.linear1(x)+ x = self.relu(x)+ x = self.linear2(x)++ olens = None+ olens = (ilens - 1) // self.k + 1+ masks = (~make_pad_mask(olens)[:, None, :]).to(x.device)+ for layer, block in enumerate(self.blocks):+ x, masks = block(x, masks)+ return x, olensdiff --git a/funasr/models/sense_voice/decoder.py b/funasr/models/sense_voice/decoder.pyindex 60af29ab8..ff933d77b 100644--- a/funasr/models/sense_voice/decoder.py+++ b/funasr/models/sense_voice/decoder.py@@ -360,6 +360,7 @@ class SenseVoiceDecoder(nn.Module):"""Score."""ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0)logp = self.forward(ys.unsqueeze(0), x.unsqueeze(0), cache=state)+ logp = torch.log_softmax(logp, dim=-1)return logp.squeeze(0)[-1, :], statediff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.pyindex 127d5a0a5..22272eefc 100644--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@ -1264,15 +1264,29 @@ class SenseVoiceSANM(nn.Module):if isinstance(task, str):task = [task]task = "".join([f"<|{x}|>" for x in task])- initial_prompt = kwargs.get("initial_prompt", f"<|startoftranscript|>{task}")++ sos = kwargs.get("model_conf").get("sos")+ if isinstance(sos, str):+ initial_prompt = kwargs.get("initial_prompt", f"<|startoftranscript|>{task}")- language = DecodingOptions.get("language", None)- language = None if language == "auto" else language+ language = DecodingOptions.get("language", None)+ language = None if language == "auto" else language- sos = f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt- sos_int = tokenizer.encode(sos, allowed_special="all")+ sos = f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt+ sos_int = tokenizer.encode(sos, allowed_special="all")+ else:+ language = DecodingOptions.get("language", None)+ language = None if language == "auto" else language+ initial_prompt = kwargs.get("initial_prompt", f"{task}")+ initial_prompt_lid = f"{initial_prompt}<|{language}|>" if language is not None else initial_prompt+ initial_prompt_lid_int = tokenizer.encode(initial_prompt_lid, allowed_special="all")+ sos_int = [sos] + initial_prompt_lid_inteos = kwargs.get("model_conf").get("eos")- eos_int = tokenizer.encode(eos, allowed_special="all")+ if isinstance(eos, str):+ eos_int = tokenizer.encode(eos, allowed_special="all")+ else:+ eos_int = [eos]+self.beam_search.sos = sos_intself.beam_search.eos = eos_int[0]@@ -1298,7 +1312,7 @@ class SenseVoiceSANM(nn.Module):self.beam_search.event_score_ga = DecodingOptions.get("gain_tokens_score", [1, 1, 1, 1])encoder_out, encoder_out_lens = self.encode(- speech[None, :, :].permute(0, 2, 1), speech_lengths+ speech[None, :, :], speech_lengths)if text_token_int is not None:diff --git a/funasr/models/sense_voice/whisper_lib/model.py b/funasr/models/sense_voice/whisper_lib/model.pyindex 8b3d3ab1c..3d0d6a819 100644--- a/funasr/models/sense_voice/whisper_lib/model.py+++ b/funasr/models/sense_voice/whisper_lib/model.py@@ -27,9 +27,24 @@ class ModelDimensions:n_text_layer: int+# class LayerNorm(nn.LayerNorm):+# def forward(self, x: Tensor) -> Tensor:+# return super().forward(x.float()).type(x.dtype)++class LayerNorm(nn.LayerNorm):- def forward(self, x: Tensor) -> Tensor:- return super().forward(x.float()).type(x.dtype)+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)++ def forward(self, input):+ output = F.layer_norm(+ input.float(),+ self.normalized_shape,+ self.weight.float() if self.weight is not None else None,+ self.bias.float() if self.bias is not None else None,+ self.eps,+ )+ return output.type_as(input)class Linear(nn.Linear):diff --git a/funasr/models/transformer/encoder.py b/funasr/models/transformer/encoder.pyindex a6a85ae75..987924f0c 100644--- a/funasr/models/transformer/encoder.py+++ b/funasr/models/transformer/encoder.py@@ -64,7 +64,7 @@ class EncoderLayer(nn.Module):stochastic_depth_rate=0.0,):"""Construct an EncoderLayer object."""- super(EncoderLayer, self).__init__()+ super().__init__()self.self_attn = self_attnself.feed_forward = feed_forwardself.norm1 = LayerNorm(size)diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.pyindex 1a553f812..ec887cc8c 100644--- a/funasr/train_utils/trainer_ds.py+++ b/funasr/train_utils/trainer_ds.py@@ -621,7 +621,6 @@ class Trainer:self.train_acc_avg = train_acc_avg.detach().cpu().item() / self.world_sizedef forward_step(self, model, batch, loss_dict={}):- dtype = torch.bfloat16with maybe_autocast(dtype=self.dtype, use_deepspeed=self.use_deepspeed):retval = model(**batch)