Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
Dev gzf exp (#1663)
zhifu gaoe971e000ad58
diff --git a/funasr/datasets/sense_voice_datasets/datasets.py b/funasr/datasets/sense_voice_datasets/datasets.pyindex 5468ea66e..4f14b35f9 100644--- a/funasr/datasets/sense_voice_datasets/datasets.py+++ b/funasr/datasets/sense_voice_datasets/datasets.py@@ -1,3 +1,5 @@+import logging+import torchimport random@@ -46,6 +48,8 @@ class SenseVoiceDataset(torch.utils.data.Dataset):self.float_pad_value = float_pad_valueself.sos = kwargs.get("sos", "<|startoftranscript|>")self.eos = kwargs.get("eos", "<|endoftext|>")+ self.batch_size = kwargs.get("batch_size")+ self.batch_type = kwargs.get("batch_type")def get_source_len(self, index):item = self.index_ds[index]@@ -124,4 +128,32 @@ class SenseVoiceDataset(torch.utils.data.Dataset):outputs[key] = torch.nn.utils.rnn.pad_sequence(data_list, batch_first=True, padding_value=pad_value)++ if self.batch_type != "example":+ b, t, _ = outputs["speech"].shape+ if b * t > self.batch_size:+ beg = torch.randint(0, 2, ()).item()+ logging.info(+ f"Warning, b * t: {b * t} > {self.batch_size}, drop half data 1st, beg:{beg}"+ )+ for key, data_list in outputs.items():+ outputs[key] = outputs[key][beg : beg + b : 2]++ b, t, _ = outputs["speech"].shape+ if b * t > self.batch_size:+ beg = torch.randint(0, 2, ()).item()+ logging.info(+ f"Warning, b * t: {b * t} > {self.batch_size}, drop half data 2nd, beg:{beg}"+ )+ for key, data_list in outputs.items():+ outputs[key] = outputs[key][beg : beg + b : 2]++ b, t, _ = outputs["speech"].shape+ if b * t > self.batch_size:+ beg = torch.randint(0, 2, ()).item()+ logging.info(+ f"Warning, b * t: {b * t} > {self.batch_size}, drop half data 3th, beg:{beg}"+ )+ for key, data_list in outputs.items():+ outputs[key] = outputs[key][beg : beg + b : 2]return outputsdiff --git a/funasr/models/conformer_rwkv/decoder.py b/funasr/models/conformer_rwkv/decoder.pyindex 5e2ac1262..4c4104966 100644--- a/funasr/models/conformer_rwkv/decoder.py+++ b/funasr/models/conformer_rwkv/decoder.py@@ -97,9 +97,7 @@ class DecoderLayer(nn.Module):from funasr.models.sense_voice.rwkv_v6 import RWKV_Tmix_x060 as RWKV_Tmix# self.attn = RWKVLayer(args=args, layer_id=layer_id)self.self_attn = RWKV_Tmix(args, layer_id=layer_id)- if args.get("datatype", "bf16") == "bf16":- self.self_attn.to(torch.bfloat16)- # self.norm1.to(torch.bfloat16)+self.args = argsself.ln0 = Noneif self.layer_id == 0 and not args.get("ln0", True):@@ -125,6 +123,10 @@ class DecoderLayer(nn.Module):nn.init.orthogonal_(self.self_attn.gate.weight, gain=0.1)nn.init.zeros_(self.self_attn.output.weight)+ if args.get("datatype", "bf16") == "bf16":+ self.self_attn.to(torch.bfloat16)+ # self.norm1.to(torch.bfloat16)+def forward(self, tgt, tgt_mask, memory, memory_mask, cache=None):"""Compute decoded features.diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.pyindex 4cbb490b4..b731bb677 100644--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@ -1,3 +1,4 @@+import loggingfrom dataclasses import dataclassfrom typing import Dictfrom typing import Iterable, Optional