Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
docs: add docstrings to 1950 functions across 276 files for API documentation
游雁78cc132a5157
diff --git a/funasr/__init__.py b/funasr/__init__.pyindex e466840f9..37a5d8e2a 100644--- a/funasr/__init__.py+++ b/funasr/__init__.py@@ -19,6 +19,12 @@ _STRICT_IMPORT = os.environ.get("FUNASR_STRICT_IMPORT") == "1"def _record_import_error(name, error):+ """Internal: record import error.++ Args:+ name: TODO.+ error: TODO.+ """_IMPORT_ERRORS[name] = f"{error.__class__.__name__}: {error}"_IMPORT_ERROR_TRACEBACKS[name] = traceback.format_exc()if _IMPORT_DEBUG:@@ -26,14 +32,22 @@ def _record_import_error(name, error):def get_import_errors():+ """Get import errors."""return dict(_IMPORT_ERRORS)def get_import_error_tracebacks():+ """Get import error tracebacks."""return dict(_IMPORT_ERROR_TRACEBACKS)def import_submodules(package, recursive=True):+ """Import submodules.++ Args:+ package: TODO.+ recursive: TODO.+ """if isinstance(package, str):try:package = importlib.import_module(package)diff --git a/funasr/auto/auto_frontend.py b/funasr/auto/auto_frontend.pyindex 562d5f856..ed7ba0444 100644--- a/funasr/auto/auto_frontend.py+++ b/funasr/auto/auto_frontend.py@@ -29,6 +29,11 @@ from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spkclass AutoFrontend:def __init__(self, **kwargs):+ """Initialize AutoFrontend.++ Args:+ **kwargs: Additional keyword arguments.+ """assert "model" in kwargsif "model_conf" not in kwargs:logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))@@ -47,6 +52,14 @@ class AutoFrontend:def __call__(self, input, input_len=None, kwargs=None, **cfg):+ """Internal: call .++ Args:+ input: Input audio/text data.+ input_len: TODO.+ kwargs: Additional keyword arguments.+ **cfg: Configuration overrides.+ """kwargs = self.kwargs if kwargs is None else kwargskwargs.update(cfg)diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.pyindex f3dbc5df5..05df315ca 100644--- a/funasr/auto/auto_model.py+++ b/funasr/auto/auto_model.py@@ -52,6 +52,7 @@ def _resolve_ncpu(config, fallback=4):def _get_import_errors():+ """Internal: get import errors."""try:import funasrexcept Exception:@@ -63,6 +64,13 @@ def _get_import_errors():def _format_unregistered_component_error(component_type, component_name, registry):+ """Internal: format unregistered component error.++ Args:+ component_type: TODO.+ component_name: TODO.+ registry: TODO.+ """registered = sorted(registry.keys())preview = ", ".join(registered[:80])if len(registered) > 80:@@ -414,6 +422,12 @@ class AutoModel:return model, kwargsdef __call__(self, *args, **cfg):+ """Internal: call .++ Args:+ *args: Variable positional arguments.+ **cfg: Configuration overrides.+ """kwargs = self.kwargsdeep_update(kwargs, cfg)res = self.model(*args, kwargs)diff --git a/funasr/auto/auto_tokenizer.py b/funasr/auto/auto_tokenizer.pyindex 59ff8893d..817dc768b 100644--- a/funasr/auto/auto_tokenizer.py+++ b/funasr/auto/auto_tokenizer.py@@ -4,4 +4,5 @@ class AutoTokenizer:"""def __init__(self):+ """Initialize AutoTokenizer."""passdiff --git a/funasr/bin/compute_audio_cmvn.py b/funasr/bin/compute_audio_cmvn.pyindex 880a24745..1cdfa5bc0 100644--- a/funasr/bin/compute_audio_cmvn.py+++ b/funasr/bin/compute_audio_cmvn.py@@ -13,6 +13,11 @@ from funasr.train_utils.set_all_random_seed import set_all_random_seed@hydra.main(config_name=None, version_base=None)def main_hydra(kwargs: DictConfig):+ """Main hydra.++ Args:+ kwargs: Additional keyword arguments.+ """if kwargs.get("debug", False):import pdb@@ -27,6 +32,11 @@ def main_hydra(kwargs: DictConfig):def main(**kwargs):+ """Main.++ Args:+ **kwargs: Additional keyword arguments.+ """print(kwargs)# set random seed# tables.print()diff --git a/funasr/bin/export.py b/funasr/bin/export.pyindex 9d0140102..e151db1fe 100644--- a/funasr/bin/export.py+++ b/funasr/bin/export.py@@ -8,7 +8,17 @@ from funasr.auto.auto_model import AutoModel@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """def to_plain_list(cfg_item):+ """To plain list.++ Args:+ cfg_item: TODO.+ """if isinstance(cfg_item, ListConfig):return OmegaConf.to_container(cfg_item, resolve=True)elif isinstance(cfg_item, DictConfig):diff --git a/funasr/bin/inference.py b/funasr/bin/inference.pyindex 39ee5c067..b4c17d5fa 100644--- a/funasr/bin/inference.py+++ b/funasr/bin/inference.py@@ -7,7 +7,17 @@ from funasr.auto.auto_model import AutoModel@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """def to_plain_list(cfg_item):+ """To plain list.++ Args:+ cfg_item: TODO.+ """if isinstance(cfg_item, ListConfig):return OmegaConf.to_container(cfg_item, resolve=True)elif isinstance(cfg_item, DictConfig):diff --git a/funasr/bin/tokenize_text.py b/funasr/bin/tokenize_text.pyindex 8c2aae050..e127e1234 100755--- a/funasr/bin/tokenize_text.py+++ b/funasr/bin/tokenize_text.py@@ -81,6 +81,26 @@ def tokenize(g2p: Optional[str],):+ """Tokenize.++ Args:+ input: Input audio/text data.+ output: TODO.+ field: TODO.+ delimiter: TODO.+ token_type: TODO.+ space_symbol: TODO.+ non_linguistic_symbols: TODO.+ bpemodel: TODO.+ log_level: TODO.+ write_vocabulary: TODO.+ vocabulary_size: Size/dimension parameter.+ remove_non_linguistic_symbols: TODO.+ cutoff: TODO.+ add_symbol: TODO.+ cleaner: TODO.+ g2p: TODO.+ """logging.basicConfig(level=log_level,format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",@@ -185,6 +205,7 @@ def tokenize(def get_parser() -> argparse.ArgumentParser:+ """Get parser."""parser = argparse.ArgumentParser(description="Tokenize texts",formatter_class=argparse.ArgumentDefaultsHelpFormatter,@@ -266,6 +287,11 @@ def get_parser() -> argparse.ArgumentParser:def main(cmd=None):+ """Main.++ Args:+ cmd: TODO.+ """print(get_commandline_args(), file=sys.stderr)parser = get_parser()args = parser.parse_args(cmd)diff --git a/funasr/bin/train.py b/funasr/bin/train.pyindex 4e3144133..d54cd0bbe 100644--- a/funasr/bin/train.py+++ b/funasr/bin/train.py@@ -40,6 +40,11 @@ from funasr import AutoModel@hydra.main(config_name=None, version_base=None)def main_hydra(kwargs: DictConfig):+ """Main hydra.++ Args:+ kwargs: Additional keyword arguments.+ """if kwargs.get("debug", False):import pdb@@ -56,6 +61,11 @@ def main_hydra(kwargs: DictConfig):def main(**kwargs):# set random seed+ """Main.++ Args:+ **kwargs: Additional keyword arguments.+ """set_all_random_seed(kwargs.get("seed", 0))torch.backends.cudnn.enabled = kwargs.get("cudnn_enabled", torch.backends.cudnn.enabled)torch.backends.cudnn.benchmark = kwargs.get("cudnn_benchmark", torch.backends.cudnn.benchmark)@@ -137,6 +147,14 @@ def main(**kwargs):min_num_params: int = int(1e8),) -> bool:# 根据自定义逻辑决定是否包装模块+ """Custom auto wrap policy.++ Args:+ module: TODO.+ recurse: TODO.+ nonwrapped_numel: TODO.+ min_num_params: TODO.+ """is_large = nonwrapped_numel >= min_num_paramsrequires_grad_uniform = len({p.requires_grad for p in module.parameters()}) == 1return is_large and requires_grad_uniformdiff --git a/funasr/bin/train_ds.py b/funasr/bin/train_ds.pyindex 4aa5f3bfd..b32881d99 100644--- a/funasr/bin/train_ds.py+++ b/funasr/bin/train_ds.py@@ -43,6 +43,11 @@ except:@hydra.main(config_name=None, version_base=None)def main_hydra(kwargs: DictConfig):+ """Main hydra.++ Args:+ kwargs: Additional keyword arguments.+ """if kwargs.get("debug", False):import pdb@@ -59,6 +64,11 @@ def main_hydra(kwargs: DictConfig):def main(**kwargs):# set random seed+ """Main.++ Args:+ **kwargs: Additional keyword arguments.+ """set_all_random_seed(kwargs.get("seed", 0))torch.backends.cudnn.enabled = kwargs.get("cudnn_enabled", torch.backends.cudnn.enabled)torch.backends.cudnn.benchmark = kwargs.get("cudnn_benchmark", torch.backends.cudnn.benchmark)diff --git a/funasr/datasets/audio_datasets/datasets.py b/funasr/datasets/audio_datasets/datasets.pyindex 68b2d3cc7..35266b050 100644--- a/funasr/datasets/audio_datasets/datasets.py+++ b/funasr/datasets/audio_datasets/datasets.py@@ -23,6 +23,18 @@ class AudioDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs,):+ """Initialize AudioDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ is_training: Boolean flag for training.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -53,17 +65,33 @@ class AudioDataset(torch.utils.data.Dataset):self.float_pad_value = float_pad_valuedef get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]# import pdb;# pdb.set_trace()@@ -97,6 +125,11 @@ class AudioDataset(torch.utils.data.Dataset):}def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:for key in sample.keys():@@ -127,10 +160,21 @@ class AudioDatasetHotword(AudioDataset):seaco_id: bool = 0,**kwargs,):+ """Initialize AudioDatasetHotword.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)self.seaco_id = seaco_iddef __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]# import pdb;# pdb.set_trace()@@ -164,6 +208,18 @@ class AudioDatasetHotword(AudioDataset):pre_index=None,pre_hwlist=None,):+ """Generate index.++ Args:+ length: TODO.+ hotword_min_length: TODO.+ hotword_max_length: TODO.+ sample_rate: TODO.+ double_rate: TODO.+ pre_prob: TODO.+ pre_index: TODO.+ pre_hwlist: TODO.+ """if length < hotword_min_length:return [-1]if random.random() < sample_rate:@@ -210,6 +266,11 @@ class AudioDatasetHotword(AudioDataset):}def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}hotword_indxs = []seaco_id = samples[0]["seaco_id"]diff --git a/funasr/datasets/audio_datasets/espnet_samplers.py b/funasr/datasets/audio_datasets/espnet_samplers.pyindex 004201e53..a411ba35f 100644--- a/funasr/datasets/audio_datasets/espnet_samplers.py+++ b/funasr/datasets/audio_datasets/espnet_samplers.py@@ -12,6 +12,12 @@ from funasr.register import tables@tables.register("batch_sampler_classes", "EspnetStyleBatchSampler")def EspnetStyleBatchSampler_fn(dataset, **kwargs):+ """Espnetstylebatchsampler fn.++ Args:+ dataset: TODO.+ **kwargs: Additional keyword arguments.+ """dataloader_args = {}batch_sampler = EspnetStyleBatchSampler(dataset, **kwargs)@@ -45,6 +51,22 @@ class EspnetStyleBatchSampler(DistributedSampler):**kwargs,):+ """Initialize EspnetStyleBatchSampler.++ Args:+ dataset: TODO.+ batch_size: Number of samples per batch.+ batch_type: TODO.+ rank: TODO.+ num_replicas: TODO.+ rank_split: TODO.+ shuffle: TODO.+ drop_last: TODO.+ is_training: Boolean flag for training.+ sort_size: Size/dimension parameter.+ start_step: TODO.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()num_replicas = dist.get_world_size()@@ -79,6 +101,7 @@ class EspnetStyleBatchSampler(DistributedSampler):# shuffle=shuffle, drop_last=drop_last)def __iter__(self):+ """Internal: iter ."""if self.shuffle:g = torch.Generator()g.manual_seed(self.epoch)@@ -158,8 +181,14 @@ class EspnetStyleBatchSampler(DistributedSampler):def __len__(self):# Calculate the number of batches per epoch for the current rank+ """Internal: len ."""return self.batch_numdef set_epoch(self, epoch):# Set the epoch for shuffling+ """Set epoch.++ Args:+ epoch: TODO.+ """self.epoch = epochdiff --git a/funasr/datasets/audio_datasets/index_ds.py b/funasr/datasets/audio_datasets/index_ds.pyindex dd7c52cd4..ff9af8533 100644--- a/funasr/datasets/audio_datasets/index_ds.py+++ b/funasr/datasets/audio_datasets/index_ds.py@@ -16,6 +16,12 @@ from funasr.register import tablesclass IndexDSJsonlRankFull(torch.utils.data.Dataset):def __init__(self, path: str, **kwargs):+ """Initialize IndexDSJsonlRankFull.++ Args:+ path: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.max_source_length = kwargs.get("max_source_length", 2048)self.min_source_length = kwargs.get("min_source_length", 0)@@ -135,17 +141,33 @@ class IndexDSJsonlRankFull(torch.utils.data.Dataset):logging.info("total_num of samplers: {}, {}".format(len(self.contents), path))def __len__(self):+ """Internal: len ."""return len(self.contents)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """data = self.contents[index]return datadef get_source_len(self, data_dict):+ """Get source len.++ Args:+ data_dict: TODO.+ """return data_dict.get("source_len", 1)def get_target_len(self, data_dict):+ """Get target len.++ Args:+ data_dict: TODO.+ """return data_dict.get("target_len", 0)diff --git a/funasr/datasets/audio_datasets/jsonl2scp.py b/funasr/datasets/audio_datasets/jsonl2scp.pyindex 3ce96ca78..cdd2f5e65 100644--- a/funasr/datasets/audio_datasets/jsonl2scp.py+++ b/funasr/datasets/audio_datasets/jsonl2scp.py@@ -11,6 +11,14 @@ import torch.distributed as distdef gen_scp_from_jsonl(jsonl_file, data_type_list, wav_scp_file, text_file):+ """Gen scp from jsonl.++ Args:+ jsonl_file: TODO.+ data_type_list: TODO.+ wav_scp_file: TODO.+ text_file: TODO.+ """wav_f = open(wav_scp_file, "w")text_f = open(text_file, "w")with open(jsonl_file, encoding="utf-8") as fin:@@ -37,6 +45,11 @@ def gen_scp_from_jsonl(jsonl_file, data_type_list, wav_scp_file, text_file):@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """kwargs = OmegaConf.to_container(cfg, resolve=True)print(kwargs)diff --git a/funasr/datasets/audio_datasets/preprocessor.py b/funasr/datasets/audio_datasets/preprocessor.pyindex b3f942156..9726ee9f5 100644--- a/funasr/datasets/audio_datasets/preprocessor.py+++ b/funasr/datasets/audio_datasets/preprocessor.py@@ -18,10 +18,23 @@ from funasr.register import tables@tables.register("preprocessor_classes", "SpeechPreprocessSpeedPerturb")class SpeechPreprocessSpeedPerturb(nn.Module):def __init__(self, speed_perturb: list = None, **kwargs):+ """Initialize SpeechPreprocessSpeedPerturb.++ Args:+ speed_perturb: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.speed_perturb = speed_perturbdef forward(self, waveform, fs, **kwargs):+ """Forward pass for training.++ Args:+ waveform: TODO.+ fs: TODO.+ **kwargs: Additional keyword arguments.+ """if self.speed_perturb is None:return waveformspeed = random.choice(self.speed_perturb)@@ -45,11 +58,25 @@ class TextPreprocessSegDict(nn.Module):split_with_space: bool = False,**kwargs):+ """Initialize TextPreprocessSegDict.++ Args:+ seg_dict: TODO.+ text_cleaner: TODO.+ split_with_space: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.text_cleaner = TextCleaner(text_cleaner)def forward(self, text, **kwargs):+ """Forward pass for training.++ Args:+ text: Text tensor or string input.+ **kwargs: Additional keyword arguments.+ """text = self.text_cleaner(text)return textdiff --git a/funasr/datasets/audio_datasets/samplers.py b/funasr/datasets/audio_datasets/samplers.pyindex f7057de11..f478db002 100644--- a/funasr/datasets/audio_datasets/samplers.py+++ b/funasr/datasets/audio_datasets/samplers.py@@ -18,6 +18,12 @@ from funasr.register import tables@tables.register("batch_sampler_classes", "RankFullLocalShuffleBatchSampler")@tables.register("batch_sampler_classes", "RankFullLocalShuffleDynamicBatchSampler")def CustomDistributedBatchSampler_fn(dataset, **kwargs):+ """Customdistributedbatchsampler fn.++ Args:+ dataset: TODO.+ **kwargs: Additional keyword arguments.+ """dataloader_args = {}batch_type = kwargs.get("batch_type", "example")if batch_type == "example":@@ -50,6 +56,18 @@ class CustomDistributedBatchSampler(Sampler):**kwargs,):+ """Initialize CustomDistributedBatchSampler.++ Args:+ dataset: TODO.+ batch_size: Number of samples per batch.+ num_replicas: TODO.+ rank: TODO.+ shuffle: TODO.+ drop_last: TODO.+ is_training: Boolean flag for training.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()num_replicas = dist.get_world_size()@@ -79,6 +97,7 @@ class CustomDistributedBatchSampler(Sampler):def __iter__(self):# Generate a list of indices+ """Internal: iter ."""if self.shuffle:g = torch.Generator()g.manual_seed(self.epoch)@@ -123,9 +142,15 @@ class CustomDistributedBatchSampler(Sampler):def __len__(self):+ """Internal: len ."""return self.num_samples // self.batch_sizedef set_epoch(self, epoch):+ """Set epoch.++ Args:+ epoch: TODO.+ """self.epoch = epoch@@ -143,6 +168,19 @@ class CustomDistributedBufferBatchSampler(Sampler):**kwargs,):+ """Initialize CustomDistributedBufferBatchSampler.++ Args:+ dataset: TODO.+ batch_size: Number of samples per batch.+ num_replicas: TODO.+ rank: TODO.+ shuffle: TODO.+ drop_last: TODO.+ is_training: Boolean flag for training.+ sort_size: Size/dimension parameter.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()num_replicas = dist.get_world_size()@@ -173,6 +211,7 @@ class CustomDistributedBufferBatchSampler(Sampler):def __iter__(self):# Generate a list of indices+ """Internal: iter ."""if self.shuffle:g = torch.Generator()g.manual_seed(self.epoch)@@ -225,6 +264,11 @@ class CustomDistributedBufferBatchSampler(Sampler):def _create_batches_from_buffer(self, buffer):# Function to convert the sorted buffer into batches+ """Internal: create batches from buffer.++ Args:+ buffer: TODO.+ """batched_buffer = [buffer[i : i + self.batch_size] for i in range(0, len(buffer), self.batch_size)]@@ -234,9 +278,15 @@ class CustomDistributedBufferBatchSampler(Sampler):def __len__(self):+ """Internal: len ."""return self.num_samples // self.batch_sizedef set_epoch(self, epoch):+ """Set epoch.++ Args:+ epoch: TODO.+ """self.epoch = epoch@@ -253,6 +303,18 @@ class CustomDistributedDynamicBatchSampler(DistributedSampler):**kwargs,):+ """Initialize CustomDistributedDynamicBatchSampler.++ Args:+ dataset: TODO.+ batch_size: Number of samples per batch.+ num_replicas: TODO.+ rank: TODO.+ shuffle: TODO.+ drop_last: TODO.+ is_training: Boolean flag for training.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()num_replicas = dist.get_world_size()@@ -274,6 +336,7 @@ class CustomDistributedDynamicBatchSampler(DistributedSampler):self.length_scale_source = kwargs.get("length_scale_source", 1.0)def __iter__(self):+ """Internal: iter ."""if self.shuffle:g = torch.Generator()g.manual_seed(self.epoch)@@ -315,9 +378,15 @@ class CustomDistributedDynamicBatchSampler(DistributedSampler):def __len__(self):+ """Internal: len ."""return 1def set_epoch(self, epoch):+ """Set epoch.++ Args:+ epoch: TODO.+ """self.epoch = epoch@@ -338,6 +407,22 @@ class CustomDistributedBufferDynamicBatchSampler(DistributedSampler):**kwargs,):+ """Initialize CustomDistributedBufferDynamicBatchSampler.++ Args:+ dataset: TODO.+ batch_size: Number of samples per batch.+ batch_type: TODO.+ num_replicas: TODO.+ rank: TODO.+ rank_split: TODO.+ shuffle: TODO.+ drop_last: TODO.+ is_training: Boolean flag for training.+ sort_size: Size/dimension parameter.+ start_step: TODO.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()num_replicas = dist.get_world_size()@@ -375,6 +460,7 @@ class CustomDistributedBufferDynamicBatchSampler(DistributedSampler):# )def __iter__(self):+ """Internal: iter ."""if self.shuffle:g = torch.Generator()g.manual_seed(self.epoch)@@ -434,9 +520,15 @@ class CustomDistributedBufferDynamicBatchSampler(DistributedSampler):def __len__(self):# Calculate the number of batches per epoch for the current rank+ """Internal: len ."""return self.batch_numdef set_epoch(self, epoch):+ """Set epoch.++ Args:+ epoch: TODO.+ """self.epoch = epoch@@ -444,6 +536,16 @@ class DistributedSamplerWarp(BatchSampler):def __init__(self, dataset, batch_size, num_replicas=None, rank=None, shuffle=True, drop_last=False):+ """Initialize DistributedSamplerWarp.++ Args:+ dataset: TODO.+ batch_size: Number of samples per batch.+ num_replicas: TODO.+ rank: TODO.+ shuffle: TODO.+ drop_last: TODO.+ """if num_replicas is None:if not torch.distributed.is_available():raise RuntimeError("Requires distributed package to be available")@@ -470,6 +572,7 @@ class DistributedSamplerWarp(BatchSampler):def __iter__(self):# If we shuffle, we need to call the set_epoch method+ """Internal: iter ."""if self.shuffle:self.sampler.set_epoch(self.epoch)@@ -477,4 +580,9 @@ class DistributedSamplerWarp(BatchSampler):return super().__iter__()def set_epoch(self, epoch):+ """Set epoch.++ Args:+ epoch: TODO.+ """self.epoch = epochdiff --git a/funasr/datasets/audio_datasets/scp2jsonl.py b/funasr/datasets/audio_datasets/scp2jsonl.pyindex 48c64d262..f1f2f710f 100644--- a/funasr/datasets/audio_datasets/scp2jsonl.py+++ b/funasr/datasets/audio_datasets/scp2jsonl.py@@ -13,6 +13,14 @@ from tqdm import tqdmdef gen_jsonl_from_wav_text_list(path, data_type_list=("source", "target"), jsonl_file_out: str = None, **kwargs):+ """Gen jsonl from wav text list.++ Args:+ path: TODO.+ data_type_list: TODO.+ jsonl_file_out: TODO.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()world_size = dist.get_world_size()@@ -73,6 +81,13 @@ def gen_jsonl_from_wav_text_list(def parse_context_length(data_list: list, data_type: str, id=0):+ """Parse context length.++ Args:+ data_list: TODO.+ data_type: TODO.+ id: TODO.+ """pbar = tqdm(total=len(data_list), dynamic_ncols=True)res = {}for i, line in enumerate(data_list):@@ -99,6 +114,11 @@ def parse_context_length(data_list: list, data_type: str, id=0):@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """kwargs = OmegaConf.to_container(cfg, resolve=True)print(kwargs)diff --git a/funasr/datasets/audio_datasets/scp2len.py b/funasr/datasets/audio_datasets/scp2len.pyindex 5d742b17c..17819057e 100644--- a/funasr/datasets/audio_datasets/scp2len.py+++ b/funasr/datasets/audio_datasets/scp2len.py@@ -13,6 +13,14 @@ from tqdm import tqdmdef gen_jsonl_from_wav_text_list(path, data_type_list=("source",), jsonl_file_out: str = None, **kwargs):+ """Gen jsonl from wav text list.++ Args:+ path: TODO.+ data_type_list: TODO.+ jsonl_file_out: TODO.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()world_size = dist.get_world_size()@@ -75,6 +83,13 @@ def gen_jsonl_from_wav_text_list(def parse_context_length(data_list: list, data_type: str, id=0):+ """Parse context length.++ Args:+ data_list: TODO.+ data_type: TODO.+ id: TODO.+ """pbar = tqdm(total=len(data_list), dynamic_ncols=True)res = {}for i, line in enumerate(data_list):@@ -97,6 +112,11 @@ def parse_context_length(data_list: list, data_type: str, id=0):@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """kwargs = OmegaConf.to_container(cfg, resolve=True)print(kwargs)diff --git a/funasr/datasets/audio_datasets/sensevoice2jsonl.py b/funasr/datasets/audio_datasets/sensevoice2jsonl.pyindex 102e82797..e4f72366b 100644--- a/funasr/datasets/audio_datasets/sensevoice2jsonl.py+++ b/funasr/datasets/audio_datasets/sensevoice2jsonl.py@@ -15,6 +15,15 @@ from tqdm import tqdmdef gen_jsonl_from_wav_text_list(path, data_type_list=("source", "target"), jsonl_file_out: str = None, model_dir: str = "iic/SenseVoiceSmall", **kwargs):+ """Gen jsonl from wav text list.++ Args:+ path: TODO.+ data_type_list: TODO.+ jsonl_file_out: TODO.+ model_dir: TODO.+ **kwargs: Additional keyword arguments.+ """try:rank = dist.get_rank()world_size = dist.get_world_size()@@ -120,6 +129,11 @@ def gen_jsonl_from_wav_text_list(dist.barrier()def contains_punctuation(s):+ """Contains punctuation.++ Args:+ s: TODO.+ """punctuations = (string.punctuation +',。、;:?!""''()【】《》〈〉「」『』〔〕[]{}~·…—–'@@ -127,6 +141,13 @@ def contains_punctuation(s):return any(char in punctuations for char in s)def parse_context_length(data_list: list, data_type: str, id=0):+ """Parse context length.++ Args:+ data_list: TODO.+ data_type: TODO.+ id: TODO.+ """pbar = tqdm(total=len(data_list), dynamic_ncols=True)res = {}for i, line in enumerate(data_list):@@ -159,6 +180,11 @@ def parse_context_length(data_list: list, data_type: str, id=0):@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """kwargs = OmegaConf.to_container(cfg, resolve=True)print(kwargs)diff --git a/funasr/datasets/audio_datasets/update_jsonl.py b/funasr/datasets/audio_datasets/update_jsonl.pyindex bc8470808..e21943736 100644--- a/funasr/datasets/audio_datasets/update_jsonl.py+++ b/funasr/datasets/audio_datasets/update_jsonl.py@@ -13,6 +13,13 @@ from concurrent.futures import ThreadPoolExecutordef gen_scp_from_jsonl(jsonl_file, jsonl_file_out, ncpu):+ """Gen scp from jsonl.++ Args:+ jsonl_file: TODO.+ jsonl_file_out: TODO.+ ncpu: TODO.+ """jsonl_file_out_f = open(jsonl_file_out, "w")with open(jsonl_file, encoding="utf-8") as fin:lines = fin.readlines()@@ -42,6 +49,12 @@ def gen_scp_from_jsonl(jsonl_file, jsonl_file_out, ncpu):def update_data(lines, i):+ """Update data.++ Args:+ lines: TODO.+ i: TODO.+ """line = lines[i]data = json.loads(line.strip())@@ -61,6 +74,13 @@ def update_data(lines, i):def update_wav_len(jsonl_file_list_in, jsonl_file_out_dir, ncpu=1):+ """Update wav len.++ Args:+ jsonl_file_list_in: TODO.+ jsonl_file_out_dir: TODO.+ ncpu: TODO.+ """os.makedirs(jsonl_file_out_dir, exist_ok=True)with open(jsonl_file_list_in, "r") as f:data_file_lists = f.readlines()@@ -76,6 +96,11 @@ def update_wav_len(jsonl_file_list_in, jsonl_file_out_dir, ncpu=1):@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """kwargs = OmegaConf.to_container(cfg, resolve=True)logging.info(kwargs)diff --git a/funasr/datasets/dataloader_entry.py b/funasr/datasets/dataloader_entry.pyindex 055e4c8a7..c868c20d2 100644--- a/funasr/datasets/dataloader_entry.py+++ b/funasr/datasets/dataloader_entry.py@@ -7,6 +7,13 @@ from funasr.register import tables# @tables.register("dataloader_classes", "DataloaderMapStyle")def DataloaderMapStyle(frontend=None, tokenizer=None, **kwargs):# dataset+ """Dataloadermapstyle.++ Args:+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ **kwargs: Additional keyword arguments.+ """logging.info("Build dataloader")dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset"))dataset_tr = dataset_class(@@ -48,6 +55,13 @@ def DataloaderMapStyle(frontend=None, tokenizer=None, **kwargs):class DataloaderMapStyle:def __init__(self, frontend=None, tokenizer=None, **kwargs):# dataset+ """Initialize DataloaderMapStyle.++ Args:+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ **kwargs: Additional keyword arguments.+ """logging.info("Build dataloader")dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset"))@@ -82,6 +96,14 @@ class DataloaderMapStyle:def build_iter(self, epoch=0, data_split_i=0, start_step=0, **kwargs):# reload dataset slice+ """Build iter.++ Args:+ epoch: TODO.+ data_split_i: TODO.+ start_step: TODO.+ **kwargs: Additional keyword arguments.+ """if self.data_split_num > 1:del self.dataset_trself.dataset_tr = self.dataset_class(@@ -119,6 +141,13 @@ class DataloaderMapStyle:@tables.register("dataloader_classes", "DataloaderIterable")def DataloaderIterable(frontend=None, tokenizer=None, **kwargs):+ """Dataloaderiterable.++ Args:+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ **kwargs: Additional keyword arguments.+ """logging.info("Build dataloader")dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "LargeDataset"))dataset_tr = dataset_class(diff --git a/funasr/datasets/fun_asr_datasets/datasets.py b/funasr/datasets/fun_asr_datasets/datasets.pyindex af8702a89..c70ff1a1b 100644--- a/funasr/datasets/fun_asr_datasets/datasets.py+++ b/funasr/datasets/fun_asr_datasets/datasets.py@@ -26,6 +26,17 @@ class FunASR(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs,):+ """Initialize FunASR.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -82,14 +93,30 @@ class FunASR(torch.utils.data.Dataset):self.min_output_non_mask_token_len = kwargs.get("min_non_mask_token_len", 6) # [eos]def get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def get_random_user_prompt(self, item, user_prompt):+ """Get random user prompt.++ Args:+ item: TODO.+ user_prompt: TODO.+ """tasks = ["语音转写:", "Speech transcription:"]language = item.get("language", None)# LID in distill data is fake@@ -112,9 +139,15 @@ class FunASR(torch.utils.data.Dataset):return user_promptdef __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """output = Nonefor idx in range(self.retry):@@ -309,6 +342,11 @@ class FunASR(torch.utils.data.Dataset):def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """for idx in range(self.retry):badcase_flag = False@@ -354,6 +392,12 @@ class FunASR(torch.utils.data.Dataset):class FunASR(torch.utils.data.Dataset): # torch.utils.data.Datasetdef __init__(self, path: str, **kwargs):+ """Initialize FunASR.++ Args:+ path: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.max_source_length = kwargs.get("max_source_length", 8000)@@ -490,15 +534,26 @@ class FunASR(torch.utils.data.Dataset): # torch.utils.data.Datasetf"\n\ntotal_num of samplers: {len(self.contents)}, total_whrs: {total_whrs:.5f}, total_token_for_llm_B: {total_token_for_llm_B:.5g}, {path}, {file_list}\n\n")def __len__(self):+ """Internal: len ."""return len(self.contents)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """data = self.contents[index]return datadef get_source_len(self, data_dict):+ """Get source len.++ Args:+ data_dict: TODO.+ """source_len = data_dict.get("source_len", -1)if source_len < 0:source_len = len(data_dict["system"]) + len(data_dict["user"])@@ -506,4 +561,9 @@ class FunASR(torch.utils.data.Dataset): # torch.utils.data.Datasetdef get_target_len(self, data_dict):+ """Get target len.++ Args:+ data_dict: TODO.+ """return 0diff --git a/funasr/datasets/fun_asr_datasets/multicontext_prompt.py b/funasr/datasets/fun_asr_datasets/multicontext_prompt.pyindex 6d46dceb2..1f8dd69dc 100644--- a/funasr/datasets/fun_asr_datasets/multicontext_prompt.py+++ b/funasr/datasets/fun_asr_datasets/multicontext_prompt.py@@ -33,6 +33,16 @@ class MultiContextPrompt:use_asr_hotwords=True,use_multi_lingual_prompt=True,**kwargs):+ """Initialize MultiContextPrompt.++ Args:+ use_hist: TODO.+ use_one_pass_result: TODO.+ use_hotwords: TODO.+ use_asr_hotwords: TODO.+ use_multi_lingual_prompt: TODO.+ **kwargs: Additional keyword arguments.+ """self.use_hist = use_histself.use_one_pass_result = use_one_pass_resultself.use_hotwords = use_hotwords@@ -60,11 +70,21 @@ class MultiContextPrompt:self.min_neg_hotwords_num = kwargs.get("min_neg_hotwords_num", 0)def get_hotwords_list(self, hotwords_file):+ """Get hotwords list.++ Args:+ hotwords_file: TODO.+ """with open(hotwords_file, "r") as f:hotwords_list = f.read().strip().split("\n")return hotwords_list, len(hotwords_list)def detect_language(self, text):+ """Detect language.++ Args:+ text: Text tensor or string input.+ """if isinstance(text, list):text = " ".join(text)@@ -95,6 +115,11 @@ class MultiContextPrompt:def hotwords_sampling(self, hotwords):# hotwords_list = hotwords.split(", ")+ """Hotwords sampling.++ Args:+ hotwords: TODO.+ """hotwords_list = hotwordsselected_hotwords = []if self.max_neg_hotwords_num > -1:@@ -112,6 +137,12 @@ class MultiContextPrompt:return selected_hotwords, selected_hotwords_numdef get_prompt(self, item, language):+ """Get prompt.++ Args:+ item: TODO.+ language: Language identifier.+ """template = self.CONTEXT_TEMPLATES[language]prompt = template['header']@@ -157,6 +188,12 @@ class MultiContextPrompt:return promptdef get_inference_prompt(self, item, language="zh"):+ """Get inference prompt.++ Args:+ item: TODO.+ language: Language identifier.+ """template = self.CONTEXT_TEMPLATES[language]prompt = template['header']@@ -233,6 +270,15 @@ class MultiContextPromptNew:use_hotwords=True,use_multi_lingual_prompt=True,**kwargs):+ """Initialize MultiContextPromptNew.++ Args:+ use_hist: TODO.+ use_one_pass_result: TODO.+ use_hotwords: TODO.+ use_multi_lingual_prompt: TODO.+ **kwargs: Additional keyword arguments.+ """self.use_hist = use_histself.use_one_pass_result = use_one_pass_resultself.use_hotwords = use_hotwords@@ -244,6 +290,11 @@ class MultiContextPromptNew:def hotwords_sampling(self, hotwords):+ """Hotwords sampling.++ Args:+ hotwords: TODO.+ """hotwords_list = hotwords.split(", ")if self.max_hotwords_num > 0:max_hotwords_num = min(self.max_hotwords_num, len(hotwords_list))@@ -261,6 +312,12 @@ class MultiContextPromptNew:return hotwords_list, selected_hotwords_numdef get_prompt(self, item, language):+ """Get prompt.++ Args:+ item: TODO.+ language: Language identifier.+ """template = self.CONTEXT_TEMPLATES[language]prompt = template['header']@@ -287,6 +344,13 @@ class MultiContextPromptNew:return promptdef get_inference_prompt(self, hist_context="", one_pass_result="", hotwords=""):+ """Get inference prompt.++ Args:+ hist_context: TODO.+ one_pass_result: TODO.+ hotwords: TODO.+ """language = 'zh' if self.use_multi_lingual_prompt and np.random.rand() < 0.5 else 'en'template = self.CONTEXT_TEMPLATES[language]diff --git a/funasr/datasets/kws_datasets/datasets.py b/funasr/datasets/kws_datasets/datasets.pyindex 4679295cb..24a9ae75d 100644--- a/funasr/datasets/kws_datasets/datasets.py+++ b/funasr/datasets/kws_datasets/datasets.py@@ -21,6 +21,18 @@ class KwsMTDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs,):+ """Initialize KwsMTDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ is_training: Boolean flag for training.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -53,17 +65,33 @@ class KwsMTDataset(torch.utils.data.Dataset):self.float_pad_value = float_pad_valuedef get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]# import pdb;# pdb.set_trace()@@ -111,6 +139,11 @@ class KwsMTDataset(torch.utils.data.Dataset):def collator(self, samples: list=None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:for key in sample.keys():diff --git a/funasr/datasets/llm_datasets/datasets.py b/funasr/datasets/llm_datasets/datasets.pyindex 61cadedd1..f1ae4a524 100644--- a/funasr/datasets/llm_datasets/datasets.py+++ b/funasr/datasets/llm_datasets/datasets.py@@ -21,6 +21,17 @@ class AudioLLMNARDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs):+ """Initialize AudioLLMNARDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -52,17 +63,33 @@ class AudioLLMNARDataset(torch.utils.data.Dataset):self.int_pad_value = self.IGNORE_INDEXdef get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]source = item["source"]data_src = load_audio_text_image_video(source, fs=self.fs)@@ -143,6 +170,11 @@ class AudioLLMNARDataset(torch.utils.data.Dataset):}def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:for key in sample.keys():@@ -180,6 +212,17 @@ class AudioLLMDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs):+ """Initialize AudioLLMDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -211,17 +254,33 @@ class AudioLLMDataset(torch.utils.data.Dataset):self.int_pad_value = self.IGNORE_INDEXdef get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]# import pdb;# pdb.set_trace()@@ -278,6 +337,11 @@ class AudioLLMDataset(torch.utils.data.Dataset):}def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:for key in sample.keys():@@ -315,6 +379,17 @@ class AudioLLMARDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs):+ """Initialize AudioLLMARDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -346,17 +421,33 @@ class AudioLLMARDataset(torch.utils.data.Dataset):self.int_pad_value = self.IGNORE_INDEXdef get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]# import pdb;# pdb.set_trace()@@ -413,6 +504,11 @@ class AudioLLMARDataset(torch.utils.data.Dataset):}def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:for key in sample.keys():diff --git a/funasr/datasets/llm_datasets/preprocessor.py b/funasr/datasets/llm_datasets/preprocessor.pyindex b99255e38..59edf67f0 100644--- a/funasr/datasets/llm_datasets/preprocessor.py+++ b/funasr/datasets/llm_datasets/preprocessor.py@@ -19,10 +19,21 @@ from funasr.register import tables@tables.register("preprocessor_classes", "TextPreprocessRemovePunctuation")class TextPreprocessRemovePunctuation(nn.Module):def __init__(self, **kwargs):+ """Initialize TextPreprocessRemovePunctuation.++ Args:+ **kwargs: Additional keyword arguments.+ """super().__init__()def forward(self, text, **kwargs):# 定义英文标点符号+ """Forward pass for training.++ Args:+ text: Text tensor or string input.+ **kwargs: Additional keyword arguments.+ """en_punct = string.punctuation# 定义中文标点符号(部分常用的)cn_punct = "。?!,、;:“”‘’()《》【】…—~·"diff --git a/funasr/datasets/llm_datasets_qwenaudio/datasets.py b/funasr/datasets/llm_datasets_qwenaudio/datasets.pyindex 569665a1d..ca9fbbceb 100644--- a/funasr/datasets/llm_datasets_qwenaudio/datasets.py+++ b/funasr/datasets/llm_datasets_qwenaudio/datasets.py@@ -21,6 +21,17 @@ class AudioLLMQwenAudioDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs):+ """Initialize AudioLLMQwenAudioDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -54,17 +65,33 @@ class AudioLLMQwenAudioDataset(torch.utils.data.Dataset):self.answer_template = "{}"def get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]source = item["source"]data_src = load_audio_text_image_video(source, fs=self.fs)@@ -138,6 +165,11 @@ class AudioLLMQwenAudioDataset(torch.utils.data.Dataset):}def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:for key in sample.keys():diff --git a/funasr/datasets/llm_datasets_vicuna/datasets.py b/funasr/datasets/llm_datasets_vicuna/datasets.pyindex cde29a94a..dd5ce96f9 100644--- a/funasr/datasets/llm_datasets_vicuna/datasets.py+++ b/funasr/datasets/llm_datasets_vicuna/datasets.py@@ -21,6 +21,17 @@ class AudioLLMVicunaDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs):+ """Initialize AudioLLMVicunaDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -54,17 +65,33 @@ class AudioLLMVicunaDataset(torch.utils.data.Dataset):self.answer_template = "{}"def get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """item = self.index_ds[index]source = item["source"]data_src = load_audio_text_image_video(source, fs=self.fs)@@ -138,6 +165,11 @@ class AudioLLMVicunaDataset(torch.utils.data.Dataset):}def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:for key in sample.keys():diff --git a/funasr/datasets/openai_datasets/datasets.py b/funasr/datasets/openai_datasets/datasets.pyindex d670708a3..c373c424c 100644--- a/funasr/datasets/openai_datasets/datasets.py+++ b/funasr/datasets/openai_datasets/datasets.py@@ -23,6 +23,17 @@ class OpenAIDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs,):+ """Initialize OpenAIDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -68,20 +79,36 @@ class OpenAIDataset(torch.utils.data.Dataset):self.audio_encoder_downsample_rate = kwargs.get("audio_encoder_downsample_rate", 4)def get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):# import pdb;# pdb.set_trace()+ """Internal: getitem .++ Args:+ index: TODO.+ """output = Nonefor idx in range(self.retry):@@ -191,6 +218,11 @@ class OpenAIDataset(torch.utils.data.Dataset):def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """for idx in range(self.retry):badcase_flag = False@@ -245,6 +277,17 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs,):+ """Initialize OpenAIDatasetMultiTurn.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -290,14 +333,25 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):self.max_source_length = kwargs.get("max_source_length", 3000)def get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):@@ -305,6 +359,11 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):## pdb.set_trace()+ """Internal: getitem .++ Args:+ index: TODO.+ """output = Nonefor idx in range(self.retry):@@ -440,6 +499,11 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):def collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """for idx in range(self.retry):badcase_flag = Falsediff --git a/funasr/datasets/openai_datasets/index_ds.py b/funasr/datasets/openai_datasets/index_ds.pyindex 010d2d5d7..4b6e1091d 100644--- a/funasr/datasets/openai_datasets/index_ds.py+++ b/funasr/datasets/openai_datasets/index_ds.py@@ -14,6 +14,12 @@ from funasr.register import tablesclass OpenAIIndexDSJsonl(torch.utils.data.Dataset): # torch.utils.data.Datasetdef __init__(self, path: str, **kwargs):+ """Initialize OpenAIIndexDSJsonl.++ Args:+ path: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.max_source_length = kwargs.get("max_source_length", 3000)@@ -89,15 +95,26 @@ class OpenAIIndexDSJsonl(torch.utils.data.Dataset): # torch.utils.data.Datasetlogging.info("total_num of samplers: {}, {}".format(len(self.contents), path))def __len__(self):+ """Internal: len ."""return len(self.contents)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """data = self.contents[index]return datadef get_source_len(self, data_dict):+ """Get source len.++ Args:+ data_dict: TODO.+ """source_len = data_dict.get("source_len", -1)if source_len < 0:source_len = len(data_dict["system"]) + len(data_dict["user"])@@ -105,6 +122,11 @@ class OpenAIIndexDSJsonl(torch.utils.data.Dataset): # torch.utils.data.Datasetdef get_target_len(self, data_dict):+ """Get target len.++ Args:+ data_dict: TODO.+ """return 0diff --git a/funasr/datasets/sense_voice_datasets/datasets.py b/funasr/datasets/sense_voice_datasets/datasets.pyindex 2e0266ef1..8e1f45961 100644--- a/funasr/datasets/sense_voice_datasets/datasets.py+++ b/funasr/datasets/sense_voice_datasets/datasets.py@@ -24,6 +24,17 @@ class SenseVoiceDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs,):+ """Initialize SenseVoiceDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -61,18 +72,34 @@ class SenseVoiceDataset(torch.utils.data.Dataset):self.permute = Truedef get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """output = Nonefor idx in range(self.retry):if idx == 0:@@ -153,6 +180,11 @@ class SenseVoiceDataset(torch.utils.data.Dataset):return outputdef collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:if sample is None:@@ -209,6 +241,12 @@ class SenseVoiceDataset(torch.utils.data.Dataset):return outputsdef _filter_badcase(self, outputs, i=0):+ """Internal: filter badcase.++ Args:+ outputs: TODO.+ i: TODO.+ """b, t, _ = outputs["speech"].shapeif b * t > self.batch_size * 1.25:@@ -247,6 +285,17 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):float_pad_value: float = 0.0,**kwargs,):+ """Initialize SenseVoiceCTCDataset.++ Args:+ path: TODO.+ index_ds: TODO.+ frontend: Audio frontend for feature extraction.+ tokenizer: Tokenizer instance for text encoding/decoding.+ int_pad_value: TODO.+ float_pad_value: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()index_ds_class = tables.index_ds_classes.get(index_ds)self.index_ds = index_ds_class(path, **kwargs)@@ -284,18 +333,34 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):self.permute = Truedef get_source_len(self, index):+ """Get source len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_source_len(item)def get_target_len(self, index):+ """Get target len.++ Args:+ index: TODO.+ """item = self.index_ds[index]return self.index_ds.get_target_len(item)def __len__(self):+ """Internal: len ."""return len(self.index_ds)def __getitem__(self, index):+ """Internal: getitem .++ Args:+ index: TODO.+ """output = Nonefor idx in range(self.retry):if idx == 0:@@ -357,6 +422,11 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):return outputdef collator(self, samples: list = None):+ """Collator.++ Args:+ samples: TODO.+ """outputs = {}for sample in samples:if sample is None:@@ -410,6 +480,12 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):return outputsdef _filter_badcase(self, outputs, i=0):+ """Internal: filter badcase.++ Args:+ outputs: TODO.+ i: TODO.+ """b, t, _ = outputs["speech"].shapeif b * t > self.batch_size * 1.25:diff --git a/funasr/download/download_dataset_from_hub.py b/funasr/download/download_dataset_from_hub.pyindex 2419ce869..26082e79f 100644--- a/funasr/download/download_dataset_from_hub.py+++ b/funasr/download/download_dataset_from_hub.py@@ -1,8 +1,14 @@def download_dataset():+ """Download dataset."""passdef download_dataset_from_ms(**kwargs):+ """Download dataset from ms.++ Args:+ **kwargs: Additional keyword arguments.+ """from modelscope.msdatasets import MsDatasetdataset_name = kwargs.get("dataset_name", "speech_asr/speech_asr_aishell1_trainsets")diff --git a/funasr/download/download_model_from_hub.py b/funasr/download/download_model_from_hub.pyindex 0cdab34c2..322775519 100644--- a/funasr/download/download_model_from_hub.py+++ b/funasr/download/download_model_from_hub.py@@ -42,6 +42,11 @@ def download_model(**kwargs):def download_from_ms(**kwargs):+ """Download from ms.++ Args:+ **kwargs: Additional keyword arguments.+ """model_or_path = kwargs.get("model")if model_or_path in name_maps_ms:model_or_path = name_maps_ms[model_or_path]@@ -115,6 +120,11 @@ def download_from_ms(**kwargs):def download_from_hf(**kwargs):+ """Download from hf.++ Args:+ **kwargs: Additional keyword arguments.+ """model_or_path = kwargs.get("model")if model_or_path in name_maps_hf:model_or_path = name_maps_hf[model_or_path]@@ -180,6 +190,13 @@ def download_from_hf(**kwargs):def add_file_root_path(model_or_path: str, file_path_metas: dict, cfg={}):+ """Add file root path.++ Args:+ model_or_path: TODO.+ file_path_metas: TODO.+ cfg: Configuration overrides.+ """if isinstance(file_path_metas, dict):if isinstance(cfg, list):cfg.append({})diff --git a/funasr/download/file.py b/funasr/download/file.pyindex a8a13f874..0375f0c87 100644--- a/funasr/download/file.py+++ b/funasr/download/file.py@@ -12,6 +12,11 @@ from urllib.parse import urlparsedef download_from_url(url):+ """Download from url.++ Args:+ url: TODO.+ """result = urlparse(url)file_path = Noneif result.scheme is not None and len(result.scheme) > 0:@@ -38,18 +43,41 @@ class Storage(metaclass=ABCMeta):@abstractmethoddef read(self, filepath: str):+ """Read.++ Args:+ filepath: TODO.+ """pass@abstractmethoddef read_text(self, filepath: str):+ """Read text.++ Args:+ filepath: TODO.+ """pass@abstractmethoddef write(self, obj: bytes, filepath: Union[str, Path]) -> None:+ """Write.++ Args:+ obj: TODO.+ filepath: TODO.+ """pass@abstractmethoddef write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:+ """Write text.++ Args:+ obj: TODO.+ filepath: TODO.+ encoding: TODO.+ """pass@@ -133,11 +161,21 @@ class HTTPStorage(Storage):def read(self, url):# TODO @wenmeng.zwm add progress bar if file is too large+ """Read.++ Args:+ url: TODO.+ """r = requests.get(url)r.raise_for_status()return r.contentdef read_text(self, url):+ """Read text.++ Args:+ url: TODO.+ """r = requests.get(url)r.raise_for_status()return r.text@@ -169,9 +207,22 @@ class HTTPStorage(Storage):os.remove(f.name)def write(self, obj: bytes, url: Union[str, Path]) -> None:+ """Write.++ Args:+ obj: TODO.+ url: TODO.+ """raise NotImplementedError("write is not supported by HTTP Storage")def write_text(self, obj: str, url: Union[str, Path], encoding: str = "utf-8") -> None:+ """Write text.++ Args:+ obj: TODO.+ url: TODO.+ encoding: TODO.+ """raise NotImplementedError("write_text is not supported by HTTP Storage")@@ -180,12 +231,28 @@ class OSSStorage(Storage):def __init__(self, oss_config_file=None):# read from config file or env var+ """Initialize OSSStorage.++ Args:+ oss_config_file: TODO.+ """raise NotImplementedError("OSSStorage.__init__ to be implemented in the future")def read(self, filepath):+ """Read.++ Args:+ filepath: TODO.+ """raise NotImplementedError("OSSStorage.read to be implemented in the future")def read_text(self, filepath, encoding="utf-8"):+ """Read text.++ Args:+ filepath: TODO.+ encoding: TODO.+ """raise NotImplementedError("OSSStorage.read_text to be implemented in the future")@contextlib.contextmanager@@ -215,9 +282,22 @@ class OSSStorage(Storage):os.remove(f.name)def write(self, obj: bytes, filepath: Union[str, Path]) -> None:+ """Write.++ Args:+ obj: TODO.+ filepath: TODO.+ """raise NotImplementedError("OSSStorage.write to be implemented in the future")def write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:+ """Write text.++ Args:+ obj: TODO.+ filepath: TODO.+ encoding: TODO.+ """raise NotImplementedError("OSSStorage.write_text to be implemented in the future")@@ -234,6 +314,11 @@ class File(object):@staticmethoddef _get_storage(uri):+ """Internal: get storage.++ Args:+ uri: TODO.+ """assert isinstance(uri, str), f"uri should be str type, but got {type(uri)}"if "://" not in uri:diff --git a/funasr/download/runtime_sdk_download_tool.py b/funasr/download/runtime_sdk_download_tool.pyindex fb0ff028d..98c5826df 100644--- a/funasr/download/runtime_sdk_download_tool.py+++ b/funasr/download/runtime_sdk_download_tool.py@@ -6,6 +6,7 @@ from funasr.utils.type_utils import str2booldef main():+ """Main."""parser = argparse.ArgumentParser()parser.add_argument("--model-name", type=str, required=True)parser.add_argument("--export-dir", type=str, required=True)diff --git a/funasr/frontends/default.py b/funasr/frontends/default.pyindex 6ea88d6f6..8ef283d40 100644--- a/funasr/frontends/default.py+++ b/funasr/frontends/default.py@@ -45,6 +45,26 @@ class DefaultFrontend(nn.Module):use_channel: int = None,**kwargs,):+ """Initialize DefaultFrontend.++ Args:+ fs: TODO.+ n_fft: TODO.+ win_length: TODO.+ hop_length: TODO.+ window: TODO.+ center: TODO.+ normalized: TODO.+ onesided: TODO.+ n_mels: TODO.+ fmin: TODO.+ fmax: TODO.+ htk: TODO.+ frontend_conf: Configuration dict for frontend.+ apply_stft: TODO.+ use_channel: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# Deepcopy (In general, dict shouldn't be used as default arg)@@ -84,11 +104,18 @@ class DefaultFrontend(nn.Module):self.frontend_type = "default"def output_size(self) -> int:+ """Output size."""return self.n_melsdef forward(self, input: torch.Tensor, input_lengths: Union[torch.Tensor, list]) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """if isinstance(input_lengths, list):input_lengths = torch.tensor(input_lengths)if input.dtype == torch.float64:@@ -131,6 +158,12 @@ class DefaultFrontend(nn.Module):return input_feats, feats_lensdef _compute_stft(self, input: torch.Tensor, input_lengths: torch.Tensor) -> torch.Tensor:+ """Internal: compute stft.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """input_stft, feats_lens = self.stft(input, input_lengths)assert input_stft.dim() >= 4, input_stft.shape@@ -172,6 +205,31 @@ class MultiChannelFrontend(nn.Module):cmvn_file: str = None,mc: bool = True,):+ """Initialize MultiChannelFrontend.++ Args:+ fs: TODO.+ n_fft: TODO.+ win_length: TODO.+ hop_length: TODO.+ frame_length: TODO.+ frame_shift: TODO.+ window: TODO.+ center: TODO.+ normalized: TODO.+ onesided: TODO.+ n_mels: TODO.+ fmin: TODO.+ fmax: TODO.+ htk: TODO.+ frontend_conf: Configuration dict for frontend.+ apply_stft: TODO.+ use_channel: TODO.+ lfr_m: TODO.+ lfr_n: TODO.+ cmvn_file: TODO.+ mc: TODO.+ """super().__init__()# Deepcopy (In general, dict shouldn't be used as default arg)frontend_conf = copy.deepcopy(frontend_conf)@@ -230,12 +288,19 @@ class MultiChannelFrontend(nn.Module):self.frontend_type = "multichannelfrontend"def output_size(self) -> int:+ """Output size."""return self.n_melsdef forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:# 1. Domain-conversion: e.g. Stft: time -> time-freq+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """if self.stft is not None:input_stft, feats_lens = self._compute_stft(input, input_lengths)else:@@ -305,6 +370,12 @@ class MultiChannelFrontend(nn.Module):return input_feats, feats_lensdef _compute_stft(self, input: torch.Tensor, input_lengths: torch.Tensor) -> torch.Tensor:+ """Internal: compute stft.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """input_stft, feats_lens = self.stft(input, input_lengths)assert input_stft.dim() >= 4, input_stft.shape@@ -317,6 +388,11 @@ class MultiChannelFrontend(nn.Module):return input_stft, feats_lensdef _load_cmvn(self, cmvn_file):+ """Internal: load cmvn.++ Args:+ cmvn_file: TODO.+ """with open(cmvn_file, "r", encoding="utf-8") as f:lines = f.readlines()means_list = []diff --git a/funasr/frontends/eend_ola_feature.py b/funasr/frontends/eend_ola_feature.pyindex 0c79a3bb8..aead92daf 100644--- a/funasr/frontends/eend_ola_feature.py+++ b/funasr/frontends/eend_ola_feature.py@@ -8,6 +8,12 @@ import numpy as npdef transform(Y, dtype=np.float32):+ """Transform.++ Args:+ Y: TODO.+ dtype: TODO.+ """Y = np.abs(Y)n_fft = 2 * (Y.shape[1] - 1)sr = 8000@@ -21,12 +27,25 @@ def transform(Y, dtype=np.float32):def subsample(Y, T, subsampling=1):+ """Subsample.++ Args:+ Y: TODO.+ T: TODO.+ subsampling: TODO.+ """Y_ss = Y[::subsampling]T_ss = T[::subsampling]return Y_ss, T_ssdef splice(Y, context_size=0):+ """Splice.++ Args:+ Y: TODO.+ context_size: Size/dimension parameter.+ """Y_pad = np.pad(Y, [(context_size, context_size), (0, 0)], "constant")Y_spliced = np.lib.stride_tricks.as_strided(np.ascontiguousarray(Y_pad),@@ -38,6 +57,13 @@ def splice(Y, context_size=0):def stft(data, frame_size=1024, frame_shift=256):+ """Stft.++ Args:+ data: TODO.+ frame_size: Size/dimension parameter.+ frame_shift: TODO.+ """fft_size = 1 << (frame_size - 1).bit_length()if len(data) % frame_shift == 0:return librosa.stft(data, n_fft=fft_size, win_length=frame_size, hop_length=frame_shift).T[diff --git a/funasr/frontends/fused.py b/funasr/frontends/fused.pyindex 1da1e9f73..8a82125f1 100644--- a/funasr/frontends/fused.py+++ b/funasr/frontends/fused.py@@ -9,6 +9,14 @@ from typing import Tupleclass FusedFrontends(nn.Module):def __init__(self, frontends=None, align_method="linear_projection", proj_dim=100, fs=16000):+ """Initialize FusedFrontends.++ Args:+ frontends: TODO.+ align_method: TODO.+ proj_dim: Size/dimension parameter.+ fs: TODO.+ """super().__init__()self.align_method = align_method # fusing method : linear_projection only for nowself.proj_dim = proj_dim # dim of the projection done on each frontend@@ -96,6 +104,7 @@ class FusedFrontends(nn.Module):self.projection_layers = self.projection_layers.to(torch.device(dev))def output_size(self) -> int:+ """Output size."""return len(self.frontends) * self.proj_dimdef forward(@@ -103,6 +112,12 @@ class FusedFrontends(nn.Module):) -> Tuple[torch.Tensor, torch.Tensor]:# step 0 : get all frontends features+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """self.feats = []for frontend in self.frontends:with torch.no_grad():diff --git a/funasr/frontends/s3prl.py b/funasr/frontends/s3prl.pyindex 5c358cca6..36f1dc7a9 100644--- a/funasr/frontends/s3prl.py+++ b/funasr/frontends/s3prl.py@@ -15,6 +15,11 @@ from funasr.models.transformer.utils.nets_utils import pad_listdef base_s3prl_setup(args):+ """Base s3prl setup.++ Args:+ args: TODO.+ """args.upstream_feature_selection = getattr(args, "upstream_feature_selection", None)args.upstream_model_config = getattr(args, "upstream_model_config", None)args.upstream_refresh = getattr(args, "upstream_refresh", False)@@ -35,6 +40,14 @@ class S3prlFrontend(nn.Module):download_dir: str = None,multilayer_feature: bool = False,):+ """Initialize S3prlFrontend.++ Args:+ fs: TODO.+ frontend_conf: Configuration dict for frontend.+ download_dir: TODO.+ multilayer_feature: TODO.+ """super().__init__()if isinstance(fs, str):fs = humanfriendly.parse_size(fs)@@ -112,11 +125,18 @@ class S3prlFrontend(nn.Module):return tiled_featuredef output_size(self) -> int:+ """Output size."""return self.output_dimdef forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """wavs = [wav[: input_lengths[i]] for i, wav in enumerate(input)]self.upstream.eval()with torch.no_grad():@@ -135,5 +155,6 @@ class S3prlFrontend(nn.Module):return input_feats, feats_lensdef reload_pretrained_parameters(self):+ """Reload pretrained parameters."""self.upstream.load_state_dict(self.pretrained_params)logging.info("Pretrained S3PRL frontend model parameters reloaded!")diff --git a/funasr/frontends/utils/beamformer.py b/funasr/frontends/utils/beamformer.pyindex d19abe0ab..222847d61 100644--- a/funasr/frontends/utils/beamformer.py+++ b/funasr/frontends/utils/beamformer.py@@ -78,5 +78,11 @@ def get_mvdr_vector(def apply_beamforming_vector(beamform_vector: ComplexTensor, mix: ComplexTensor) -> ComplexTensor:# (..., C) x (..., C, T) -> (..., T)+ """Apply beamforming vector.++ Args:+ beamform_vector: TODO.+ mix: TODO.+ """es = FC.einsum("...c,...ct->...t", [beamform_vector.conj(), mix])return esdiff --git a/funasr/frontends/utils/complex_utils.py b/funasr/frontends/utils/complex_utils.pyindex 06c69363b..d7e2714a9 100644--- a/funasr/frontends/utils/complex_utils.py+++ b/funasr/frontends/utils/complex_utils.py@@ -23,6 +23,12 @@ def new_complex_like(ref: Union[torch.Tensor, ComplexTensor],real_imag: Tuple[torch.Tensor, torch.Tensor],):+ """New complex like.++ Args:+ ref: TODO.+ real_imag: TODO.+ """if isinstance(ref, ComplexTensor):return ComplexTensor(*real_imag)elif is_torch_complex_tensor(ref):@@ -32,14 +38,29 @@ def new_complex_like(def is_torch_complex_tensor(c):+ """Is torch complex tensor.++ Args:+ c: TODO.+ """return not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c)def is_complex(c):+ """Is complex.++ Args:+ c: TODO.+ """return isinstance(c, ComplexTensor) or is_torch_complex_tensor(c)def to_double(c):+ """To double.++ Args:+ c: TODO.+ """if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c):return c.to(dtype=torch.complex128)else:@@ -47,6 +68,11 @@ def to_double(c):def to_float(c):+ """To float.++ Args:+ c: TODO.+ """if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c):return c.to(dtype=torch.complex64)else:@@ -54,6 +80,13 @@ def to_float(c):def cat(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs):+ """Cat.++ Args:+ seq: TODO.+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """if not isinstance(seq, (list, tuple)):raise TypeError("cat(): argument 'tensors' (position 1) must be tuple of Tensors, " "not Tensor"@@ -65,6 +98,13 @@ def cat(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs):def complex_norm(c: Union[torch.Tensor, ComplexTensor], dim=-1, keepdim=False) -> torch.Tensor:+ """Complex norm.++ Args:+ c: TODO.+ dim: TODO.+ keepdim: TODO.+ """if not is_complex(c):raise TypeError("Input is not a complex tensor.")if is_torch_complex_tensor(c):@@ -77,6 +117,12 @@ def einsum(equation, *operands):# NOTE: Do not mix ComplexTensor and torch.complex in the input!# NOTE (wangyou): Until PyTorch 1.9.0, torch.einsum does not support# mixed input with complex and real tensors.+ """Einsum.++ Args:+ equation: TODO.+ *operands: Variable positional arguments.+ """if len(operands) == 1:if isinstance(operands[0], (tuple, list)):operands = operands[0]@@ -109,6 +155,11 @@ def einsum(equation, *operands):def inverse(c: Union[torch.Tensor, ComplexTensor]) -> Union[torch.Tensor, ComplexTensor]:+ """Inverse.++ Args:+ c: TODO.+ """if isinstance(c, ComplexTensor):return c.inverse2()else:@@ -121,6 +172,12 @@ def matmul(# NOTE: Do not mix ComplexTensor and torch.complex in the input!# NOTE (wangyou): Until PyTorch 1.9.0, torch.matmul does not support# multiplication between complex and real tensors.+ """Matmul.++ Args:+ a: TODO.+ b: TODO.+ """if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor):return FC.matmul(a, b)elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)):@@ -141,10 +198,21 @@ def matmul(def trace(a: Union[torch.Tensor, ComplexTensor]):# NOTE (wangyou): until PyTorch 1.9.0, torch.trace does not# support bacth processing. Use FC.trace() as fallback.+ """Trace.++ Args:+ a: TODO.+ """return FC.trace(a)def reverse(a: Union[torch.Tensor, ComplexTensor], dim=0):+ """Reverse.++ Args:+ a: TODO.+ dim: TODO.+ """if isinstance(a, ComplexTensor):return FC.reverse(a, dim=dim)else:@@ -174,6 +242,13 @@ def solve(b: Union[torch.Tensor, ComplexTensor], a: Union[torch.Tensor, ComplexTdef stack(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs):+ """Stack.++ Args:+ seq: TODO.+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """if not isinstance(seq, (list, tuple)):raise TypeError("stack(): argument 'tensors' (position 1) must be tuple of Tensors, " "not Tensor"diff --git a/funasr/frontends/utils/dnn_beamformer.py b/funasr/frontends/utils/dnn_beamformer.pyindex 73b6bea42..2ba20d34e 100644--- a/funasr/frontends/utils/dnn_beamformer.py+++ b/funasr/frontends/utils/dnn_beamformer.py@@ -36,6 +36,20 @@ class DNN_Beamformer(torch.nn.Module):ref_channel: int = -1,beamformer_type="mvdr",):+ """Initialize DNN_Beamformer.++ Args:+ bidim: TODO.+ btype: TODO.+ blayers: TODO.+ bunits: TODO.+ bprojs: TODO.+ bnmask: TODO.+ dropout_rate: TODO.+ badim: TODO.+ ref_channel: TODO.+ beamformer_type: TODO.+ """super().__init__()self.mask = MaskEstimator(btype, bidim, blayers, bunits, bprojs, dropout_rate, nmask=bnmask)self.ref = AttentionReference(bidim, badim)@@ -69,6 +83,14 @@ class DNN_Beamformer(torch.nn.Module):def apply_beamforming(data, ilens, psd_speech, psd_noise):# u: (B, C)+ """Apply beamforming.++ Args:+ data: TODO.+ ilens: TODO.+ psd_speech: TODO.+ psd_noise: TODO.+ """if self.ref_channel < 0:u, _ = self.ref(psd_speech, ilens)else:@@ -126,6 +148,12 @@ class DNN_Beamformer(torch.nn.Module):class AttentionReference(torch.nn.Module):def __init__(self, bidim, att_dim):+ """Initialize AttentionReference.++ Args:+ bidim: TODO.+ att_dim: Size/dimension parameter.+ """super().__init__()self.mlp_psd = torch.nn.Linear(bidim, att_dim)self.gvec = torch.nn.Linear(att_dim, 1)diff --git a/funasr/frontends/utils/dnn_wpe.py b/funasr/frontends/utils/dnn_wpe.pyindex 917133953..a03d266a8 100644--- a/funasr/frontends/utils/dnn_wpe.py+++ b/funasr/frontends/utils/dnn_wpe.py@@ -23,6 +23,21 @@ class DNN_WPE(torch.nn.Module):iterations: int = 1,normalization: bool = False,):+ """Initialize DNN_WPE.++ Args:+ wtype: TODO.+ widim: TODO.+ wlayers: TODO.+ wunits: TODO.+ wprojs: TODO.+ dropout_rate: TODO.+ taps: TODO.+ delay: TODO.+ use_dnn_mask: TODO.+ iterations: TODO.+ normalization: TODO.+ """super().__init__()self.iterations = iterationsself.taps = tapsdiff --git a/funasr/frontends/utils/feature_transform.py b/funasr/frontends/utils/feature_transform.pyindex ab1486be9..49f9b32d4 100644--- a/funasr/frontends/utils/feature_transform.py+++ b/funasr/frontends/utils/feature_transform.py@@ -25,6 +25,19 @@ class FeatureTransform(torch.nn.Module):uttmvn_norm_means: bool = True,uttmvn_norm_vars: bool = False,):+ """Initialize FeatureTransform.++ Args:+ fs: TODO.+ n_fft: TODO.+ n_mels: TODO.+ fmin: TODO.+ fmax: TODO.+ stats_file: TODO.+ apply_uttmvn: TODO.+ uttmvn_norm_means: TODO.+ uttmvn_norm_vars: TODO.+ """super().__init__()self.apply_uttmvn = apply_uttmvn@@ -44,6 +57,12 @@ class FeatureTransform(torch.nn.Module):self, x: ComplexTensor, ilens: Union[torch.LongTensor, np.ndarray, List[int]]) -> Tuple[torch.Tensor, torch.LongTensor]:# (B, T, F) or (B, T, C, F)+ """Forward pass for training.++ Args:+ x: TODO.+ ilens: TODO.+ """if x.dim() not in (3, 4):raise ValueError(f"Input dim must be 3 or 4: {x.dim()}")if not torch.is_tensor(ilens):@@ -103,6 +122,17 @@ class LogMel(torch.nn.Module):htk: bool = False,norm=1,):+ """Initialize LogMel.++ Args:+ fs: TODO.+ n_fft: TODO.+ n_mels: TODO.+ fmin: TODO.+ fmax: TODO.+ htk: TODO.+ norm: TODO.+ """super().__init__()_mel_options = dict(@@ -116,12 +146,19 @@ class LogMel(torch.nn.Module):self.register_buffer("melmat", torch.from_numpy(melmat.T).float())def extra_repr(self):+ """Extra repr."""return ", ".join(f"{k}={v}" for k, v in self.mel_options.items())def forward(self, feat: torch.Tensor, ilens: torch.LongTensor) -> Tuple[torch.Tensor, torch.LongTensor]:# feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2)+ """Forward pass for training.++ Args:+ feat: TODO.+ ilens: TODO.+ """mel_feat = torch.matmul(feat, self.melmat)logmel_feat = (mel_feat + 1e-20).log()@@ -151,6 +188,14 @@ class GlobalMVN(torch.nn.Module):norm_vars: bool = True,eps: float = 1.0e-20,):+ """Initialize GlobalMVN.++ Args:+ stats_file: TODO.+ norm_means: TODO.+ norm_vars: TODO.+ eps: TODO.+ """super().__init__()self.norm_means = norm_meansself.norm_vars = norm_vars@@ -170,6 +215,7 @@ class GlobalMVN(torch.nn.Module):self.register_buffer("scale", torch.from_numpy(1 / std.astype(np.float32)))def extra_repr(self):+ """Extra repr."""return (f"stats_file={self.stats_file}, "f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"@@ -179,6 +225,12 @@ class GlobalMVN(torch.nn.Module):self, x: torch.Tensor, ilens: torch.LongTensor) -> Tuple[torch.Tensor, torch.LongTensor]:# feat: (B, T, D)+ """Forward pass for training.++ Args:+ x: TODO.+ ilens: TODO.+ """if self.norm_means:x += self.bias.type_as(x)x.masked_fill(make_pad_mask(ilens, x, 1), 0.0)@@ -190,17 +242,31 @@ class GlobalMVN(torch.nn.Module):class UtteranceMVN(torch.nn.Module):def __init__(self, norm_means: bool = True, norm_vars: bool = False, eps: float = 1.0e-20):+ """Initialize UtteranceMVN.++ Args:+ norm_means: TODO.+ norm_vars: TODO.+ eps: TODO.+ """super().__init__()self.norm_means = norm_meansself.norm_vars = norm_varsself.eps = epsdef extra_repr(self):+ """Extra repr."""return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"def forward(self, x: torch.Tensor, ilens: torch.LongTensor) -> Tuple[torch.Tensor, torch.LongTensor]:+ """Forward pass for training.++ Args:+ x: TODO.+ ilens: TODO.+ """return utterance_mvn(x, ilens, norm_means=self.norm_means, norm_vars=self.norm_vars, eps=self.eps)@@ -244,6 +310,12 @@ def utterance_mvn(def feature_transform_for(args, n_fft):+ """Feature transform for.++ Args:+ args: TODO.+ n_fft: TODO.+ """return FeatureTransform(# Mel options,fs=args.fbank_fs,diff --git a/funasr/frontends/utils/frontend.py b/funasr/frontends/utils/frontend.pyindex a2a6cc120..924e8c2a7 100644--- a/funasr/frontends/utils/frontend.py+++ b/funasr/frontends/utils/frontend.py@@ -37,6 +37,29 @@ class Frontend(nn.Module):ref_channel: int = -1,bdropout_rate=0.0,):+ """Initialize Frontend.++ Args:+ idim: TODO.+ use_wpe: TODO.+ wtype: TODO.+ wlayers: TODO.+ wunits: TODO.+ wprojs: TODO.+ wdropout_rate: TODO.+ taps: TODO.+ delay: TODO.+ use_dnn_mask_for_wpe: TODO.+ use_beamformer: TODO.+ btype: TODO.+ blayers: TODO.+ bunits: TODO.+ bprojs: TODO.+ bnmask: TODO.+ badim: TODO.+ ref_channel: TODO.+ bdropout_rate: TODO.+ """super().__init__()self.use_beamformer = use_beamformer@@ -88,6 +111,12 @@ class Frontend(nn.Module):def forward(self, x: ComplexTensor, ilens: Union[torch.LongTensor, numpy.ndarray, List[int]]) -> Tuple[ComplexTensor, torch.LongTensor, Optional[ComplexTensor]]:+ """Forward pass for training.++ Args:+ x: TODO.+ ilens: TODO.+ """assert len(x) == len(ilens), (len(x), len(ilens))# (B, T, F) or (B, T, C, F)if x.dim() not in (3, 4):@@ -126,6 +155,12 @@ class Frontend(nn.Module):def frontend_for(args, idim):+ """Frontend for.++ Args:+ args: TODO.+ idim: TODO.+ """return Frontend(idim=idim,# WPE optionsdiff --git a/funasr/frontends/utils/log_mel.py b/funasr/frontends/utils/log_mel.pyindex 611cf1eb3..dd66b7fcc 100644--- a/funasr/frontends/utils/log_mel.py+++ b/funasr/frontends/utils/log_mel.py@@ -30,6 +30,17 @@ class LogMel(torch.nn.Module):htk: bool = False,log_base: float = None,):+ """Initialize LogMel.++ Args:+ fs: TODO.+ n_fft: TODO.+ n_mels: TODO.+ fmin: TODO.+ fmax: TODO.+ htk: TODO.+ log_base: TODO.+ """super().__init__()fmin = 0 if fmin is None else fmin@@ -51,6 +62,7 @@ class LogMel(torch.nn.Module):self.register_buffer("melmat", torch.from_numpy(melmat.T).float())def extra_repr(self):+ """Extra repr."""return ", ".join(f"{k}={v}" for k, v in self.mel_options.items())def forward(@@ -59,6 +71,12 @@ class LogMel(torch.nn.Module):ilens: torch.Tensor = None,) -> Tuple[torch.Tensor, torch.Tensor]:# feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2)+ """Forward pass for training.++ Args:+ feat: TODO.+ ilens: TODO.+ """mel_feat = torch.matmul(feat, self.melmat)mel_feat = torch.clamp(mel_feat, min=1e-10)diff --git a/funasr/frontends/utils/mask_estimator.py b/funasr/frontends/utils/mask_estimator.pyindex 29a21d97a..12b1294cb 100644--- a/funasr/frontends/utils/mask_estimator.py+++ b/funasr/frontends/utils/mask_estimator.py@@ -12,6 +12,17 @@ from funasr.models.language_model.rnn.encoders import RNNPclass MaskEstimator(torch.nn.Module):def __init__(self, type, idim, layers, units, projs, dropout, nmask=1):+ """Initialize MaskEstimator.++ Args:+ type: TODO.+ idim: TODO.+ layers: TODO.+ units: TODO.+ projs: TODO.+ dropout: TODO.+ nmask: TODO.+ """super().__init__()subsample = np.ones(layers + 1, dtype=np.int32)diff --git a/funasr/frontends/utils/stft.py b/funasr/frontends/utils/stft.pyindex 381f1e986..be2d4a3f1 100644--- a/funasr/frontends/utils/stft.py+++ b/funasr/frontends/utils/stft.py@@ -32,6 +32,17 @@ class Stft(torch.nn.Module):normalized: bool = False,onesided: bool = True,):+ """Initialize Stft.++ Args:+ n_fft: TODO.+ win_length: TODO.+ hop_length: TODO.+ window: TODO.+ center: TODO.+ normalized: TODO.+ onesided: TODO.+ """super().__init__()self.n_fft = n_fftif win_length is None:@@ -48,6 +59,7 @@ class Stft(torch.nn.Module):self.window = windowdef extra_repr(self):+ """Extra repr."""return (f"n_fft={self.n_fft}, "f"win_length={self.win_length}, "diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.pyindex c733d6663..072bebf82 100644--- a/funasr/frontends/wav_frontend.py+++ b/funasr/frontends/wav_frontend.py@@ -13,6 +13,11 @@ from funasr.register import tablesdef load_cmvn(cmvn_file):+ """Load cmvn.++ Args:+ cmvn_file: TODO.+ """with open(cmvn_file, "r", encoding="utf-8") as f:lines = f.readlines()means_list = []@@ -56,6 +61,13 @@ def apply_cmvn(inputs, cmvn): # noqadef apply_lfr(inputs, lfr_m, lfr_n):+ """Apply lfr.++ Args:+ inputs: TODO.+ lfr_m: TODO.+ lfr_n: TODO.+ """LFR_inputs = []T = inputs.shape[0]T_lfr = int(np.ceil(T / lfr_n))@@ -96,6 +108,24 @@ class WavFrontend(nn.Module):upsacle_samples: bool = True,**kwargs,):+ """Initialize WavFrontend.++ Args:+ cmvn_file: TODO.+ fs: TODO.+ window: TODO.+ n_mels: TODO.+ frame_length: TODO.+ frame_shift: TODO.+ filter_length_min: TODO.+ filter_length_max: TODO.+ lfr_m: TODO.+ lfr_n: TODO.+ dither: TODO.+ snip_edges: TODO.+ upsacle_samples: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.fs = fsself.window = window@@ -113,6 +143,7 @@ class WavFrontend(nn.Module):self.cmvn = None if self.cmvn_file is None else load_cmvn(self.cmvn_file)def output_size(self) -> int:+ """Output size."""return self.n_mels * self.lfr_mdef forward(@@ -121,6 +152,13 @@ class WavFrontend(nn.Module):input_lengths,**kwargs,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ **kwargs: Additional keyword arguments.+ """batch_size = input.size(0)feats = []feats_lens = []@@ -160,6 +198,12 @@ class WavFrontend(nn.Module):def forward_fbank(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward fbank.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """batch_size = input.size(0)feats = []feats_lens = []@@ -190,6 +234,12 @@ class WavFrontend(nn.Module):def forward_lfr_cmvn(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward lfr cmvn.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """batch_size = input.size(0)feats = []feats_lens = []@@ -229,6 +279,24 @@ class WavFrontendOnline(nn.Module):upsacle_samples: bool = True,**kwargs,):+ """Initialize WavFrontendOnline.++ Args:+ cmvn_file: TODO.+ fs: TODO.+ window: TODO.+ n_mels: TODO.+ frame_length: TODO.+ frame_shift: TODO.+ filter_length_min: TODO.+ filter_length_max: TODO.+ lfr_m: TODO.+ lfr_n: TODO.+ dither: TODO.+ snip_edges: TODO.+ upsacle_samples: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.fs = fsself.window = window@@ -254,6 +322,7 @@ class WavFrontendOnline(nn.Module):# self.lfr_splice_cache = []def output_size(self) -> int:+ """Output size."""return self.n_mels * self.lfr_m@staticmethod@@ -311,6 +380,13 @@ class WavFrontendOnline(nn.Module):def compute_frame_num(sample_length: int, frame_sample_length: int, frame_shift_sample_length: int) -> int:+ """Compute frame num.++ Args:+ sample_length: TODO.+ frame_sample_length: TODO.+ frame_shift_sample_length: TODO.+ """frame_num = int((sample_length - frame_sample_length) / frame_shift_sample_length + 1)return frame_num if frame_num >= 1 and sample_length >= frame_sample_length else 0@@ -321,6 +397,14 @@ class WavFrontendOnline(nn.Module):cache: dict = None,**kwargs,) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """Forward fbank.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}batch_size = input.size(0)@@ -383,6 +467,15 @@ class WavFrontendOnline(nn.Module):cache: dict = None,**kwargs,):+ """Forward lfr cmvn.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ is_final: Whether this is the final chunk in streaming.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}batch_size = input.size(0)@@ -410,6 +503,13 @@ class WavFrontendOnline(nn.Module):return feats_pad, feats_lens, lfr_splice_frame_idxsdef forward(self, input: torch.Tensor, input_lengths: torch.Tensor, **kwargs):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ **kwargs: Additional keyword arguments.+ """is_final = kwargs.get("is_final", False)cache = kwargs.get("cache", {})if len(cache) == 0:@@ -492,6 +592,11 @@ class WavFrontendOnline(nn.Module):return feats, feats_lengthsdef init_cache(self, cache: dict = None):+ """Init cache.++ Args:+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}cache["reserve_waveforms"] = torch.empty(0)@@ -515,6 +620,16 @@ class WavFrontendMel23(nn.Module):lfr_n: int = 1,**kwargs,):+ """Initialize WavFrontendMel23.++ Args:+ fs: TODO.+ frame_length: TODO.+ frame_shift: TODO.+ lfr_m: TODO.+ lfr_n: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.fs = fsself.frame_length = frame_length@@ -524,11 +639,18 @@ class WavFrontendMel23(nn.Module):self.n_mels = 23def output_size(self) -> int:+ """Output size."""return self.n_mels * (2 * self.lfr_m + 1)def forward(self, input: torch.Tensor, input_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ """batch_size = input.size(0)feats = []feats_lens = []diff --git a/funasr/frontends/whisper_frontend.py b/funasr/frontends/whisper_frontend.pyindex 1bd8aecdd..6fd10f696 100644--- a/funasr/frontends/whisper_frontend.py+++ b/funasr/frontends/whisper_frontend.py@@ -23,6 +23,16 @@ class WhisperFrontend(nn.Module):permute: bool = False,**kwargs,):+ """Initialize WhisperFrontend.++ Args:+ fs: TODO.+ whisper_model: Whisper Model instance.+ do_pad_trim: TODO.+ n_mels: TODO.+ permute: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()assert fs == 16000self.fs = fs@@ -55,6 +65,7 @@ class WhisperFrontend(nn.Module):# assert whisper_model in whisper.available_models()def output_size(self) -> int:+ """Output size."""return self.n_melsdef log_mel_spectrogram(@@ -62,6 +73,12 @@ class WhisperFrontend(nn.Module):audio: torch.Tensor,ilens: torch.Tensor = None,) -> torch.Tensor:+ """Log mel spectrogram.++ Args:+ audio: TODO.+ ilens: TODO.+ """window = torch.hann_window(self.win_length).to(audio.device)stft = torch.stft(audio, self.n_fft, self.hop_length, window=window, return_complex=True)@@ -94,6 +111,13 @@ class WhisperFrontend(nn.Module):input_lengths: torch.Tensor,**kwargs,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ input_lengths: Lengths of input.+ **kwargs: Additional keyword arguments.+ """batch_size = input.size(0)feats = []feats_lens = []diff --git a/funasr/losses/label_smoothing_loss.py b/funasr/losses/label_smoothing_loss.pyindex ffc38dae7..42d5dd127 100644--- a/funasr/losses/label_smoothing_loss.py+++ b/funasr/losses/label_smoothing_loss.py@@ -66,11 +66,24 @@ class LabelSmoothingLoss(nn.Module):class SequenceBinaryCrossEntropy(nn.Module):def __init__(self, normalize_length=False, criterion=nn.BCEWithLogitsLoss(reduction="none")):+ """Initialize SequenceBinaryCrossEntropy.++ Args:+ normalize_length: TODO.+ criterion: TODO.+ """super().__init__()self.normalize_length = normalize_lengthself.criterion = criteriondef forward(self, pred, label, lengths):+ """Forward pass for training.++ Args:+ pred: TODO.+ label: TODO.+ lengths: TODO.+ """pad_mask = make_pad_mask(lengths, maxlen=pred.shape[1]).to(pred.device)loss = self.criterion(pred, label)denom = (~pad_mask).sum() if self.normalize_length else pred.shape[0]diff --git a/funasr/metrics/compute_eer.py b/funasr/metrics/compute_eer.pyindex 5180b1669..ea0e3a2f3 100644--- a/funasr/metrics/compute_eer.py+++ b/funasr/metrics/compute_eer.py@@ -31,6 +31,12 @@ def _compute_eer(label, pred, positive_label=1):def compute_eer(trials_path, scores_path):+ """Compute eer.++ Args:+ trials_path: TODO.+ scores_path: TODO.+ """labels = []for one_line in open(trials_path, "r"):labels.append(one_line.strip().rsplit(" ", 1)[-1] == "target")@@ -46,6 +52,7 @@ def compute_eer(trials_path, scores_path):def main():+ """Main."""parser = argparse.ArgumentParser()parser.add_argument("trials", help="trial list")parser.add_argument("scores", help="score file, normalized to [0, 1]")diff --git a/funasr/metrics/compute_min_dcf.py b/funasr/metrics/compute_min_dcf.pyindex 472f7b3e2..a1e3427db 100644--- a/funasr/metrics/compute_min_dcf.py+++ b/funasr/metrics/compute_min_dcf.py@@ -16,6 +16,7 @@ import sys, argparse, osdef GetArgs():+ """Getargs."""parser = argparse.ArgumentParser(description="Compute the minimum ""detection cost function along with the threshold at which it occurs. "@@ -61,6 +62,11 @@ def GetArgs():def CheckArgs(args):+ """Checkargs.++ Args:+ args: TODO.+ """if args.c_fa <= 0:raise Exception("--c-fa must be greater than 0")if args.c_miss <= 0:@@ -77,6 +83,12 @@ def ComputeErrorRates(scores, labels):# Sort the scores from smallest to largest, and also get the corresponding# indexes of the sorted scores. We will treat the sorted scores as the# thresholds at which the the error-rates are evaluated.+ """Computeerrorrates.++ Args:+ scores: TODO.+ labels: TODO.+ """sorted_indexes, thresholds = zip(*sorted([(index, threshold) for index, threshold in enumerate(scores)], key=itemgetter(1)))@@ -113,6 +125,16 @@ def ComputeErrorRates(scores, labels):# Computes the minimum of the detection cost function. The comments refer to# equations in Section 3 of the NIST 2016 Speaker Recognition Evaluation Plan.def ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa):+ """Computemindcf.++ Args:+ fnrs: TODO.+ fprs: TODO.+ thresholds: TODO.+ p_target: TODO.+ c_miss: TODO.+ c_fa: TODO.+ """min_c_det = float("inf")min_c_det_threshold = thresholds[0]for i in range(0, len(fnrs)):@@ -129,6 +151,15 @@ def ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa):def compute_min_dcf(scores_filename, trials_filename, c_miss=1, c_fa=1, p_target=0.01):+ """Compute min dcf.++ Args:+ scores_filename: TODO.+ trials_filename: TODO.+ c_miss: TODO.+ c_fa: TODO.+ p_target: TODO.+ """scores_file = open(scores_filename, "r").readlines()trials_file = open(trials_filename, "r").readlines()c_miss = c_miss@@ -162,6 +193,7 @@ def compute_min_dcf(scores_filename, trials_filename, c_miss=1, c_fa=1, p_targetdef main():+ """Main."""args = GetArgs()mindcf, threshold = compute_min_dcf(args.scores_filename, args.trials_filename, args.c_miss, args.c_fa, args.p_targetdiff --git a/funasr/metrics/wer.py b/funasr/metrics/wer.pyindex 6bac30324..c32a262fa 100755--- a/funasr/metrics/wer.py+++ b/funasr/metrics/wer.py@@ -11,6 +11,14 @@ def compute_wer(cer_file,cn_postprocess=False,):+ """Compute wer.++ Args:+ ref_file: TODO.+ hyp_file: TODO.+ cer_file: TODO.+ cn_postprocess: TODO.+ """rst = {"Wrd": 0,"Corr": 0,@@ -114,6 +122,12 @@ def compute_wer(def compute_wer_by_line(hyp, ref):+ """Compute wer by line.++ Args:+ hyp: TODO.+ ref: TODO.+ """hyp = list(map(lambda x: x.lower(), hyp))ref = list(map(lambda x: x.lower(), ref))@@ -187,6 +201,11 @@ def compute_wer_by_line(hyp, ref):def print_cer_detail(rst):+ """Print cer detail.++ Args:+ rst: TODO.+ """return ("("+ "nwords="@@ -208,6 +227,11 @@ def print_cer_detail(rst):@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """ref_file = cfg.get("ref_file", None)hyp_file = cfg.get("hyp_file", None)cer_file = cfg.get("cer_file", None)diff --git a/funasr/models/bat/model.py b/funasr/models/bat/model.pyindex ec2244337..5d64f752e 100644--- a/funasr/models/bat/model.py+++ b/funasr/models/bat/model.py@@ -29,6 +29,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yielddiff --git a/funasr/models/bicif_paraformer/cif_predictor.py b/funasr/models/bicif_paraformer/cif_predictor.pyindex 77d4e9659..95c100f79 100644--- a/funasr/models/bicif_paraformer/cif_predictor.py+++ b/funasr/models/bicif_paraformer/cif_predictor.py@@ -1,544 +1,659 @@-#!/usr/bin/env python3-# -*- encoding: utf-8 -*--# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.-# MIT License (https://opensource.org/licenses/MIT)--import torch--from funasr.register import tables-from funasr.models.transformer.utils.nets_utils import make_pad_mask---class mae_loss(torch.nn.Module):-- def __init__(self, normalize_length=False):- super(mae_loss, self).__init__()- self.normalize_length = normalize_length- self.criterion = torch.nn.L1Loss(reduction="sum")-- def forward(self, token_length, pre_token_length):- loss_token_normalizer = token_length.size(0)- if self.normalize_length:- loss_token_normalizer = token_length.sum().type(torch.float32)- loss = self.criterion(token_length, pre_token_length)- loss = loss / loss_token_normalizer- return loss---def cif(hidden, alphas, threshold):- batch_size, len_time, hidden_size = hidden.size()-- # loop varss- integrate = torch.zeros([batch_size], device=hidden.device)- frame = torch.zeros([batch_size, hidden_size], device=hidden.device)- # intermediate vars along time- list_fires = []- list_frames = []-- for t in range(len_time):- alpha = alphas[:, t]- distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate-- integrate += alpha- list_fires.append(integrate)-- fire_place = integrate >= threshold- integrate = torch.where(- fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate- )- cur = torch.where(fire_place, distribution_completion, alpha)- remainds = alpha - cur-- frame += cur[:, None] * hidden[:, t, :]- list_frames.append(frame)- frame = torch.where(- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame- )-- fires = torch.stack(list_fires, 1)- frames = torch.stack(list_frames, 1)- list_ls = []- len_labels = torch.round(alphas.sum(-1)).int()- max_label_len = len_labels.max()- for b in range(batch_size):- fire = fires[b, :]- l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze(-1))- pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)- list_ls.append(torch.cat([l, pad_l], 0))- return torch.stack(list_ls, 0), fires---def cif_wo_hidden(alphas, threshold):- batch_size, len_time = alphas.size()-- # loop varss- integrate = torch.zeros([batch_size], device=alphas.device)- # intermediate vars along time- list_fires = []-- for t in range(len_time):- alpha = alphas[:, t]-- integrate += alpha- list_fires.append(integrate)-- fire_place = integrate >= threshold- integrate = torch.where(- fire_place,- integrate - torch.ones([batch_size], device=alphas.device) * threshold,- integrate,- )-- fires = torch.stack(list_fires, 1)- return fires--[email protected]("predictor_classes", "CifPredictorV3")-class CifPredictorV3(torch.nn.Module):- def __init__(- self,- idim,- l_order,- r_order,- threshold=1.0,- dropout=0.1,- smooth_factor=1.0,- noise_threshold=0,- tail_threshold=0.0,- tf2torch_tensor_name_prefix_torch="predictor",- tf2torch_tensor_name_prefix_tf="seq2seq/cif",- smooth_factor2=1.0,- noise_threshold2=0,- upsample_times=5,- upsample_type="cnn",- use_cif1_cnn=True,- tail_mask=True,- ):- super(CifPredictorV3, self).__init__()-- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)- self.cif_output = torch.nn.Linear(idim, 1)- self.dropout = torch.nn.Dropout(p=dropout)- self.threshold = threshold- self.smooth_factor = smooth_factor- self.noise_threshold = noise_threshold- self.tail_threshold = tail_threshold- self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch- self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf-- self.upsample_times = upsample_times- self.upsample_type = upsample_type- self.use_cif1_cnn = use_cif1_cnn- if self.upsample_type == "cnn":- self.upsample_cnn = torch.nn.ConvTranspose1d(- idim, idim, self.upsample_times, self.upsample_times- )- self.cif_output2 = torch.nn.Linear(idim, 1)- elif self.upsample_type == "cnn_blstm":- self.upsample_cnn = torch.nn.ConvTranspose1d(- idim, idim, self.upsample_times, self.upsample_times- )- self.blstm = torch.nn.LSTM(- idim, idim, 1, bias=True, batch_first=True, dropout=0.0, bidirectional=True- )- self.cif_output2 = torch.nn.Linear(idim * 2, 1)- elif self.upsample_type == "cnn_attn":- self.upsample_cnn = torch.nn.ConvTranspose1d(- idim, idim, self.upsample_times, self.upsample_times- )- from funasr.models.transformer.encoder import EncoderLayer as TransformerEncoderLayer- from funasr.models.transformer.attention import MultiHeadedAttention- from funasr.models.transformer.positionwise_feed_forward import PositionwiseFeedForward-- positionwise_layer_args = (- idim,- idim * 2,- 0.1,- )- self.self_attn = TransformerEncoderLayer(- idim,- MultiHeadedAttention(4, idim, 0.1),- PositionwiseFeedForward(*positionwise_layer_args),- 0.1,- True, # normalize_before,- False, # concat_after,- )- self.cif_output2 = torch.nn.Linear(idim, 1)- self.smooth_factor2 = smooth_factor2- self.noise_threshold2 = noise_threshold2-- def forward(- self,- hidden,- target_label=None,- mask=None,- ignore_id=-1,- mask_chunk_predictor=None,- target_label_length=None,- ):- h = hidden- context = h.transpose(1, 2)- queries = self.pad(context)- output = torch.relu(self.cif_conv1d(queries))-- # alphas2 is an extra head for timestamp prediction- if not self.use_cif1_cnn:- _output = context- else:- _output = output- if self.upsample_type == "cnn":- output2 = self.upsample_cnn(_output)- output2 = output2.transpose(1, 2)- elif self.upsample_type == "cnn_blstm":- output2 = self.upsample_cnn(_output)- output2 = output2.transpose(1, 2)- output2, (_, _) = self.blstm(output2)- elif self.upsample_type == "cnn_attn":- output2 = self.upsample_cnn(_output)- output2 = output2.transpose(1, 2)- output2, _ = self.self_attn(output2, mask)-- alphas2 = torch.sigmoid(self.cif_output2(output2))- alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)- # repeat the mask in T demension to match the upsampled length- if mask is not None:- mask2 = (- mask.repeat(1, self.upsample_times, 1)- .transpose(-1, -2)- .reshape(alphas2.shape[0], -1)- )- mask2 = mask2.unsqueeze(-1)- alphas2 = alphas2 * mask2- alphas2 = alphas2.squeeze(-1)- token_num2 = alphas2.sum(-1)-- output = output.transpose(1, 2)-- output = self.cif_output(output)- alphas = torch.sigmoid(output)- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)- if mask is not None:- mask = mask.transpose(-1, -2).float()- alphas = alphas * mask- if mask_chunk_predictor is not None:- alphas = alphas * mask_chunk_predictor- alphas = alphas.squeeze(-1)- mask = mask.squeeze(-1)- if target_label_length is not None:- target_length = target_label_length- elif target_label is not None:- target_length = (target_label != ignore_id).float().sum(-1)- else:- target_length = None- token_num = alphas.sum(-1)-- if target_length is not None:- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))- elif self.tail_threshold > 0.0:- hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=mask)-- acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)- if target_length is None and self.tail_threshold > 0.0:- token_num_int = torch.max(token_num).type(torch.int32).item()- acoustic_embeds = acoustic_embeds[:, :token_num_int, :]- return acoustic_embeds, token_num, alphas, cif_peak, token_num2-- def get_upsample_timestamp(self, hidden, mask=None, token_num=None):- h = hidden- b = hidden.shape[0]- context = h.transpose(1, 2)- queries = self.pad(context)- output = torch.relu(self.cif_conv1d(queries))-- # alphas2 is an extra head for timestamp prediction- if not self.use_cif1_cnn:- _output = context- else:- _output = output- if self.upsample_type == "cnn":- output2 = self.upsample_cnn(_output)- output2 = output2.transpose(1, 2)- elif self.upsample_type == "cnn_blstm":- output2 = self.upsample_cnn(_output)- output2 = output2.transpose(1, 2)- output2, (_, _) = self.blstm(output2)- elif self.upsample_type == "cnn_attn":- output2 = self.upsample_cnn(_output)- output2 = output2.transpose(1, 2)- output2, _ = self.self_attn(output2, mask)- alphas2 = torch.sigmoid(self.cif_output2(output2))- alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)- # repeat the mask in T demension to match the upsampled length- if mask is not None:- mask2 = (- mask.repeat(1, self.upsample_times, 1)- .transpose(-1, -2)- .reshape(alphas2.shape[0], -1)- )- mask2 = mask2.unsqueeze(-1)- alphas2 = alphas2 * mask2- alphas2 = alphas2.squeeze(-1)- _token_num = alphas2.sum(-1)- if token_num is not None:- alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))- # re-downsample- ds_alphas = alphas2.reshape(b, -1, self.upsample_times).sum(-1)- ds_cif_peak = cif_wo_hidden(ds_alphas, self.threshold - 1e-4)- # upsampled alphas and cif_peak- us_alphas = alphas2- us_cif_peak = cif_wo_hidden(us_alphas, self.threshold - 1e-4)- return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak-- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):- b, t, d = hidden.size()- tail_threshold = self.tail_threshold- if mask is not None:- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)- ones_t = torch.ones_like(zeros_t)- mask_1 = torch.cat([mask, zeros_t], dim=1)- mask_2 = torch.cat([ones_t, mask], dim=1)- mask = mask_2 - mask_1- tail_threshold = mask * tail_threshold- alphas = torch.cat([alphas, zeros_t], dim=1)- alphas = torch.add(alphas, tail_threshold)- else:- tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)- tail_threshold = torch.reshape(tail_threshold, (1, 1))- alphas = torch.cat([alphas, tail_threshold], dim=1)- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)- hidden = torch.cat([hidden, zeros], dim=1)- token_num = alphas.sum(dim=-1)- token_num_floor = torch.floor(token_num)-- return hidden, alphas, token_num_floor-- def gen_frame_alignments(- self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None- ):- batch_size, maximum_length = alphas.size()- int_type = torch.int32-- is_training = self.training- if is_training:- token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)- else:- token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)-- max_token_num = torch.max(token_num).item()-- alphas_cumsum = torch.cumsum(alphas, dim=1)- alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)- alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)-- index = torch.ones([batch_size, max_token_num], dtype=int_type)- index = torch.cumsum(index, dim=1)- index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)-- index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)- index_div_bool_zeros = index_div.eq(0)- index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1- index_div_bool_zeros_count = torch.clamp(- index_div_bool_zeros_count, 0, encoder_sequence_length.max()- )- token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)- index_div_bool_zeros_count *= token_num_mask-- index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(- 1, 1, maximum_length- )- ones = torch.ones_like(index_div_bool_zeros_count_tile)- zeros = torch.zeros_like(index_div_bool_zeros_count_tile)- ones = torch.cumsum(ones, dim=2)- cond = index_div_bool_zeros_count_tile == ones- index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)-- index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)- index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)- index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)- predictor_mask = (- (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))- .type(int_type)- .to(encoder_sequence_length.device)- )- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask-- predictor_alignments = index_div_bool_zeros_count_tile_out- predictor_alignments_length = predictor_alignments.sum(-1).type(- encoder_sequence_length.dtype- )- return predictor_alignments.detach(), predictor_alignments_length.detach()--[email protected]("predictor_classes", "CifPredictorV3Export")-class CifPredictorV3Export(torch.nn.Module):- def __init__(self, model, **kwargs):- super().__init__()-- self.pad = model.pad- self.cif_conv1d = model.cif_conv1d- self.cif_output = model.cif_output- self.threshold = model.threshold- self.smooth_factor = model.smooth_factor- self.noise_threshold = model.noise_threshold- self.tail_threshold = model.tail_threshold-- self.upsample_times = model.upsample_times- self.upsample_cnn = model.upsample_cnn- self.blstm = model.blstm- self.cif_output2 = model.cif_output2- self.smooth_factor2 = model.smooth_factor2- self.noise_threshold2 = model.noise_threshold2-- def forward(- self,- hidden: torch.Tensor,- mask: torch.Tensor,- ):- h = hidden- context = h.transpose(1, 2)- queries = self.pad(context)- output = torch.relu(self.cif_conv1d(queries))- output = output.transpose(1, 2)-- output = self.cif_output(output)- alphas = torch.sigmoid(output)- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)- mask = mask.transpose(-1, -2).float()- alphas = alphas * mask- alphas = alphas.squeeze(-1)- token_num = alphas.sum(-1)-- mask = mask.squeeze(-1)- hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)- acoustic_embeds, cif_peak = cif_export(hidden, alphas, self.threshold)-- return acoustic_embeds, token_num, alphas, cif_peak-- def get_upsample_timestmap(self, hidden, mask=None, token_num=None):- h = hidden- b = hidden.shape[0]- context = h.transpose(1, 2)-- # generate alphas2- _output = context- output2 = self.upsample_cnn(_output)- output2 = output2.transpose(1, 2)- output2, (_, _) = self.blstm(output2)- alphas2 = torch.sigmoid(self.cif_output2(output2))- alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)-- mask = (- mask.repeat(1, self.upsample_times, 1).transpose(-1, -2).reshape(alphas2.shape[0], -1)- )- mask = mask.unsqueeze(-1)- alphas2 = alphas2 * mask- alphas2 = alphas2.squeeze(-1)- _token_num = alphas2.sum(-1)- alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))- # upsampled alphas and cif_peak- us_alphas = alphas2- us_cif_peak = cif_wo_hidden_export(us_alphas, self.threshold - 1e-4)- return us_alphas, us_cif_peak-- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):- b, t, d = hidden.size()- tail_threshold = self.tail_threshold-- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)- ones_t = torch.ones_like(zeros_t)-- mask_1 = torch.cat([mask, zeros_t], dim=1)- mask_2 = torch.cat([ones_t, mask], dim=1)- mask = mask_2 - mask_1- tail_threshold = mask * tail_threshold- alphas = torch.cat([alphas, zeros_t], dim=1)- alphas = torch.add(alphas, tail_threshold)-- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)- hidden = torch.cat([hidden, zeros], dim=1)- token_num = alphas.sum(dim=-1)- token_num_floor = torch.floor(token_num)-- return hidden, alphas, token_num_floor---def cif_export(hidden, alphas, threshold: float):- batch_size, len_time, hidden_size = hidden.size()- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)-- # loop varss- integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)- frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)- # intermediate vars along time- list_fires = []- list_frames = []-- for t in range(len_time):- alpha = alphas[:, t]- distribution_completion = (- torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate- )-- integrate += alpha- list_fires.append(integrate)-- fire_place = integrate >= threshold- integrate = torch.where(- fire_place,- integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),- integrate,- )- cur = torch.where(fire_place, distribution_completion, alpha)- remainds = alpha - cur-- frame += cur[:, None] * hidden[:, t, :]- list_frames.append(frame)- frame = torch.where(- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame- )-- fires = torch.stack(list_fires, 1)- frames = torch.stack(list_frames, 1)-- fire_idxs = fires >= threshold- frame_fires = torch.zeros_like(hidden)- max_label_len = frames[0, fire_idxs[0]].size(0)- for b in range(batch_size):- frame_fire = frames[b, fire_idxs[b]]- frame_len = frame_fire.size(0)- frame_fires[b, :frame_len, :] = frame_fire-- if frame_len >= max_label_len:- max_label_len = frame_len- frame_fires = frame_fires[:, :max_label_len, :]- return frame_fires, fires---def cif_wo_hidden_export(alphas, threshold: float):- batch_size, len_time = alphas.size()-- # loop varss- integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=alphas.device)- # intermediate vars along time- list_fires = []-- for t in range(len_time):- alpha = alphas[:, t]-- integrate += alpha- list_fires.append(integrate)-- fire_place = integrate >= threshold- integrate = torch.where(- fire_place,- integrate - torch.ones([batch_size], device=alphas.device) * threshold,- integrate,- )-- fires = torch.stack(list_fires, 1)- return fires-+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++import torch++from funasr.register import tables+from funasr.models.transformer.utils.nets_utils import make_pad_mask+++class mae_loss(torch.nn.Module):++ def __init__(self, normalize_length=False):+ """Initialize mae_loss.++ Args:+ normalize_length: TODO.+ """+ super(mae_loss, self).__init__()+ self.normalize_length = normalize_length+ self.criterion = torch.nn.L1Loss(reduction="sum")++ def forward(self, token_length, pre_token_length):+ """Forward pass for training.++ Args:+ token_length: TODO.+ pre_token_length: TODO.+ """+ loss_token_normalizer = token_length.size(0)+ if self.normalize_length:+ loss_token_normalizer = token_length.sum().type(torch.float32)+ loss = self.criterion(token_length, pre_token_length)+ loss = loss / loss_token_normalizer+ return loss+++def cif(hidden, alphas, threshold):+ """Cif.++ Args:+ hidden: TODO.+ alphas: TODO.+ threshold: TODO.+ """+ batch_size, len_time, hidden_size = hidden.size()++ # loop varss+ integrate = torch.zeros([batch_size], device=hidden.device)+ frame = torch.zeros([batch_size, hidden_size], device=hidden.device)+ # intermediate vars along time+ list_fires = []+ list_frames = []++ for t in range(len_time):+ alpha = alphas[:, t]+ distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate++ integrate += alpha+ list_fires.append(integrate)++ fire_place = integrate >= threshold+ integrate = torch.where(+ fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate+ )+ cur = torch.where(fire_place, distribution_completion, alpha)+ remainds = alpha - cur++ frame += cur[:, None] * hidden[:, t, :]+ list_frames.append(frame)+ frame = torch.where(+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame+ )++ fires = torch.stack(list_fires, 1)+ frames = torch.stack(list_frames, 1)+ list_ls = []+ len_labels = torch.round(alphas.sum(-1)).int()+ max_label_len = len_labels.max()+ for b in range(batch_size):+ fire = fires[b, :]+ l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze(-1))+ pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)+ list_ls.append(torch.cat([l, pad_l], 0))+ return torch.stack(list_ls, 0), fires+++def cif_wo_hidden(alphas, threshold):+ """Cif wo hidden.++ Args:+ alphas: TODO.+ threshold: TODO.+ """+ batch_size, len_time = alphas.size()++ # loop varss+ integrate = torch.zeros([batch_size], device=alphas.device)+ # intermediate vars along time+ list_fires = []++ for t in range(len_time):+ alpha = alphas[:, t]++ integrate += alpha+ list_fires.append(integrate)++ fire_place = integrate >= threshold+ integrate = torch.where(+ fire_place,+ integrate - torch.ones([batch_size], device=alphas.device) * threshold,+ integrate,+ )++ fires = torch.stack(list_fires, 1)+ return fires++[email protected]("predictor_classes", "CifPredictorV3")+class CifPredictorV3(torch.nn.Module):+ def __init__(+ self,+ idim,+ l_order,+ r_order,+ threshold=1.0,+ dropout=0.1,+ smooth_factor=1.0,+ noise_threshold=0,+ tail_threshold=0.0,+ tf2torch_tensor_name_prefix_torch="predictor",+ tf2torch_tensor_name_prefix_tf="seq2seq/cif",+ smooth_factor2=1.0,+ noise_threshold2=0,+ upsample_times=5,+ upsample_type="cnn",+ use_cif1_cnn=True,+ tail_mask=True,+ ):+ """Initialize CifPredictorV3.++ Args:+ idim: TODO.+ l_order: TODO.+ r_order: TODO.+ threshold: TODO.+ dropout: TODO.+ smooth_factor: TODO.+ noise_threshold: TODO.+ tail_threshold: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ smooth_factor2: TODO.+ noise_threshold2: TODO.+ upsample_times: TODO.+ upsample_type: TODO.+ use_cif1_cnn: TODO.+ tail_mask: TODO.+ """+ super(CifPredictorV3, self).__init__()++ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)+ self.cif_output = torch.nn.Linear(idim, 1)+ self.dropout = torch.nn.Dropout(p=dropout)+ self.threshold = threshold+ self.smooth_factor = smooth_factor+ self.noise_threshold = noise_threshold+ self.tail_threshold = tail_threshold+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf++ self.upsample_times = upsample_times+ self.upsample_type = upsample_type+ self.use_cif1_cnn = use_cif1_cnn+ if self.upsample_type == "cnn":+ self.upsample_cnn = torch.nn.ConvTranspose1d(+ idim, idim, self.upsample_times, self.upsample_times+ )+ self.cif_output2 = torch.nn.Linear(idim, 1)+ elif self.upsample_type == "cnn_blstm":+ self.upsample_cnn = torch.nn.ConvTranspose1d(+ idim, idim, self.upsample_times, self.upsample_times+ )+ self.blstm = torch.nn.LSTM(+ idim, idim, 1, bias=True, batch_first=True, dropout=0.0, bidirectional=True+ )+ self.cif_output2 = torch.nn.Linear(idim * 2, 1)+ elif self.upsample_type == "cnn_attn":+ self.upsample_cnn = torch.nn.ConvTranspose1d(+ idim, idim, self.upsample_times, self.upsample_times+ )+ from funasr.models.transformer.encoder import EncoderLayer as TransformerEncoderLayer+ from funasr.models.transformer.attention import MultiHeadedAttention+ from funasr.models.transformer.positionwise_feed_forward import PositionwiseFeedForward++ positionwise_layer_args = (+ idim,+ idim * 2,+ 0.1,+ )+ self.self_attn = TransformerEncoderLayer(+ idim,+ MultiHeadedAttention(4, idim, 0.1),+ PositionwiseFeedForward(*positionwise_layer_args),+ 0.1,+ True, # normalize_before,+ False, # concat_after,+ )+ self.cif_output2 = torch.nn.Linear(idim, 1)+ self.smooth_factor2 = smooth_factor2+ self.noise_threshold2 = noise_threshold2++ def forward(+ self,+ hidden,+ target_label=None,+ mask=None,+ ignore_id=-1,+ mask_chunk_predictor=None,+ target_label_length=None,+ ):+ """Forward pass for training.++ Args:+ hidden: TODO.+ target_label: TODO.+ mask: TODO.+ ignore_id: TODO.+ mask_chunk_predictor: TODO.+ target_label_length: TODO.+ """+ h = hidden+ context = h.transpose(1, 2)+ queries = self.pad(context)+ output = torch.relu(self.cif_conv1d(queries))++ # alphas2 is an extra head for timestamp prediction+ if not self.use_cif1_cnn:+ _output = context+ else:+ _output = output+ if self.upsample_type == "cnn":+ output2 = self.upsample_cnn(_output)+ output2 = output2.transpose(1, 2)+ elif self.upsample_type == "cnn_blstm":+ output2 = self.upsample_cnn(_output)+ output2 = output2.transpose(1, 2)+ output2, (_, _) = self.blstm(output2)+ elif self.upsample_type == "cnn_attn":+ output2 = self.upsample_cnn(_output)+ output2 = output2.transpose(1, 2)+ output2, _ = self.self_attn(output2, mask)++ alphas2 = torch.sigmoid(self.cif_output2(output2))+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)+ # repeat the mask in T demension to match the upsampled length+ if mask is not None:+ mask2 = (+ mask.repeat(1, self.upsample_times, 1)+ .transpose(-1, -2)+ .reshape(alphas2.shape[0], -1)+ )+ mask2 = mask2.unsqueeze(-1)+ alphas2 = alphas2 * mask2+ alphas2 = alphas2.squeeze(-1)+ token_num2 = alphas2.sum(-1)++ output = output.transpose(1, 2)++ output = self.cif_output(output)+ alphas = torch.sigmoid(output)+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)+ if mask is not None:+ mask = mask.transpose(-1, -2).float()+ alphas = alphas * mask+ if mask_chunk_predictor is not None:+ alphas = alphas * mask_chunk_predictor+ alphas = alphas.squeeze(-1)+ mask = mask.squeeze(-1)+ if target_label_length is not None:+ target_length = target_label_length+ elif target_label is not None:+ target_length = (target_label != ignore_id).float().sum(-1)+ else:+ target_length = None+ token_num = alphas.sum(-1)++ if target_length is not None:+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))+ elif self.tail_threshold > 0.0:+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=mask)++ acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)+ if target_length is None and self.tail_threshold > 0.0:+ token_num_int = torch.max(token_num).type(torch.int32).item()+ acoustic_embeds = acoustic_embeds[:, :token_num_int, :]+ return acoustic_embeds, token_num, alphas, cif_peak, token_num2++ def get_upsample_timestamp(self, hidden, mask=None, token_num=None):+ """Get upsample timestamp.++ Args:+ hidden: TODO.+ mask: TODO.+ token_num: TODO.+ """+ h = hidden+ b = hidden.shape[0]+ context = h.transpose(1, 2)+ queries = self.pad(context)+ output = torch.relu(self.cif_conv1d(queries))++ # alphas2 is an extra head for timestamp prediction+ if not self.use_cif1_cnn:+ _output = context+ else:+ _output = output+ if self.upsample_type == "cnn":+ output2 = self.upsample_cnn(_output)+ output2 = output2.transpose(1, 2)+ elif self.upsample_type == "cnn_blstm":+ output2 = self.upsample_cnn(_output)+ output2 = output2.transpose(1, 2)+ output2, (_, _) = self.blstm(output2)+ elif self.upsample_type == "cnn_attn":+ output2 = self.upsample_cnn(_output)+ output2 = output2.transpose(1, 2)+ output2, _ = self.self_attn(output2, mask)+ alphas2 = torch.sigmoid(self.cif_output2(output2))+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)+ # repeat the mask in T demension to match the upsampled length+ if mask is not None:+ mask2 = (+ mask.repeat(1, self.upsample_times, 1)+ .transpose(-1, -2)+ .reshape(alphas2.shape[0], -1)+ )+ mask2 = mask2.unsqueeze(-1)+ alphas2 = alphas2 * mask2+ alphas2 = alphas2.squeeze(-1)+ _token_num = alphas2.sum(-1)+ if token_num is not None:+ alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))+ # re-downsample+ ds_alphas = alphas2.reshape(b, -1, self.upsample_times).sum(-1)+ ds_cif_peak = cif_wo_hidden(ds_alphas, self.threshold - 1e-4)+ # upsampled alphas and cif_peak+ us_alphas = alphas2+ us_cif_peak = cif_wo_hidden(us_alphas, self.threshold - 1e-4)+ return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak++ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):+ """Tail process fn.++ Args:+ hidden: TODO.+ alphas: TODO.+ token_num: TODO.+ mask: TODO.+ """+ b, t, d = hidden.size()+ tail_threshold = self.tail_threshold+ if mask is not None:+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)+ ones_t = torch.ones_like(zeros_t)+ mask_1 = torch.cat([mask, zeros_t], dim=1)+ mask_2 = torch.cat([ones_t, mask], dim=1)+ mask = mask_2 - mask_1+ tail_threshold = mask * tail_threshold+ alphas = torch.cat([alphas, zeros_t], dim=1)+ alphas = torch.add(alphas, tail_threshold)+ else:+ tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)+ tail_threshold = torch.reshape(tail_threshold, (1, 1))+ alphas = torch.cat([alphas, tail_threshold], dim=1)+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)+ hidden = torch.cat([hidden, zeros], dim=1)+ token_num = alphas.sum(dim=-1)+ token_num_floor = torch.floor(token_num)++ return hidden, alphas, token_num_floor++ def gen_frame_alignments(+ self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None+ ):+ """Gen frame alignments.++ Args:+ alphas: TODO.+ encoder_sequence_length: TODO.+ """+ batch_size, maximum_length = alphas.size()+ int_type = torch.int32++ is_training = self.training+ if is_training:+ token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)+ else:+ token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)++ max_token_num = torch.max(token_num).item()++ alphas_cumsum = torch.cumsum(alphas, dim=1)+ alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)+ alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)++ index = torch.ones([batch_size, max_token_num], dtype=int_type)+ index = torch.cumsum(index, dim=1)+ index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)++ index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)+ index_div_bool_zeros = index_div.eq(0)+ index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1+ index_div_bool_zeros_count = torch.clamp(+ index_div_bool_zeros_count, 0, encoder_sequence_length.max()+ )+ token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)+ index_div_bool_zeros_count *= token_num_mask++ index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(+ 1, 1, maximum_length+ )+ ones = torch.ones_like(index_div_bool_zeros_count_tile)+ zeros = torch.zeros_like(index_div_bool_zeros_count_tile)+ ones = torch.cumsum(ones, dim=2)+ cond = index_div_bool_zeros_count_tile == ones+ index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)++ index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)+ index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)+ index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)+ predictor_mask = (+ (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))+ .type(int_type)+ .to(encoder_sequence_length.device)+ )+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask++ predictor_alignments = index_div_bool_zeros_count_tile_out+ predictor_alignments_length = predictor_alignments.sum(-1).type(+ encoder_sequence_length.dtype+ )+ return predictor_alignments.detach(), predictor_alignments_length.detach()++[email protected]("predictor_classes", "CifPredictorV3Export")+class CifPredictorV3Export(torch.nn.Module):+ def __init__(self, model, **kwargs):+ """Initialize CifPredictorV3Export.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """+ super().__init__()++ self.pad = model.pad+ self.cif_conv1d = model.cif_conv1d+ self.cif_output = model.cif_output+ self.threshold = model.threshold+ self.smooth_factor = model.smooth_factor+ self.noise_threshold = model.noise_threshold+ self.tail_threshold = model.tail_threshold++ self.upsample_times = model.upsample_times+ self.upsample_cnn = model.upsample_cnn+ self.blstm = model.blstm+ self.cif_output2 = model.cif_output2+ self.smooth_factor2 = model.smooth_factor2+ self.noise_threshold2 = model.noise_threshold2++ def forward(+ self,+ hidden: torch.Tensor,+ mask: torch.Tensor,+ ):+ """Forward pass for training.++ Args:+ hidden: TODO.+ mask: TODO.+ """+ h = hidden+ context = h.transpose(1, 2)+ queries = self.pad(context)+ output = torch.relu(self.cif_conv1d(queries))+ output = output.transpose(1, 2)++ output = self.cif_output(output)+ alphas = torch.sigmoid(output)+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)+ mask = mask.transpose(-1, -2).float()+ alphas = alphas * mask+ alphas = alphas.squeeze(-1)+ token_num = alphas.sum(-1)++ mask = mask.squeeze(-1)+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)+ acoustic_embeds, cif_peak = cif_export(hidden, alphas, self.threshold)++ return acoustic_embeds, token_num, alphas, cif_peak++ def get_upsample_timestmap(self, hidden, mask=None, token_num=None):+ """Get upsample timestmap.++ Args:+ hidden: TODO.+ mask: TODO.+ token_num: TODO.+ """+ h = hidden+ b = hidden.shape[0]+ context = h.transpose(1, 2)++ # generate alphas2+ _output = context+ output2 = self.upsample_cnn(_output)+ output2 = output2.transpose(1, 2)+ output2, (_, _) = self.blstm(output2)+ alphas2 = torch.sigmoid(self.cif_output2(output2))+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)++ mask = (+ mask.repeat(1, self.upsample_times, 1).transpose(-1, -2).reshape(alphas2.shape[0], -1)+ )+ mask = mask.unsqueeze(-1)+ alphas2 = alphas2 * mask+ alphas2 = alphas2.squeeze(-1)+ _token_num = alphas2.sum(-1)+ alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))+ # upsampled alphas and cif_peak+ us_alphas = alphas2+ us_cif_peak = cif_wo_hidden_export(us_alphas, self.threshold - 1e-4)+ return us_alphas, us_cif_peak++ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):+ """Tail process fn.++ Args:+ hidden: TODO.+ alphas: TODO.+ token_num: TODO.+ mask: TODO.+ """+ b, t, d = hidden.size()+ tail_threshold = self.tail_threshold++ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)+ ones_t = torch.ones_like(zeros_t)++ mask_1 = torch.cat([mask, zeros_t], dim=1)+ mask_2 = torch.cat([ones_t, mask], dim=1)+ mask = mask_2 - mask_1+ tail_threshold = mask * tail_threshold+ alphas = torch.cat([alphas, zeros_t], dim=1)+ alphas = torch.add(alphas, tail_threshold)++ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)+ hidden = torch.cat([hidden, zeros], dim=1)+ token_num = alphas.sum(dim=-1)+ token_num_floor = torch.floor(token_num)++ return hidden, alphas, token_num_floor+++def cif_export(hidden, alphas, threshold: float):+ """Cif export.++ Args:+ hidden: TODO.+ alphas: TODO.+ threshold: TODO.+ """+ batch_size, len_time, hidden_size = hidden.size()+ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)++ # loop varss+ integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)+ frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)+ # intermediate vars along time+ list_fires = []+ list_frames = []++ for t in range(len_time):+ alpha = alphas[:, t]+ distribution_completion = (+ torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate+ )++ integrate += alpha+ list_fires.append(integrate)++ fire_place = integrate >= threshold+ integrate = torch.where(+ fire_place,+ integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),+ integrate,+ )+ cur = torch.where(fire_place, distribution_completion, alpha)+ remainds = alpha - cur++ frame += cur[:, None] * hidden[:, t, :]+ list_frames.append(frame)+ frame = torch.where(+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame+ )++ fires = torch.stack(list_fires, 1)+ frames = torch.stack(list_frames, 1)++ fire_idxs = fires >= threshold+ frame_fires = torch.zeros_like(hidden)+ max_label_len = frames[0, fire_idxs[0]].size(0)+ for b in range(batch_size):+ frame_fire = frames[b, fire_idxs[b]]+ frame_len = frame_fire.size(0)+ frame_fires[b, :frame_len, :] = frame_fire++ if frame_len >= max_label_len:+ max_label_len = frame_len+ frame_fires = frame_fires[:, :max_label_len, :]+ return frame_fires, fires+++def cif_wo_hidden_export(alphas, threshold: float):+ """Cif wo hidden export.++ Args:+ alphas: TODO.+ threshold: TODO.+ """+ batch_size, len_time = alphas.size()++ # loop varss+ integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=alphas.device)+ # intermediate vars along time+ list_fires = []++ for t in range(len_time):+ alpha = alphas[:, t]++ integrate += alpha+ list_fires.append(integrate)++ fire_place = integrate >= threshold+ integrate = torch.where(+ fire_place,+ integrate - torch.ones([batch_size], device=alphas.device) * threshold,+ integrate,+ )++ fires = torch.stack(list_fires, 1)+ return fires+diff --git a/funasr/models/bicif_paraformer/export_meta.py b/funasr/models/bicif_paraformer/export_meta.pyindex 75171f443..0bf291bde 100644--- a/funasr/models/bicif_paraformer/export_meta.py+++ b/funasr/models/bicif_paraformer/export_meta.py@@ -10,6 +10,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")model.encoder = encoder_class(model.encoder, onnx=is_onnx)@@ -41,6 +47,12 @@ def export_forward(speech_lengths: torch.Tensor,):# a. To device+ """Export forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """batch = {"speech": speech, "speech_lengths": speech_lengths}enc, enc_len = self.encoder(**batch)@@ -58,20 +70,24 @@ def export_forward(def export_dummy_inputs(self):+ """Export dummy inputs."""speech = torch.randn(2, 30, 560)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)return (speech, speech_lengths)def export_input_names(self):+ """Export input names."""return ["speech", "speech_lengths"]def export_output_names(self):+ """Export output names."""return ["logits", "token_num", "us_alphas", "us_cif_peak"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"},"speech_lengths": {@@ -84,4 +100,5 @@ def export_dynamic_axes(self):def export_name(self):+ """Export name."""return "model.onnx"diff --git a/funasr/models/bicif_paraformer/model.py b/funasr/models/bicif_paraformer/model.pyindex 4db9c76cd..4944e5d36 100644--- a/funasr/models/bicif_paraformer/model.py+++ b/funasr/models/bicif_paraformer/model.py@@ -31,6 +31,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -49,6 +54,12 @@ class BiCifParaformer(Paraformer):*args,**kwargs,):+ """Initialize BiCifParaformer.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def _calc_pre2_loss(@@ -58,6 +69,14 @@ class BiCifParaformer(Paraformer):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc pre2 loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -80,6 +99,14 @@ class BiCifParaformer(Paraformer):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -124,6 +151,12 @@ class BiCifParaformer(Paraformer):return loss_att, acc_att, cer_att, wer_att, loss_predef calc_predictor(self, encoder_out, encoder_out_lens):+ """Calc predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -133,6 +166,13 @@ class BiCifParaformer(Paraformer):return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_indexdef calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num):+ """Calc predictor timestamp.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ token_num: TODO.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -230,6 +270,16 @@ class BiCifParaformer(Paraformer):):# init beamsearch+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != Noneis_use_lm = (kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None@@ -369,6 +419,11 @@ class BiCifParaformer(Paraformer):return results, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelif "max_seq_len" not in kwargs:diff --git a/funasr/models/branchformer/cgmlp.py b/funasr/models/branchformer/cgmlp.pyindex 17a10166e..3e7c69014 100644--- a/funasr/models/branchformer/cgmlp.py+++ b/funasr/models/branchformer/cgmlp.py@@ -23,6 +23,15 @@ class ConvolutionalSpatialGatingUnit(torch.nn.Module):use_linear_after_conv: bool,gate_activation: str,):+ """Initialize ConvolutionalSpatialGatingUnit.++ Args:+ size: TODO.+ kernel_size: Size/dimension parameter.+ dropout_rate: TODO.+ use_linear_after_conv: TODO.+ gate_activation: TODO.+ """super().__init__()n_channels = size // 2 # split input channels@@ -48,6 +57,7 @@ class ConvolutionalSpatialGatingUnit(torch.nn.Module):self.dropout = torch.nn.Dropout(dropout_rate)def espnet_initialization_fn(self):+ """Espnet initialization fn."""torch.nn.init.normal_(self.conv.weight, std=1e-6)torch.nn.init.ones_(self.conv.bias)if self.linear is not None:@@ -93,6 +103,16 @@ class ConvolutionalGatingMLP(torch.nn.Module):use_linear_after_conv: bool,gate_activation: str,):+ """Initialize ConvolutionalGatingMLP.++ Args:+ size: TODO.+ linear_units: TODO.+ kernel_size: Size/dimension parameter.+ dropout_rate: TODO.+ use_linear_after_conv: TODO.+ gate_activation: TODO.+ """super().__init__()self.channel_proj1 = torch.nn.Sequential(@@ -108,6 +128,12 @@ class ConvolutionalGatingMLP(torch.nn.Module):self.channel_proj2 = torch.nn.Linear(linear_units // 2, size)def forward(self, x, mask):+ """Forward pass for training.++ Args:+ x: TODO.+ mask: TODO.+ """if isinstance(x, tuple):xs_pad, pos_emb = xelse:diff --git a/funasr/models/branchformer/encoder.py b/funasr/models/branchformer/encoder.pyindex a15a3f282..95aff772d 100644--- a/funasr/models/branchformer/encoder.py+++ b/funasr/models/branchformer/encoder.py@@ -73,6 +73,18 @@ class BranchformerEncoderLayer(torch.nn.Module):attn_branch_drop_rate: float = 0.0,stochastic_depth_rate: float = 0.0,):+ """Initialize BranchformerEncoderLayer.++ Args:+ size: TODO.+ attn: TODO.+ cgmlp: TODO.+ dropout_rate: TODO.+ merge_method: TODO.+ cgmlp_weight: TODO.+ attn_branch_drop_rate: TODO.+ stochastic_depth_rate: TODO.+ """super().__init__()assert (attn is not None) or (cgmlp is not None), "At least one branch should be valid"@@ -305,6 +317,33 @@ class BranchformerEncoder(nn.Module):padding_idx: int = -1,stochastic_depth_rate: Union[float, List[float]] = 0.0,):+ """Initialize BranchformerEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ use_attn: TODO.+ attention_heads: TODO.+ attention_layer_type: TODO.+ pos_enc_layer_type: TODO.+ rel_pos_type: TODO.+ use_cgmlp: TODO.+ cgmlp_linear_units: TODO.+ cgmlp_conv_kernel: TODO.+ use_linear_after_conv: TODO.+ gate_activation: TODO.+ merge_method: TODO.+ cgmlp_weight: TODO.+ attn_branch_drop_rate: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ zero_triu: TODO.+ padding_idx: TODO.+ stochastic_depth_rate: TODO.+ """super().__init__()self._output_size = output_size@@ -472,6 +511,7 @@ class BranchformerEncoder(nn.Module):self.after_norm = LayerNorm(output_size)def output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(diff --git a/funasr/models/branchformer/fastformer.py b/funasr/models/branchformer/fastformer.pyindex ede94dc30..435c486b0 100644--- a/funasr/models/branchformer/fastformer.py+++ b/funasr/models/branchformer/fastformer.py@@ -20,6 +20,13 @@ class FastSelfAttention(torch.nn.Module):attention_heads,dropout_rate,):+ """Initialize FastSelfAttention.++ Args:+ size: TODO.+ attention_heads: TODO.+ dropout_rate: TODO.+ """super().__init__()if size % attention_heads != 0:raise ValueError(@@ -37,9 +44,15 @@ class FastSelfAttention(torch.nn.Module):self.dropout = torch.nn.Dropout(dropout_rate)def espnet_initialization_fn(self):+ """Espnet initialization fn."""self.apply(self.init_weights)def init_weights(self, module):+ """Init weights.++ Args:+ module: TODO.+ """if isinstance(module, torch.nn.Linear):module.weight.data.normal_(mean=0.0, std=0.02)if isinstance(module, torch.nn.Linear) and module.bias is not None:diff --git a/funasr/models/branchformer/model.py b/funasr/models/branchformer/model.pyindex f7e4b3623..2dc88b23a 100644--- a/funasr/models/branchformer/model.py+++ b/funasr/models/branchformer/model.py@@ -14,4 +14,10 @@ class Branchformer(Transformer):**kwargs,):+ """Initialize Branchformer.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)diff --git a/funasr/models/campplus/cluster_backend.py b/funasr/models/campplus/cluster_backend.pyindex 9e7042153..998ccd679 100644--- a/funasr/models/campplus/cluster_backend.py+++ b/funasr/models/campplus/cluster_backend.py@@ -19,12 +19,25 @@ class SpectralCluster:"""def __init__(self, min_num_spks=1, max_num_spks=15, pval=0.022):+ """Initialize SpectralCluster.++ Args:+ min_num_spks: TODO.+ max_num_spks: TODO.+ pval: TODO.+ """self.min_num_spks = min_num_spksself.max_num_spks = max_num_spksself.pval = pvaldef __call__(self, X, oracle_num=None):# Similarity matrix computation+ """Internal: call .++ Args:+ X: TODO.+ oracle_num: TODO.+ """sim_mat = self.get_sim_mat(X)# Refining similarity matrix with pval@@ -46,10 +59,20 @@ class SpectralCluster:def get_sim_mat(self, X):# Cosine similarities+ """Get sim mat.++ Args:+ X: TODO.+ """M = sklearn.metrics.pairwise.cosine_similarity(X, X)return Mdef p_pruning(self, A):+ """P pruning.++ Args:+ A: TODO.+ """if A.shape[0] * self.pval < 6:pval = 6.0 / A.shape[0]else:@@ -67,6 +90,11 @@ class SpectralCluster:return Adef get_laplacian(self, M):+ """Get laplacian.++ Args:+ M: TODO.+ """M[np.diag_indices(M.shape[0])] = 0D = np.sum(np.abs(M), axis=1)D = np.diag(D)@@ -74,6 +102,12 @@ class SpectralCluster:return Ldef get_spec_embs(self, L, k_oracle=None):+ """Get spec embs.++ Args:+ L: TODO.+ k_oracle: TODO.+ """lambdas, eig_vecs = scipy.linalg.eigh(L)if k_oracle is not None:@@ -88,10 +122,21 @@ class SpectralCluster:return emb, num_of_spkdef cluster_embs(self, emb, k):+ """Cluster embs.++ Args:+ emb: TODO.+ k: TODO.+ """_, labels, _ = k_means(emb, k)return labelsdef getEigenGaps(self, eig_vals):+ """Geteigengaps.++ Args:+ eig_vals: TODO.+ """eig_vals_gap_list = []for i in range(len(eig_vals) - 1):gap = float(eig_vals[i + 1]) - float(eig_vals[i])@@ -109,6 +154,15 @@ class UmapHdbscan:def __init__(self, n_neighbors=20, n_components=60, min_samples=10, min_cluster_size=10, metric="cosine"):+ """Initialize UmapHdbscan.++ Args:+ n_neighbors: TODO.+ n_components: TODO.+ min_samples: TODO.+ min_cluster_size: Size/dimension parameter.+ metric: TODO.+ """self.n_neighbors = n_neighborsself.n_components = n_componentsself.min_samples = min_samples@@ -116,6 +170,11 @@ class UmapHdbscan:self.metric = metricdef __call__(self, X):+ """Internal: call .++ Args:+ X: TODO.+ """import umap.umap_ as umapumap_X = umap.UMAP(@@ -140,6 +199,11 @@ class ClusterBackend(torch.nn.Module):"""def __init__(self, merge_thr=0.78):+ """Initialize ClusterBackend.++ Args:+ merge_thr: TODO.+ """super().__init__()self.model_config = {"merge_thr": merge_thr}# self.other_config = kwargs@@ -149,6 +213,12 @@ class ClusterBackend(torch.nn.Module):def forward(self, X, **params):# clustering and return the labels+ """Forward pass for training.++ Args:+ X: TODO.+ **params: Additional keyword arguments.+ """k = params["oracle_num"] if "oracle_num" in params else Noneassert len(X.shape) == 2, "modelscope error: the shape of input should be [N, C]"if X.shape[0] < 20:@@ -166,6 +236,13 @@ class ClusterBackend(torch.nn.Module):def merge_by_cos(self, labels, embs, cos_thr):# merge the similar speakers by cosine similarity+ """Merge by cos.++ Args:+ labels: TODO.+ embs: TODO.+ cos_thr: TODO.+ """assert cos_thr > 0 and cos_thr <= 1while True:spk_num = labels.max() + 1diff --git a/funasr/models/campplus/components.py b/funasr/models/campplus/components.pyindex 263b94334..c026cd4d5 100644--- a/funasr/models/campplus/components.py+++ b/funasr/models/campplus/components.py@@ -13,6 +13,13 @@ class BasicResBlock(torch.nn.Module):expansion = 1def __init__(self, in_planes, planes, stride=1):+ """Initialize BasicResBlock.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ """super(BasicResBlock, self).__init__()self.conv1 = torch.nn.Conv2d(in_planes, planes, kernel_size=3, stride=(stride, 1), padding=1, bias=False@@ -35,6 +42,11 @@ class BasicResBlock(torch.nn.Module):)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """out = F.relu(self.bn1(self.conv1(x)))out = self.bn2(self.conv2(out))out += self.shortcut(x)@@ -44,6 +56,14 @@ class BasicResBlock(torch.nn.Module):class FCM(torch.nn.Module):def __init__(self, block=BasicResBlock, num_blocks=[2, 2], m_channels=32, feat_dim=80):+ """Initialize FCM.++ Args:+ block: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ """super(FCM, self).__init__()self.in_planes = m_channelsself.conv1 = torch.nn.Conv2d(1, m_channels, kernel_size=3, stride=1, padding=1, bias=False)@@ -59,6 +79,14 @@ class FCM(torch.nn.Module):self.out_channels = m_channels * (feat_dim // 8)def _make_layer(self, block, planes, num_blocks, stride):+ """Internal: make layer.++ Args:+ block: TODO.+ planes: TODO.+ num_blocks: TODO.+ stride: TODO.+ """strides = [stride] + [1] * (num_blocks - 1)layers = []for stride in strides:@@ -67,6 +95,11 @@ class FCM(torch.nn.Module):return torch.nn.Sequential(*layers)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = x.unsqueeze(1)out = F.relu(self.bn1(self.conv1(x)))out = self.layer1(out)@@ -79,6 +112,12 @@ class FCM(torch.nn.Module):def get_nonlinear(config_str, channels):+ """Get nonlinear.++ Args:+ config_str: TODO.+ channels: TODO.+ """nonlinear = torch.nn.Sequential()for name in config_str.split("-"):if name == "relu":@@ -95,6 +134,15 @@ def get_nonlinear(config_str, channels):def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2):+ """Statistics pooling.++ Args:+ x: TODO.+ dim: TODO.+ keepdim: TODO.+ unbiased: TODO.+ eps: TODO.+ """mean = x.mean(dim=dim)std = x.std(dim=dim, unbiased=unbiased)stats = torch.cat([mean, std], dim=-1)@@ -105,6 +153,11 @@ def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2):class StatsPool(torch.nn.Module):def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """return statistics_pooling(x)@@ -120,6 +173,18 @@ class TDNNLayer(torch.nn.Module):bias=False,config_str="batchnorm-relu",):+ """Initialize TDNNLayer.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ padding: TODO.+ dilation: TODO.+ bias: TODO.+ config_str: TODO.+ """super(TDNNLayer, self).__init__()if padding < 0:assert (@@ -138,6 +203,11 @@ class TDNNLayer(torch.nn.Module):self.nonlinear = get_nonlinear(config_str, out_channels)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = self.linear(x)x = self.nonlinear(x)return x@@ -147,6 +217,18 @@ class CAMLayer(torch.nn.Module):def __init__(self, bn_channels, out_channels, kernel_size, stride, padding, dilation, bias, reduction=2):+ """Initialize CAMLayer.++ Args:+ bn_channels: TODO.+ out_channels: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ padding: TODO.+ dilation: TODO.+ bias: TODO.+ reduction: TODO.+ """super(CAMLayer, self).__init__()self.linear_local = torch.nn.Conv1d(bn_channels,@@ -163,6 +245,11 @@ class CAMLayer(torch.nn.Module):self.sigmoid = torch.nn.Sigmoid()def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """y = self.linear_local(x)context = x.mean(-1, keepdim=True) + self.seg_pooling(x)context = self.relu(self.linear1(context))@@ -170,6 +257,13 @@ class CAMLayer(torch.nn.Module):return y * mdef seg_pooling(self, x, seg_len=100, stype="avg"):+ """Seg pooling.++ Args:+ x: TODO.+ seg_len: TODO.+ stype: TODO.+ """if stype == "avg":seg = F.avg_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True)elif stype == "max":@@ -195,6 +289,19 @@ class CAMDenseTDNNLayer(torch.nn.Module):config_str="batchnorm-relu",memory_efficient=False,):+ """Initialize CAMDenseTDNNLayer.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ bn_channels: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ dilation: TODO.+ bias: TODO.+ config_str: TODO.+ memory_efficient: TODO.+ """super(CAMDenseTDNNLayer, self).__init__()assert kernel_size % 2 == 1, "Expect equal paddings, but got even kernel size ({})".format(kernel_size@@ -215,9 +322,19 @@ class CAMDenseTDNNLayer(torch.nn.Module):)def bn_function(self, x):+ """Bn function.++ Args:+ x: TODO.+ """return self.linear1(self.nonlinear1(x))def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if self.training and self.memory_efficient:x = cp.checkpoint(self.bn_function, x)else:@@ -240,6 +357,20 @@ class CAMDenseTDNNBlock(torch.nn.ModuleList):config_str="batchnorm-relu",memory_efficient=False,):+ """Initialize CAMDenseTDNNBlock.++ Args:+ num_layers: TODO.+ in_channels: TODO.+ out_channels: TODO.+ bn_channels: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ dilation: TODO.+ bias: TODO.+ config_str: TODO.+ memory_efficient: TODO.+ """super(CAMDenseTDNNBlock, self).__init__()for i in range(num_layers):layer = CAMDenseTDNNLayer(@@ -256,6 +387,11 @@ class CAMDenseTDNNBlock(torch.nn.ModuleList):self.add_module("tdnnd%d" % (i + 1), layer)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """for layer in self:x = torch.cat([x, layer(x)], dim=1)return x@@ -263,11 +399,24 @@ class CAMDenseTDNNBlock(torch.nn.ModuleList):class TransitLayer(torch.nn.Module):def __init__(self, in_channels, out_channels, bias=True, config_str="batchnorm-relu"):+ """Initialize TransitLayer.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ bias: TODO.+ config_str: TODO.+ """super(TransitLayer, self).__init__()self.nonlinear = get_nonlinear(config_str, in_channels)self.linear = torch.nn.Conv1d(in_channels, out_channels, 1, bias=bias)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = self.nonlinear(x)x = self.linear(x)return x@@ -275,11 +424,24 @@ class TransitLayer(torch.nn.Module):class DenseLayer(torch.nn.Module):def __init__(self, in_channels, out_channels, bias=False, config_str="batchnorm-relu"):+ """Initialize DenseLayer.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ bias: TODO.+ config_str: TODO.+ """super(DenseLayer, self).__init__()self.linear = torch.nn.Conv1d(in_channels, out_channels, 1, bias=bias)self.nonlinear = get_nonlinear(config_str, out_channels)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if len(x.shape) == 2:x = self.linear(x.unsqueeze(dim=-1)).squeeze(dim=-1)else:diff --git a/funasr/models/campplus/model.py b/funasr/models/campplus/model.pyindex 6730ad589..12b739137 100644--- a/funasr/models/campplus/model.py+++ b/funasr/models/campplus/model.py@@ -31,6 +31,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -56,6 +61,19 @@ class CAMPPlus(torch.nn.Module):output_level="segment",**kwargs,):+ """Initialize CAMPPlus.++ Args:+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ growth_rate: TODO.+ bn_size: Size/dimension parameter.+ init_channels: TODO.+ config_str: TODO.+ memory_efficient: TODO.+ output_level: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.head = FCM(feat_dim=feat_dim)diff --git a/funasr/models/campplus/utils.py b/funasr/models/campplus/utils.pyindex 4addd0a8a..e9f5eb4a1 100644--- a/funasr/models/campplus/utils.py+++ b/funasr/models/campplus/utils.py@@ -22,6 +22,11 @@ from funasr.models.transformer.utils.nets_utils import pad_listdef check_audio_list(audio: list):+ """Check audio list.++ Args:+ audio: TODO.+ """audio_dur = 0for i in range(len(audio)):seg = audio[i]@@ -38,6 +43,11 @@ def check_audio_list(audio: list):def sv_preprocess(inputs: Union[np.ndarray, list]):+ """Sv preprocess.++ Args:+ inputs: TODO.+ """output = []for i in range(len(inputs)):if isinstance(inputs[i], str):@@ -64,12 +74,23 @@ def sv_preprocess(inputs: Union[np.ndarray, list]):def sv_chunk(vad_segments: list, fs=16000) -> list:+ """Sv chunk.++ Args:+ vad_segments: TODO.+ fs: TODO.+ """config = {"seg_dur": 1.5,"seg_shift": 0.75,}def seg_chunk(seg_data):+ """Seg chunk.++ Args:+ seg_data: TODO.+ """seg_st = seg_data[0]data = seg_data[2]chunk_len = int(config["seg_dur"] * fs)@@ -96,6 +117,11 @@ def sv_chunk(vad_segments: list, fs=16000) -> list:def extract_feature(audio):+ """Extract feature.++ Args:+ audio: TODO.+ """features = []feature_times = []feature_lengths = []@@ -114,6 +140,14 @@ def extract_feature(audio):def postprocess(segments: list, vad_segments: list, labels: np.ndarray, embeddings: np.ndarray) -> list:+ """Postprocess.++ Args:+ segments: TODO.+ vad_segments: TODO.+ labels: TODO.+ embeddings: TODO.+ """assert len(segments) == len(labels)labels = correct_labels(labels)distribute_res = []@@ -130,6 +164,12 @@ def postprocess(spk_embs = np.stack(spk_embs)def is_overlapped(t1, t2):+ """Is overlapped.++ Args:+ t1: TODO.+ t2: TODO.+ """if t1 > t2 + 1e-4:return Truereturn False@@ -148,6 +188,11 @@ def postprocess(def correct_labels(labels):+ """Correct labels.++ Args:+ labels: TODO.+ """labels_id = 0id2id = {}new_labels = []@@ -160,6 +205,11 @@ def correct_labels(labels):def merge_seque(distribute_res):+ """Merge seque.++ Args:+ distribute_res: TODO.+ """res = [distribute_res[0]]for i in range(1, len(distribute_res)):if distribute_res[i][2] != res[-1][2] or distribute_res[i][0] > res[-1][1]:@@ -171,6 +221,12 @@ def merge_seque(distribute_res):def smooth(res, mindur=0.7):# if only one segment, return directly+ """Smooth.++ Args:+ res: TODO.+ mindur: TODO.+ """if len(res) < 2:return res# short segments are assigned to nearest speakers.@@ -193,6 +249,12 @@ def smooth(res, mindur=0.7):def distribute_spk(sentence_list, sd_time_list):+ """Distribute spk.++ Args:+ sentence_list: TODO.+ sd_time_list: TODO.+ """sd_time_list = [(spk_st * 1000, spk_ed * 1000, spk) for spk_st, spk_ed, spk in sd_time_list]for d in sentence_list:sentence_start = d['start']@@ -220,18 +282,41 @@ class Storage(metaclass=ABCMeta):@abstractmethoddef read(self, filepath: str):+ """Read.++ Args:+ filepath: TODO.+ """pass@abstractmethoddef read_text(self, filepath: str):+ """Read text.++ Args:+ filepath: TODO.+ """pass@abstractmethoddef write(self, obj: bytes, filepath: Union[str, Path]) -> None:+ """Write.++ Args:+ obj: TODO.+ filepath: TODO.+ """pass@abstractmethoddef write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:+ """Write text.++ Args:+ obj: TODO.+ filepath: TODO.+ encoding: TODO.+ """pass@@ -315,11 +400,21 @@ class HTTPStorage(Storage):def read(self, url):# TODO @wenmeng.zwm add progress bar if file is too large+ """Read.++ Args:+ url: TODO.+ """r = requests.get(url)r.raise_for_status()return r.contentdef read_text(self, url):+ """Read text.++ Args:+ url: TODO.+ """r = requests.get(url)r.raise_for_status()return r.text@@ -351,9 +446,22 @@ class HTTPStorage(Storage):os.remove(f.name)def write(self, obj: bytes, url: Union[str, Path]) -> None:+ """Write.++ Args:+ obj: TODO.+ url: TODO.+ """raise NotImplementedError("write is not supported by HTTP Storage")def write_text(self, obj: str, url: Union[str, Path], encoding: str = "utf-8") -> None:+ """Write text.++ Args:+ obj: TODO.+ url: TODO.+ encoding: TODO.+ """raise NotImplementedError("write_text is not supported by HTTP Storage")@@ -362,12 +470,28 @@ class OSSStorage(Storage):def __init__(self, oss_config_file=None):# read from config file or env var+ """Initialize OSSStorage.++ Args:+ oss_config_file: TODO.+ """raise NotImplementedError("OSSStorage.__init__ to be implemented in the future")def read(self, filepath):+ """Read.++ Args:+ filepath: TODO.+ """raise NotImplementedError("OSSStorage.read to be implemented in the future")def read_text(self, filepath, encoding="utf-8"):+ """Read text.++ Args:+ filepath: TODO.+ encoding: TODO.+ """raise NotImplementedError("OSSStorage.read_text to be implemented in the future")@contextlib.contextmanager@@ -397,9 +521,22 @@ class OSSStorage(Storage):os.remove(f.name)def write(self, obj: bytes, filepath: Union[str, Path]) -> None:+ """Write.++ Args:+ obj: TODO.+ filepath: TODO.+ """raise NotImplementedError("OSSStorage.write to be implemented in the future")def write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:+ """Write text.++ Args:+ obj: TODO.+ filepath: TODO.+ encoding: TODO.+ """raise NotImplementedError("OSSStorage.write_text to be implemented in the future")@@ -416,6 +553,11 @@ class File(object):@staticmethoddef _get_storage(uri):+ """Internal: get storage.++ Args:+ uri: TODO.+ """assert isinstance(uri, str), f"uri should be str type, but got {type(uri)}"if "://" not in uri:diff --git a/funasr/models/conformer/encoder.py b/funasr/models/conformer/encoder.pyindex 7c939b47c..7b1e61946 100644--- a/funasr/models/conformer/encoder.py+++ b/funasr/models/conformer/encoder.py@@ -347,6 +347,35 @@ class ConformerEncoder(nn.Module):interctc_use_conditioning: bool = False,stochastic_depth_rate: Union[float, List[float]] = 0.0,):+ """Initialize ConformerEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ macaron_style: TODO.+ rel_pos_type: TODO.+ pos_enc_layer_type: TODO.+ selfattention_layer_type: TODO.+ activation_type: TODO.+ use_cnn_module: TODO.+ zero_triu: TODO.+ cnn_module_kernel: TODO.+ padding_idx: TODO.+ interctc_layer_idx: TODO.+ interctc_use_conditioning: TODO.+ stochastic_depth_rate: TODO.+ """super().__init__()self._output_size = output_size@@ -524,6 +553,7 @@ class ConformerEncoder(nn.Module):self.conditioning_layer = Nonedef output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(@@ -1004,6 +1034,7 @@ class ConformerChunkEncoder(torch.nn.Module):self.time_reduction_factor = time_reduction_factordef output_size(self) -> int:+ """Output size."""return self._output_sizedef get_encoder_input_raw_size(self, size: int, hop_length: int) -> int:@@ -1181,6 +1212,15 @@ class ConformerChunkEncoder(torch.nn.Module):left_context: int = 32,right_context: int = 0,) -> torch.Tensor:+ """Simu chunk forward.++ Args:+ x: TODO.+ x_len: TODO.+ chunk_size: Size/dimension parameter.+ left_context: TODO.+ right_context: TODO.+ """short_status, limit_size = check_short_utt(self.embed.subsampling_factor, x.size(1))if short_status:diff --git a/funasr/models/conformer/model.py b/funasr/models/conformer/model.pyindex d7e77b155..71710e2da 100644--- a/funasr/models/conformer/model.py+++ b/funasr/models/conformer/model.py@@ -16,4 +16,10 @@ class Conformer(Transformer):**kwargs,):+ """Initialize Conformer.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)diff --git a/funasr/models/conformer_rwkv/decoder.py b/funasr/models/conformer_rwkv/decoder.pyindex 4c4104966..24805e502 100644--- a/funasr/models/conformer_rwkv/decoder.py+++ b/funasr/models/conformer_rwkv/decoder.py@@ -31,6 +31,11 @@ from funasr.register import tablesclass LayerNorm(nn.LayerNorm):def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """return super().forward(x.float()).type(x.dtype)@@ -216,6 +221,18 @@ class BaseTransformerDecoder(nn.Module, BatchScorerInterface):pos_enc_class=PositionalEncoding,normalize_before: bool = True,):+ """Initialize BaseTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ """super().__init__()attention_dim = encoder_output_size@@ -395,6 +412,25 @@ class TransformerRWKVDecoder(BaseTransformerDecoder):concat_after: bool = False,**kwargs,):+ """Initialize TransformerRWKVDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,diff --git a/funasr/models/conformer_rwkv/model.py b/funasr/models/conformer_rwkv/model.pyindex d7e77b155..71710e2da 100644--- a/funasr/models/conformer_rwkv/model.py+++ b/funasr/models/conformer_rwkv/model.py@@ -16,4 +16,10 @@ class Conformer(Transformer):**kwargs,):+ """Initialize Conformer.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)diff --git a/funasr/models/contextual_paraformer/decoder.py b/funasr/models/contextual_paraformer/decoder.pyindex ba2ce9ade..74d378a7d 100644--- a/funasr/models/contextual_paraformer/decoder.py+++ b/funasr/models/contextual_paraformer/decoder.py@@ -59,6 +59,15 @@ class ContextualDecoderLayer(torch.nn.Module):cache=None,):# tgt = self.dropout(tgt)+ """Forward pass for training.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """if isinstance(tgt, Tuple):tgt, _ = tgtresidual = tgt@@ -103,6 +112,15 @@ class ContextualBiasDecoder(torch.nn.Module):self.normalize_before = normalize_beforedef forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Forward pass for training.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """x = tgtif self.src_attn is not None:if self.normalize_before:@@ -139,6 +157,27 @@ class ContextualParaformerDecoder(ParaformerSANMDecoder):kernel_size: int = 21,sanm_shfit: int = 0,):+ """Initialize ContextualParaformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ att_layer_num: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -322,6 +361,15 @@ class ContextualParaformerDecoderExport(torch.nn.Module):onnx: bool = True,**kwargs,):+ """Initialize ContextualParaformerDecoderExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ model_name: TODO.+ onnx: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()from funasr.utils.torch_function import sequence_mask@@ -386,6 +434,11 @@ class ContextualParaformerDecoderExport(torch.nn.Module):self.dropout = self.model.dropoutdef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -404,6 +457,15 @@ class ContextualParaformerDecoderExport(torch.nn.Module):bias_embed: torch.Tensor,):+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ bias_embed: TODO.+ """tgt = ys_in_padtgt_mask = self.make_pad_mask(ys_in_lens)tgt_mask, _ = self.prepare_mask(tgt_mask)diff --git a/funasr/models/contextual_paraformer/export_meta.py b/funasr/models/contextual_paraformer/export_meta.pyindex 9d3a63b8a..325e768e8 100644--- a/funasr/models/contextual_paraformer/export_meta.py+++ b/funasr/models/contextual_paraformer/export_meta.py@@ -12,12 +12,19 @@ from funasr.models.seaco_paraformer.export_meta import ContextualEmbedderExportclass ContextualEmbedderExport2(ContextualEmbedderExport):def __init__(self, model, **kwargs):+ """Initialize ContextualEmbedderExport2.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """super().__init__(model)self.embedding = model.bias_embedmodel.bias_encoder.batch_first = Falseself.bias_encoder = model.bias_encoderdef export_dummy_inputs(self):+ """Export dummy inputs."""hotword = torch.tensor([[10, 11, 12, 13, 14, 10, 11, 12, 13, 14],@@ -34,6 +41,12 @@ class ContextualEmbedderExport2(ContextualEmbedderExport):def export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")@@ -87,6 +100,13 @@ def export_backbone_forward(speech_lengths: torch.Tensor,bias_embed: torch.Tensor,):+ """Export backbone forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ bias_embed: TODO.+ """batch = {"speech": speech, "speech_lengths": speech_lengths}enc, enc_len = self.encoder(**batch)@@ -101,6 +121,7 @@ def export_backbone_forward(def export_backbone_dummy_inputs(self):+ """Export backbone dummy inputs."""speech = torch.randn(2, 30, self.feats_dim)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)bias_embed = torch.randn(2, 1, 512)@@ -108,14 +129,17 @@ def export_backbone_dummy_inputs(self):def export_backbone_input_names(self):+ """Export backbone input names."""return ["speech", "speech_lengths", "bias_embed"]def export_backbone_output_names(self):+ """Export backbone output names."""return ["logits", "token_num"]def export_backbone_dynamic_axes(self):+ """Export backbone dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"},"speech_lengths": {@@ -127,4 +151,5 @@ def export_backbone_dynamic_axes(self):def export_backbone_name(self):+ """Export backbone name."""return "model.onnx"diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.pyindex fd8822025..3f80be7f9 100644--- a/funasr/models/contextual_paraformer/model.py+++ b/funasr/models/contextual_paraformer/model.py@@ -34,6 +34,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -50,6 +55,12 @@ class ContextualParaformer(Paraformer):*args,**kwargs,):+ """Initialize ContextualParaformer.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)self.target_buffer_length = kwargs.get("target_buffer_length", -1)@@ -165,6 +176,16 @@ class ContextualParaformer(Paraformer):hotword_pad: torch.Tensor,hotword_lengths: torch.Tensor,):+ """Internal: calc att clas loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ hotword_pad: TODO.+ hotword_lengths: Lengths of hotword.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -251,6 +272,16 @@ class ContextualParaformer(Paraformer):pre_acoustic_embeds,contextual_info,):+ """Sampler.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ pre_acoustic_embeds: TODO.+ contextual_info: TODO.+ """tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)@@ -304,6 +335,16 @@ class ContextualParaformer(Paraformer):hw_list=None,clas_scale=1.0,):+ """Cal decoder with predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ sematic_embeds: TODO.+ ys_pad_lens: Lengths of ys_pad.+ hw_list: TODO.+ clas_scale: TODO.+ """if hw_list is None:hw_list = [torch.Tensor([1]).long().to(encoder_out.device)] # empty hotword listhw_list_pad = pad_list(hw_list, 0)@@ -352,6 +393,16 @@ class ContextualParaformer(Paraformer):):# init beamsearch+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != Noneis_use_lm = (kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None@@ -479,7 +530,19 @@ class ContextualParaformer(Paraformer):return results, meta_datadef generate_hotwords_list(self, hotword_list_or_file, tokenizer=None, frontend=None):+ """Generate hotwords list.++ Args:+ hotword_list_or_file: TODO.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ """def load_seg_dict(seg_dict_file):+ """Load seg dict.++ Args:+ seg_dict_file: TODO.+ """seg_dict = {}assert isinstance(seg_dict_file, str)with open(seg_dict_file, "r", encoding="utf8") as f:@@ -492,6 +555,12 @@ class ContextualParaformer(Paraformer):return seg_dictdef seg_tokenize(txt, seg_dict):+ """Seg tokenize.++ Args:+ txt: TODO.+ seg_dict: TODO.+ """pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$")out_txt = ""for word in txt:@@ -589,6 +658,11 @@ class ContextualParaformer(Paraformer):self,**kwargs,):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """if "max_seq_len" not in kwargs:kwargs["max_seq_len"] = 512from .export_meta import export_rebuild_modeldiff --git a/funasr/models/ct_transformer/export_meta.py b/funasr/models/ct_transformer/export_meta.pyindex c222008ce..cbba5c597 100644--- a/funasr/models/ct_transformer/export_meta.py+++ b/funasr/models/ct_transformer/export_meta.py@@ -10,6 +10,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")model.encoder = encoder_class(model.encoder, onnx=is_onnx)@@ -39,6 +45,7 @@ def export_forward(self, inputs: torch.Tensor, text_lengths: torch.Tensor):def export_dummy_inputs(self):+ """Export dummy inputs."""length = 120text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)).type(torch.int32)text_lengths = torch.tensor([length - 20, length], dtype=torch.int32)@@ -46,14 +53,17 @@ def export_dummy_inputs(self):def export_input_names(self):+ """Export input names."""return ["inputs", "text_lengths"]def export_output_names(self):+ """Export output names."""return ["logits"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"inputs": {0: "batch_size", 1: "feats_length"},"text_lengths": {@@ -64,4 +74,5 @@ def export_dynamic_axes(self):def export_name(self):+ """Export name."""return "model.onnx"diff --git a/funasr/models/ct_transformer/model.py b/funasr/models/ct_transformer/model.pyindex ae2d318bc..ab70258a0 100644--- a/funasr/models/ct_transformer/model.py+++ b/funasr/models/ct_transformer/model.py@@ -28,6 +28,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -65,6 +70,23 @@ class CTTransformer(torch.nn.Module):sentence_end_id: int = 3,**kwargs,):+ """Initialize CTTransformer.++ Args:+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ vocab_size: Size/dimension parameter.+ punc_list: TODO.+ punc_weight: TODO.+ embed_unit: TODO.+ att_unit: TODO.+ dropout_rate: TODO.+ ignore_id: TODO.+ sos: TODO.+ eos: TODO.+ sentence_end_id: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()punc_size = len(punc_list)@@ -103,6 +125,7 @@ class CTTransformer(torch.nn.Module):return y, Nonedef with_vad(self):+ """With vad."""return Falsedef score(self, y: torch.Tensor, state: Any, x: torch.Tensor) -> Tuple[torch.Tensor, Any]:@@ -245,6 +268,16 @@ class CTTransformer(torch.nn.Module):vad_indexes: Optional[torch.Tensor] = None,vad_indexes_lengths: Optional[torch.Tensor] = None,):+ """Forward pass for training.++ Args:+ text: Text tensor or string input.+ punc: TODO.+ text_lengths: Length of each text sample.+ punc_lengths: Lengths of punc.+ vad_indexes: TODO.+ vad_indexes_lengths: Lengths of vad_indexes.+ """nll, y_lengths = self.nll(text, punc, text_lengths, punc_lengths, vad_indexes=vad_indexes)ntokens = y_lengths.sum()loss = nll.sum() / ntokens@@ -263,6 +296,16 @@ class CTTransformer(torch.nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """assert len(data_in) == 1if not data_in[0] or (isinstance(data_in[0], str) and not data_in[0].strip()):meta_data = {"batch_data_time": -1}@@ -427,6 +470,11 @@ class CTTransformer(torch.nn.Module):def export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelmodels = export_rebuild_model(model=self, **kwargs)diff --git a/funasr/models/ct_transformer/utils.py b/funasr/models/ct_transformer/utils.pyindex b6e11e7af..465ca4225 100644--- a/funasr/models/ct_transformer/utils.py+++ b/funasr/models/ct_transformer/utils.py@@ -7,6 +7,12 @@ import redef split_to_mini_sentence(words: list, word_limit: int = 20):+ """Split to mini sentence.++ Args:+ words: TODO.+ word_limit: TODO.+ """assert word_limit > 1if len(words) <= word_limit:return [words]@@ -21,6 +27,13 @@ def split_to_mini_sentence(words: list, word_limit: int = 20):def split_words(text: str, jieba_usr_dict=None, **kwargs):+ """Split words.++ Args:+ text: Text tensor or string input.+ jieba_usr_dict: TODO.+ **kwargs: Additional keyword arguments.+ """if jieba_usr_dict:input_list = text.split()token_list_all = []@@ -80,6 +93,11 @@ def split_words(text: str, jieba_usr_dict=None, **kwargs):def isEnglish(text: str):+ """Isenglish.++ Args:+ text: Text tensor or string input.+ """if re.search("^[a-zA-Z']+$", text):return Trueelse:@@ -87,6 +105,11 @@ def isEnglish(text: str):def join_chinese_and_english(input_list):+ """Join chinese and english.++ Args:+ input_list: TODO.+ """line = ""for token in input_list:if isEnglish(token):diff --git a/funasr/models/ct_transformer_streaming/attention.py b/funasr/models/ct_transformer_streaming/attention.pyindex 97e770bbb..3b9de079e 100644--- a/funasr/models/ct_transformer_streaming/attention.py+++ b/funasr/models/ct_transformer_streaming/attention.py@@ -9,9 +9,23 @@ from funasr.models.sanm.attention import MultiHeadedAttentionSANMclass MultiHeadedAttentionSANMwithMask(MultiHeadedAttentionSANM):def __init__(self, *args, **kwargs):+ """Initialize MultiHeadedAttentionSANMwithMask.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None):+ """Forward pass for training.++ Args:+ x: TODO.+ mask: TODO.+ mask_shfit_chunk: TODO.+ mask_att_chunk_encoder: TODO.+ """q_h, k_h, v_h, v = self.forward_qkv(x)fsmn_memory = self.forward_fsmn(v, mask[0], mask_shfit_chunk)q_h = q_h * self.d_k ** (-0.5)diff --git a/funasr/models/ct_transformer_streaming/encoder.py b/funasr/models/ct_transformer_streaming/encoder.pyindex a61319aa4..f0a4877fe 100644--- a/funasr/models/ct_transformer_streaming/encoder.py+++ b/funasr/models/ct_transformer_streaming/encoder.py@@ -201,6 +201,30 @@ class SANMVadEncoder(torch.nn.Module):sanm_shfit: int = 0,selfattention_layer_type: str = "sanm",):+ """Initialize SANMVadEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ padding_idx: TODO.+ interctc_layer_idx: TODO.+ interctc_use_conditioning: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ selfattention_layer_type: TODO.+ """super().__init__()self._output_size = output_size@@ -325,6 +349,7 @@ class SANMVadEncoder(torch.nn.Module):self.dropout = torch.nn.Dropout(dropout_rate)def output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(@@ -420,6 +445,12 @@ class EncoderLayerSANMExport(torch.nn.Module):def forward(self, x, mask):+ """Forward pass for training.++ Args:+ x: TODO.+ mask: TODO.+ """residual = xx = self.norm1(x)x = self.self_attn(x, mask)@@ -443,6 +474,15 @@ class SANMVadEncoderExport(torch.nn.Module):model_name="encoder",onnx: bool = True,):+ """Initialize SANMVadEncoderExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ feats_dim: Size/dimension parameter.+ model_name: TODO.+ onnx: TODO.+ """super().__init__()self.embed = model.embedself.model = model@@ -466,6 +506,12 @@ class SANMVadEncoderExport(torch.nn.Module):self.model.encoders[i] = EncoderLayerSANMExport(d)def prepare_mask(self, mask, sub_masks):+ """Prepare mask.++ Args:+ mask: TODO.+ sub_masks: TODO.+ """mask_3d_btd = mask[:, :, None]mask_4d_bhlt = (1 - sub_masks) * -10000.0@@ -478,6 +524,14 @@ class SANMVadEncoderExport(torch.nn.Module):vad_masks: torch.Tensor,sub_masks: torch.Tensor,):+ """Forward pass for training.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ vad_masks: TODO.+ sub_masks: TODO.+ """speech = speech * self._output_size**0.5mask = self.make_pad_mask(speech_lengths)vad_masks = self.prepare_mask(mask, vad_masks)@@ -503,4 +557,5 @@ class SANMVadEncoderExport(torch.nn.Module):return xs_pad, speech_lengthsdef get_output_size(self):+ """Get output size."""return self.model.encoders[0].sizediff --git a/funasr/models/ct_transformer_streaming/export_meta.py b/funasr/models/ct_transformer_streaming/export_meta.pyindex ba0283e5d..f6ca99bb2 100644--- a/funasr/models/ct_transformer_streaming/export_meta.py+++ b/funasr/models/ct_transformer_streaming/export_meta.py@@ -10,6 +10,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")model.encoder = encoder_class(model.encoder, onnx=is_onnx)@@ -46,6 +52,7 @@ def export_forward(def export_dummy_inputs(self):+ """Export dummy inputs."""length = 120text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)).type(torch.int32)text_lengths = torch.tensor([length], dtype=torch.int32)@@ -56,14 +63,17 @@ def export_dummy_inputs(self):def export_input_names(self):+ """Export input names."""return ["inputs", "text_lengths", "vad_masks", "sub_masks"]def export_output_names(self):+ """Export output names."""return ["logits"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"inputs": {1: "feats_length"},"vad_masks": {2: "feats_length1", 3: "feats_length2"},@@ -73,4 +83,5 @@ def export_dynamic_axes(self):def export_name(self):+ """Export name."""return "model.onnx"diff --git a/funasr/models/ct_transformer_streaming/model.py b/funasr/models/ct_transformer_streaming/model.pyindex 4394821d5..f3c63d5af 100644--- a/funasr/models/ct_transformer_streaming/model.py+++ b/funasr/models/ct_transformer_streaming/model.py@@ -21,6 +21,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -37,6 +42,12 @@ class CTTransformerStreaming(CTTransformer):*args,**kwargs,):+ """Initialize CTTransformerStreaming.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def punc_forward(@@ -56,6 +67,7 @@ class CTTransformerStreaming(CTTransformer):return y, Nonedef with_vad(self):+ """With vad."""return Truedef inference(@@ -68,6 +80,17 @@ class CTTransformerStreaming(CTTransformer):cache: dict = None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}assert len(data_in) == 1@@ -189,6 +212,11 @@ class CTTransformerStreaming(CTTransformer):def export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelmodels = export_rebuild_model(model=self, **kwargs)diff --git a/funasr/models/ctc/ctc.py b/funasr/models/ctc/ctc.pyindex 8eb64d159..f372d71f3 100644--- a/funasr/models/ctc/ctc.py+++ b/funasr/models/ctc/ctc.py@@ -25,6 +25,17 @@ class CTC(torch.nn.Module):ignore_nan_grad: bool = True,extra_linear: bool = True,):+ """Initialize CTC.++ Args:+ odim: TODO.+ encoder_output_size: Size/dimension parameter.+ dropout_rate: TODO.+ ctc_type: TODO.+ reduce: TODO.+ ignore_nan_grad: TODO.+ extra_linear: TODO.+ """super().__init__()eprojs = encoder_output_sizeself.dropout_rate = dropout_rate@@ -51,6 +62,14 @@ class CTC(torch.nn.Module):self.reduce = reducedef loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor:+ """Loss fn.++ Args:+ th_pred: TODO.+ th_target: TODO.+ th_ilen: TODO.+ th_olen: TODO.+ """if self.ctc_type == "builtin":th_pred = th_pred.log_softmax(2)loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen)diff --git a/funasr/models/ctc/model.py b/funasr/models/ctc/model.pyindex e493c3b3a..439a23197 100644--- a/funasr/models/ctc/model.py+++ b/funasr/models/ctc/model.py@@ -37,6 +37,25 @@ class Transformer(nn.Module):**kwargs,):+ """Initialize Transformer.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ ctc_conf: Configuration dict for ctc.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ length_normalized_loss: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -149,6 +168,14 @@ class Transformer(nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -169,6 +196,16 @@ class Transformer(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")diff --git a/funasr/models/data2vec/data2vec.py b/funasr/models/data2vec/data2vec.pyindex ed58d1c7c..c7386e35d 100644--- a/funasr/models/data2vec/data2vec.py+++ b/funasr/models/data2vec/data2vec.py@@ -27,6 +27,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -42,6 +47,15 @@ class Data2VecPretrainModel(nn.Module):preencoder=None,):+ """Initialize Data2VecPretrainModel.++ Args:+ frontend: Audio frontend for feature extraction.+ specaug: TODO.+ normalize: TODO.+ encoder: TODO.+ preencoder: TODO.+ """super().__init__()self.frontend = frontend@@ -91,6 +105,12 @@ class Data2VecPretrainModel(nn.Module):def collect_feats(self, speech: torch.Tensor, speech_lengths: torch.Tensor) -> Dict[str, torch.Tensor]:+ """Collect feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """feats, feats_lengths = self._extract_feats(speech, speech_lengths)return {"feats": feats, "feats_lengths": feats_lengths}@@ -130,6 +150,12 @@ class Data2VecPretrainModel(nn.Module):def _extract_feats(self, speech: torch.Tensor, speech_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Internal: extract feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """assert speech_lengths.dim() == 1, speech_lengths.shape# for data-parallel@@ -147,7 +173,13 @@ class Data2VecPretrainModel(nn.Module):return feats, feats_lengthsdef set_num_updates(self, num_updates):+ """Set num updates.++ Args:+ num_updates: TODO.+ """self.num_updates = num_updatesdef get_num_updates(self):+ """Get num updates."""return self.num_updatesdiff --git a/funasr/models/data2vec/data2vec_encoder.py b/funasr/models/data2vec/data2vec_encoder.pyindex f591dd6a3..09d161b62 100644--- a/funasr/models/data2vec/data2vec_encoder.py+++ b/funasr/models/data2vec/data2vec_encoder.py@@ -22,6 +22,14 @@ from funasr.models.transformer.utils.nets_utils import make_pad_maskdef get_annealed_rate(start, end, curr_step, total_steps):+ """Get annealed rate.++ Args:+ start: TODO.+ end: TODO.+ curr_step: TODO.+ total_steps: TODO.+ """r = end - startpct_remaining = 1 - curr_step / total_stepsreturn end - r * pct_remaining@@ -95,6 +103,63 @@ class Data2VecEncoder(nn.Module):# FP16 optimizationrequired_seq_len_multiple: int = 2,):+ """Initialize Data2VecEncoder.++ Args:+ input_size: Size/dimension parameter.+ extractor_mode: TODO.+ conv_feature_layers: TODO.+ layer_type: TODO.+ layer_norm_first: TODO.+ encoder_layers: TODO.+ encoder_embed_dim: Size/dimension parameter.+ encoder_ffn_embed_dim: Size/dimension parameter.+ encoder_attention_heads: TODO.+ activation_fn: TODO.+ dropout: TODO.+ attention_dropout: TODO.+ activation_dropout: TODO.+ encoder_layerdrop: TODO.+ dropout_input: TODO.+ dropout_features: TODO.+ feature_grad_mult: TODO.+ mask_prob: TODO.+ mask_length: TODO.+ mask_selection: TODO.+ mask_other: TODO.+ no_mask_overlap: TODO.+ mask_min_space: TODO.+ require_same_masks: TODO.+ mask_dropout: TODO.+ mask_channel_length: TODO.+ mask_channel_prob: TODO.+ mask_channel_before: TODO.+ mask_channel_selection: TODO.+ mask_channel_other: TODO.+ no_mask_channel_overlap: TODO.+ mask_channel_min_space: TODO.+ conv_pos: TODO.+ conv_pos_groups: TODO.+ pos_conv_depth: TODO.+ max_positions: TODO.+ average_top_k_layers: TODO.+ layer_norm_target_layer: TODO.+ instance_norm_target_layer: TODO.+ instance_norm_targets: TODO.+ layer_norm_targets: TODO.+ batch_norm_target_layer: TODO.+ group_norm_target_layer: TODO.+ ema_decay: TODO.+ ema_end_decay: TODO.+ ema_anneal_end_step: TODO.+ ema_transformer_only: TODO.+ ema_layers_only: TODO.+ min_target_var: TODO.+ min_pred_var: TODO.+ loss_beta: TODO.+ loss_scale: TODO.+ required_seq_len_multiple: TODO.+ """super().__init__()# ConvFeatureExtractionModel@@ -206,6 +271,7 @@ class Data2VecEncoder(nn.Module):logging.info("Data2VecEncoder settings: {}".format(self.__dict__))def make_ema_teacher(self):+ """Make ema teacher."""skip_keys = set()if self.ema_layers_only:self.ema_transformer_only = True@@ -220,6 +286,11 @@ class Data2VecEncoder(nn.Module):)def set_num_updates(self, num_updates):+ """Set num updates.++ Args:+ num_updates: TODO.+ """if self.ema is None and self.final_proj is not None:logging.info("Making EMA Teacher")self.make_ema_teacher()@@ -247,6 +318,14 @@ class Data2VecEncoder(nn.Module):mask_indices=None,mask_channel_indices=None,):+ """Apply mask.++ Args:+ x: TODO.+ padding_mask: TODO.+ mask_indices: TODO.+ mask_channel_indices: TODO.+ """B, T, C = x.shapeif self.mask_channel_prob > 0 and self.mask_channel_before:@@ -313,6 +392,13 @@ class Data2VecEncoder(nn.Module):"""def _conv_out_length(input_length, kernel_size, stride):+ """Internal: conv out length.++ Args:+ input_length: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ """return torch.floor((input_length - kernel_size).to(torch.float32) / stride + 1)conv_cfg_list = eval(self.conv_feature_layers)@@ -336,6 +422,18 @@ class Data2VecEncoder(nn.Module):padding_count=None,):# create padding_mask by ilens+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ mask: TODO.+ features_only: TODO.+ layer: TODO.+ mask_indices: TODO.+ mask_channel_indices: TODO.+ padding_count: TODO.+ """if ilens is not None:padding_mask = make_pad_mask(lengths=ilens).to(xs_pad.device)else:@@ -526,6 +624,11 @@ class Data2VecEncoder(nn.Module):@staticmethoddef compute_var(y):+ """Compute var.++ Args:+ y: TODO.+ """y = y.view(-1, y.size(-1))if dist.is_initialized():zc = torch.tensor(y.size(0)).cuda()@@ -542,6 +645,14 @@ class Data2VecEncoder(nn.Module):return torch.sqrt(y.var(dim=0) + 1e-6).mean()def extract_features(self, xs_pad, ilens, mask=False, layer=None):+ """Extract features.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ mask: TODO.+ layer: TODO.+ """res = self.forward(xs_pad,ilens,@@ -552,6 +663,11 @@ class Data2VecEncoder(nn.Module):return resdef remove_pretraining_modules(self, last_layer=None):+ """Remove pretraining modules.++ Args:+ last_layer: TODO.+ """self.final_proj = Noneself.ema = Noneif last_layer is not None:@@ -560,4 +676,5 @@ class Data2VecEncoder(nn.Module):)def output_size(self) -> int:+ """Output size."""return self.encoder_embed_dimdiff --git a/funasr/models/data2vec/data_utils.py b/funasr/models/data2vec/data_utils.pyindex f486edf5a..8d87a70a8 100644--- a/funasr/models/data2vec/data_utils.py+++ b/funasr/models/data2vec/data_utils.py@@ -90,6 +90,14 @@ def compute_mask_indices(mask_idc = []def arrange(s, e, length, keep_length):+ """Arrange.++ Args:+ s: TODO.+ e: TODO.+ length: TODO.+ keep_length: TODO.+ """span_start = np.random.randint(s, e - length)mask_idc.extend(span_start + i for i in range(length))diff --git a/funasr/models/data2vec/ema_module.py b/funasr/models/data2vec/ema_module.pyindex 3d7c21e98..bf60dbe4b 100644--- a/funasr/models/data2vec/ema_module.py+++ b/funasr/models/data2vec/ema_module.py@@ -59,6 +59,11 @@ class EMAModule:state_dict = self.model.state_dict()def _to_float(t):+ """Internal: to float.++ Args:+ t: TODO.+ """return t.float() if torch.is_floating_point(t) else tfor param_key in state_dict:@@ -74,9 +79,15 @@ class EMAModule:self.build_fp32_params(state_dict)def set_decay(self, decay):+ """Set decay.++ Args:+ decay: TODO.+ """self.decay = decaydef get_decay(self):+ """Get decay."""return self.decaydef _step_internal(self, new_model):@@ -115,6 +126,11 @@ class EMAModule:self.restore(ema_state_dict, build_fp32_params=False)def step(self, new_model):+ """Step.++ Args:+ new_model: New Model instance.+ """self._step_internal(new_model)def reverse(self, model):diff --git a/funasr/models/data2vec/grad_multiply.py b/funasr/models/data2vec/grad_multiply.pyindex 08d15f55d..6041e0305 100644--- a/funasr/models/data2vec/grad_multiply.py+++ b/funasr/models/data2vec/grad_multiply.py@@ -9,10 +9,23 @@ import torchclass GradMultiply(torch.autograd.Function):@staticmethoddef forward(ctx, x, scale):+ """Forward pass for training.++ Args:+ ctx: TODO.+ x: TODO.+ scale: TODO.+ """ctx.scale = scaleres = x.new(x)return res@staticmethoddef backward(ctx, grad):+ """Backward.++ Args:+ ctx: TODO.+ grad: TODO.+ """return grad * ctx.scale, Nonediff --git a/funasr/models/data2vec/multihead_attention.py b/funasr/models/data2vec/multihead_attention.pyindex 85bd0387d..7b977b067 100644--- a/funasr/models/data2vec/multihead_attention.py+++ b/funasr/models/data2vec/multihead_attention.py@@ -17,12 +17,24 @@ from funasr.models.data2vec.quant_noise import quant_noiseclass FairseqDropout(nn.Module):def __init__(self, p, module_name=None):+ """Initialize FairseqDropout.++ Args:+ p: TODO.+ module_name: TODO.+ """super().__init__()self.p = pself.module_name = module_nameself.apply_during_inference = Falsedef forward(self, x, inplace: bool = False):+ """Forward pass for training.++ Args:+ x: TODO.+ inplace: TODO.+ """if self.p > 0 and (self.training or self.apply_during_inference):return F.dropout(x, p=self.p, training=True, inplace=inplace)else:@@ -35,6 +47,14 @@ class FairseqDropout(nn.Module):retain_dropout_modules: Optional[List[str]] = None,**kwargs,):+ """Make generation fast .++ Args:+ name: TODO.+ retain_dropout: TODO.+ retain_dropout_modules: TODO.+ **kwargs: Additional keyword arguments.+ """if retain_dropout:if retain_dropout_modules is not None and self.module_name is None:logging.warning(@@ -72,6 +92,22 @@ class MultiheadAttention(nn.Module):q_noise=0.0,qn_block_size=8,):+ """Initialize MultiheadAttention.++ Args:+ embed_dim: Size/dimension parameter.+ num_heads: TODO.+ kdim: TODO.+ vdim: TODO.+ dropout: TODO.+ bias: TODO.+ add_bias_kv: TODO.+ add_zero_attn: TODO.+ self_attention: TODO.+ encoder_decoder_attention: TODO.+ q_noise: TODO.+ qn_block_size: Size/dimension parameter.+ """super().__init__()self.embed_dim = embed_dimself.kdim = kdim if kdim is not None else embed_dim@@ -122,9 +158,11 @@ class MultiheadAttention(nn.Module):self.skip_embed_dim_check = Falsedef prepare_for_onnx_export_(self):+ """Prepare for onnx export ."""self.onnx_trace = Truedef reset_parameters(self):+ """Reset parameters."""if self.qkv_same_dim:# Empirically observed the convergence to be much better with# the scaled initialization@@ -145,6 +183,11 @@ class MultiheadAttention(nn.Module):nn.init.xavier_normal_(self.bias_v)def _get_reserve_head_index(self, num_heads_to_keep: int):+ """Internal: get reserve head index.++ Args:+ num_heads_to_keep: TODO.+ """k_proj_heads_norm = []q_proj_heads_norm = []v_proj_heads_norm = []@@ -178,6 +221,11 @@ class MultiheadAttention(nn.Module):return reserve_head_indexdef _adaptive_prune_heads(self, reserve_head_index: List[Tuple[int, int]]):+ """Internal: adaptive prune heads.++ Args:+ reserve_head_index: TODO.+ """new_q_weight = []new_q_bias = []new_k_weight = []@@ -236,6 +284,7 @@ class MultiheadAttention(nn.Module):self.v_proj.out_features = self.embed_dimdef _set_skip_embed_dim_check(self):+ """Internal: set skip embed dim check."""self.skip_embed_dim_check = Truedef forward(@@ -506,6 +555,15 @@ class MultiheadAttention(nn.Module):static_kv: bool,) -> Optional[Tensor]:# saved key padding masks have shape (bsz, seq_len)+ """Internal: append prev key padding mask.++ Args:+ key_padding_mask: TODO.+ prev_key_padding_mask: TODO.+ batch_size: Number of samples per batch.+ src_len: TODO.+ static_kv: TODO.+ """if prev_key_padding_mask is not None and static_kv:new_key_padding_mask = prev_key_padding_maskelif prev_key_padding_mask is not None and key_padding_mask is not None:@@ -562,6 +620,11 @@ class MultiheadAttention(nn.Module):def _get_input_buffer(self, incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]]) -> Dict[str, Optional[Tensor]]:+ """Internal: get input buffer.++ Args:+ incremental_state: TODO.+ """result = self.get_incremental_state(incremental_state, "attn_state")if result is not None:return result@@ -574,12 +637,32 @@ class MultiheadAttention(nn.Module):incremental_state: Dict[str, Dict[str, Optional[Tensor]]],buffer: Dict[str, Optional[Tensor]],):+ """Internal: set input buffer.++ Args:+ incremental_state: TODO.+ buffer: TODO.+ """return self.set_incremental_state(incremental_state, "attn_state", buffer)def apply_sparse_mask(self, attn_weights, tgt_len: int, src_len: int, bsz: int):+ """Apply sparse mask.++ Args:+ attn_weights: TODO.+ tgt_len: TODO.+ src_len: TODO.+ bsz: TODO.+ """return attn_weightsdef upgrade_state_dict_named(self, state_dict, name):+ """Upgrade state dict named.++ Args:+ state_dict: TODO.+ name: TODO.+ """prefix = name + "." if name != "" else ""items_to_add = {}keys_to_remove = []diff --git a/funasr/models/data2vec/quant_noise.py b/funasr/models/data2vec/quant_noise.pyindex 50b2c1528..42b9674f6 100644--- a/funasr/models/data2vec/quant_noise.py+++ b/funasr/models/data2vec/quant_noise.py@@ -57,6 +57,12 @@ def quant_noise(module, p, block_size):def _forward_pre_hook(mod, input):# no noise for evaluation+ """Internal: forward pre hook.++ Args:+ mod: TODO.+ input: Input audio/text data.+ """if mod.training:if not is_conv:# gather weight and sizesdiff --git a/funasr/models/data2vec/utils.py b/funasr/models/data2vec/utils.pyindex b4c160358..f8b55b73f 100644--- a/funasr/models/data2vec/utils.py+++ b/funasr/models/data2vec/utils.py@@ -14,9 +14,20 @@ from funasr.models.data2vec.multihead_attention import MultiheadAttentionclass Fp32LayerNorm(nn.LayerNorm):def __init__(self, *args, **kwargs):+ """Initialize Fp32LayerNorm.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = F.layer_norm(input.float(),self.normalized_shape,@@ -29,9 +40,20 @@ class Fp32LayerNorm(nn.LayerNorm):class Fp32GroupNorm(nn.GroupNorm):def __init__(self, *args, **kwargs):+ """Initialize Fp32GroupNorm.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = F.group_norm(input.float(),self.num_groups,@@ -44,10 +66,20 @@ class Fp32GroupNorm(nn.GroupNorm):class TransposeLast(nn.Module):def __init__(self, deconstruct_idx=None):+ """Initialize TransposeLast.++ Args:+ deconstruct_idx: TODO.+ """super().__init__()self.deconstruct_idx = deconstruct_idxdef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if self.deconstruct_idx is not None:x = x[self.deconstruct_idx]return x.transpose(-2, -1)@@ -55,6 +87,12 @@ class TransposeLast(nn.Module):class SamePad(nn.Module):def __init__(self, kernel_size, causal=False):+ """Initialize SamePad.++ Args:+ kernel_size: Size/dimension parameter.+ causal: TODO.+ """super().__init__()if causal:self.remove = kernel_size - 1@@ -62,6 +100,11 @@ class SamePad(nn.Module):self.remove = 1 if kernel_size % 2 == 0 else 0def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if self.remove > 0:x = x[:, :, : -self.remove]return x@@ -69,6 +112,14 @@ class SamePad(nn.Module):def pad_to_multiple(x, multiple, dim=-1, value=0):# Inspired from https://github.com/lucidrains/local-attention/blob/master/local_attention/local_attention.py#L41+ """Pad to multiple.++ Args:+ x: TODO.+ multiple: TODO.+ dim: TODO.+ value: TODO.+ """if x is None:return None, 0tsz = x.size(dim)@@ -82,16 +133,27 @@ def pad_to_multiple(x, multiple, dim=-1, value=0):def gelu_accurate(x):+ """Gelu accurate.++ Args:+ x: TODO.+ """if not hasattr(gelu_accurate, "_a"):gelu_accurate._a = math.sqrt(2 / math.pi)return 0.5 * x * (1 + torch.tanh(gelu_accurate._a * (x + 0.044715 * torch.pow(x, 3))))def gelu(x: torch.Tensor) -> torch.Tensor:+ """Gelu.++ Args:+ x: TODO.+ """return torch.nn.functional.gelu(x.float()).type_as(x)def get_available_activation_fns():+ """Get available activation fns."""return ["relu","gelu",@@ -138,6 +200,11 @@ def init_bert_params(module):def normal_(data):# with FSDP, module params will be on CUDA, so we cast them back to CPU# so that the RNG is consistent with and without FSDP+ """Normal .++ Args:+ data: TODO.+ """data.copy_(data.cpu().normal_(mean=0.0, std=0.02).to(data.device))if isinstance(module, nn.Linear):diff --git a/funasr/models/data2vec/wav2vec2.py b/funasr/models/data2vec/wav2vec2.pyindex a3e837ad3..ad1d86566 100644--- a/funasr/models/data2vec/wav2vec2.py+++ b/funasr/models/data2vec/wav2vec2.py@@ -25,6 +25,15 @@ class ConvFeatureExtractionModel(nn.Module):conv_bias: bool = False,in_d: int = 1,):+ """Initialize ConvFeatureExtractionModel.++ Args:+ conv_layers: TODO.+ dropout: TODO.+ mode: TODO.+ conv_bias: TODO.+ in_d: TODO.+ """super().__init__()assert mode in {"default", "layer_norm"}@@ -38,7 +47,19 @@ class ConvFeatureExtractionModel(nn.Module):is_group_norm=False,conv_bias=False,):+ """Block.++ Args:+ n_in: TODO.+ n_out: TODO.+ k: TODO.+ stride: TODO.+ is_layer_norm: Boolean flag for layer norm.+ is_group_norm: Boolean flag for group norm.+ conv_bias: TODO.+ """def make_conv():+ """Make conv."""conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias)nn.init.kaiming_normal_(conv.weight)return conv@@ -87,6 +108,11 @@ class ConvFeatureExtractionModel(nn.Module):in_d = dimdef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if len(x.shape) == 2:x = x.unsqueeze(1)else:@@ -98,6 +124,13 @@ class ConvFeatureExtractionModel(nn.Module):def make_conv_pos(e, k, g):+ """Make conv pos.++ Args:+ e: TODO.+ k: TODO.+ g: TODO.+ """pos_conv = nn.Conv1d(e,e,@@ -118,6 +151,7 @@ def make_conv_pos(e, k, g):class TransformerEncoder(nn.Module):def build_encoder_layer(self):+ """Build encoder layer."""if self.layer_type == "transformer":layer = TransformerSentenceEncoderLayer(embedding_dim=self.embedding_dim,@@ -154,6 +188,26 @@ class TransformerEncoder(nn.Module):encoder_layerdrop,max_positions,):+ """Initialize TransformerEncoder.++ Args:+ dropout: TODO.+ encoder_embed_dim: Size/dimension parameter.+ required_seq_len_multiple: TODO.+ pos_conv_depth: TODO.+ conv_pos: TODO.+ conv_pos_groups: TODO.+ layer_type: TODO.+ encoder_layers: TODO.+ encoder_ffn_embed_dim: Size/dimension parameter.+ encoder_attention_heads: TODO.+ attention_dropout: TODO.+ activation_dropout: TODO.+ activation_fn: TODO.+ layer_norm_first: TODO.+ encoder_layerdrop: TODO.+ max_positions: TODO.+ """super().__init__()# position@@ -165,6 +219,14 @@ class TransformerEncoder(nn.Module):k = max(3, conv_pos // num_layers)def make_conv_block(e, k, g, l):+ """Make conv block.++ Args:+ e: TODO.+ k: TODO.+ g: TODO.+ l: TODO.+ """return nn.Sequential(*[nn.Sequential(@@ -210,6 +272,13 @@ class TransformerEncoder(nn.Module):self.apply(utils.init_bert_params)def forward(self, x, padding_mask=None, layer=None):+ """Forward pass for training.++ Args:+ x: TODO.+ padding_mask: TODO.+ layer: TODO.+ """x, layer_results = self.extract_features(x, padding_mask, layer)if self.layer_norm_first and layer is None:@@ -225,6 +294,14 @@ class TransformerEncoder(nn.Module):min_layer=0,):+ """Extract features.++ Args:+ x: TODO.+ padding_mask: TODO.+ tgt_layer: TODO.+ min_layer: TODO.+ """if padding_mask is not None:x[padding_mask] = 0@@ -272,6 +349,13 @@ class TransformerEncoder(nn.Module):x = x[:, :-pad_length]def undo_pad(a, b, c):+ """Undo pad.++ Args:+ a: TODO.+ b: TODO.+ c: TODO.+ """return (a[:-pad_length],b[:-pad_length] if b is not None else b,@@ -309,6 +393,18 @@ class TransformerSentenceEncoderLayer(nn.Module):layer_norm_first: bool = False,) -> None:+ """Initialize TransformerSentenceEncoderLayer.++ Args:+ embedding_dim: Size/dimension parameter.+ ffn_embedding_dim: Size/dimension parameter.+ num_attention_heads: TODO.+ dropout: TODO.+ attention_dropout: TODO.+ activation_dropout: TODO.+ activation_fn: TODO.+ layer_norm_first: TODO.+ """super().__init__()# Initialize parametersself.embedding_dim = embedding_dimdiff --git a/funasr/models/e_branchformer/encoder.py b/funasr/models/e_branchformer/encoder.pyindex adc6642f4..1e32fb84c 100644--- a/funasr/models/e_branchformer/encoder.py+++ b/funasr/models/e_branchformer/encoder.py@@ -68,6 +68,17 @@ class EBranchformerEncoderLayer(torch.nn.Module):dropout_rate: float,merge_conv_kernel: int = 3,):+ """Initialize EBranchformerEncoderLayer.++ Args:+ size: TODO.+ attn: TODO.+ cgmlp: TODO.+ feed_forward: TODO.+ feed_forward_macaron: TODO.+ dropout_rate: TODO.+ merge_conv_kernel: TODO.+ """super().__init__()self.size = size@@ -210,6 +221,37 @@ class EBranchformerEncoder(nn.Module):interctc_layer_idx=None,interctc_use_conditioning: bool = False,):+ """Initialize EBranchformerEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ attention_layer_type: TODO.+ pos_enc_layer_type: TODO.+ rel_pos_type: TODO.+ cgmlp_linear_units: TODO.+ cgmlp_conv_kernel: TODO.+ use_linear_after_conv: TODO.+ gate_activation: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ zero_triu: TODO.+ padding_idx: TODO.+ layer_drop_rate: TODO.+ max_pos_emb_len: TODO.+ use_ffn: TODO.+ macaron_ffn: TODO.+ ffn_activation_type: TODO.+ linear_units: TODO.+ positionwise_layer_type: TODO.+ merge_conv_kernel: TODO.+ interctc_layer_idx: TODO.+ interctc_use_conditioning: TODO.+ """super().__init__()self._output_size = output_size@@ -375,6 +417,7 @@ class EBranchformerEncoder(nn.Module):self.conditioning_layer = Nonedef output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(diff --git a/funasr/models/e_branchformer/model.py b/funasr/models/e_branchformer/model.pyindex 9853f7352..e8f791afa 100644--- a/funasr/models/e_branchformer/model.py+++ b/funasr/models/e_branchformer/model.py@@ -14,4 +14,10 @@ class EBranchformer(Transformer):**kwargs,):+ """Initialize EBranchformer.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)diff --git a/funasr/models/e_paraformer/decoder.py b/funasr/models/e_paraformer/decoder.pyindex 7edd91a2a..f784e0914 100644--- a/funasr/models/e_paraformer/decoder.py+++ b/funasr/models/e_paraformer/decoder.py@@ -121,6 +121,15 @@ class DecoderLayerSANM(torch.nn.Module):return x, tgt_mask, memory, memory_mask, cachedef get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Get attn mat.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt = self.feed_forward(tgt)@@ -257,6 +266,35 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):tf2torch_tensor_name_prefix_torch: str = "decoder",tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",):+ """Initialize ParaformerSANMDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ wo_input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ att_layer_num: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ lora_list: TODO.+ lora_rank: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ chunk_multiply_factor: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -426,6 +464,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):ys_in_lens: torch.Tensor,):+ """Forward asf2.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -444,6 +490,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):ys_in_lens: torch.Tensor,):+ """Forward asf6.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -592,6 +646,11 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):class DecoderLayerSANMExport(torch.nn.Module):def __init__(self, model):+ """Initialize DecoderLayerSANMExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.self_attn = model.self_attnself.src_attn = model.src_attn@@ -603,6 +662,15 @@ class DecoderLayerSANMExport(torch.nn.Module):def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Forward pass for training.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt = self.feed_forward(tgt)@@ -621,6 +689,15 @@ class DecoderLayerSANMExport(torch.nn.Module):return x, tgt_mask, memory, memory_mask, cachedef get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Get attn mat.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt = self.feed_forward(tgt)@@ -640,6 +717,15 @@ class DecoderLayerSANMExport(torch.nn.Module):@tables.register("decoder_classes", "ParaformerSANMDecoderExport")class ParaformerSANMDecoderExport(torch.nn.Module):def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):+ """Initialize ParaformerSANMDecoderExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ model_name: TODO.+ onnx: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# self.embed = model.embed #Embedding(model.embed, max_seq_len)@@ -673,6 +759,11 @@ class ParaformerSANMDecoderExport(torch.nn.Module):self.model_name = model_namedef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -692,6 +783,16 @@ class ParaformerSANMDecoderExport(torch.nn.Module):return_both: bool = False,):+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ return_hidden: TODO.+ return_both: TODO.+ """tgt = ys_in_padtgt_mask = self.make_pad_mask(ys_in_lens)tgt_mask, _ = self.prepare_mask(tgt_mask)@@ -728,6 +829,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):ys_in_lens: torch.Tensor,):+ """Forward asf2.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -749,6 +858,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):ys_in_lens: torch.Tensor,):+ """Forward asf6.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -829,6 +946,15 @@ class ParaformerSANMDecoderExport(torch.nn.Module):@tables.register("decoder_classes", "ParaformerSANMDecoderOnlineExport")class ParaformerSANMDecoderOnlineExport(torch.nn.Module):def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):+ """Initialize ParaformerSANMDecoderOnlineExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ model_name: TODO.+ onnx: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# self.embed = model.embed #Embedding(model.embed, max_seq_len)self.model = model@@ -863,6 +989,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):self.model_name = model_namedef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -881,6 +1012,15 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):*args,):+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ *args: Variable positional arguments.+ """tgt = ys_in_padtgt_mask = self.make_pad_mask(ys_in_lens)tgt_mask, _ = self.prepare_mask(tgt_mask)@@ -913,6 +1053,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):return x, out_cachesdef get_dummy_inputs(self, enc_size):+ """Get dummy inputs.++ Args:+ enc_size: Size/dimension parameter.+ """enc = torch.randn(2, 100, enc_size).type(torch.float32)enc_len = torch.tensor([30, 100], dtype=torch.int32)acoustic_embeds = torch.randn(2, 10, enc_size).type(torch.float32)@@ -930,6 +1075,7 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):return (enc, enc_len, acoustic_embeds, acoustic_embeds_len, *cache)def get_input_names(self):+ """Get input names."""cache_num = len(self.model.decoders)if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:cache_num += len(self.model.decoders2)@@ -938,12 +1084,14 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):]def get_output_names(self):+ """Get output names."""cache_num = len(self.model.decoders)if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:cache_num += len(self.model.decoders2)return ["logits", "sample_ids"] + ["out_cache_%d" % i for i in range(cache_num)]def get_dynamic_axes(self):+ """Get dynamic axes."""ret = {"enc": {0: "batch_size", 1: "enc_length"},"acoustic_embeds": {0: "batch_size", 1: "token_length"},@@ -1004,6 +1152,25 @@ class ParaformerSANDecoder(BaseTransformerDecoder):concat_after: bool = False,embeds_id: int = -1,):+ """Initialize ParaformerSANDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ embeds_id: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -1093,6 +1260,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):model_name="decoder",onnx: bool = True,):+ """Initialize ParaformerDecoderSANExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ model_name: TODO.+ onnx: TODO.+ """super().__init__()# self.embed = model.embed #Embedding(model.embed, max_seq_len)self.model = model@@ -1116,6 +1291,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):self.model_name = model_namedef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -1133,6 +1313,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):ys_in_lens: torch.Tensor,):+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = self.make_pad_mask(ys_in_lens)tgt_mask, _ = self.prepare_mask(tgt_mask)@@ -1151,6 +1339,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):return x, ys_in_lensdef get_dummy_inputs(self, enc_size):+ """Get dummy inputs.++ Args:+ enc_size: Size/dimension parameter.+ """tgt = torch.LongTensor([0]).unsqueeze(0)memory = torch.randn(1, 100, enc_size)pre_acoustic_embeds = torch.randn(1, 1, enc_size)@@ -1164,17 +1357,21 @@ class ParaformerDecoderSANExport(torch.nn.Module):return (tgt, memory, pre_acoustic_embeds, cache)def is_optimizable(self):+ """Is optimizable."""return Truedef get_input_names(self):+ """Get input names."""cache_num = len(self.model.decoders) + len(self.model.decoders2)return ["tgt", "memory", "pre_acoustic_embeds"] + ["cache_%d" % i for i in range(cache_num)]def get_output_names(self):+ """Get output names."""cache_num = len(self.model.decoders) + len(self.model.decoders2)return ["y"] + ["out_cache_%d" % i for i in range(cache_num)]def get_dynamic_axes(self):+ """Get dynamic axes."""ret = {"tgt": {0: "tgt_batch", 1: "tgt_length"},"memory": {0: "memory_batch", 1: "memory_length"},diff --git a/funasr/models/e_paraformer/export_meta.py b/funasr/models/e_paraformer/export_meta.pyindex db9385519..eb0678847 100644--- a/funasr/models/e_paraformer/export_meta.py+++ b/funasr/models/e_paraformer/export_meta.py@@ -9,6 +9,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """model.device = kwargs.get("device")is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")@@ -41,6 +47,12 @@ def export_forward(speech_lengths: torch.Tensor,):# a. To device+ """Export forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """batch = {"speech": speech, "speech_lengths": speech_lengths}# batch = to_device(batch, device=self.device)@@ -57,20 +69,24 @@ def export_forward(def export_dummy_inputs(self):+ """Export dummy inputs."""speech = torch.randn(2, 30, 560)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)return (speech, speech_lengths)def export_input_names(self):+ """Export input names."""return ["speech", "speech_lengths"]def export_output_names(self):+ """Export output names."""return ["logits", "token_num"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"},"speech_lengths": {@@ -83,4 +99,5 @@ def export_dynamic_axes(self):def export_name(self,):+ """Export name."""return "model.onnx"diff --git a/funasr/models/e_paraformer/model.py b/funasr/models/e_paraformer/model.pyindex 47b01901d..34ab3e375 100644--- a/funasr/models/e_paraformer/model.py+++ b/funasr/models/e_paraformer/model.py@@ -77,6 +77,37 @@ class EParaformer(torch.nn.Module):**kwargs,):+ """Initialize EParaformer.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ predictor: TODO.+ predictor_conf: Configuration dict for predictor.+ ctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ predictor_weight: TODO.+ predictor_bias: TODO.+ sampling_ratio: TODO.+ share_embedding: TODO.+ use_1st_decoder_loss: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -282,6 +313,12 @@ class EParaformer(torch.nn.Module):def calc_predictor(self, encoder_out, encoder_out_lens):+ """Calc predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -294,6 +331,14 @@ class EParaformer(torch.nn.Module):self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens):+ """Cal decoder with predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ sematic_embeds: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """decoder_outs = self.decoder(encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens)decoder_out = decoder_outs[0]decoder_out = torch.log_softmax(decoder_out, dim=-1)@@ -306,6 +351,14 @@ class EParaformer(torch.nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -364,6 +417,15 @@ class EParaformer(torch.nn.Module):def sampler(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):+ """Sampler.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ pre_acoustic_embeds: TODO.+ """tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)@@ -404,6 +466,15 @@ class EParaformer(torch.nn.Module):def sampler_with_grad(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):+ """Sampler with grad.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ pre_acoustic_embeds: TODO.+ """tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)@@ -450,6 +521,14 @@ class EParaformer(torch.nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -463,6 +542,11 @@ class EParaformer(torch.nn.Module):self,**kwargs,):+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.paraformer.search import BeamSearchParafrom funasr.models.transformer.scorers.ctc import CTCPrefixScorerfrom funasr.models.transformer.scorers.length_bonus import LengthBonus@@ -516,6 +600,16 @@ class EParaformer(torch.nn.Module):**kwargs,):# init beamsearch+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != Noneis_use_lm = (kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None@@ -662,6 +756,11 @@ class EParaformer(torch.nn.Module):return results, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelif "max_seq_len" not in kwargs:diff --git a/funasr/models/e_paraformer/pif_predictor.py b/funasr/models/e_paraformer/pif_predictor.pyindex 8d350f6ec..6cd666486 100644--- a/funasr/models/e_paraformer/pif_predictor.py+++ b/funasr/models/e_paraformer/pif_predictor.py@@ -1,107 +1,131 @@-#!/usr/bin/env python3-# -*- encoding: utf-8 -*--# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.-# Copyright 2024 Kun Zou ([email protected]). All Rights Reserved.-# MIT License (https://opensource.org/licenses/MIT)--import torch-import logging-import numpy as np--from funasr.register import tables-from funasr.train_utils.device_funcs import to_device-from funasr.models.transformer.utils.nets_utils import make_pad_mask-from torch.cuda.amp import autocast--[email protected]("predictor_classes", "PifPredictor")-class PifPredictor(torch.nn.Module):- """- Author: Kun Zou, [email protected]- E-Paraformer: A Faster and Better Parallel Transformer for Non-autoregressive End-to-End Mandarin Speech Recognition- https://www.isca-archive.org/interspeech_2024/zou24_interspeech.pdf- """- def __init__(- self,- idim,- l_order,- r_order,- threshold=1.0,- dropout=0.1,- smooth_factor=1.0,- noise_threshold=0,- sigma=0.5,- bias=0.0,- sigma_heads=4,- ):- super().__init__()-- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)- self.cif_output = torch.nn.Linear(idim, 1)- self.dropout = torch.nn.Dropout(p=dropout)- self.threshold = threshold- self.smooth_factor = smooth_factor- self.noise_threshold = noise_threshold- self.sigma = torch.nn.Parameter(torch.tensor([sigma]*sigma_heads))- self.bias = torch.nn.Parameter(torch.tensor([bias]*sigma_heads))- self.sigma_heads = sigma_heads-- def forward(- self,- hidden,- target_label=None,- mask=None,- ignore_id=-1,- mask_chunk_predictor=None,- target_label_length=None,- ):-- with autocast(False):- h = hidden- context = h.transpose(1, 2)- queries = self.pad(context)- memory = self.cif_conv1d(queries)- output = memory + context- output = self.dropout(output)- output = output.transpose(1, 2)- output = torch.relu(output)- output = self.cif_output(output)- alphas = torch.sigmoid(output)- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)- if mask is not None:- mask = mask.transpose(-1, -2).float()- alphas = alphas * mask- if mask_chunk_predictor is not None:- alphas = alphas * mask_chunk_predictor- alphas = alphas.squeeze(-1)- mask = mask.squeeze(-1)- if target_label_length is not None:- target_length = target_label_length- elif target_label is not None:- target_mask = (target_label != ignore_id).float()- target_length = target_mask.sum(-1)- else:- target_mask = None- target_length = None- token_num = alphas.sum(-1)- if target_length is not None:- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))- max_token_num = torch.max(target_length)- else:- token_num_int = token_num.round()- alphas *=(token_num_int / token_num)[:, None]- max_token_num = torch.max(token_num_int)- alignment = torch.cumsum(alphas, dim=-1)- fire_positions = (torch.arange(max_token_num) + 0.5).type_as(alphas).unsqueeze(0)- scores = - ((fire_positions[:, None, :, None] - alignment[:, None, None, :]) * self.sigma[None, :, None, None]) **2 + self.bias[None, :, None, None]- scores = scores.masked_fill(~(mask[:, None, None, :].to(torch.bool)), float("-inf"))- weights = torch.softmax(scores, dim=-1)- n_hidden = hidden.view(hidden.size(0), -1, self.sigma_heads, hidden.size(-1) // self.sigma_heads).transpose(1, 2)- acoustic_embeds = torch.matmul(weights, n_hidden).transpose(1,2).contiguous().view(hidden.size(0), -1, hidden.size(-1))-- if target_mask is not None:- acoustic_embeds *= target_mask[:, :, None]- cif_peak = None- return acoustic_embeds, token_num, alphas, cif_peak-+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# Copyright 2024 Kun Zou ([email protected]). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++import torch+import logging+import numpy as np++from funasr.register import tables+from funasr.train_utils.device_funcs import to_device+from funasr.models.transformer.utils.nets_utils import make_pad_mask+from torch.cuda.amp import autocast++[email protected]("predictor_classes", "PifPredictor")+class PifPredictor(torch.nn.Module):+ """+ Author: Kun Zou, [email protected]+ E-Paraformer: A Faster and Better Parallel Transformer for Non-autoregressive End-to-End Mandarin Speech Recognition+ https://www.isca-archive.org/interspeech_2024/zou24_interspeech.pdf+ """+ def __init__(+ self,+ idim,+ l_order,+ r_order,+ threshold=1.0,+ dropout=0.1,+ smooth_factor=1.0,+ noise_threshold=0,+ sigma=0.5,+ bias=0.0,+ sigma_heads=4,+ ):+ """Initialize PifPredictor.++ Args:+ idim: TODO.+ l_order: TODO.+ r_order: TODO.+ threshold: TODO.+ dropout: TODO.+ smooth_factor: TODO.+ noise_threshold: TODO.+ sigma: TODO.+ bias: TODO.+ sigma_heads: TODO.+ """+ super().__init__()++ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)+ self.cif_output = torch.nn.Linear(idim, 1)+ self.dropout = torch.nn.Dropout(p=dropout)+ self.threshold = threshold+ self.smooth_factor = smooth_factor+ self.noise_threshold = noise_threshold+ self.sigma = torch.nn.Parameter(torch.tensor([sigma]*sigma_heads))+ self.bias = torch.nn.Parameter(torch.tensor([bias]*sigma_heads))+ self.sigma_heads = sigma_heads++ def forward(+ self,+ hidden,+ target_label=None,+ mask=None,+ ignore_id=-1,+ mask_chunk_predictor=None,+ target_label_length=None,+ ):++ """Forward pass for training.++ Args:+ hidden: TODO.+ target_label: TODO.+ mask: TODO.+ ignore_id: TODO.+ mask_chunk_predictor: TODO.+ target_label_length: TODO.+ """+ with autocast(False):+ h = hidden+ context = h.transpose(1, 2)+ queries = self.pad(context)+ memory = self.cif_conv1d(queries)+ output = memory + context+ output = self.dropout(output)+ output = output.transpose(1, 2)+ output = torch.relu(output)+ output = self.cif_output(output)+ alphas = torch.sigmoid(output)+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)+ if mask is not None:+ mask = mask.transpose(-1, -2).float()+ alphas = alphas * mask+ if mask_chunk_predictor is not None:+ alphas = alphas * mask_chunk_predictor+ alphas = alphas.squeeze(-1)+ mask = mask.squeeze(-1)+ if target_label_length is not None:+ target_length = target_label_length+ elif target_label is not None:+ target_mask = (target_label != ignore_id).float()+ target_length = target_mask.sum(-1)+ else:+ target_mask = None+ target_length = None+ token_num = alphas.sum(-1)+ if target_length is not None:+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))+ max_token_num = torch.max(target_length)+ else:+ token_num_int = token_num.round()+ alphas *=(token_num_int / token_num)[:, None]+ max_token_num = torch.max(token_num_int)+ alignment = torch.cumsum(alphas, dim=-1)+ fire_positions = (torch.arange(max_token_num) + 0.5).type_as(alphas).unsqueeze(0)+ scores = - ((fire_positions[:, None, :, None] - alignment[:, None, None, :]) * self.sigma[None, :, None, None]) **2 + self.bias[None, :, None, None]+ scores = scores.masked_fill(~(mask[:, None, None, :].to(torch.bool)), float("-inf"))+ weights = torch.softmax(scores, dim=-1)+ n_hidden = hidden.view(hidden.size(0), -1, self.sigma_heads, hidden.size(-1) // self.sigma_heads).transpose(1, 2)+ acoustic_embeds = torch.matmul(weights, n_hidden).transpose(1,2).contiguous().view(hidden.size(0), -1, hidden.size(-1))++ if target_mask is not None:+ acoustic_embeds *= target_mask[:, :, None]+ cif_peak = None+ return acoustic_embeds, token_num, alphas, cif_peak+diff --git a/funasr/models/eend/e2e_diar_eend_ola.py b/funasr/models/eend/e2e_diar_eend_ola.pyindex cae5d1f23..a3dac8e99 100644--- a/funasr/models/eend/e2e_diar_eend_ola.py+++ b/funasr/models/eend/e2e_diar_eend_ola.py@@ -25,10 +25,21 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yielddef pad_attractor(att, max_n_speakers):+ """Pad attractor.++ Args:+ att: TODO.+ max_n_speakers: TODO.+ """C, D = att.shapeif C < max_n_speakers:att = torch.cat(@@ -38,6 +49,12 @@ def pad_attractor(att, max_n_speakers):def pad_labels(ts, out_size):+ """Pad labels.++ Args:+ ts: TODO.+ out_size: Size/dimension parameter.+ """for i, t in enumerate(ts):if t.shape[1] < out_size:ts[i] = F.pad(t, (0, out_size - t.shape[1], 0, 0), mode="constant", value=0.0)@@ -45,6 +62,12 @@ def pad_labels(ts, out_size):def pad_results(ys, out_size):+ """Pad results.++ Args:+ ys: TODO.+ out_size: Size/dimension parameter.+ """ys_padded = []for i, y in enumerate(ys):if y.shape[1] < out_size:@@ -78,6 +101,18 @@ class DiarEENDOLAModel(nn.Module):mapping_dict=None,**kwargs,):+ """Initialize DiarEENDOLAModel.++ Args:+ frontend: Audio frontend for feature extraction.+ encoder: TODO.+ encoder_decoder_attractor: TODO.+ n_units: TODO.+ max_n_speaker: TODO.+ attractor_loss_weight: TODO.+ mapping_dict: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.frontend = frontendself.enc = encoder@@ -92,6 +127,12 @@ class DiarEENDOLAModel(nn.Module):self.output_layer = nn.Linear(n_units, mapping_dict["oov"] + 1)def forward_encoder(self, xs, ilens):+ """Forward encoder.++ Args:+ xs: TODO.+ ilens: TODO.+ """xs = nn.utils.rnn.pad_sequence(xs, batch_first=True, padding_value=-1)pad_shape = xs.shapexs_mask = [torch.ones(ilen).to(xs.device) for ilen in ilens]@@ -104,6 +145,12 @@ class DiarEENDOLAModel(nn.Module):return embdef forward_post_net(self, logits, ilens):+ """Forward post net.++ Args:+ logits: TODO.+ ilens: TODO.+ """maxlen = torch.max(ilens).to(torch.int).item()logits = nn.utils.rnn.pad_sequence(logits, batch_first=True, padding_value=-1)logits = nn.utils.rnn.pack_padded_sequence(@@ -125,6 +172,13 @@ class DiarEENDOLAModel(nn.Module):) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:# Check that batch_size is unified+ """Forward pass for training.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speaker_labels: TODO.+ orders: TODO.+ """assert len(speech) == len(speaker_labels), (len(speech), len(speaker_labels))speech_lengths = torch.tensor([len(sph) for sph in speech]).to(torch.int64)speaker_labels_lengths = torch.tensor([spk.shape[-1] for spk in speaker_labels]).to(@@ -186,6 +240,15 @@ class DiarEENDOLAModel(nn.Module):threshold: float = 0.5,**kwargs,):+ """Estimate sequential.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ n_speakers: TODO.+ shuffle: TODO.+ threshold: TODO.+ **kwargs: Additional keyword arguments.+ """speech_lengths = torch.tensor([len(sph) for sph in speech]).to(torch.int64)emb = self.forward_encoder(speech, speech_lengths)if shuffle:@@ -231,6 +294,12 @@ class DiarEENDOLAModel(nn.Module):return ys, emb, attractors, raw_n_speakersdef recover_y_from_powerlabel(self, logit, n_speaker):+ """Recover y from powerlabel.++ Args:+ logit: TODO.+ n_speaker: TODO.+ """pred = torch.argmax(torch.softmax(logit, dim=-1), dim=-1)oov_index = torch.where(pred == self.mapping_dict["oov"])[0]for i in oov_index:@@ -252,6 +321,11 @@ class DiarEENDOLAModel(nn.Module):def inv_mapping_func(self, label):+ """Inv mapping func.++ Args:+ label: TODO.+ """if not isinstance(label, int):label = int(label)if label in self.mapping_dict["label2dec"].keys():@@ -261,4 +335,9 @@ class DiarEENDOLAModel(nn.Module):return numdef collect_feats(self, **batch: torch.Tensor) -> Dict[str, torch.Tensor]:+ """Collect feats.++ Args:+ **batch: Additional keyword arguments.+ """passdiff --git a/funasr/models/eend/eend_ola_dataloader.py b/funasr/models/eend/eend_ola_dataloader.pyindex 43f00fb65..d8099634b 100644--- a/funasr/models/eend/eend_ola_dataloader.py+++ b/funasr/models/eend/eend_ola_dataloader.py@@ -8,6 +8,11 @@ from torch.utils.data import Datasetdef custom_collate(batch):+ """Custom collate.++ Args:+ batch: TODO.+ """keys, speech, speaker_labels, orders = zip(*batch)speech = [torch.from_numpy(np.copy(sph)).to(torch.float32) for sph in speech]speaker_labels = [torch.from_numpy(np.copy(spk)).to(torch.float32) for spk in speaker_labels]@@ -22,6 +27,11 @@ class EENDOLADataset(Dataset):self,data_file,):+ """Initialize EENDOLADataset.++ Args:+ data_file: TODO.+ """self.data_file = data_filewith open(data_file) as f:lines = f.readlines()@@ -29,9 +39,15 @@ class EENDOLADataset(Dataset):logging.info("total samples: {}".format(len(self.samples)))def __len__(self):+ """Internal: len ."""return len(self.samples)def __getitem__(self, idx):+ """Internal: getitem .++ Args:+ idx: TODO.+ """key, speech_path, speaker_label_path = self.samples[idx]speech = kaldiio.load_mat(speech_path)speaker_label = kaldiio.load_mat(speaker_label_path).reshape(speech.shape[0], -1)@@ -44,6 +60,14 @@ class EENDOLADataset(Dataset):class EENDOLADataLoader:def __init__(self, data_file, batch_size, shuffle=True, num_workers=8):+ """Initialize EENDOLADataLoader.++ Args:+ data_file: TODO.+ batch_size: Number of samples per batch.+ shuffle: TODO.+ num_workers: TODO.+ """dataset = EENDOLADataset(data_file)self.data_loader = DataLoader(dataset,@@ -54,4 +78,9 @@ class EENDOLADataLoader:)def build_iter(self, epoch):+ """Build iter.++ Args:+ epoch: TODO.+ """return self.data_loaderdiff --git a/funasr/models/eend/encoder.py b/funasr/models/eend/encoder.pyindex 0dbd98fdf..b7e51632e 100644--- a/funasr/models/eend/encoder.py+++ b/funasr/models/eend/encoder.py@@ -7,6 +7,13 @@ from torch import nnclass MultiHeadSelfAttention(nn.Module):def __init__(self, n_units, h=8, dropout_rate=0.1):+ """Initialize MultiHeadSelfAttention.++ Args:+ n_units: TODO.+ h: TODO.+ dropout_rate: TODO.+ """super().__init__()self.linearQ = nn.Linear(n_units, n_units)self.linearK = nn.Linear(n_units, n_units)@@ -17,6 +24,13 @@ class MultiHeadSelfAttention(nn.Module):self.dropout = nn.Dropout(dropout_rate)def __call__(self, x, batch_size, x_mask):+ """Internal: call .++ Args:+ x: TODO.+ batch_size: Number of samples per batch.+ x_mask: TODO.+ """q = self.linearQ(x).view(batch_size, -1, self.h, self.d_k)k = self.linearK(x).view(batch_size, -1, self.h, self.d_k)v = self.linearV(x).view(batch_size, -1, self.h, self.d_k)@@ -33,17 +47,37 @@ class MultiHeadSelfAttention(nn.Module):class PositionwiseFeedForward(nn.Module):def __init__(self, n_units, d_units, dropout_rate):+ """Initialize PositionwiseFeedForward.++ Args:+ n_units: TODO.+ d_units: TODO.+ dropout_rate: TODO.+ """super(PositionwiseFeedForward, self).__init__()self.linear1 = nn.Linear(n_units, d_units)self.linear2 = nn.Linear(d_units, n_units)self.dropout = nn.Dropout(dropout_rate)def __call__(self, x):+ """Internal: call .++ Args:+ x: TODO.+ """return self.linear2(self.dropout(F.relu(self.linear1(x))))class PositionalEncoding(torch.nn.Module):def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False):+ """Initialize PositionalEncoding.++ Args:+ d_model: D Model instance.+ dropout_rate: TODO.+ max_len: TODO.+ reverse: TODO.+ """super(PositionalEncoding, self).__init__()self.d_model = d_modelself.reverse = reverse@@ -53,6 +87,11 @@ class PositionalEncoding(torch.nn.Module):self.extend_pe(torch.tensor(0.0).expand(1, max_len))def extend_pe(self, x):+ """Extend pe.++ Args:+ x: TODO.+ """if self.pe is not None:if self.pe.size(1) >= x.size(1):if self.pe.dtype != x.dtype or self.pe.device != x.device:@@ -73,6 +112,11 @@ class PositionalEncoding(torch.nn.Module):self.pe = pe.to(device=x.device, dtype=x.dtype)def forward(self, x: torch.Tensor):+ """Forward pass for training.++ Args:+ x: TODO.+ """self.extend_pe(x)x = x * self.xscale + self.pe[:, : x.size(1)]return self.dropout(x)@@ -89,6 +133,17 @@ class EENDOLATransformerEncoder(nn.Module):dropout_rate: float = 0.1,use_pos_emb: bool = False,):+ """Initialize EENDOLATransformerEncoder.++ Args:+ idim: TODO.+ n_layers: TODO.+ n_units: TODO.+ e_units: TODO.+ h: TODO.+ dropout_rate: TODO.+ use_pos_emb: TODO.+ """super(EENDOLATransformerEncoder, self).__init__()self.linear_in = nn.Linear(idim, n_units)self.lnorm_in = nn.LayerNorm(n_units)@@ -106,6 +161,12 @@ class EENDOLATransformerEncoder(nn.Module):self.lnorm_out = nn.LayerNorm(n_units)def __call__(self, x, x_mask=None):+ """Internal: call .++ Args:+ x: TODO.+ x_mask: TODO.+ """BT_size = x.shape[0] * x.shape[1]e = self.linear_in(x.reshape(BT_size, -1))for i in range(self.n_layers):diff --git a/funasr/models/eend/encoder_decoder_attractor.py b/funasr/models/eend/encoder_decoder_attractor.pyindex 65007912d..7961a71a4 100644--- a/funasr/models/eend/encoder_decoder_attractor.py+++ b/funasr/models/eend/encoder_decoder_attractor.py@@ -7,6 +7,13 @@ from torch import nnclass EncoderDecoderAttractor(nn.Module):def __init__(self, n_units, encoder_dropout=0.1, decoder_dropout=0.1):+ """Initialize EncoderDecoderAttractor.++ Args:+ n_units: TODO.+ encoder_dropout: TODO.+ decoder_dropout: TODO.+ """super(EncoderDecoderAttractor, self).__init__()self.enc0_dropout = nn.Dropout(encoder_dropout)self.encoder = nn.LSTM(n_units, n_units, 1, batch_first=True, dropout=encoder_dropout)@@ -16,6 +23,12 @@ class EncoderDecoderAttractor(nn.Module):self.n_units = n_unitsdef forward_core(self, xs, zeros):+ """Forward core.++ Args:+ xs: TODO.+ zeros: TODO.+ """ilens = torch.from_numpy(np.array([x.shape[0] for x in xs])).to(torch.int64)xs = [self.enc0_dropout(x) for x in xs]xs = nn.utils.rnn.pad_sequence(xs, batch_first=True, padding_value=-1)@@ -36,6 +49,12 @@ class EncoderDecoderAttractor(nn.Module):return attractorsdef forward(self, xs, n_speakers):+ """Forward pass for training.++ Args:+ xs: TODO.+ n_speakers: TODO.+ """zeros = [torch.zeros(n_spk + 1, self.n_units).to(torch.float32).to(xs[0].device)for n_spk in n_speakers@@ -56,6 +75,12 @@ class EncoderDecoderAttractor(nn.Module):return loss, attractorsdef estimate(self, xs, max_n_speakers=15):+ """Estimate.++ Args:+ xs: TODO.+ max_n_speakers: TODO.+ """zeros = [torch.zeros(max_n_speakers, self.n_units).to(torch.float32).to(xs[0].device) for _ in xs]diff --git a/funasr/models/eend/utils/feature.py b/funasr/models/eend/utils/feature.pyindex 1dd55dc05..d62bb4104 100644--- a/funasr/models/eend/utils/feature.py+++ b/funasr/models/eend/utils/feature.py@@ -12,6 +12,13 @@ def get_input_dim(context_size,transform_type,):+ """Get input dim.++ Args:+ frame_size: Size/dimension parameter.+ context_size: Size/dimension parameter.+ transform_type: TODO.+ """if transform_type.startswith("logmel23"):frame_size = 23elif transform_type.startswith("logmel"):@@ -155,6 +162,13 @@ def stft(data, frame_size=1024, frame_shift=256):def _count_frames(data_len, size, shift):# HACK: Assuming librosa.stft(..., center=True)+ """Internal: count frames.++ Args:+ data_len: TODO.+ size: TODO.+ shift: TODO.+ """n_frames = 1 + int(data_len / shift)if data_len % shift == 0:n_frames = n_frames - 1diff --git a/funasr/models/eend/utils/kaldi_data.py b/funasr/models/eend/utils/kaldi_data.pyindex 59e7a16f8..25f658b80 100644--- a/funasr/models/eend/utils/kaldi_data.py+++ b/funasr/models/eend/utils/kaldi_data.py@@ -26,6 +26,11 @@ def load_segments(segments_file):def load_segments_hash(segments_file):+ """Load segments hash.++ Args:+ segments_file: TODO.+ """ret = {}if not os.path.exists(segments_file):return None@@ -36,6 +41,11 @@ def load_segments_hash(segments_file):def load_segments_rechash(segments_file):+ """Load segments rechash.++ Args:+ segments_file: TODO.+ """ret = {}if not os.path.exists(segments_file):return None@@ -140,6 +150,11 @@ def extract_segments(wavs, segments=None):class KaldiData:def __init__(self, data_dir):+ """Initialize KaldiData.++ Args:+ data_dir: TODO.+ """self.data_dir = data_dirself.segments = load_segments_rechash(os.path.join(self.data_dir, "segments"))self.utt2spk = load_utt2spk(os.path.join(self.data_dir, "utt2spk"))@@ -148,5 +163,12 @@ class KaldiData:self.spk2utt = load_spk2utt(os.path.join(self.data_dir, "spk2utt"))def load_wav(self, recid, start=0, end=None):+ """Load wav.++ Args:+ recid: TODO.+ start: TODO.+ end: TODO.+ """data, rate = load_wav(self.wavs[recid], start, end)return data, ratediff --git a/funasr/models/eend/utils/losses.py b/funasr/models/eend/utils/losses.pyindex 82fb57358..09086161e 100644--- a/funasr/models/eend/utils/losses.py+++ b/funasr/models/eend/utils/losses.py@@ -5,6 +5,12 @@ from scipy.optimize import linear_sum_assignmentdef standard_loss(ys, ts):+ """Standard loss.++ Args:+ ys: TODO.+ ts: TODO.+ """losses = [F.binary_cross_entropy(torch.sigmoid(y), t) * len(y) for y, t in zip(ys, ts)]loss = torch.sum(torch.stack(losses))n_frames = (@@ -17,6 +23,12 @@ def standard_loss(ys, ts):def fast_batch_pit_n_speaker_loss(ys, ts):+ """Fast batch pit n speaker loss.++ Args:+ ys: TODO.+ ts: TODO.+ """with torch.no_grad():bs = len(ys)indices = []@@ -35,6 +47,12 @@ def fast_batch_pit_n_speaker_loss(ys, ts):def cal_power_loss(logits, power_ts):+ """Cal power loss.++ Args:+ logits: TODO.+ power_ts: TODO.+ """losses = [F.cross_entropy(input=logit, target=power_t.to(torch.long)) * len(logit)for logit, power_t in zip(logits, power_ts)diff --git a/funasr/models/eend/utils/power.py b/funasr/models/eend/utils/power.pyindex b6b32c556..9db0a92d1 100644--- a/funasr/models/eend/utils/power.py+++ b/funasr/models/eend/utils/power.py@@ -7,6 +7,12 @@ from itertools import permutationsdef generate_mapping_dict(max_speaker_num=6, max_olp_speaker_num=3):+ """Generate mapping dict.++ Args:+ max_speaker_num: TODO.+ max_olp_speaker_num: TODO.+ """all_kinds = []all_kinds.append(0)for i in range(max_olp_speaker_num):@@ -32,6 +38,12 @@ def generate_mapping_dict(max_speaker_num=6, max_olp_speaker_num=3):def raw_dec_trans(x, max_speaker_num):+ """Raw dec trans.++ Args:+ x: TODO.+ max_speaker_num: TODO.+ """num_list = []for i in range(max_speaker_num):num_list.append(x[:, i])@@ -45,6 +57,12 @@ def raw_dec_trans(x, max_speaker_num):def mapping_func(num, mapping_dict):+ """Mapping func.++ Args:+ num: TODO.+ mapping_dict: TODO.+ """if num in mapping_dict["dec2label"].keys():label = mapping_dict["dec2label"][num]else:@@ -53,6 +71,13 @@ def mapping_func(num, mapping_dict):def dec_trans(x, max_speaker_num, mapping_dict):+ """Dec trans.++ Args:+ x: TODO.+ max_speaker_num: TODO.+ mapping_dict: TODO.+ """num_list = []for i in range(max_speaker_num):num_list.append(x[:, i])@@ -67,6 +92,14 @@ def dec_trans(x, max_speaker_num, mapping_dict):def create_powerlabel(label, mapping_dict, max_speaker_num=6, max_olp_speaker_num=3):+ """Create powerlabel.++ Args:+ label: TODO.+ mapping_dict: TODO.+ max_speaker_num: TODO.+ max_olp_speaker_num: TODO.+ """T, C = label.shapepadding_label = np.zeros((T, max_speaker_num))padding_label[:, :C] = label@@ -76,6 +109,15 @@ def create_powerlabel(label, mapping_dict, max_speaker_num=6, max_olp_speaker_nudef generate_perm_pse(label, n_speaker, mapping_dict, max_speaker_num, max_olp_speaker_num=3):+ """Generate perm pse.++ Args:+ label: TODO.+ n_speaker: TODO.+ mapping_dict: TODO.+ max_speaker_num: TODO.+ max_olp_speaker_num: TODO.+ """perms = np.array(list(permutations(range(n_speaker)))).astype(np.float32)perms = torch.from_numpy(perms).to(label.device).to(torch.int64)perm_labels = [label[:, perm] for perm in perms]@@ -91,6 +133,16 @@ def generate_perm_pse(label, n_speaker, mapping_dict, max_speaker_num, max_olp_sdef generate_min_pse(label, n_speaker, mapping_dict, max_speaker_num, pse_logit, max_olp_speaker_num=3):+ """Generate min pse.++ Args:+ label: TODO.+ n_speaker: TODO.+ mapping_dict: TODO.+ max_speaker_num: TODO.+ pse_logit: TODO.+ max_olp_speaker_num: TODO.+ """perm_labels, perm_pse_labels = generate_perm_pse(label, n_speaker, mapping_dict, max_speaker_num, max_olp_speaker_num=max_olp_speaker_num)diff --git a/funasr/models/eend/utils/report.py b/funasr/models/eend/utils/report.pyindex eb1bdacd1..d8a6abf60 100644--- a/funasr/models/eend/utils/report.py+++ b/funasr/models/eend/utils/report.py@@ -17,6 +17,12 @@ metrics = [def recover_prediction(y, n_speaker):+ """Recover prediction.++ Args:+ y: TODO.+ n_speaker: TODO.+ """if n_speaker <= 1:return yelif n_speaker == 2:@@ -72,6 +78,13 @@ def recover_prediction(y, n_speaker):class PowerReporter:def __init__(self, valid_data_loader, mapping_dict, max_n_speaker):+ """Initialize PowerReporter.++ Args:+ valid_data_loader: TODO.+ mapping_dict: TODO.+ max_n_speaker: TODO.+ """valid_data_loader_cp = copy.deepcopy(valid_data_loader)self.valid_data_loader = valid_data_loader_cpdel valid_data_loader@@ -79,9 +92,23 @@ class PowerReporter:self.max_n_speaker = max_n_speakerdef report(self, model, eidx, device):+ """Report.++ Args:+ model: Model instance or model name.+ eidx: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """self.report_val(model, eidx, device)def report_val(self, model, eidx, device):+ """Report val.++ Args:+ model: Model instance or model name.+ eidx: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """model.eval()ud_valid_start = time.time()valid_res, valid_loss, stats_keys, vad_valid_accuracy = self.report_core(@@ -105,6 +132,12 @@ class PowerReporter:print("Valid cost time ... ", ud_valid)def inv_mapping_func(self, label, mapping_dict):+ """Inv mapping func.++ Args:+ label: TODO.+ mapping_dict: TODO.+ """if not isinstance(label, int):label = int(label)if label in mapping_dict["label2dec"].keys():@@ -114,6 +147,13 @@ class PowerReporter:return numdef report_core(self, model, data_loader, device):+ """Report core.++ Args:+ model: Model instance or model name.+ data_loader: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """res = {}for item in metrics:res[item[0]] = 0.0@@ -162,6 +202,13 @@ class PowerReporter:return res, loss_s, stats.keys(), vad_accdef calc_diarization_error(self, decisions, label, label_delay=0):+ """Calc diarization error.++ Args:+ decisions: TODO.+ label: TODO.+ label_delay: TODO.+ """label = label[: len(label) - label_delay, ...]n_ref = torch.sum(label, dim=-1)n_sys = torch.sum(decisions, dim=-1)diff --git a/funasr/models/emotion2vec/audio.py b/funasr/models/emotion2vec/audio.pyindex 260957343..0e83277e5 100644--- a/funasr/models/emotion2vec/audio.py+++ b/funasr/models/emotion2vec/audio.py@@ -32,6 +32,16 @@ class AudioEncoder(ModalitySpecificEncoder):alibi_biases: Dict,):+ """Initialize AudioEncoder.++ Args:+ modality_cfg: TODO.+ embed_dim: Size/dimension parameter.+ make_block: TODO.+ norm_layer: TODO.+ layer_norm_first: TODO.+ alibi_biases: TODO.+ """self.feature_enc_layers = eval(modality_cfg.feature_encoder_spec)feature_embed_dim = self.feature_enc_layers[-1][0]@@ -108,12 +118,25 @@ class AudioEncoder(ModalitySpecificEncoder):)def convert_padding_mask(self, x, padding_mask):+ """Convert padding mask.++ Args:+ x: TODO.+ padding_mask: TODO.+ """def get_feat_extract_output_lengths(input_lengths: torch.LongTensor):"""Computes the output length of the convolutional layers"""def _conv_out_length(input_length, kernel_size, stride):+ """Internal: conv out length.++ Args:+ input_length: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ """return torch.floor((input_length - kernel_size) / stride + 1)for i in range(len(self.feature_enc_layers)):@@ -148,6 +171,7 @@ class AudioEncoder(ModalitySpecificEncoder):return padding_maskdef reset_parameters(self):+ """Reset parameters."""super().reset_parameters()for mod in self.project_features.children():if isinstance(mod, nn.Linear):diff --git a/funasr/models/emotion2vec/base.py b/funasr/models/emotion2vec/base.pyindex 1b4301ea3..7a9659355 100644--- a/funasr/models/emotion2vec/base.py+++ b/funasr/models/emotion2vec/base.py@@ -39,6 +39,19 @@ class ModalitySpecificEncoder(nn.Module):decoder: nn.Module,get_alibi_bias: Optional[Callable[[int, int, str, str], torch.Tensor]],):+ """Initialize ModalitySpecificEncoder.++ Args:+ modality_cfg: TODO.+ embed_dim: Size/dimension parameter.+ local_encoder: TODO.+ project_features: TODO.+ fixed_positional_encoder: TODO.+ relative_positional_encoder: TODO.+ context_encoder: TODO.+ decoder: TODO.+ get_alibi_bias: TODO.+ """super().__init__()self.modality_cfg = modality_cfg@@ -102,6 +115,12 @@ class ModalitySpecificEncoder(nn.Module):self.get_alibi_bias = partial(_learned_alibi_bias, alibi_bias=self.alibi_bias)def upgrade_state_dict_named(self, state_dict, name):+ """Upgrade state dict named.++ Args:+ state_dict: TODO.+ name: TODO.+ """k = f"{name}.alibi_scale"if k in state_dict and state_dict[k].dim() == 4:state_dict[k] = state_dict[k].unsqueeze(0)@@ -109,9 +128,21 @@ class ModalitySpecificEncoder(nn.Module):return state_dictdef convert_padding_mask(self, x, padding_mask):+ """Convert padding mask.++ Args:+ x: TODO.+ padding_mask: TODO.+ """return padding_maskdef decoder_input(self, x, mask_info: MaskInfo):+ """Decoder input.++ Args:+ x: TODO.+ mask_info: TODO.+ """inp_drop = self.modality_cfg.decoder.input_dropoutif inp_drop > 0:x = F.dropout(x, inp_drop, training=self.training, inplace=True)@@ -144,6 +175,11 @@ class ModalitySpecificEncoder(nn.Module):return x, mask_infodef local_features(self, features):+ """Local features.++ Args:+ features: TODO.+ """if self.local_grad_mult > 0:if self.local_grad_mult == 1.0:x = self.local_encoder(features)@@ -167,6 +203,17 @@ class ModalitySpecificEncoder(nn.Module):precomputed_mask=None,):+ """Contextualized features.++ Args:+ x: TODO.+ padding_mask: TODO.+ mask: TODO.+ remove_masked: TODO.+ clone_batch: TODO.+ mask_seeds: TODO.+ precomputed_mask: TODO.+ """if padding_mask is not None:padding_mask = self.convert_padding_mask(x, padding_mask)@@ -290,6 +337,17 @@ class ModalitySpecificEncoder(nn.Module):mask_seeds: Optional[torch.Tensor] = None,precomputed_mask=None,):+ """Forward pass for training.++ Args:+ features: TODO.+ padding_mask: TODO.+ mask: TODO.+ remove_masked: TODO.+ clone_batch: TODO.+ mask_seeds: TODO.+ precomputed_mask: TODO.+ """x = self.local_features(features)return self.contextualized_features(x,@@ -302,6 +360,7 @@ class ModalitySpecificEncoder(nn.Module):)def reset_parameters(self):+ """Reset parameters."""passdef compute_mask(@@ -312,6 +371,15 @@ class ModalitySpecificEncoder(nn.Module):apply,precomputed_mask,):+ """Compute mask.++ Args:+ x: TODO.+ padding_mask: TODO.+ mask_seed: TODO.+ apply: TODO.+ precomputed_mask: TODO.+ """if precomputed_mask is not None:mask = precomputed_maskmask_info = self.make_maskinfo(x, mask)@@ -362,6 +430,13 @@ class ModalitySpecificEncoder(nn.Module):return x, mask_infodef make_maskinfo(self, x, mask, shape=None):+ """Make maskinfo.++ Args:+ x: TODO.+ mask: TODO.+ shape: TODO.+ """if shape is None:B, T, D = x.shapeelse:@@ -392,6 +467,12 @@ class ModalitySpecificEncoder(nn.Module):return mask_infodef apply_mask(self, x, mask_info):+ """Apply mask.++ Args:+ x: TODO.+ mask_info: TODO.+ """cfg = self.modality_cfgB, T, C = x.shape@@ -417,11 +498,24 @@ class ModalitySpecificEncoder(nn.Module):return xdef remove_pretraining_modules(self, keep_decoder=False):+ """Remove pretraining modules.++ Args:+ keep_decoder: TODO.+ """if not keep_decoder:self.decoder = Nonedef get_annealed_rate(start, end, curr_step, total_steps):+ """Get annealed rate.++ Args:+ start: TODO.+ end: TODO.+ curr_step: TODO.+ total_steps: TODO.+ """if curr_step >= total_steps:return endr = end - start@@ -431,6 +525,13 @@ def get_annealed_rate(start, end, curr_step, total_steps):# adapted from MAEdef random_masking(x, mask_ratio, mask_seed: Optional[MaskSeed]):+ """Random masking.++ Args:+ x: TODO.+ mask_ratio: TODO.+ mask_seed: TODO.+ """N, L, D = x.shape # batch, length, dimlen_keep = int(L * (1 - mask_ratio))@@ -463,6 +564,12 @@ def random_masking(x, mask_ratio, mask_seed: Optional[MaskSeed]):def gather_unmasked(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor:+ """Gather unmasked.++ Args:+ x: TODO.+ mask_info: TODO.+ """return torch.gather(x,dim=1,@@ -471,6 +578,12 @@ def gather_unmasked(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor:def gather_unmasked_mask(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor:+ """Gather unmasked mask.++ Args:+ x: TODO.+ mask_info: TODO.+ """return torch.gather(x,dim=1,@@ -484,8 +597,26 @@ def get_alibi(dims: int = 1,distance: str = "manhattan",):+ """Get alibi.++ Args:+ max_positions: TODO.+ attention_heads: TODO.+ dims: TODO.+ distance: TODO.+ """def get_slopes(n):+ """Get slopes.++ Args:+ n: TODO.+ """def get_slopes_power_of_2(n):+ """Get slopes power of 2.++ Args:+ n: TODO.+ """start = 2 ** (-(2 ** -(math.log2(n) - 3)))ratio = startreturn [start * ratio**i for i in range(n)]@@ -552,6 +683,18 @@ def get_alibi_bias(dims=1,distance="manhattan",):+ """Get alibi bias.++ Args:+ alibi_biases: TODO.+ batch_size: Number of samples per batch.+ time_steps: TODO.+ heads: TODO.+ dtype: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ dims: TODO.+ distance: TODO.+ """cache_key = f"{dims}_{heads}_{distance}"buffered = alibi_biases.get(cache_key, None)@@ -589,6 +732,17 @@ def _learned_alibi_bias(dtype,device,):+ """Internal: learned alibi bias.++ Args:+ alibi_bias: TODO.+ batch_size: Number of samples per batch.+ time_steps: TODO.+ heads: TODO.+ scale: TODO.+ dtype: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """assert alibi_bias.size(1) == heads, alibi_bias.shapeassert alibi_bias.dtype == dtype, alibi_bias.dtypeassert alibi_bias.device == device, alibi_bias.device@@ -602,6 +756,12 @@ def _learned_alibi_bias(def masked_alibi(alibi_bias, mask_info):+ """Masked alibi.++ Args:+ alibi_bias: TODO.+ mask_info: TODO.+ """H = alibi_bias.size(1)orig_bias = alibi_biasdiff --git a/funasr/models/emotion2vec/export_meta.py b/funasr/models/emotion2vec/export_meta.pyindex f0c5f434b..3bd53d64c 100644--- a/funasr/models/emotion2vec/export_meta.py+++ b/funasr/models/emotion2vec/export_meta.py@@ -9,6 +9,12 @@ import torch.nn.functional as Fdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """model.device = kwargs.get("device")# store original forward since self.extract_features is calling it@@ -25,6 +31,11 @@ def export_rebuild_model(model, **kwargs):def export_forward(self, x: torch.Tensor):+ """Export forward.++ Args:+ x: TODO.+ """with torch.no_grad():if self.cfg.normalize:mean = torch.mean(x, dim=1, keepdim=True)@@ -44,18 +55,22 @@ def export_forward(self, x: torch.Tensor):def export_dummy_inputs(self):+ """Export dummy inputs."""return (torch.randn(1, 16000),)def export_input_names(self):+ """Export input names."""return ["input"]def export_output_names(self):+ """Export output names."""return ["output"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"input": {0: "batch_size",@@ -66,4 +81,5 @@ def export_dynamic_axes(self):def export_name(self):+ """Export name."""return "emotion2vec"diff --git a/funasr/models/emotion2vec/fairseq_modules.py b/funasr/models/emotion2vec/fairseq_modules.pyindex 209dedaa3..3f7887225 100644--- a/funasr/models/emotion2vec/fairseq_modules.py+++ b/funasr/models/emotion2vec/fairseq_modules.py@@ -6,11 +6,25 @@ import numpy as npdef LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True, export=False):+ """Layernorm.++ Args:+ normalized_shape: TODO.+ eps: TODO.+ elementwise_affine: TODO.+ export: TODO.+ """return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine)class SamePad(nn.Module):def __init__(self, kernel_size, causal=False):+ """Initialize SamePad.++ Args:+ kernel_size: Size/dimension parameter.+ causal: TODO.+ """super().__init__()if causal:self.remove = kernel_size - 1@@ -18,6 +32,11 @@ class SamePad(nn.Module):self.remove = 1 if kernel_size % 2 == 0 else 0def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if self.remove > 0:x = x[:, :, : -self.remove]return x@@ -25,10 +44,20 @@ class SamePad(nn.Module):class TransposeLast(nn.Module):def __init__(self, deconstruct_idx=None):+ """Initialize TransposeLast.++ Args:+ deconstruct_idx: TODO.+ """super().__init__()self.deconstruct_idx = deconstruct_idxdef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if self.deconstruct_idx is not None:x = x[self.deconstruct_idx]return x.transpose(-2, -1)@@ -36,9 +65,20 @@ class TransposeLast(nn.Module):class Fp32LayerNorm(nn.LayerNorm):def __init__(self, *args, **kwargs):+ """Initialize Fp32LayerNorm.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = F.layer_norm(input.float(),self.normalized_shape,@@ -51,9 +91,20 @@ class Fp32LayerNorm(nn.LayerNorm):class Fp32GroupNorm(nn.GroupNorm):def __init__(self, *args, **kwargs):+ """Initialize Fp32GroupNorm.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = F.group_norm(input.float(),self.num_groups,@@ -72,6 +123,14 @@ class ConvFeatureExtractionModel(nn.Module):mode: str = "default",conv_bias: bool = False,):+ """Initialize ConvFeatureExtractionModel.++ Args:+ conv_layers: TODO.+ dropout: TODO.+ mode: TODO.+ conv_bias: TODO.+ """super().__init__()assert mode in {"default", "layer_norm"}@@ -85,7 +144,19 @@ class ConvFeatureExtractionModel(nn.Module):is_group_norm=False,conv_bias=False,):+ """Block.++ Args:+ n_in: TODO.+ n_out: TODO.+ k: TODO.+ stride: TODO.+ is_layer_norm: Boolean flag for layer norm.+ is_group_norm: Boolean flag for group norm.+ conv_bias: TODO.+ """def make_conv():+ """Make conv."""conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias)nn.init.kaiming_normal_(conv.weight)return conv@@ -137,6 +208,11 @@ class ConvFeatureExtractionModel(nn.Module):def forward(self, x):# BxT -> BxCxT+ """Forward pass for training.++ Args:+ x: TODO.+ """x = x.unsqueeze(1)for conv in self.conv_layers:@@ -225,6 +301,14 @@ def compute_mask_indices(mask_idc = []def arrange(s, e, length, keep_length):+ """Arrange.++ Args:+ s: TODO.+ e: TODO.+ length: TODO.+ keep_length: TODO.+ """span_start = np.random.randint(s, e - length)mask_idc.extend(span_start + i for i in range(length))@@ -279,20 +363,45 @@ def compute_mask_indices(class GradMultiply(torch.autograd.Function):@staticmethoddef forward(ctx, x, scale):+ """Forward pass for training.++ Args:+ ctx: TODO.+ x: TODO.+ scale: TODO.+ """ctx.scale = scaleres = x.new(x)return res@staticmethoddef backward(ctx, grad):+ """Backward.++ Args:+ ctx: TODO.+ grad: TODO.+ """return grad * ctx.scale, Nonedef is_xla_tensor(tensor):+ """Is xla tensor.++ Args:+ tensor: TODO.+ """return torch.is_tensor(tensor) and tensor.device.type == "xla"def index_put(tensor, indices, value):+ """Index put.++ Args:+ tensor: TODO.+ indices: TODO.+ value: TODO.+ """if is_xla_tensor(tensor):for _ in range(indices.dim(), tensor.dim()):indices = indices.unsqueeze(-1)diff --git a/funasr/models/emotion2vec/model.py b/funasr/models/emotion2vec/model.pyindex 75ede4c0c..3ecace8eb 100644--- a/funasr/models/emotion2vec/model.py+++ b/funasr/models/emotion2vec/model.py@@ -28,6 +28,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -40,6 +45,11 @@ class Emotion2vec(torch.nn.Module):"""def __init__(self, **kwargs):+ """Initialize Emotion2vec.++ Args:+ **kwargs: Additional keyword arguments.+ """super().__init__()# import pdb; pdb.set_trace()cfg = OmegaConf.create(kwargs["model_conf"])@@ -50,6 +60,13 @@ class Emotion2vec(torch.nn.Module):)def make_block(drop_path, dim=None, heads=None):+ """Make block.++ Args:+ drop_path: TODO.+ dim: TODO.+ heads: TODO.+ """return AltBlock(cfg.get("embed_dim") if dim is None else dim,cfg.get("num_heads") if heads is None else heads,@@ -116,6 +133,21 @@ class Emotion2vec(torch.nn.Module):**kwargs,):+ """Forward pass for training.++ Args:+ source: TODO.+ target: TODO.+ id: TODO.+ mode: TODO.+ padding_mask: TODO.+ mask: TODO.+ features_only: TODO.+ force_remove_masked: TODO.+ remove_extra_tokens: TODO.+ precomputed_mask: TODO.+ **kwargs: Additional keyword arguments.+ """feature_extractor = self.modality_encoders["AUDIO"]mask_seeds = None@@ -180,6 +212,15 @@ class Emotion2vec(torch.nn.Module):def extract_features(self, source, mode=None, padding_mask=None, mask=False, remove_extra_tokens=True):+ """Extract features.++ Args:+ source: TODO.+ mode: TODO.+ padding_mask: TODO.+ mask: TODO.+ remove_extra_tokens: TODO.+ """res = self.forward(source,mode=mode,@@ -205,6 +246,16 @@ class Emotion2vec(torch.nn.Module):# channel = sf.info(source_file).channels# assert sr == 16e3, "Sample rate should be 16kHz, but got {}in file {}".format(sr, source_file)# assert channel == 1, "Channel should be 1, but got {} in file {}".format(channel, source_file)+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """granularity = kwargs.get("granularity", "utterance")extract_embedding = kwargs.get("extract_embedding", True)if self.proj is None:@@ -267,6 +318,11 @@ class Emotion2vec(torch.nn.Module):return results, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelmodels = export_rebuild_model(model=self, **kwargs)diff --git a/funasr/models/emotion2vec/modules.py b/funasr/models/emotion2vec/modules.pyindex 6e3640948..2cb8db6e5 100644--- a/funasr/models/emotion2vec/modules.py+++ b/funasr/models/emotion2vec/modules.py@@ -38,10 +38,21 @@ class D2vDecoderConfig:class FixedPositionalEncoder(nn.Module):def __init__(self, pos_embed):+ """Initialize FixedPositionalEncoder.++ Args:+ pos_embed: TODO.+ """super().__init__()self.positions = pos_embeddef forward(self, x, padding_mask):+ """Forward pass for training.++ Args:+ x: TODO.+ padding_mask: TODO.+ """return self.positions@@ -52,17 +63,37 @@ class TextFeatPositionalEncoder(nn.Module):"""def __init__(self, pos_encoder):+ """Initialize TextFeatPositionalEncoder.++ Args:+ pos_encoder: TODO.+ """super().__init__()self.pos_encoder = pos_encoderdef forward(self, x, padding_mask):# assume padded token embeddings are 0s# TODO: consider using padding_mask as input+ """Forward pass for training.++ Args:+ x: TODO.+ padding_mask: TODO.+ """return self.pos_encoder(x[..., 0])class BlockEncoder(nn.Module):def __init__(self, blocks, norm_layer, layer_norm_first, layerdrop, dropout):+ """Initialize BlockEncoder.++ Args:+ blocks: TODO.+ norm_layer: TODO.+ layer_norm_first: TODO.+ layerdrop: TODO.+ dropout: TODO.+ """super().__init__()self.blocks = blocksself.norm = norm_layer@@ -71,6 +102,14 @@ class BlockEncoder(nn.Module):self.dropout = nn.Dropout(dropout, inplace=True)def forward(self, x, padding_mask, alibi_bias, alibi_scale):+ """Forward pass for training.++ Args:+ x: TODO.+ padding_mask: TODO.+ alibi_bias: TODO.+ alibi_scale: TODO.+ """if self.norm is not None and not self.layer_norm_first:x = self.norm(x)@@ -94,16 +133,30 @@ class DecoderBase(nn.Module):decoder_cfg: D2vDecoderConfigdef __init__(self, cfg: D2vDecoderConfig):+ """Initialize DecoderBase.++ Args:+ cfg: Configuration overrides.+ """super().__init__()self.decoder_cfg = cfgdef reset_parameters(self):+ """Reset parameters."""for mod in self.proj.modules():if isinstance(mod, nn.Linear):mod.reset_parameters()def add_residual(self, x, residual, i, mask_info):+ """Add residual.++ Args:+ x: TODO.+ residual: TODO.+ i: TODO.+ mask_info: TODO.+ """if (residual is Noneor not self.decoder_cfg.decoder_residual@@ -118,9 +171,20 @@ class DecoderBase(nn.Module):class Decoder1d(DecoderBase):def __init__(self, cfg: D2vDecoderConfig, input_dim):+ """Initialize Decoder1d.++ Args:+ cfg: Configuration overrides.+ input_dim: Size/dimension parameter.+ """super().__init__(cfg)def make_block(in_dim):+ """Make block.++ Args:+ in_dim: Size/dimension parameter.+ """block = [nn.Conv1d(in_dim,@@ -160,6 +224,12 @@ class Decoder1d(DecoderBase):def forward(self, x, mask_info):+ """Forward pass for training.++ Args:+ x: TODO.+ mask_info: TODO.+ """x = x.transpose(1, 2)residual = x@@ -193,6 +263,25 @@ class AltBlock(nn.Module):ffn_targets=False,cosine_attention=False,):+ """Initialize AltBlock.++ Args:+ dim: TODO.+ num_heads: TODO.+ mlp_ratio: TODO.+ qkv_bias: TODO.+ qk_scale: TODO.+ drop: TODO.+ attn_drop: TODO.+ mlp_drop: TODO.+ post_mlp_drop: TODO.+ drop_path: TODO.+ act_layer: TODO.+ norm_layer: TODO.+ layer_norm_first: TODO.+ ffn_targets: TODO.+ cosine_attention: TODO.+ """super().__init__()self.layer_norm_first = layer_norm_first@@ -223,6 +312,13 @@ class AltBlock(nn.Module):self.post_mlp_dropout = nn.Dropout(post_mlp_drop, inplace=False)def forward(self, x, padding_mask=None, alibi_bias=None):+ """Forward pass for training.++ Args:+ x: TODO.+ padding_mask: TODO.+ alibi_bias: TODO.+ """if self.layer_norm_first:x = x + self.drop_path(self.attn(self.norm1(x), padding_mask, alibi_bias))r = x = self.mlp(self.norm2(x))@@ -253,6 +349,17 @@ class AltAttention(nn.Module):proj_drop=0.0,cosine_attention=False,):+ """Initialize AltAttention.++ Args:+ dim: TODO.+ num_heads: TODO.+ qkv_bias: TODO.+ qk_scale: TODO.+ attn_drop: TODO.+ proj_drop: TODO.+ cosine_attention: TODO.+ """super().__init__()self.num_heads = num_headshead_dim = dim // num_heads@@ -271,6 +378,13 @@ class AltAttention(nn.Module):)def forward(self, x, padding_mask=None, alibi_bias=None):+ """Forward pass for training.++ Args:+ x: TODO.+ padding_mask: TODO.+ alibi_bias: TODO.+ """B, N, C = x.shapeqkv = (self.qkv(x)diff --git a/funasr/models/emotion2vec/timm_modules.py b/funasr/models/emotion2vec/timm_modules.pyindex 416d2cbaf..04d9cffc8 100644--- a/funasr/models/emotion2vec/timm_modules.py+++ b/funasr/models/emotion2vec/timm_modules.py@@ -28,20 +28,42 @@ class DropPath(nn.Module):"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""def __init__(self, drop_prob: float = 0.0, scale_by_keep: bool = True):+ """Initialize DropPath.++ Args:+ drop_prob: TODO.+ scale_by_keep: TODO.+ """super(DropPath, self).__init__()self.drop_prob = drop_probself.scale_by_keep = scale_by_keepdef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)def extra_repr(self):+ """Extra repr."""return f"drop_prob={round(self.drop_prob,3):0.3f}"# From PyTorch internalsdef _ntuple(n):+ """Internal: ntuple.++ Args:+ n: TODO.+ """def parse(x):+ """Parse.++ Args:+ x: TODO.+ """if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):return tuple(x)return tuple(repeat(x, n))@@ -70,6 +92,18 @@ class Mlp(nn.Module):drop=0.0,use_conv=False,):+ """Initialize Mlp.++ Args:+ in_features: TODO.+ hidden_features: TODO.+ out_features: TODO.+ act_layer: TODO.+ norm_layer: TODO.+ bias: TODO.+ drop: TODO.+ use_conv: TODO.+ """super().__init__()out_features = out_features or in_featureshidden_features = hidden_features or in_features@@ -85,6 +119,11 @@ class Mlp(nn.Module):self.drop2 = nn.Dropout(drop_probs[1])def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = self.fc1(x)x = self.act(x)x = self.drop1(x)diff --git a/funasr/models/eres2net/eres2net.py b/funasr/models/eres2net/eres2net.pyindex de6c2bbdd..fe6a65941 100644--- a/funasr/models/eres2net/eres2net.py+++ b/funasr/models/eres2net/eres2net.py@@ -23,9 +23,15 @@ from funasr.models.eres2net.fusion import AFFclass ReLU(nn.Hardtanh):def __init__(self, inplace=False):+ """Initialize ReLU.++ Args:+ inplace: TODO.+ """super(ReLU, self).__init__(0, 20, inplace)def __repr__(self):+ """Internal: repr ."""inplace_str = "inplace" if self.inplace else ""return self.__class__.__name__ + " (" + inplace_str + ")"@@ -44,6 +50,15 @@ class BasicBlockERes2Net(nn.Module):expansion = 2def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):+ """Initialize BasicBlockERes2Net.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockERes2Net, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -74,6 +89,11 @@ class BasicBlockERes2Net(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -106,6 +126,15 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):expansion = 2def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):+ """Initialize BasicBlockERes2Net_diff_AFF.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockERes2Net_diff_AFF, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -141,6 +170,11 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -182,6 +216,18 @@ class ERes2Net(nn.Module):pooling_func="TSTP",two_emb_layer=False,):+ """Initialize ERes2Net.++ Args:+ block: TODO.+ block_fuse: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ pooling_func: TODO.+ two_emb_layer: TODO.+ """super(ERes2Net, self).__init__()self.in_planes = m_channelsself.feat_dim = feat_dim@@ -223,6 +269,14 @@ class ERes2Net(nn.Module):self.seg_2 = nn.Identity()def _make_layer(self, block, planes, num_blocks, stride):+ """Internal: make layer.++ Args:+ block: TODO.+ planes: TODO.+ num_blocks: TODO.+ stride: TODO.+ """strides = [stride] + [1] * (num_blocks - 1)layers = []for stride in strides:@@ -231,6 +285,11 @@ class ERes2Net(nn.Module):return nn.Sequential(*layers)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)x = x.unsqueeze_(1)out = F.relu(self.bn1(self.conv1(x)))@@ -260,6 +319,15 @@ class BasicBlockRes2Net(nn.Module):expansion = 2def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):+ """Initialize BasicBlockRes2Net.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockRes2Net, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -289,6 +357,11 @@ class BasicBlockRes2Net(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -330,6 +403,17 @@ class Res2Net(nn.Module):pooling_func="TSTP",two_emb_layer=False,):+ """Initialize Res2Net.++ Args:+ block: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ pooling_func: TODO.+ two_emb_layer: TODO.+ """super(Res2Net, self).__init__()self.in_planes = m_channelsself.feat_dim = feat_dim@@ -355,6 +439,14 @@ class Res2Net(nn.Module):self.seg_2 = nn.Identity()def _make_layer(self, block, planes, num_blocks, stride):+ """Internal: make layer.++ Args:+ block: TODO.+ planes: TODO.+ num_blocks: TODO.+ stride: TODO.+ """strides = [stride] + [1] * (num_blocks - 1)layers = []for stride in strides:@@ -363,6 +455,11 @@ class Res2Net(nn.Module):return nn.Sequential(*layers)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)x = x.unsqueeze_(1)diff --git a/funasr/models/eres2net/eres2net_aug.py b/funasr/models/eres2net/eres2net_aug.pyindex f700b9c98..36217892f 100644--- a/funasr/models/eres2net/eres2net_aug.py+++ b/funasr/models/eres2net/eres2net_aug.py@@ -23,9 +23,15 @@ from funasr.models.eres2net.fusion import AFFclass ReLU(nn.Hardtanh):def __init__(self, inplace=False):+ """Initialize ReLU.++ Args:+ inplace: TODO.+ """super(ReLU, self).__init__(0, 20, inplace)def __repr__(self):+ """Internal: repr ."""inplace_str = "inplace" if self.inplace else ""return self.__class__.__name__ + " (" + inplace_str + ")"@@ -44,6 +50,15 @@ class BasicBlockERes2Net(nn.Module):expansion = 4def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3):+ """Initialize BasicBlockERes2Net.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockERes2Net, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -74,6 +89,11 @@ class BasicBlockERes2Net(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -106,6 +126,15 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):expansion = 4def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3):+ """Initialize BasicBlockERes2Net_diff_AFF.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockERes2Net_diff_AFF, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -142,6 +171,11 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -183,6 +217,18 @@ class ERes2NetAug(nn.Module):pooling_func="TSTP",two_emb_layer=False,):+ """Initialize ERes2NetAug.++ Args:+ block: TODO.+ block_fuse: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ pooling_func: TODO.+ two_emb_layer: TODO.+ """super(ERes2NetAug, self).__init__()self.in_planes = m_channelsself.feat_dim = feat_dim@@ -221,6 +267,14 @@ class ERes2NetAug(nn.Module):self.seg_2 = nn.Identity()def _make_layer(self, block, planes, num_blocks, stride):+ """Internal: make layer.++ Args:+ block: TODO.+ planes: TODO.+ num_blocks: TODO.+ stride: TODO.+ """strides = [stride] + [1] * (num_blocks - 1)layers = []for stride in strides:@@ -229,6 +283,11 @@ class ERes2NetAug(nn.Module):return nn.Sequential(*layers)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)x = x.unsqueeze_(1)diff --git a/funasr/models/eres2net/eres2netv2.py b/funasr/models/eres2net/eres2netv2.pyindex 4720ab8c9..44b3b0dc8 100644--- a/funasr/models/eres2net/eres2netv2.py+++ b/funasr/models/eres2net/eres2netv2.py@@ -14,9 +14,15 @@ from funasr.models.eres2net.fusion import AFFclass ReLU(nn.Hardtanh):def __init__(self, inplace=False):+ """Initialize ReLU.++ Args:+ inplace: TODO.+ """super(ReLU, self).__init__(0, 20, inplace)def __repr__(self):+ """Internal: repr ."""inplace_str = "inplace" if self.inplace else ""return self.__class__.__name__ + " (" + inplace_str + ")"@@ -24,6 +30,16 @@ class ReLU(nn.Hardtanh):class BasicBlockERes2NetV2(nn.Module):def __init__(self, in_planes, planes, stride=1, baseWidth=26, scale=2, expansion=2):+ """Initialize BasicBlockERes2NetV2.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ expansion: TODO.+ """super(BasicBlockERes2NetV2, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = nn.Conv2d(in_planes, width * scale, kernel_size=1, stride=stride, bias=False)@@ -53,6 +69,11 @@ class BasicBlockERes2NetV2(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -84,6 +105,16 @@ class BasicBlockERes2NetV2(nn.Module):class BasicBlockERes2NetV2AFF(nn.Module):def __init__(self, in_planes, planes, stride=1, baseWidth=26, scale=2, expansion=2):+ """Initialize BasicBlockERes2NetV2AFF.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ expansion: TODO.+ """super(BasicBlockERes2NetV2AFF, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = nn.Conv2d(in_planes, width * scale, kernel_size=1, stride=stride, bias=False)@@ -118,6 +149,11 @@ class BasicBlockERes2NetV2AFF(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -161,6 +197,21 @@ class ERes2NetV2(nn.Module):pooling_func="TSTP",two_emb_layer=False,):+ """Initialize ERes2NetV2.++ Args:+ block: TODO.+ block_fuse: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ baseWidth: TODO.+ scale: TODO.+ expansion: TODO.+ pooling_func: TODO.+ two_emb_layer: TODO.+ """super(ERes2NetV2, self).__init__()self.in_planes = m_channelsself.feat_dim = feat_dim@@ -195,6 +246,14 @@ class ERes2NetV2(nn.Module):self.seg_2 = nn.Identity()def _make_layer(self, block, planes, num_blocks, stride):+ """Internal: make layer.++ Args:+ block: TODO.+ planes: TODO.+ num_blocks: TODO.+ stride: TODO.+ """strides = [stride] + [1] * (num_blocks - 1)layers = []for stride in strides:@@ -205,6 +264,11 @@ class ERes2NetV2(nn.Module):return nn.Sequential(*layers)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)x = x.unsqueeze_(1)out = F.relu(self.bn1(self.conv1(x)))diff --git a/funasr/models/eres2net/fusion.py b/funasr/models/eres2net/fusion.pyindex fbe699efb..74886e293 100644--- a/funasr/models/eres2net/fusion.py+++ b/funasr/models/eres2net/fusion.py@@ -8,6 +8,12 @@ import torch.nn as nnclass AFF(nn.Module):def __init__(self, channels=64, r=4):+ """Initialize AFF.++ Args:+ channels: TODO.+ r: TODO.+ """super(AFF, self).__init__()inter_channels = int(channels // r)@@ -20,6 +26,12 @@ class AFF(nn.Module):)def forward(self, x, ds_y):+ """Forward pass for training.++ Args:+ x: TODO.+ ds_y: TODO.+ """xa = torch.cat((x, ds_y), dim=1)x_att = self.local_att(xa)x_att = 1.0 + torch.tanh(x_att)diff --git a/funasr/models/eres2net/model.py b/funasr/models/eres2net/model.pyindex e2549a2b7..ebf624ecc 100644--- a/funasr/models/eres2net/model.py+++ b/funasr/models/eres2net/model.py@@ -32,6 +32,20 @@ class ERes2NetV2SV(torch.nn.Module):two_emb_layer=False,**kwargs,):+ """Initialize ERes2NetV2SV.++ Args:+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ m_channels: TODO.+ baseWidth: TODO.+ scale: TODO.+ expansion: TODO.+ num_blocks: TODO.+ pooling_func: TODO.+ two_emb_layer: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.model = ERes2NetV2(feat_dim=feat_dim,@@ -56,6 +70,11 @@ class ERes2NetV2SV(torch.nn.Module):self._load_pretrained(init_param)def _load_pretrained(self, path):+ """Internal: load pretrained.++ Args:+ path: TODO.+ """state_dict = torch.load(path, map_location="cpu")if "state_dict" in state_dict:state_dict = state_dict["state_dict"]@@ -65,6 +84,11 @@ class ERes2NetV2SV(torch.nn.Module):logging.info(f"ERes2NetV2 loaded pretrained weights from {path}")def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """return self.model(x)def inference(@@ -76,6 +100,16 @@ class ERes2NetV2SV(torch.nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}time1 = time.perf_counter()audio_sample_list = load_audio_text_image_video(diff --git a/funasr/models/fsmn_kws/encoder.py b/funasr/models/fsmn_kws/encoder.pyindex 2c316871f..ff4e2892d 100755--- a/funasr/models/fsmn_kws/encoder.py+++ b/funasr/models/fsmn_kws/encoder.py@@ -11,6 +11,11 @@ from funasr.register import tablesdef toKaldiMatrix(np_mat):+ """Tokaldimatrix.++ Args:+ np_mat: TODO.+ """np.set_printoptions(threshold=np.inf, linewidth=np.nan)out_str = str(np_mat)out_str = out_str.replace('[', '')@@ -21,17 +26,29 @@ def toKaldiMatrix(np_mat):class LinearTransform(nn.Module):def __init__(self, input_dim, output_dim):+ """Initialize LinearTransform.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ """super(LinearTransform, self).__init__()self.input_dim = input_dimself.output_dim = output_dimself.linear = nn.Linear(input_dim, output_dim, bias=False)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = self.linear(input)return outputdef to_kaldi_net(self):+ """To kaldi net."""re_str = ''re_str += '<LinearTransform> %d %d\n' % (self.output_dim,self.input_dim)@@ -44,6 +61,11 @@ class LinearTransform(nn.Module):return re_strdef to_pytorch_net(self, fread):+ """To pytorch net.++ Args:+ fread: TODO.+ """linear_line = fread.readline()linear_split = linear_line.strip().split()assert len(linear_split) == 3@@ -74,17 +96,29 @@ class LinearTransform(nn.Module):class AffineTransform(nn.Module):def __init__(self, input_dim, output_dim):+ """Initialize AffineTransform.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ """super(AffineTransform, self).__init__()self.input_dim = input_dimself.output_dim = output_dimself.linear = nn.Linear(input_dim, output_dim)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = self.linear(input)return outputdef to_kaldi_net(self):+ """To kaldi net."""re_str = ''re_str += '<AffineTransform> %d %d\n' % (self.output_dim,self.input_dim)@@ -101,6 +135,11 @@ class AffineTransform(nn.Module):return re_strdef to_pytorch_net(self, fread):+ """To pytorch net.++ Args:+ fread: TODO.+ """affine_line = fread.readline()affine_split = affine_line.strip().split()assert len(affine_split) == 3@@ -143,21 +182,38 @@ class AffineTransform(nn.Module):class RectifiedLinear(nn.Module):def __init__(self, input_dim, output_dim):+ """Initialize RectifiedLinear.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ """super(RectifiedLinear, self).__init__()self.dim = input_dimself.relu = nn.ReLU()self.dropout = nn.Dropout(0.1)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """out = self.relu(input)return outdef to_kaldi_net(self):+ """To kaldi net."""re_str = ''re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)return re_strdef to_pytorch_net(self, fread):+ """To pytorch net.++ Args:+ fread: TODO.+ """line = fread.readline()splits = line.strip().split()assert len(splits) == 3@@ -178,6 +234,16 @@ class FSMNBlock(nn.Module):lstride=1,rstride=1,):+ """Initialize FSMNBlock.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ """super(FSMNBlock, self).__init__()self.dim = input_dim@@ -202,6 +268,12 @@ class FSMNBlock(nn.Module):self.conv_right = Nonedef forward(self, input: torch.Tensor, cache: torch.Tensor = None):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ cache: State cache dict for streaming inference.+ """x = torch.unsqueeze(input, 1)x_per = x.permute(0, 3, 2, 1) # B D T C@@ -228,6 +300,7 @@ class FSMNBlock(nn.Module):return output, cachedef to_kaldi_net(self):+ """To kaldi net."""re_str = ''re_str += '<Fsmn> %d %d\n' % (self.dim, self.dim)re_str += '<LearnRateCoef> %d <LOrder> %d <ROrder> %d <LStride> %d <RStride> %d <MaxNorm> 0\n' % (@@ -246,6 +319,11 @@ class FSMNBlock(nn.Module):return re_strdef to_pytorch_net(self, fread):+ """To pytorch net.++ Args:+ fread: TODO.+ """fsmn_line = fread.readline()fsmn_split = fsmn_line.strip().split()assert len(fsmn_split) == 3@@ -320,6 +398,17 @@ class BasicBlock(nn.Module):rstride: int,stack_layer: int,):+ """Initialize BasicBlock.++ Args:+ linear_dim: Size/dimension parameter.+ proj_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ stack_layer: TODO.+ """super(BasicBlock, self).__init__()self.lorder = lorderself.rorder = rorder@@ -332,6 +421,12 @@ class BasicBlock(nn.Module):self.relu = RectifiedLinear(linear_dim, linear_dim)def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor] = None):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ cache: State cache dict for streaming inference.+ """x1 = self.linear(input) # B T Dif cache is not None:@@ -348,6 +443,7 @@ class BasicBlock(nn.Module):return x4def to_kaldi_net(self):+ """To kaldi net."""re_str = ''re_str += self.linear.to_kaldi_net()re_str += self.fsmn_block.to_kaldi_net()@@ -357,6 +453,11 @@ class BasicBlock(nn.Module):return re_strdef to_pytorch_net(self, fread):+ """To pytorch net.++ Args:+ fread: TODO.+ """self.linear.to_pytorch_net(fread)self.fsmn_block.to_pytorch_net(fread)self.affine.to_pytorch_net(fread)@@ -368,6 +469,11 @@ class BasicBlock_export(nn.Module):self,model,):+ """Initialize BasicBlock_export.++ Args:+ model: Model instance or model name.+ """super(BasicBlock_export, self).__init__()self.linear = model.linearself.fsmn_block = model.fsmn_block@@ -375,6 +481,12 @@ class BasicBlock_export(nn.Module):self.relu = model.reludef forward(self, input: torch.Tensor, in_cache: torch.Tensor):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ in_cache: TODO.+ """x = self.linear(input) # B T D# cache_layer_name = 'cache_layer_{}'.format(self.stack_layer)# if cache_layer_name not in in_cache:@@ -387,15 +499,27 @@ class BasicBlock_export(nn.Module):class FsmnStack(nn.Sequential):def __init__(self, *args):+ """Initialize FsmnStack.++ Args:+ *args: Variable positional arguments.+ """super(FsmnStack, self).__init__(*args)def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor]):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ cache: State cache dict for streaming inference.+ """x = inputfor module in self._modules.values():x = module(x, cache)return xdef to_kaldi_net(self):+ """To kaldi net."""re_str = ''for module in self._modules.values():re_str += module.to_kaldi_net()@@ -403,6 +527,11 @@ class FsmnStack(nn.Sequential):return re_strdef to_pytorch_net(self, fread):+ """To pytorch net.++ Args:+ fread: TODO.+ """for module in self._modules.values():module.to_pytorch_net(fread)@@ -436,6 +565,22 @@ class FSMNConvert(nn.Module):output_dim: int,use_softmax: bool = True,):+ """Initialize FSMNConvert.++ Args:+ input_dim: Size/dimension parameter.+ input_affine_dim: Size/dimension parameter.+ fsmn_layers: TODO.+ linear_dim: Size/dimension parameter.+ proj_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ output_affine_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ use_softmax: TODO.+ """super().__init__()self.input_dim = input_dim@@ -463,6 +608,7 @@ class FSMNConvert(nn.Module):self.softmax = nn.Softmax(dim=-1)def output_size(self) -> int:+ """Output size."""return self.output_dimdef forward(@@ -491,6 +637,7 @@ class FSMNConvert(nn.Module):return x6def to_kaldi_net(self):+ """To kaldi net."""re_str = ''re_str += '<Nnet>\n're_str += self.in_linear1.to_kaldi_net()@@ -508,6 +655,11 @@ class FSMNConvert(nn.Module):return re_strdef to_pytorch_net(self, kaldi_file):+ """To pytorch net.++ Args:+ kaldi_file: TODO.+ """with open(kaldi_file, 'r', encoding='utf8') as fread:fread = open(kaldi_file, 'r')nnet_start_line = fread.readline()diff --git a/funasr/models/fsmn_kws/model.py b/funasr/models/fsmn_kws/model.pyindex 066730f1c..40d6bd98e 100644--- a/funasr/models/fsmn_kws/model.py+++ b/funasr/models/fsmn_kws/model.py@@ -48,6 +48,24 @@ class FsmnKWS(torch.nn.Module):blank_id: int = 0,**kwargs,):+ """Initialize FsmnKWS.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -158,6 +176,14 @@ class FsmnKWS(torch.nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -177,6 +203,16 @@ class FsmnKWS(torch.nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """keywords = kwargs.get("keywords")from funasr.utils.kws_utils import KwsCtcPrefixDecoderself.kws_decoder = KwsCtcPrefixDecoder(@@ -257,6 +293,19 @@ class FsmnKWSConvert(torch.nn.Module):blank_id: int = 0,**kwargs,):+ """Initialize FsmnKWSConvert.++ Args:+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ blank_id: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()encoder_class = tables.encoder_classes.get(encoder)@@ -278,8 +327,14 @@ class FsmnKWSConvert(torch.nn.Module):self.error_calculator = Nonedef to_kaldi_net(self):+ """To kaldi net."""return self.encoder.to_kaldi_net()def to_pytorch_net(self, kaldi_file):+ """To pytorch net.++ Args:+ kaldi_file: TODO.+ """return self.encoder.to_pytorch_net(kaldi_file)diff --git a/funasr/models/fsmn_kws_mt/encoder.py b/funasr/models/fsmn_kws_mt/encoder.pyindex 4b36d839a..b46dfd9eb 100755--- a/funasr/models/fsmn_kws_mt/encoder.py+++ b/funasr/models/fsmn_kws_mt/encoder.py@@ -42,6 +42,23 @@ class FSMNMT(nn.Module):output_dim2: int,use_softmax: bool = True,):+ """Initialize FSMNMT.++ Args:+ input_dim: Size/dimension parameter.+ input_affine_dim: Size/dimension parameter.+ fsmn_layers: TODO.+ linear_dim: Size/dimension parameter.+ proj_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ output_affine_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ output_dim2: TODO.+ use_softmax: TODO.+ """super().__init__()self.input_dim = input_dim@@ -68,9 +85,11 @@ class FSMNMT(nn.Module):self.softmax = nn.Softmax(dim=-1)def output_size(self) -> int:+ """Output size."""return self.output_dimdef output_size2(self) -> int:+ """Output size2."""return self.output_dim2def forward(@@ -121,6 +140,23 @@ class FSMNMTConvert(nn.Module):output_dim2: int,use_softmax: bool = True,):+ """Initialize FSMNMTConvert.++ Args:+ input_dim: Size/dimension parameter.+ input_affine_dim: Size/dimension parameter.+ fsmn_layers: TODO.+ linear_dim: Size/dimension parameter.+ proj_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ output_affine_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ output_dim2: TODO.+ use_softmax: TODO.+ """super().__init__()self.input_dim = input_dim@@ -147,12 +183,15 @@ class FSMNMTConvert(nn.Module):self.softmax = nn.Softmax(dim=-1)def output_size(self) -> int:+ """Output size."""return self.output_dimdef output_size2(self) -> int:+ """Output size2."""return self.output_dim2def to_kaldi_net(self):+ """To kaldi net."""re_str = ''re_str += '<Nnet>\n're_str += self.in_linear1.to_kaldi_net()@@ -170,6 +209,7 @@ class FSMNMTConvert(nn.Module):return re_strdef to_kaldi_net2(self):+ """To kaldi net2."""re_str = ''re_str += '<Nnet>\n're_str += self.in_linear1.to_kaldi_net()@@ -187,6 +227,11 @@ class FSMNMTConvert(nn.Module):return re_strdef to_pytorch_net(self, kaldi_file):+ """To pytorch net.++ Args:+ kaldi_file: TODO.+ """with open(kaldi_file, 'r', encoding='utf8') as fread:fread = open(kaldi_file, 'r')nnet_start_line = fread.readline()diff --git a/funasr/models/fsmn_kws_mt/model.py b/funasr/models/fsmn_kws_mt/model.pyindex 3fa728a21..c8ca992c0 100644--- a/funasr/models/fsmn_kws_mt/model.py+++ b/funasr/models/fsmn_kws_mt/model.py@@ -46,6 +46,22 @@ class FsmnKWSMT(torch.nn.Module):blank_id: int = 0,**kwargs,):+ """Initialize FsmnKWSMT.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ ctc_conf: Configuration dict for ctc.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -173,6 +189,14 @@ class FsmnKWSMT(torch.nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -190,6 +214,14 @@ class FsmnKWSMT(torch.nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc2 loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc2(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -209,6 +241,16 @@ class FsmnKWSMT(torch.nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """keywords = kwargs.get("keywords")from funasr.utils.kws_utils import KwsCtcPrefixDecoderself.kws_decoder = KwsCtcPrefixDecoder(@@ -317,6 +359,17 @@ class FsmnKWSMTConvert(torch.nn.Module):blank_id: int = 0,**kwargs,):+ """Initialize FsmnKWSMTConvert.++ Args:+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ input_size: Size/dimension parameter.+ blank_id: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()encoder_class = tables.encoder_classes.get(encoder)@@ -328,10 +381,17 @@ class FsmnKWSMTConvert(torch.nn.Module):self.error_calculator = Nonedef to_kaldi_net(self):+ """To kaldi net."""return self.encoder.to_kaldi_net()def to_kaldi_net2(self):+ """To kaldi net2."""return self.encoder.to_kaldi_net2()def to_pytorch_net(self, kaldi_file):+ """To pytorch net.++ Args:+ kaldi_file: TODO.+ """return self.encoder.to_pytorch_net(kaldi_file)diff --git a/funasr/models/fsmn_vad_streaming/encoder.py b/funasr/models/fsmn_vad_streaming/encoder.pyindex 14c2f5f10..9e44eea10 100755--- a/funasr/models/fsmn_vad_streaming/encoder.py+++ b/funasr/models/fsmn_vad_streaming/encoder.py@@ -13,12 +13,23 @@ from funasr.register import tablesclass LinearTransform(nn.Module):def __init__(self, input_dim, output_dim):+ """Initialize LinearTransform.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ """super(LinearTransform, self).__init__()self.input_dim = input_dimself.output_dim = output_dimself.linear = nn.Linear(input_dim, output_dim, bias=False)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = self.linear(input)return output@@ -27,12 +38,23 @@ class LinearTransform(nn.Module):class AffineTransform(nn.Module):def __init__(self, input_dim, output_dim):+ """Initialize AffineTransform.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ """super(AffineTransform, self).__init__()self.input_dim = input_dimself.output_dim = output_dimself.linear = nn.Linear(input_dim, output_dim)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = self.linear(input)return output@@ -41,12 +63,23 @@ class AffineTransform(nn.Module):class RectifiedLinear(nn.Module):def __init__(self, input_dim, output_dim):+ """Initialize RectifiedLinear.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ """super(RectifiedLinear, self).__init__()self.dim = input_dimself.relu = nn.ReLU()self.dropout = nn.Dropout(0.1)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """out = self.relu(input)return out@@ -62,6 +95,16 @@ class FSMNBlock(nn.Module):lstride=1,rstride=1,):+ """Initialize FSMNBlock.++ Args:+ input_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ """super(FSMNBlock, self).__init__()self.dim = input_dim@@ -86,6 +129,12 @@ class FSMNBlock(nn.Module):self.conv_right = Nonedef forward(self, input: torch.Tensor, cache: torch.Tensor = None):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ cache: State cache dict for streaming inference.+ """x = torch.unsqueeze(input, 1)x_per = x.permute(0, 3, 2, 1) # B D T C@@ -123,6 +172,17 @@ class BasicBlock(nn.Module):rstride: int,stack_layer: int,):+ """Initialize BasicBlock.++ Args:+ linear_dim: Size/dimension parameter.+ proj_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ stack_layer: TODO.+ """super(BasicBlock, self).__init__()self.lorder = lorderself.rorder = rorder@@ -135,6 +195,12 @@ class BasicBlock(nn.Module):self.relu = RectifiedLinear(linear_dim, linear_dim)def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor] = None):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ cache: State cache dict for streaming inference.+ """x1 = self.linear(input) # B T Dif cache is not None:@@ -156,6 +222,11 @@ class BasicBlock_export(nn.Module):self,model,):+ """Initialize BasicBlock_export.++ Args:+ model: Model instance or model name.+ """super(BasicBlock_export, self).__init__()self.linear = model.linearself.fsmn_block = model.fsmn_block@@ -163,6 +234,12 @@ class BasicBlock_export(nn.Module):self.relu = model.reludef forward(self, input: torch.Tensor, in_cache: torch.Tensor):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ in_cache: TODO.+ """x = self.linear(input) # B T D# cache_layer_name = 'cache_layer_{}'.format(self.stack_layer)# if cache_layer_name not in in_cache:@@ -175,9 +252,20 @@ class BasicBlock_export(nn.Module):class FsmnStack(nn.Sequential):def __init__(self, *args):+ """Initialize FsmnStack.++ Args:+ *args: Variable positional arguments.+ """super(FsmnStack, self).__init__(*args)def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor]):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ cache: State cache dict for streaming inference.+ """x = inputfor module in self._modules.values():x = module(x, cache)@@ -213,6 +301,22 @@ class FSMN(nn.Module):output_dim: int,use_softmax: bool = True,):+ """Initialize FSMN.++ Args:+ input_dim: Size/dimension parameter.+ input_affine_dim: Size/dimension parameter.+ fsmn_layers: TODO.+ linear_dim: Size/dimension parameter.+ proj_dim: Size/dimension parameter.+ lorder: TODO.+ rorder: TODO.+ lstride: TODO.+ rstride: TODO.+ output_affine_dim: Size/dimension parameter.+ output_dim: Size/dimension parameter.+ use_softmax: TODO.+ """super().__init__()self.input_dim = input_dim@@ -240,9 +344,11 @@ class FSMN(nn.Module):self.softmax = nn.Softmax(dim=-1)def fuse_modules(self):+ """Fuse modules."""passdef output_size(self) -> int:+ """Output size."""return self.output_dimdef forward(@@ -278,6 +384,12 @@ class FSMNExport(nn.Module):model,**kwargs,):+ """Initialize FSMNExport.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """super().__init__()# self.input_dim = input_dim@@ -310,6 +422,7 @@ class FSMNExport(nn.Module):self.fsmn[i] = BasicBlock_export(d)def fuse_modules(self):+ """Fuse modules."""passdef forward(diff --git a/funasr/models/fsmn_vad_streaming/export_meta.py b/funasr/models/fsmn_vad_streaming/export_meta.pyindex f76016931..7865cc5a9 100644--- a/funasr/models/fsmn_vad_streaming/export_meta.py+++ b/funasr/models/fsmn_vad_streaming/export_meta.py@@ -9,6 +9,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")model.encoder = encoder_class(model.encoder, onnx=is_onnx)@@ -25,12 +31,25 @@ def export_rebuild_model(model, **kwargs):def export_forward(self, feats: torch.Tensor, *args, **kwargs):+ """Export forward.++ Args:+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """scores, out_caches = self.encoder(feats, *args)return scores, out_cachesdef export_dummy_inputs(self, data_in=None, frame=30):+ """Export dummy inputs.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ frame: TODO.+ """if data_in is None:speech = torch.randn(1, frame, self.encoder_conf.get("input_dim"))else:@@ -46,14 +65,17 @@ def export_dummy_inputs(self, data_in=None, frame=30):def export_input_names(self):+ """Export input names."""return ["speech", "in_cache0", "in_cache1", "in_cache2", "in_cache3"]def export_output_names(self):+ """Export output names."""return ["logits", "out_cache0", "out_cache1", "out_cache2", "out_cache3"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"speech": {1: "feats_length"},}@@ -62,4 +84,5 @@ def export_dynamic_axes(self):def export_name(self,):+ """Export name."""return "model.onnx"diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.pyindex d4c91c8d5..3acd481e1 100644--- a/funasr/models/fsmn_vad_streaming/model.py+++ b/funasr/models/fsmn_vad_streaming/model.py@@ -86,6 +86,40 @@ class VADXOptions:frame_length_ms: int = 25,**kwargs,):+ """Initialize VADXOptions.++ Args:+ sample_rate: TODO.+ detect_mode: TODO.+ snr_mode: TODO.+ max_end_silence_time: TODO.+ max_start_silence_time: TODO.+ do_start_point_detection: TODO.+ do_end_point_detection: TODO.+ window_size_ms: TODO.+ sil_to_speech_time_thres: TODO.+ speech_to_sil_time_thres: TODO.+ speech_2_noise_ratio: TODO.+ do_extend: TODO.+ lookback_time_start_point: TODO.+ lookahead_time_end_point: TODO.+ max_single_segment_time: TODO.+ nn_eval_block_size: Size/dimension parameter.+ dcd_block_size: Size/dimension parameter.+ snr_thres: TODO.+ noise_frame_num_used_for_snr: TODO.+ decibel_thres: TODO.+ speech_noise_thres: TODO.+ fe_prior_thres: TODO.+ silence_pdf_num: TODO.+ sil_pdf_ids: TODO.+ speech_noise_thresh_low: TODO.+ speech_noise_thresh_high: TODO.+ output_frame_probs: TODO.+ frame_in_ms: TODO.+ frame_length_ms: TODO.+ **kwargs: Additional keyword arguments.+ """self.sample_rate = sample_rateself.detect_mode = detect_modeself.snr_mode = snr_mode@@ -125,6 +159,7 @@ class E2EVadSpeechBufWithDoa(object):"""def __init__(self):+ """Initialize E2EVadSpeechBufWithDoa."""self.start_ms = 0self.end_ms = 0self.buffer = []@@ -133,6 +168,7 @@ class E2EVadSpeechBufWithDoa(object):self.doa = 0def Reset(self):+ """Reset."""self.start_ms = 0self.end_ms = 0self.buffer = []@@ -149,6 +185,7 @@ class E2EVadFrameProb(object):"""def __init__(self):+ """Initialize E2EVadFrameProb."""self.noise_prob = 0.0self.speech_prob = 0.0self.score = 0.0@@ -170,6 +207,14 @@ class WindowDetector(object):speech_to_sil_time: int,frame_size_ms: int,):+ """Initialize WindowDetector.++ Args:+ window_size_ms: TODO.+ sil_to_speech_time: TODO.+ speech_to_sil_time: TODO.+ frame_size_ms: TODO.+ """self.window_size_ms = window_size_msself.sil_to_speech_time = sil_to_speech_timeself.speech_to_sil_time = speech_to_sil_time@@ -190,6 +235,7 @@ class WindowDetector(object):self.hydre_frame_count = 0def Reset(self) -> None:+ """Reset."""self.cur_win_pos = 0self.win_sum = 0self.win_state = [0] * self.win_size_frame@@ -200,11 +246,19 @@ class WindowDetector(object):self.hydre_frame_count = 0def GetWinSize(self) -> int:+ """Getwinsize."""return int(self.win_size_frame)def DetectOneFrame(self, frameState: FrameState, frame_count: int, cache: dict = None) -> AudioChangeState:+ """Detectoneframe.++ Args:+ frameState: TODO.+ frame_count: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}cur_frame_state = FrameState.kFrameStateSil@@ -240,6 +294,7 @@ class WindowDetector(object):return AudioChangeState.kChangeStateInvaliddef FrameSizeMs(self) -> int:+ """Framesizems."""return int(self.frame_size_ms)@@ -250,6 +305,13 @@ class Stats(object):max_end_sil_frame_cnt_thresh,speech_noise_thres,):+ """Initialize Stats.++ Args:+ sil_pdf_ids: TODO.+ max_end_sil_frame_cnt_thresh: TODO.+ speech_noise_thres: TODO.+ """self.data_buf_start_frame = 0self.frm_cnt = 0self.latest_confirmed_speech_frame = 0@@ -301,6 +363,14 @@ class FsmnVADStreaming(nn.Module):vad_post_args: Dict[str, Any] = None,**kwargs,):+ """Initialize FsmnVADStreaming.++ Args:+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ vad_post_args: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.vad_opts = VADXOptions(**kwargs)@@ -310,6 +380,11 @@ class FsmnVADStreaming(nn.Module):self.encoder_conf = encoder_confdef ResetDetection(self, cache: dict = None):+ """Resetdetection.++ Args:+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}cache["stats"].continous_silence_frame_count = 0@@ -335,6 +410,11 @@ class FsmnVADStreaming(nn.Module):cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :]def ComputeDecibel(self, cache: dict = None) -> None:+ """Computedecibel.++ Args:+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000)@@ -361,6 +441,12 @@ class FsmnVADStreaming(nn.Module):def ComputeScores(self, feats: torch.Tensor, cache: dict = None) -> None:+ """Computescores.++ Args:+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}scores = self.encoder(feats, cache=cache["encoder"]) # return B * T * D@@ -375,6 +461,12 @@ class FsmnVADStreaming(nn.Module):cache["stats"].scores = torch.cat((cache["stats"].scores, scores), dim=1)def PopDataBufTillFrame(self, frame_idx: int, cache: dict = None) -> None: # need check again+ """Popdatabuftillframe.++ Args:+ frame_idx: TODO.+ cache: State cache dict for streaming inference.+ """while cache["stats"].data_buf_start_frame < frame_idx:if len(cache["stats"].data_buf) >= int(self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000@@ -396,6 +488,16 @@ class FsmnVADStreaming(nn.Module):end_point_is_sent_end: bool,cache: dict = None,) -> None:+ """Popdatatooutputbuf.++ Args:+ start_frm: TODO.+ frm_cnt: TODO.+ first_frm_is_start_point: TODO.+ last_frm_is_end_point: TODO.+ end_point_is_sent_end: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}self.PopDataBufTillFrame(start_frm, cache=cache)@@ -448,6 +550,12 @@ class FsmnVADStreaming(nn.Module):cur_seg.contain_seg_end_point = Truedef OnSilenceDetected(self, valid_frame: int, cache: dict = None):+ """Onsilencedetected.++ Args:+ valid_frame: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}cache["stats"].lastest_confirmed_silence_frame = valid_frame@@ -458,12 +566,25 @@ class FsmnVADStreaming(nn.Module):# passdef OnVoiceDetected(self, valid_frame: int, cache: dict = None) -> None:+ """Onvoicedetected.++ Args:+ valid_frame: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}cache["stats"].latest_confirmed_speech_frame = valid_frameself.PopDataToOutputBuf(valid_frame, 1, False, False, False, cache=cache)def OnVoiceStart(self, start_frame: int, fake_result: bool = False, cache: dict = None) -> None:+ """Onvoicestart.++ Args:+ start_frame: TODO.+ fake_result: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}if self.vad_opts.do_start_point_detection:@@ -484,6 +605,14 @@ class FsmnVADStreaming(nn.Module):def OnVoiceEnd(self, end_frame: int, fake_result: bool, is_last_frame: bool, cache: dict = None) -> None:+ """Onvoiceend.++ Args:+ end_frame: TODO.+ fake_result: TODO.+ is_last_frame: Boolean flag for last frame.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}for t in range(cache["stats"].latest_confirmed_speech_frame + 1, end_frame):@@ -504,6 +633,13 @@ class FsmnVADStreaming(nn.Module):def MaybeOnVoiceEndIfLastFrame(self, is_final_frame: bool, cur_frm_idx: int, cache: dict = None) -> None:+ """Maybeonvoiceendiflastframe.++ Args:+ is_final_frame: Boolean flag for final frame.+ cur_frm_idx: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}if is_final_frame:@@ -511,11 +647,21 @@ class FsmnVADStreaming(nn.Module):cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetecteddef GetLatency(self, cache: dict = None) -> int:+ """Getlatency.++ Args:+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}return int(self.LatencyFrmNumAtStartPoint(cache=cache) * self.vad_opts.frame_in_ms)def LatencyFrmNumAtStartPoint(self, cache: dict = None) -> int:+ """Latencyfrmnumatstartpoint.++ Args:+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}vad_latency = cache["windows_detector"].GetWinSize()@@ -524,6 +670,12 @@ class FsmnVADStreaming(nn.Module):return vad_latencydef GetFrameState(self, t: int, cache: dict = None):+ """Getframestate.++ Args:+ t: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}frame_state = FrameState.kFrameStateInvalid@@ -590,6 +742,15 @@ class FsmnVADStreaming(nn.Module):is_final: bool = False,**kwargs,):+ """Forward pass for training.++ Args:+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).+ waveform: TODO.+ cache: State cache dict for streaming inference.+ is_final: Whether this is the final chunk in streaming.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}# if len(cache) == 0:@@ -657,6 +818,12 @@ class FsmnVADStreaming(nn.Module):return segmentsdef init_cache(self, cache: dict = None, **kwargs):+ """Init cache.++ Args:+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}@@ -696,6 +863,17 @@ class FsmnVADStreaming(nn.Module):cache: dict = None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}if len(cache) == 0:@@ -791,12 +969,22 @@ class FsmnVADStreaming(nn.Module):def export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelmodels = export_rebuild_model(model=self, **kwargs)return modelsdef DetectCommonFrames(self, cache: dict = None) -> int:+ """Detectcommonframes.++ Args:+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected:@@ -811,6 +999,11 @@ class FsmnVADStreaming(nn.Module):return 0def DetectLastFrames(self, cache: dict = None) -> int:+ """Detectlastframes.++ Args:+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected:@@ -830,6 +1023,14 @@ class FsmnVADStreaming(nn.Module):def DetectOneFrame(self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool, cache: dict = None) -> None:+ """Detectoneframe.++ Args:+ cur_frm_state: TODO.+ cur_frm_idx: TODO.+ is_final_frame: Boolean flag for final frame.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}tmp_cur_frm_state = FrameState.kFrameStateInvaliddiff --git a/funasr/models/fun_asr_nano/ctc.py b/funasr/models/fun_asr_nano/ctc.pyindex f2d592759..6b07a02a4 100644--- a/funasr/models/fun_asr_nano/ctc.py+++ b/funasr/models/fun_asr_nano/ctc.py@@ -21,6 +21,16 @@ class CTC(torch.nn.Module):blank_id: int = 0,**kwargs,):+ """Initialize CTC.++ Args:+ odim: TODO.+ encoder_output_size: Size/dimension parameter.+ dropout_rate: TODO.+ reduce: TODO.+ blank_id: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()eprojs = encoder_output_sizeself.dropout_rate = dropout_ratediff --git a/funasr/models/fun_asr_nano/model.py b/funasr/models/fun_asr_nano/model.pyindex f78d0e755..b7e6ad5de 100644--- a/funasr/models/fun_asr_nano/model.py+++ b/funasr/models/fun_asr_nano/model.py@@ -57,6 +57,19 @@ class FunASRNano(nn.Module):length_normalized_loss: bool = False,**kwargs,):+ """Initialize FunASRNano.++ Args:+ audio_encoder: TODO.+ audio_encoder_conf: Configuration dict for audio_encoder.+ audio_adaptor: TODO.+ audio_adaptor_conf: Configuration dict for audio_adaptor.+ llm: TODO.+ llm_conf: Configuration dict for llm.+ input_size: Size/dimension parameter.+ length_normalized_loss: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# audio encoder@@ -189,6 +202,18 @@ class FunASRNano(nn.Module):fbank_mask: torch.Tensor = None,**kwargs,):+ """Forward pass for training.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ input_ids: TODO.+ attention_mask: TODO.+ labels_ids: TODO.+ fbank_beg: TODO.+ fbank_mask: TODO.+ **kwargs: Additional keyword arguments.+ """batch_size, token_num = input_ids.shapestats = {}input_ids[input_ids < 0] = 0@@ -290,17 +315,35 @@ class FunASRNano(nn.Module):return loss, stats, weightdef forward_export(self, speech, speech_lengths, **kwargs):+ """Forward export.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ **kwargs: Additional keyword arguments.+ """x, olens = self.audio_encoder(speech, speech_lengths)encoder_out, encoder_out_lens = self.audio_adaptor(x, olens)return encoder_out, encoder_out_lensdef encode(self, speech, speech_lengths):# audio encoder+ """Encode.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)return encoder_out, encoder_out_lensdef data_template(self, data):+ """Data template.++ Args:+ data: TODO.+ """system, user, assistant = [], [], []for i, item in enumerate(data):role = item["role"]@@ -326,6 +369,15 @@ class FunASRNano(nn.Module):return contentsdef data_load_speech(self, contents: dict, tokenizer, frontend, meta_data={}, **kwargs):+ """Data load speech.++ Args:+ contents: TODO.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ meta_data: TODO.+ **kwargs: Additional keyword arguments.+ """system = contents["system"]user = contents["user"]assistant = contents["assistant"]@@ -487,6 +539,16 @@ class FunASRNano(nn.Module):frontend=None,**kwargs,):+ """Inference prepare.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}if len(data_in) > 1:@@ -568,6 +630,13 @@ class FunASRNano(nn.Module):return inputs_embeds, contents, batch, source_ids, meta_datadef get_prompt(self, hotwords: list[str], language: str = None, itn: bool = True):+ """Get prompt.++ Args:+ hotwords: TODO.+ language: Language identifier.+ itn: TODO.+ """if len(hotwords) > 0:hotwords = ", ".join(hotwords)prompt = f"请结合上下文信息,更加准确地完成语音转写任务。如果没有相关信息,我们会留空。\n\n\n**上下文信息:**\n\n\n"@@ -583,6 +652,12 @@ class FunASRNano(nn.Module):return prompt + ":"def generate_chatml(self, prompt: str, data: Union[str, torch.Tensor]):+ """Generate chatml.++ Args:+ prompt: TODO.+ data: TODO.+ """if isinstance(data, str):return [{"role": "system", "content": "You are a helpful assistant."},@@ -609,6 +684,16 @@ class FunASRNano(nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """prompt = self.get_prompt(kwargs.get("hotwords", []), kwargs.get("language", None), kwargs.get("itn", True))@@ -638,6 +723,16 @@ class FunASRNano(nn.Module):frontend=None,**kwargs,):+ """Inference llm.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """inputs_embeds, contents, batch, source_ids, meta_data = self.inference_prepare(data_in, data_lengths, key, tokenizer, frontend, **kwargs)@@ -759,6 +854,12 @@ class FunASRNano(nn.Module):@staticmethoddef from_pretrained(model: str = None, **kwargs):+ """From pretrained.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """from funasr import AutoModelmodel, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)diff --git a/funasr/models/fun_asr_nano/tools/cn_tn.py b/funasr/models/fun_asr_nano/tools/cn_tn.pyindex 4aaccd040..d76bc43f8 100644--- a/funasr/models/fun_asr_nano/tools/cn_tn.py+++ b/funasr/models/fun_asr_nano/tools/cn_tn.py@@ -408,14 +408,22 @@ class ChineseChar(object):"""def __init__(self, simplified, traditional):+ """Initialize ChineseChar.++ Args:+ simplified: TODO.+ traditional: TODO.+ """self.simplified = simplifiedself.traditional = traditional# self.__repr__ = self.__str__def __str__(self):+ """Internal: str ."""return self.simplified or self.traditional or Nonedef __repr__(self):+ """Internal: repr ."""return self.__str__()@@ -427,16 +435,34 @@ class ChineseNumberUnit(ChineseChar):"""def __init__(self, power, simplified, traditional, big_s, big_t):+ """Initialize ChineseNumberUnit.++ Args:+ power: TODO.+ simplified: TODO.+ traditional: TODO.+ big_s: TODO.+ big_t: TODO.+ """super(ChineseNumberUnit, self).__init__(simplified, traditional)self.power = powerself.big_s = big_sself.big_t = big_tdef __str__(self):+ """Internal: str ."""return "10^{}".format(self.power)@classmethoddef create(cls, index, value, numbering_type=NUMBERING_TYPES[1], small_unit=False):+ """Create.++ Args:+ index: TODO.+ value: TODO.+ numbering_type: TODO.+ small_unit: TODO.+ """if small_unit:return ChineseNumberUnit(power=index + 1,@@ -479,6 +505,17 @@ class ChineseNumberDigit(ChineseChar):"""def __init__(self, value, simplified, traditional, big_s, big_t, alt_s=None, alt_t=None):+ """Initialize ChineseNumberDigit.++ Args:+ value: TODO.+ simplified: TODO.+ traditional: TODO.+ big_s: TODO.+ big_t: TODO.+ alt_s: TODO.+ alt_t: TODO.+ """super(ChineseNumberDigit, self).__init__(simplified, traditional)self.value = valueself.big_s = big_s@@ -487,10 +524,17 @@ class ChineseNumberDigit(ChineseChar):self.alt_t = alt_tdef __str__(self):+ """Internal: str ."""return str(self.value)@classmethoddef create(cls, i, v):+ """Create.++ Args:+ i: TODO.+ v: TODO.+ """return ChineseNumberDigit(i, v[0], v[1], v[2], v[3])@@ -500,6 +544,14 @@ class ChineseMath(ChineseChar):"""def __init__(self, simplified, traditional, symbol, expression=None):+ """Initialize ChineseMath.++ Args:+ simplified: TODO.+ traditional: TODO.+ symbol: TODO.+ expression: TODO.+ """super(ChineseMath, self).__init__(simplified, traditional)self.symbol = symbolself.expression = expression@@ -527,11 +579,19 @@ class MathSymbol(object):"""def __init__(self, positive, negative, point):+ """Initialize MathSymbol.++ Args:+ positive: TODO.+ negative: TODO.+ point: TODO.+ """self.positive = positiveself.negative = negativeself.point = pointdef __iter__(self):+ """Internal: iter ."""for v in self.__dict__.values():yield v@@ -600,7 +660,19 @@ def create_system(numbering_type=NUMBERING_TYPES[1]):def chn2num(chinese_string, numbering_type=NUMBERING_TYPES[1]):+ """Chn2num.++ Args:+ chinese_string: TODO.+ numbering_type: TODO.+ """def get_symbol(char, system):+ """Get symbol.++ Args:+ char: TODO.+ system: TODO.+ """for u in system.units:if char in [u.traditional, u.simplified, u.big_s, u.big_t]:return u@@ -619,6 +691,12 @@ def chn2num(chinese_string, numbering_type=NUMBERING_TYPES[1]):return mdef string2symbols(chinese_string, system):+ """String2symbols.++ Args:+ chinese_string: TODO.+ system: TODO.+ """int_string, dec_string = chinese_string, ""for p in [system.math.point.simplified, system.math.point.traditional]:if p in chinese_string:@@ -705,7 +783,26 @@ def num2chn(use_zeros=True,use_units=True,):+ """Num2chn.++ Args:+ number_string: TODO.+ numbering_type: TODO.+ big: TODO.+ traditional: TODO.+ alt_zero: TODO.+ alt_one: TODO.+ alt_two: TODO.+ use_zeros: TODO.+ use_units: TODO.+ """def get_value(value_string, use_zeros=True):+ """Get value.++ Args:+ value_string: TODO.+ use_zeros: TODO.+ """striped_string = value_string.lstrip("0")# record nothing if all zeros@@ -818,13 +915,21 @@ class Cardinal:"""def __init__(self, cardinal=None, chntext=None):+ """Initialize Cardinal.++ Args:+ cardinal: TODO.+ chntext: TODO.+ """self.cardinal = cardinalself.chntext = chntextdef chntext2cardinal(self):+ """Chntext2cardinal."""return chn2num(self.chntext)def cardinal2chntext(self):+ """Cardinal2chntext."""return num2chn(self.cardinal)@@ -834,6 +939,12 @@ class Digit:"""def __init__(self, digit=None, chntext=None):+ """Initialize Digit.++ Args:+ digit: TODO.+ chntext: TODO.+ """self.digit = digitself.chntext = chntext@@ -841,6 +952,7 @@ class Digit:# return chn2num(self.chntext)def digit2chntext(self):+ """Digit2chntext."""return num2chn(self.digit, alt_two=False, use_units=False)@@ -850,6 +962,13 @@ class TelePhone:"""def __init__(self, telephone=None, raw_chntext=None, chntext=None):+ """Initialize TelePhone.++ Args:+ telephone: TODO.+ raw_chntext: TODO.+ chntext: TODO.+ """self.telephone = telephoneself.raw_chntext = raw_chntextself.chntext = chntext@@ -862,6 +981,11 @@ class TelePhone:# return self.telephonedef telephone2chntext(self, fixed=False):+ """Telephone2chntext.++ Args:+ fixed: TODO.+ """if fixed:sil_parts = self.telephone.split("-")self.raw_chntext = "<SIL>".join([num2chn(part, alt_two=False, use_units=False) for part in sil_parts])@@ -879,14 +1003,22 @@ class Fraction:"""def __init__(self, fraction=None, chntext=None):+ """Initialize Fraction.++ Args:+ fraction: TODO.+ chntext: TODO.+ """self.fraction = fractionself.chntext = chntextdef chntext2fraction(self):+ """Chntext2fraction."""denominator, numerator = self.chntext.split("分之")return chn2num(numerator) + "/" + chn2num(denominator)def fraction2chntext(self):+ """Fraction2chntext."""numerator, denominator = self.fraction.split("/")return num2chn(denominator) + "分之" + num2chn(numerator)@@ -897,6 +1029,12 @@ class Date:"""def __init__(self, date=None, chntext=None):+ """Initialize Date.++ Args:+ date: TODO.+ chntext: TODO.+ """self.date = dateself.chntext = chntext@@ -925,6 +1063,7 @@ class Date:# return self.datedef date2chntext(self):+ """Date2chntext."""date = self.datetry:year, other = date.strip().split("年", 1)@@ -955,6 +1094,12 @@ class Money:"""def __init__(self, money=None, chntext=None):+ """Initialize Money.++ Args:+ money: TODO.+ chntext: TODO.+ """self.money = moneyself.chntext = chntext@@ -962,6 +1107,7 @@ class Money:# return self.moneydef money2chntext(self):+ """Money2chntext."""money = self.moneypattern = re.compile(r"(\d+(\.\d+)?)")matchers = pattern.findall(money)@@ -978,17 +1124,30 @@ class Percentage:"""def __init__(self, percentage=None, chntext=None):+ """Initialize Percentage.++ Args:+ percentage: TODO.+ chntext: TODO.+ """self.percentage = percentageself.chntext = chntextdef chntext2percentage(self):+ """Chntext2percentage."""return chn2num(self.chntext.strip().strip("百分之")) + "%"def percentage2chntext(self):+ """Percentage2chntext."""return "百分之" + num2chn(self.percentage.strip().strip("%"))def normalize_nsw(raw_text):+ """Normalize nsw.++ Args:+ raw_text: TODO.+ """text = "^" + raw_text + "$"# 规范化日期@@ -1111,6 +1270,11 @@ def remove_erhua(text):def remove_space(text):+ """Remove space.++ Args:+ text: Text tensor or string input.+ """tokens = text.split()new = []for k, t in enumerate(tokens):@@ -1132,6 +1296,17 @@ class TextNorm:check_chars: bool = False,remove_space: bool = False,):+ """Initialize TextNorm.++ Args:+ to_banjiao: TODO.+ to_upper: TODO.+ to_lower: TODO.+ remove_fillers: TODO.+ remove_erhua: TODO.+ check_chars: TODO.+ remove_space: TODO.+ """self.to_banjiao = to_banjiaoself.to_upper = to_upperself.to_lower = to_lower@@ -1141,6 +1316,11 @@ class TextNorm:self.remove_space = remove_spacedef __call__(self, text):+ """Internal: call .++ Args:+ text: Text tensor or string input.+ """if self.to_banjiao:text = text.translate(QJ2BJ_TRANSFORM)diff --git a/funasr/models/fun_asr_nano/tools/format5res.py b/funasr/models/fun_asr_nano/tools/format5res.pyindex 7f1f92873..3b11bb83a 100644--- a/funasr/models/fun_asr_nano/tools/format5res.py+++ b/funasr/models/fun_asr_nano/tools/format5res.py@@ -7,6 +7,13 @@ import sysdef scoreformat(name, line, flag=1):+ """Scoreformat.++ Args:+ name: TODO.+ line: TODO.+ flag: TODO.+ """newline = ""for i in range(0, len(line)):curr = line[i]@@ -47,6 +54,11 @@ def scoreformat(name, line, flag=1):def recoformat(line):+ """Recoformat.++ Args:+ line: TODO.+ """newline = ""en_flag = 0 # 0: no-english 1 : english 2: formerfor i in range(0, len(line)):@@ -86,6 +98,11 @@ def recoformat(line):def numbersingle(line):+ """Numbersingle.++ Args:+ line: TODO.+ """chnu = ["零", "一", "二", "两", "三", "四", "五", "六", "七", "八", "九", "点"]newline = ""for id in range(len(line)):@@ -118,6 +135,11 @@ def numbersingle(line):def ch_number2digit(line):+ """Ch number2digit.++ Args:+ line: TODO.+ """number_flag = 0zero_flag = 0bits = {@@ -224,6 +246,11 @@ def ch_number2digit(line):def special(line):+ """Special.++ Args:+ line: TODO.+ """newline = ""for e in range(len(line)):if ord(line[e]) == 247:@@ -255,6 +282,11 @@ def special(line):def all_convert(content):+ """All convert.++ Args:+ content: TODO.+ """content = recoformat(content)content = numbersingle(content)content = ch_number2digit(content)diff --git a/funasr/models/fun_asr_nano/tools/scp2jsonl.py b/funasr/models/fun_asr_nano/tools/scp2jsonl.pyindex 5d269cb18..9b04d0a44 100644--- a/funasr/models/fun_asr_nano/tools/scp2jsonl.py+++ b/funasr/models/fun_asr_nano/tools/scp2jsonl.py@@ -15,10 +15,20 @@ from omegaconf import DictConfig, OmegaConf, ListConfigclass LineProcessor:def __init__(self, tokenizer):+ """Initialize LineProcessor.++ Args:+ tokenizer: Tokenizer instance for text encoding/decoding.+ """self.tokenizer = tokenizerself.lock = threading.Lock()def process_line(self, line_pair: Tuple[str, str]) -> Optional[Dict]:+ """Process line.++ Args:+ line_pair: TODO.+ """line1, line2 = line_pairline1, line2 = line1.strip(), line2.strip()@@ -67,7 +77,17 @@ class LineProcessor:@hydra.main(config_name=None, version_base=None)def main_hydra(cfg: DictConfig):+ """Main hydra.++ Args:+ cfg: Configuration overrides.+ """def to_plain_list(cfg_item):+ """To plain list.++ Args:+ cfg_item: TODO.+ """if isinstance(cfg_item, ListConfig):return OmegaConf.to_container(cfg_item, resolve=True)elif isinstance(cfg_item, DictConfig):diff --git a/funasr/models/fun_asr_nano/tools/utils.py b/funasr/models/fun_asr_nano/tools/utils.pyindex 7125bca5c..bc388cbb3 100644--- a/funasr/models/fun_asr_nano/tools/utils.py+++ b/funasr/models/fun_asr_nano/tools/utils.py@@ -7,6 +7,14 @@ import torchaudio.functional as Fdef load_audio(wav_path, rate: int = None, offset: float = 0, duration: float = None):+ """Load audio.++ Args:+ wav_path: TODO.+ rate: TODO.+ offset: TODO.+ duration: TODO.+ """with sf.SoundFile(wav_path) as f:start_frame = int(offset * f.samplerate)if duration is None:@@ -29,6 +37,13 @@ def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float =def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0):+ """Forced align.++ Args:+ log_probs: TODO.+ targets: TODO.+ blank: TODO.+ """items = []try:# The current version only supports batch_size==1.diff --git a/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py b/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.pyindex 632b52906..3bb63d6e9 100644--- a/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py+++ b/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py@@ -18,6 +18,11 @@ english_normalizer = EnglishTextNormalizer()def is_only_chinese_and_english(s):# 定义正则表达式模式,匹配中文字符范围和英文字母(包括大小写)+ """Is only chinese and english.++ Args:+ s: TODO.+ """pattern = r"^[\u4e00-\u9fa5A-Za-z0-9,\.!\?:;,。!?:;、%\'\s\-\~]+$"# 使用正则表达式进行匹配return re.match(pattern, s) is not None@@ -25,6 +30,11 @@ def is_only_chinese_and_english(s):def is_only_english(s):# 定义正则表达式模式,匹配中文字符范围和英文字母(包括大小写)+ """Is only english.++ Args:+ s: TODO.+ """pattern = r"^[A-Za-z0-9,\.!\?:;,。!?:;、%\'\s\-\~]+$"# 使用正则表达式进行匹配return re.match(pattern, s) is not None@@ -32,12 +42,24 @@ def is_only_english(s):def is_number(s):# 定义正则表达式模式,匹配中文字符范围和英文字母(包括大小写)+ """Is number.++ Args:+ s: TODO.+ """pattern = r"^[0-9,\.!\?:;,。!?:;、%\'\s]+$"# 使用正则表达式进行匹配return re.match(pattern, s) is not Nonedef safe_ja_g2p(text, kana=True, max_length=100):+ """Safe ja g2p.++ Args:+ text: Text tensor or string input.+ kana: TODO.+ max_length: TODO.+ """if len(text) > max_length:# 如果文本过长,分段处理parts = []@@ -57,6 +79,13 @@ def safe_ja_g2p(text, kana=True, max_length=100):def normalize_text(srcfn, dstfn, kana=False):+ """Normalize text.++ Args:+ srcfn: TODO.+ dstfn: TODO.+ kana: TODO.+ """with open(srcfn, "r") as f_read, open(dstfn, "w") as f_write:all_lines = f_read.readlines()for line in all_lines:diff --git a/funasr/models/glm_asr/model.py b/funasr/models/glm_asr/model.pyindex bc5673232..54bef5143 100644--- a/funasr/models/glm_asr/model.py+++ b/funasr/models/glm_asr/model.py@@ -14,6 +14,11 @@ from funasr.register import tablesclass GLMASR(nn.Module):def __init__(self, **kwargs):+ """Initialize GLMASR.++ Args:+ **kwargs: Additional keyword arguments.+ """super().__init__()model_path = kwargs.get("model_path", kwargs.get("model", "zai-org/GLM-ASR-Nano-2512"))device = kwargs.get("device", "cuda:0")@@ -43,6 +48,13 @@ class GLMASR(nn.Module):logging.info(f"GLM-ASR model loaded from {model_path}")def _resolve_model_path(self, model_path, hub, kwargs):+ """Internal: resolve model path.++ Args:+ model_path: TODO.+ hub: TODO.+ kwargs: Additional keyword arguments.+ """if os.path.exists(model_path):return model_path@@ -59,6 +71,11 @@ class GLMASR(nn.Module):return model_pathdef forward(self, **kwargs):+ """Forward pass for training.++ Args:+ **kwargs: Additional keyword arguments.+ """raise NotImplementedError("GLMASR only supports inference mode")def inference(@@ -70,6 +87,16 @@ class GLMASR(nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}time1 = time.perf_counter()diff --git a/funasr/models/language_model/rnn/attentions.py b/funasr/models/language_model/rnn/attentions.pyindex f5c450c10..d9f146ce8 100644--- a/funasr/models/language_model/rnn/attentions.py+++ b/funasr/models/language_model/rnn/attentions.py@@ -45,6 +45,7 @@ class NoAtt(torch.nn.Module):"""No attention"""def __init__(self):+ """Initialize NoAtt."""super(NoAtt, self).__init__()self.h_length = Noneself.enc_h = None@@ -98,6 +99,14 @@ class AttDot(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, han_mode=False):+ """Initialize AttDot.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ han_mode: TODO.+ """super(AttDot, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim)@@ -174,6 +183,14 @@ class AttAdd(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, han_mode=False):+ """Initialize AttAdd.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ han_mode: TODO.+ """super(AttAdd, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -258,6 +275,16 @@ class AttLoc(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False):+ """Initialize AttLoc.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ aconv_chans: TODO.+ aconv_filts: TODO.+ han_mode: TODO.+ """super(AttLoc, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -381,6 +408,14 @@ class AttCov(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, han_mode=False):+ """Initialize AttCov.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ han_mode: TODO.+ """super(AttCov, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -481,6 +516,17 @@ class AttLoc2D(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, att_win, aconv_chans, aconv_filts, han_mode=False):+ """Initialize AttLoc2D.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ att_win: TODO.+ aconv_chans: TODO.+ aconv_filts: TODO.+ han_mode: TODO.+ """super(AttLoc2D, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -597,6 +643,16 @@ class AttLocRec(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False):+ """Initialize AttLocRec.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ aconv_chans: TODO.+ aconv_filts: TODO.+ han_mode: TODO.+ """super(AttLocRec, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -716,6 +772,16 @@ class AttCovLoc(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False):+ """Initialize AttCovLoc.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ aconv_chans: TODO.+ aconv_filts: TODO.+ han_mode: TODO.+ """super(AttCovLoc, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -827,6 +893,16 @@ class AttMultiHeadDot(torch.nn.Module):"""def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False):+ """Initialize AttMultiHeadDot.++ Args:+ eprojs: TODO.+ dunits: TODO.+ aheads: TODO.+ att_dim_k: TODO.+ att_dim_v: TODO.+ han_mode: TODO.+ """super(AttMultiHeadDot, self).__init__()self.mlp_q = torch.nn.ModuleList()self.mlp_k = torch.nn.ModuleList()@@ -935,6 +1011,16 @@ class AttMultiHeadAdd(torch.nn.Module):"""def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False):+ """Initialize AttMultiHeadAdd.++ Args:+ eprojs: TODO.+ dunits: TODO.+ aheads: TODO.+ att_dim_k: TODO.+ att_dim_v: TODO.+ han_mode: TODO.+ """super(AttMultiHeadAdd, self).__init__()self.mlp_q = torch.nn.ModuleList()self.mlp_k = torch.nn.ModuleList()@@ -1055,6 +1141,18 @@ class AttMultiHeadLoc(torch.nn.Module):aconv_filts,han_mode=False,):+ """Initialize AttMultiHeadLoc.++ Args:+ eprojs: TODO.+ dunits: TODO.+ aheads: TODO.+ att_dim_k: TODO.+ att_dim_v: TODO.+ aconv_chans: TODO.+ aconv_filts: TODO.+ han_mode: TODO.+ """super(AttMultiHeadLoc, self).__init__()self.mlp_q = torch.nn.ModuleList()self.mlp_k = torch.nn.ModuleList()@@ -1205,6 +1303,18 @@ class AttMultiHeadMultiResLoc(torch.nn.Module):aconv_filts,han_mode=False,):+ """Initialize AttMultiHeadMultiResLoc.++ Args:+ eprojs: TODO.+ dunits: TODO.+ aheads: TODO.+ att_dim_k: TODO.+ att_dim_v: TODO.+ aconv_chans: TODO.+ aconv_filts: TODO.+ han_mode: TODO.+ """super(AttMultiHeadMultiResLoc, self).__init__()self.mlp_q = torch.nn.ModuleList()self.mlp_k = torch.nn.ModuleList()@@ -1333,6 +1443,15 @@ class AttForward(torch.nn.Module):"""def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts):+ """Initialize AttForward.++ Args:+ eprojs: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ aconv_chans: TODO.+ aconv_filts: TODO.+ """super(AttForward, self).__init__()self.mlp_enc = torch.nn.Linear(eprojs, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -1460,6 +1579,16 @@ class AttForwardTA(torch.nn.Module):"""def __init__(self, eunits, dunits, att_dim, aconv_chans, aconv_filts, odim):+ """Initialize AttForwardTA.++ Args:+ eunits: TODO.+ dunits: TODO.+ att_dim: Size/dimension parameter.+ aconv_chans: TODO.+ aconv_filts: TODO.+ odim: TODO.+ """super(AttForwardTA, self).__init__()self.mlp_enc = torch.nn.Linear(eunits, att_dim)self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)@@ -1483,6 +1612,7 @@ class AttForwardTA(torch.nn.Module):self.trans_agent_prob = 0.5def reset(self):+ """Reset."""self.h_length = Noneself.enc_h = Noneself.pre_compute_enc_h = Nonediff --git a/funasr/models/language_model/rnn/decoders.py b/funasr/models/language_model/rnn/decoders.pyindex cdae9e376..ec9f92b26 100644--- a/funasr/models/language_model/rnn/decoders.py+++ b/funasr/models/language_model/rnn/decoders.py@@ -66,6 +66,27 @@ class Decoder(torch.nn.Module, ScorerInterface):num_encs=1,):+ """Initialize Decoder.++ Args:+ eprojs: TODO.+ odim: TODO.+ dtype: TODO.+ dlayers: TODO.+ dunits: TODO.+ sos: TODO.+ eos: TODO.+ att: TODO.+ verbose: TODO.+ char_list: TODO.+ labeldist: TODO.+ lsm_weight: TODO.+ sampling_probability: TODO.+ dropout: TODO.+ context_residual: TODO.+ replace_sos: TODO.+ num_encs: TODO.+ """torch.nn.Module.__init__(self)self.dtype = dtypeself.dunits = dunits@@ -124,9 +145,23 @@ class Decoder(torch.nn.Module, ScorerInterface):self.logzero = -10000000000.0def zero_state(self, hs_pad):+ """Zero state.++ Args:+ hs_pad: TODO.+ """return hs_pad.new_zeros(hs_pad.size(0), self.dunits)def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev):+ """Rnn forward.++ Args:+ ey: TODO.+ z_list: TODO.+ c_list: TODO.+ z_prev: TODO.+ c_prev: TODO.+ """if self.dtype == "lstm":z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0]))for i in six.moves.range(1, self.dlayers):@@ -609,6 +644,19 @@ class Decoder(torch.nn.Module, ScorerInterface):):# to support mutiple encoder asr mode, in single encoder mode,# convert torch.Tensor to List of torch.Tensor+ """Recognize beam batch.++ Args:+ h: TODO.+ hlens: TODO.+ lpz: TODO.+ recog_args: TODO.+ char_list: TODO.+ rnnlm: TODO.+ normalize_score: TODO.+ strm_idx: TODO.+ lang_ids: TODO.+ """if self.num_encs == 1:h = [h]hlens = [hlens]@@ -1009,6 +1057,11 @@ class Decoder(torch.nn.Module, ScorerInterface):@staticmethoddef _get_last_yseq(exp_yseq):+ """Internal: get last yseq.++ Args:+ exp_yseq: TODO.+ """last = []for y_seq in exp_yseq:last.append(y_seq[-1])@@ -1016,6 +1069,12 @@ class Decoder(torch.nn.Module, ScorerInterface):@staticmethoddef _append_ids(yseq, ids):+ """Internal: append ids.++ Args:+ yseq: TODO.+ ids: TODO.+ """if isinstance(ids, list):for i, j in enumerate(ids):yseq[i].append(j)@@ -1026,6 +1085,12 @@ class Decoder(torch.nn.Module, ScorerInterface):@staticmethoddef _index_select_list(yseq, lst):+ """Internal: index select list.++ Args:+ yseq: TODO.+ lst: TODO.+ """new_yseq = []for i in lst:new_yseq.append(yseq[i][:])@@ -1033,6 +1098,13 @@ class Decoder(torch.nn.Module, ScorerInterface):@staticmethoddef _index_select_lm_state(rnnlm_state, dim, vidx):+ """Internal: index select lm state.++ Args:+ rnnlm_state: TODO.+ dim: TODO.+ vidx: TODO.+ """if isinstance(rnnlm_state, dict):new_state = {}for k, v in rnnlm_state.items():@@ -1047,6 +1119,11 @@ class Decoder(torch.nn.Module, ScorerInterface):def init_state(self, x):# to support mutiple encoder asr mode, in single encoder mode,# convert torch.Tensor to List of torch.Tensor+ """Init state.++ Args:+ x: TODO.+ """if self.num_encs == 1:x = [x]@@ -1075,6 +1152,13 @@ class Decoder(torch.nn.Module, ScorerInterface):def score(self, yseq, state, x):# to support mutiple encoder asr mode, in single encoder mode,# convert torch.Tensor to List of torch.Tensor+ """Score.++ Args:+ yseq: TODO.+ state: TODO.+ x: TODO.+ """if self.num_encs == 1:x = [x]@@ -1124,6 +1208,16 @@ class Decoder(torch.nn.Module, ScorerInterface):def decoder_for(args, odim, sos, eos, att, labeldist):+ """Decoder for.++ Args:+ args: TODO.+ odim: TODO.+ sos: TODO.+ eos: TODO.+ att: TODO.+ labeldist: TODO.+ """return Decoder(args.eprojs,odim,diff --git a/funasr/models/language_model/rnn/encoders.py b/funasr/models/language_model/rnn/encoders.pyindex fff376650..906b3c9fb 100644--- a/funasr/models/language_model/rnn/encoders.py+++ b/funasr/models/language_model/rnn/encoders.py@@ -25,6 +25,17 @@ class RNNP(torch.nn.Module):"""def __init__(self, idim, elayers, cdim, hdim, subsample, dropout, typ="blstm"):+ """Initialize RNNP.++ Args:+ idim: TODO.+ elayers: TODO.+ cdim: TODO.+ hdim: TODO.+ subsample: TODO.+ dropout: TODO.+ typ: TODO.+ """super(RNNP, self).__init__()bidir = typ[0] == "b"for i in six.moves.range(elayers):@@ -100,6 +111,16 @@ class RNN(torch.nn.Module):"""def __init__(self, idim, elayers, cdim, hdim, dropout, typ="blstm"):+ """Initialize RNN.++ Args:+ idim: TODO.+ elayers: TODO.+ cdim: TODO.+ hdim: TODO.+ dropout: TODO.+ typ: TODO.+ """super(RNN, self).__init__()bidir = typ[0] == "b"self.nbrnn = (@@ -176,6 +197,11 @@ class VGG2L(torch.nn.Module):"""def __init__(self, in_channel=1):+ """Initialize VGG2L.++ Args:+ in_channel: TODO.+ """super(VGG2L, self).__init__()# CNN layer (VGG motivated)self.conv1_1 = torch.nn.Conv2d(in_channel, 64, 3, stride=1, padding=1)@@ -243,6 +269,18 @@ class Encoder(torch.nn.Module):"""def __init__(self, etype, idim, elayers, eunits, eprojs, subsample, dropout, in_channel=1):+ """Initialize Encoder.++ Args:+ etype: TODO.+ idim: TODO.+ elayers: TODO.+ eunits: TODO.+ eprojs: TODO.+ subsample: TODO.+ dropout: TODO.+ in_channel: TODO.+ """super(Encoder, self).__init__()typ = etype.lstrip("vgg").rstrip("p")if typ not in ["lstm", "gru", "blstm", "bgru"]:diff --git a/funasr/models/language_model/seq_rnn_lm.py b/funasr/models/language_model/seq_rnn_lm.pyindex baf63bc30..86462205d 100644--- a/funasr/models/language_model/seq_rnn_lm.py+++ b/funasr/models/language_model/seq_rnn_lm.py@@ -27,6 +27,18 @@ class SequentialRNNLM(AbsLM):rnn_type: str = "lstm",ignore_id: int = 0,):+ """Initialize SequentialRNNLM.++ Args:+ vocab_size: Size/dimension parameter.+ unit: TODO.+ nhid: TODO.+ nlayers: TODO.+ dropout_rate: TODO.+ tie_weights: TODO.+ rnn_type: TODO.+ ignore_id: TODO.+ """super().__init__()ninp = unit@@ -87,6 +99,12 @@ class SequentialRNNLM(AbsLM):def forward(self, input: torch.Tensor, hidden: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ hidden: TODO.+ """emb = self.drop(self.encoder(input))output, hidden = self.rnn(emb, hidden)output = self.drop(output)diff --git a/funasr/models/language_model/transformer_lm.py b/funasr/models/language_model/transformer_lm.pyindex 3b90e3dc8..919309e51 100644--- a/funasr/models/language_model/transformer_lm.py+++ b/funasr/models/language_model/transformer_lm.py@@ -23,12 +23,30 @@ class TransformerLM(AbsLM):layer: int = 4,dropout_rate: float = 0.5,):+ """Initialize TransformerLM.++ Args:+ vocab_size: Size/dimension parameter.+ pos_enc: TODO.+ embed_unit: TODO.+ att_unit: TODO.+ head: TODO.+ unit: TODO.+ layer: TODO.+ dropout_rate: TODO.+ """super().__init__()if pos_enc == "sinusoidal":pos_enc_class = PositionalEncodingelif pos_enc is None:def pos_enc_class(*args, **kwargs):+ """Pos enc class.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """return nn.Sequential() # indentityelse:@@ -48,6 +66,11 @@ class TransformerLM(AbsLM):self.decoder = nn.Linear(att_unit, vocab_size)def _target_mask(self, ys_in_pad):+ """Internal: target mask.++ Args:+ ys_in_pad: TODO.+ """ys_mask = ys_in_pad != 0m = subsequent_mask(ys_mask.size(-1), device=ys_mask.device).unsqueeze(0)return ys_mask.unsqueeze(-2) & mdiff --git a/funasr/models/lcbnet/encoder.py b/funasr/models/lcbnet/encoder.pyindex f5f249776..01138db91 100644--- a/funasr/models/lcbnet/encoder.py+++ b/funasr/models/lcbnet/encoder.py@@ -167,6 +167,21 @@ class TransformerTextEncoder(nn.Module):normalize_before: bool = True,concat_after: bool = False,):+ """Initialize TransformerTextEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ """super().__init__()self._output_size = output_size@@ -198,6 +213,7 @@ class TransformerTextEncoder(nn.Module):self.after_norm = LayerNorm(output_size)def output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(@@ -365,6 +381,16 @@ class ConvPredictor(nn.Module):attention_dropout_rate=0.1,linear_units=2048,):+ """Initialize ConvPredictor.++ Args:+ size: TODO.+ l_order: TODO.+ r_order: TODO.+ attention_heads: TODO.+ attention_dropout_rate: TODO.+ linear_units: TODO.+ """super().__init__()self.atten = MultiHeadedAttention(attention_heads, size, attention_dropout_rate)self.norm1 = LayerNorm(size)@@ -376,6 +402,12 @@ class ConvPredictor(nn.Module):def forward(self, text_enc, asr_enc):# stage1 cross-attention+ """Forward pass for training.++ Args:+ text_enc: TODO.+ asr_enc: TODO.+ """residual = text_enctext_enc = residual + self.atten(text_enc, asr_enc, asr_enc, None)diff --git a/funasr/models/lcbnet/model.py b/funasr/models/lcbnet/model.pyindex 7b2038e6f..c964c58aa 100644--- a/funasr/models/lcbnet/model.py+++ b/funasr/models/lcbnet/model.py@@ -74,6 +74,45 @@ class LCBNet(nn.Module):**kwargs,):+ """Initialize LCBNet.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ text_encoder: TODO.+ text_encoder_conf: Configuration dict for text_encoder.+ bias_predictor: TODO.+ bias_predictor_conf: Configuration dict for bias_predictor.+ fusion_encoder: TODO.+ fusion_encoder_conf: Configuration dict for fusion_encoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ interctc_weight: TODO.+ select_num: TODO.+ select_length: TODO.+ insert_blank: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -300,6 +339,14 @@ class LCBNet(nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -331,6 +378,14 @@ class LCBNet(nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -344,6 +399,11 @@ class LCBNet(nn.Module):self,**kwargs,):+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.transformer.search import BeamSearchfrom funasr.models.transformer.scorers.ctc import CTCPrefixScorerfrom funasr.models.transformer.scorers.length_bonus import LengthBonus@@ -395,6 +455,16 @@ class LCBNet(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")diff --git a/funasr/models/llm_asr/adaptor.py b/funasr/models/llm_asr/adaptor.pyindex 43482134e..b849422e4 100644--- a/funasr/models/llm_asr/adaptor.py+++ b/funasr/models/llm_asr/adaptor.py@@ -9,6 +9,15 @@ from funasr.register import tables@tables.register("adaptor_classes", "Linear")class Linear(nn.Module):def __init__(self, downsample_rate, encoder_dim, llm_dim, ffn_dim: int = 2048, **kwargs):+ """Initialize Linear.++ Args:+ downsample_rate: TODO.+ encoder_dim: Size/dimension parameter.+ llm_dim: Size/dimension parameter.+ ffn_dim: Size/dimension parameter.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.k = downsample_rateself.encoder_dim = encoder_dim@@ -18,6 +27,11 @@ class Linear(nn.Module):self.linear2 = nn.Linear(ffn_dim, self.llm_dim)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """batch_size, seq_len, dim = x.size()num_frames_to_discard = seq_len % self.kif num_frames_to_discard > 0:@@ -35,6 +49,15 @@ class Linear(nn.Module):@tables.register("adaptor_classes", "QFormer")class EncoderProjectorQFormer(nn.Module):def __init__(self, downsample_rate, encoder_dim, llm_dim, ffn_dim: int = 2048, **kwargs):+ """Initialize EncoderProjectorQFormer.++ Args:+ downsample_rate: TODO.+ encoder_dim: Size/dimension parameter.+ llm_dim: Size/dimension parameter.+ ffn_dim: Size/dimension parameter.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.encoder_dim = encoder_dimself.llm_dim = llm_dim@@ -56,6 +79,11 @@ class EncoderProjectorQFormer(nn.Module):self.second_stride = 0.333333def split_frames(self, speech_embeds):+ """Split frames.++ Args:+ speech_embeds: TODO.+ """B, T, C = speech_embeds.shapekernel = round(T * self.second_per_frame / 30.0)stride = round(T * self.second_stride / 30.0)@@ -71,6 +99,11 @@ class EncoderProjectorQFormer(nn.Module):return speech_embeds, speech_attsdef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """B, T, C = x.size()encoder_out_feat, attention_mask = self.split_frames(x)query = self.query.expand(encoder_out_feat.shape[0], -1, -1)@@ -94,6 +127,15 @@ class Transformer(nn.Module):def __init__(self, downsample_rate=2, encoder_dim=1280, llm_dim=4096, ffn_dim: int = 2048, **kwargs):+ """Initialize Transformer.++ Args:+ downsample_rate: TODO.+ encoder_dim: Size/dimension parameter.+ llm_dim: Size/dimension parameter.+ ffn_dim: Size/dimension parameter.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.k = downsample_rateself.encoder_dim = encoder_dim@@ -129,6 +171,12 @@ class Transformer(nn.Module):def forward(self, x, ilens=None):+ """Forward pass for training.++ Args:+ x: TODO.+ ilens: TODO.+ """batch_size, seq_len, dim = x.size()# num_frames_to_discard = seq_len % self.kchunk_num = (seq_len - 1) // self.k + 1diff --git a/funasr/models/llm_asr/model.py b/funasr/models/llm_asr/model.pyindex fa1d2c3c2..3bc77299b 100644--- a/funasr/models/llm_asr/model.py+++ b/funasr/models/llm_asr/model.py@@ -64,6 +64,39 @@ class LLMASR(nn.Module):**kwargs,):+ """Initialize LLMASR.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ audio_encoder: TODO.+ audio_encoder_conf: Configuration dict for audio_encoder.+ audio_adaptor: TODO.+ audio_adaptor_conf: Configuration dict for audio_adaptor.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ llm: TODO.+ llm_conf: Configuration dict for llm.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -224,6 +257,13 @@ class LLMASR(nn.Module):speech_lengths: torch.Tensor,**kwargs,):+ """Encode.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ **kwargs: Additional keyword arguments.+ """speech = speech.permute(0, 2, 1)res = self.audio_encoder(speech)if isinstance(res, (list, tuple)):@@ -242,6 +282,16 @@ class LLMASR(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """prompt = kwargs.get("prompt", "Transcribe speech to text.")if kwargs.get("batch_size", 1) > 1:@@ -385,6 +435,39 @@ class LLMASR2(nn.Module):**kwargs,):+ """Initialize LLMASR2.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ audio_encoder: TODO.+ audio_encoder_conf: Configuration dict for audio_encoder.+ audio_adaptor: TODO.+ audio_adaptor_conf: Configuration dict for audio_adaptor.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ llm: TODO.+ llm_conf: Configuration dict for llm.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# audio encoder@@ -566,11 +649,22 @@ class LLMASR2(nn.Module):def encode(self, speech, speech_lengths):# audio encoder+ """Encode.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """encoder_out, encoder_out_lens = self.audio_encoder(speech.permute(0, 2, 1), speech_lengths)return encoder_out, encoder_out_lensdef data_template(self, data):+ """Data template.++ Args:+ data: TODO.+ """system, user, assistant = [], [], []for i, item in enumerate(data):role = item["role"]@@ -594,6 +688,15 @@ class LLMASR2(nn.Module):def data_load_speech(self, contents: dict, tokenizer, frontend, meta_data={}, **kwargs):+ """Data load speech.++ Args:+ contents: TODO.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ meta_data: TODO.+ **kwargs: Additional keyword arguments.+ """system = contents["system"]user = contents["user"]assistant = contents["assistant"]@@ -720,6 +823,16 @@ class LLMASR2(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}prompt = kwargs.get("prompt", None)@@ -836,10 +949,22 @@ class LLMASR3(LLMASR2):**kwargs,):+ """Initialize LLMASR3.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def encode(self, speech, speech_lengths):# audio encoder+ """Encode.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)return encoder_out, encoder_out_lens@@ -884,6 +1009,39 @@ class LLMASR4(nn.Module):**kwargs,):+ """Initialize LLMASR4.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ audio_encoder: TODO.+ audio_encoder_conf: Configuration dict for audio_encoder.+ audio_adaptor: TODO.+ audio_adaptor_conf: Configuration dict for audio_adaptor.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ llm: TODO.+ llm_conf: Configuration dict for llm.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# audio encoder@@ -1083,11 +1241,22 @@ class LLMASR4(nn.Module):def encode(self, speech, speech_lengths):# audio encoder+ """Encode.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """encoder_out, encoder_out_lens = self.audio_encoder(speech.permute(0, 2, 1), speech_lengths)return encoder_out, encoder_out_lensdef data_template(self, data):+ """Data template.++ Args:+ data: TODO.+ """system, user, assistant = [], [], []for i, item in enumerate(data):role = item["role"]@@ -1111,6 +1280,15 @@ class LLMASR4(nn.Module):def data_load_speech(self, contents: dict, tokenizer, frontend, meta_data={}, **kwargs):+ """Data load speech.++ Args:+ contents: TODO.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ meta_data: TODO.+ **kwargs: Additional keyword arguments.+ """system = contents["system"]user = contents["user"]assistant = contents["assistant"]@@ -1258,6 +1436,16 @@ class LLMASR4(nn.Module):**kwargs,):+ """Inference prepare.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}prompt = kwargs.get("prompt", None)@@ -1340,6 +1528,16 @@ class LLMASR4(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """inputs_embeds, contents, batch, source_ids, meta_data = self.inference_prepare(data_in, data_lengths, key, tokenizer, frontend, **kwargs)diff --git a/funasr/models/llm_asr_nar/adaptor.py b/funasr/models/llm_asr_nar/adaptor.pyindex 0d73a1a0e..54ed8bbf1 100644--- a/funasr/models/llm_asr_nar/adaptor.py+++ b/funasr/models/llm_asr_nar/adaptor.py@@ -7,6 +7,15 @@ from funasr.register import tables@tables.register("adaptor_classes", "Linear")class Linear(nn.Module):def __init__(self, downsample_rate, encoder_dim, llm_dim, ffn_dim: int = 2048, **kwargs):+ """Initialize Linear.++ Args:+ downsample_rate: TODO.+ encoder_dim: Size/dimension parameter.+ llm_dim: Size/dimension parameter.+ ffn_dim: Size/dimension parameter.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.k = downsample_rateself.encoder_dim = encoder_dim@@ -16,6 +25,11 @@ class Linear(nn.Module):self.linear2 = nn.Linear(ffn_dim, self.llm_dim)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """batch_size, seq_len, dim = x.size()num_frames_to_discard = seq_len % self.kif num_frames_to_discard > 0:diff --git a/funasr/models/llm_asr_nar/model.py b/funasr/models/llm_asr_nar/model.pyindex 192c1990b..b0d4f10d3 100644--- a/funasr/models/llm_asr_nar/model.py+++ b/funasr/models/llm_asr_nar/model.py@@ -62,6 +62,39 @@ class LLMASRNAR(nn.Module):**kwargs,):+ """Initialize LLMASRNAR.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ llm: TODO.+ llm_conf: Configuration dict for llm.+ adaptor: TODO.+ adaptor_conf: Configuration dict for adaptor.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -224,6 +257,13 @@ class LLMASRNAR(nn.Module):**kwargs,):+ """Encode.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ **kwargs: Additional keyword arguments.+ """audio_mask = kwargs.get("audio_mask", None)audio_token_lengths = audio_mask.sum(-1) if audio_mask is not None else Nonetext_token_int = kwargs.get("text_token_int", None)@@ -252,6 +292,16 @@ class LLMASRNAR(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """prompt = kwargs.get("prompt", "Transcribe speech to text.")if kwargs.get("batch_size", 1) > 1:@@ -408,6 +458,40 @@ class LLMASRNARPrompt(nn.Module):**kwargs,):+ """Initialize LLMASRNARPrompt.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ llm: TODO.+ llm_conf: Configuration dict for llm.+ adaptor: TODO.+ adaptor_conf: Configuration dict for adaptor.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ predictor_weight: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -611,6 +695,13 @@ class LLMASRNARPrompt(nn.Module):**kwargs,):+ """Encode.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ **kwargs: Additional keyword arguments.+ """audio_mask = kwargs.get("audio_mask", None)audio_token_lengths = audio_mask.sum(-1) if audio_mask is not None else Nonetext_token_int = kwargs.get("text_token_int", None)@@ -642,6 +733,14 @@ class LLMASRNARPrompt(nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -661,6 +760,16 @@ class LLMASRNARPrompt(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """prompt = kwargs.get("prompt", "Transcribe speech to text.")if kwargs.get("batch_size", 1) > 1:diff --git a/funasr/models/lora/layers.py b/funasr/models/lora/layers.pyindex b91253d56..9880b0f58 100644--- a/funasr/models/lora/layers.py+++ b/funasr/models/lora/layers.py@@ -18,6 +18,14 @@ class LoRALayer:lora_dropout: float,merge_weights: bool,):+ """Initialize LoRALayer.++ Args:+ r: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ merge_weights: TODO.+ """self.r = rself.lora_alpha = lora_alpha# Optional dropout@@ -41,6 +49,16 @@ class Embedding(nn.Embedding, LoRALayer):merge_weights: bool = True,**kwargs):+ """Initialize Embedding.++ Args:+ num_embeddings: TODO.+ embedding_dim: Size/dimension parameter.+ r: TODO.+ lora_alpha: TODO.+ merge_weights: TODO.+ **kwargs: Additional keyword arguments.+ """nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs)LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=0, merge_weights=merge_weights@@ -55,6 +73,7 @@ class Embedding(nn.Embedding, LoRALayer):self.reset_parameters()def reset_parameters(self):+ """Reset parameters."""nn.Embedding.reset_parameters(self)if hasattr(self, "lora_A"):# initialize A the same way as the default for nn.Linear and B to zero@@ -62,6 +81,11 @@ class Embedding(nn.Embedding, LoRALayer):nn.init.normal_(self.lora_B)def train(self, mode: bool = True):+ """Train.++ Args:+ mode: TODO.+ """nn.Embedding.train(self, mode)if self.merge_weights and self.merged:# Make sure that the weights are not merged@@ -70,6 +94,7 @@ class Embedding(nn.Embedding, LoRALayer):self.merged = Falsedef eval(self):+ """Eval."""nn.Linear.eval(self)if self.merge_weights and not self.merged:# Merge the weights and mark it@@ -78,6 +103,11 @@ class Embedding(nn.Embedding, LoRALayer):self.merged = Truedef forward(self, x: torch.Tensor):+ """Forward pass for training.++ Args:+ x: TODO.+ """if self.r > 0 and not self.merged:result = nn.Embedding.forward(self, x)if self.r > 0:@@ -109,6 +139,18 @@ class Linear(nn.Linear, LoRALayer):merge_weights: bool = True,**kwargs):+ """Initialize Linear.++ Args:+ in_features: TODO.+ out_features: TODO.+ r: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ fan_in_fan_out: TODO.+ merge_weights: TODO.+ **kwargs: Additional keyword arguments.+ """nn.Linear.__init__(self, in_features, out_features, **kwargs)LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights@@ -127,6 +169,7 @@ class Linear(nn.Linear, LoRALayer):self.weight.data = self.weight.data.Tdef reset_parameters(self):+ """Reset parameters."""nn.Linear.reset_parameters(self)if hasattr(self, "lora_A"):# initialize A the same way as the default for nn.Linear and B to zero@@ -134,7 +177,17 @@ class Linear(nn.Linear, LoRALayer):nn.init.zeros_(self.lora_B)def train(self, mode: bool = True):+ """Train.++ Args:+ mode: TODO.+ """def T(w):+ """T.++ Args:+ w: TODO.+ """return w.T if self.fan_in_fan_out else wnn.Linear.train(self, mode)@@ -145,7 +198,13 @@ class Linear(nn.Linear, LoRALayer):self.merged = Falsedef eval(self):+ """Eval."""def T(w):+ """T.++ Args:+ w: TODO.+ """return w.T if self.fan_in_fan_out else wnn.Linear.eval(self)@@ -156,7 +215,17 @@ class Linear(nn.Linear, LoRALayer):self.merged = Truedef forward(self, x: torch.Tensor):+ """Forward pass for training.++ Args:+ x: TODO.+ """def T(w):+ """T.++ Args:+ w: TODO.+ """return w.T if self.fan_in_fan_out else wif self.r > 0 and not self.merged:@@ -182,6 +251,19 @@ class MergedLinear(nn.Linear, LoRALayer):merge_weights: bool = True,**kwargs):+ """Initialize MergedLinear.++ Args:+ in_features: TODO.+ out_features: TODO.+ r: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ enable_lora: TODO.+ fan_in_fan_out: TODO.+ merge_weights: TODO.+ **kwargs: Additional keyword arguments.+ """nn.Linear.__init__(self, in_features, out_features, **kwargs)LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights@@ -211,6 +293,7 @@ class MergedLinear(nn.Linear, LoRALayer):self.weight.data = self.weight.data.Tdef reset_parameters(self):+ """Reset parameters."""nn.Linear.reset_parameters(self)if hasattr(self, "lora_A"):# initialize A the same way as the default for nn.Linear and B to zero@@ -218,6 +301,11 @@ class MergedLinear(nn.Linear, LoRALayer):nn.init.zeros_(self.lora_B)def zero_pad(self, x):+ """Zero pad.++ Args:+ x: TODO.+ """result = x.new_zeros((*x.shape[:-1], self.out_features))result = result.view(-1, self.out_features)result[:, self.lora_ind] = x.reshape(@@ -226,7 +314,17 @@ class MergedLinear(nn.Linear, LoRALayer):return result.view((*x.shape[:-1], self.out_features))def train(self, mode: bool = True):+ """Train.++ Args:+ mode: TODO.+ """def T(w):+ """T.++ Args:+ w: TODO.+ """return w.T if self.fan_in_fan_out else wnn.Linear.train(self, mode)@@ -242,7 +340,13 @@ class MergedLinear(nn.Linear, LoRALayer):self.merged = Falsedef eval(self):+ """Eval."""def T(w):+ """T.++ Args:+ w: TODO.+ """return w.T if self.fan_in_fan_out else wnn.Linear.eval(self)@@ -258,7 +362,17 @@ class MergedLinear(nn.Linear, LoRALayer):self.merged = Truedef forward(self, x: torch.Tensor):+ """Forward pass for training.++ Args:+ x: TODO.+ """def T(w):+ """T.++ Args:+ w: TODO.+ """return w.T if self.fan_in_fan_out else wif self.merged:@@ -289,6 +403,18 @@ class Conv2d(nn.Conv2d, LoRALayer):merge_weights: bool = True,**kwargs):+ """Initialize Conv2d.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ kernel_size: Size/dimension parameter.+ r: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ merge_weights: TODO.+ **kwargs: Additional keyword arguments.+ """nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, **kwargs)LoRALayer.__init__(self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights@@ -308,6 +434,7 @@ class Conv2d(nn.Conv2d, LoRALayer):self.reset_parameters()def reset_parameters(self):+ """Reset parameters."""nn.Conv2d.reset_parameters(self)if hasattr(self, "lora_A"):# initialize A the same way as the default for nn.Linear and B to zero@@ -315,6 +442,11 @@ class Conv2d(nn.Conv2d, LoRALayer):nn.init.zeros_(self.lora_B)def train(self, mode: bool = True):+ """Train.++ Args:+ mode: TODO.+ """nn.Conv2d.train(self, mode)if self.merge_weights and self.merged:# Make sure that the weights are not merged@@ -322,6 +454,7 @@ class Conv2d(nn.Conv2d, LoRALayer):self.merged = Falsedef eval(self):+ """Eval."""nn.Conv2d.eval(self)if self.merge_weights and not self.merged:# Merge the weights and mark it@@ -329,6 +462,11 @@ class Conv2d(nn.Conv2d, LoRALayer):self.merged = Truedef forward(self, x: torch.Tensor):+ """Forward pass for training.++ Args:+ x: TODO.+ """if self.r > 0 and not self.merged:return F.conv2d(x,diff --git a/funasr/models/lora/utils.py b/funasr/models/lora/utils.pyindex 670c1dcc0..586341367 100644--- a/funasr/models/lora/utils.py+++ b/funasr/models/lora/utils.py@@ -11,6 +11,12 @@ from .layers import LoRALayerdef mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None:+ """Mark only lora as trainable.++ Args:+ model: Model instance or model name.+ bias: TODO.+ """for n, p in model.named_parameters():if "lora_" not in n and "cif" not in n:p.requires_grad = False@@ -29,6 +35,12 @@ def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None:def lora_state_dict(model: nn.Module, bias: str = "none") -> Dict[str, torch.Tensor]:+ """Lora state dict.++ Args:+ model: Model instance or model name.+ bias: TODO.+ """my_state_dict = model.state_dict()if bias == "none":return {k: my_state_dict[k] for k in my_state_dict if "lora_" in k}diff --git a/funasr/models/mfcca/e2e_asr_mfcca.py b/funasr/models/mfcca/e2e_asr_mfcca.pyindex 681d1f61f..f6cbb089f 100644--- a/funasr/models/mfcca/e2e_asr_mfcca.py+++ b/funasr/models/mfcca/e2e_asr_mfcca.py@@ -30,6 +30,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -67,6 +72,29 @@ class MFCCA(FunASRModel):sym_blank: str = "<blank>",preencoder: Optional[AbsPreEncoder] = None,):+ """Initialize MFCCA.++ Args:+ vocab_size: Size/dimension parameter.+ token_list: TODO.+ frontend: Audio frontend for feature extraction.+ specaug: TODO.+ normalize: TODO.+ encoder: TODO.+ decoder: TODO.+ ctc: TODO.+ rnnt_decoder: TODO.+ ctc_weight: TODO.+ ignore_id: TODO.+ lsm_weight: TODO.+ mask_ratio: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ preencoder: TODO.+ """assert 0.0 <= ctc_weight <= 1.0, ctc_weightassert rnnt_decoder is None, "Not implemented"@@ -198,6 +226,14 @@ class MFCCA(FunASRModel):text: torch.Tensor,text_lengths: torch.Tensor,) -> Dict[str, torch.Tensor]:+ """Collect feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ text: Text tensor or string input.+ text_lengths: Length of each text sample.+ """feats, feats_lengths, channel_size = self._extract_feats(speech, speech_lengths)return {"feats": feats, "feats_lengths": feats_lengths}@@ -246,6 +282,12 @@ class MFCCA(FunASRModel):def _extract_feats(self, speech: torch.Tensor, speech_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Internal: extract feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """assert speech_lengths.dim() == 1, speech_lengths.shape# for data-parallelspeech = speech[:, : speech_lengths.max()]@@ -268,6 +310,14 @@ class MFCCA(FunASRModel):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -299,6 +349,14 @@ class MFCCA(FunASRModel):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """if encoder_out.dim() == 4:encoder_out = encoder_out.mean(1)loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)@@ -317,4 +375,12 @@ class MFCCA(FunASRModel):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc rnnt loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """raise NotImplementedErrordiff --git a/funasr/models/mfcca/mfcca_encoder.py b/funasr/models/mfcca/mfcca_encoder.pyindex a0bb58ee0..aa16d08f6 100644--- a/funasr/models/mfcca/mfcca_encoder.py+++ b/funasr/models/mfcca/mfcca_encoder.py@@ -159,6 +159,32 @@ class MFCCAEncoder(AbsEncoder):cnn_module_kernel: int = 31,padding_idx: int = -1,):+ """Initialize MFCCAEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ macaron_style: TODO.+ rel_pos_type: TODO.+ pos_enc_layer_type: TODO.+ selfattention_layer_type: TODO.+ activation_type: TODO.+ use_cnn_module: TODO.+ zero_triu: TODO.+ cnn_module_kernel: TODO.+ padding_idx: TODO.+ """super().__init__()self._output_size = output_size@@ -319,6 +345,7 @@ class MFCCAEncoder(AbsEncoder):self.conv4 = torch.nn.Conv2d(16, 1, [5, 7], stride=[1, 1], padding=(2, 3))def output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(diff --git a/funasr/models/monotonic_aligner/model.py b/funasr/models/monotonic_aligner/model.pyindex 4754d1fb3..be1284b75 100644--- a/funasr/models/monotonic_aligner/model.py+++ b/funasr/models/monotonic_aligner/model.py@@ -44,6 +44,22 @@ class MonotonicAligner(torch.nn.Module):length_normalized_loss: bool = False,**kwargs,):+ """Initialize MonotonicAligner.++ Args:+ input_size: Size/dimension parameter.+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ predictor: TODO.+ predictor_conf: Configuration dict for predictor.+ predictor_bias: TODO.+ length_normalized_loss: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -116,6 +132,13 @@ class MonotonicAligner(torch.nn.Module):return loss, stats, weightdef calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num):+ """Calc predictor timestamp.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ token_num: TODO.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -162,6 +185,16 @@ class MonotonicAligner(torch.nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}# extract fbank featstime1 = time.perf_counter()diff --git a/funasr/models/mossformer/e2e_ss.py b/funasr/models/mossformer/e2e_ss.pyindex 40d30ca89..9f765d274 100644--- a/funasr/models/mossformer/e2e_ss.py+++ b/funasr/models/mossformer/e2e_ss.py@@ -45,6 +45,19 @@ class MossFormer(FunASRModel):use_global_pos_enc=True,max_length=20000,):+ """Initialize MossFormer.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ num_blocks: TODO.+ kernel_size: Size/dimension parameter.+ norm: TODO.+ num_spks: TODO.+ skip_around_intra: TODO.+ use_global_pos_enc: TODO.+ max_length: TODO.+ """super(MossFormer, self).__init__()self.num_spks = num_spks# Encoding@@ -72,6 +85,11 @@ class MossFormer(FunASRModel):)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """x = self.enc(input)mask = self.mask_net(x)x = torch.stack([x] * self.num_spks)diff --git a/funasr/models/mossformer/mossformer.py b/funasr/models/mossformer/mossformer.pyindex 30aab481a..9237c5857 100644--- a/funasr/models/mossformer/mossformer.py+++ b/funasr/models/mossformer/mossformer.py@@ -5,24 +5,54 @@ from einops import rearrangedef identity(t, *args, **kwargs):+ """Identity.++ Args:+ t: TODO.+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """return tdef append_dims(x, num_dims):+ """Append dims.++ Args:+ x: TODO.+ num_dims: TODO.+ """if num_dims <= 0:return xreturn x.view(*x.shape, *((1,) * num_dims))def exists(val):+ """Exists.++ Args:+ val: TODO.+ """return val is not Nonedef default(val, d):+ """Default.++ Args:+ val: TODO.+ d: TODO.+ """return val if exists(val) else ddef padding_to_multiple_of(n, mult):+ """Padding to multiple of.++ Args:+ n: TODO.+ mult: TODO.+ """remainder = n % multif remainder == 0:return 0@@ -33,10 +63,20 @@ class Transpose(nn.Module):"""Wrapper class of torch.transpose() for Sequential module."""def __init__(self, shape: tuple):+ """Initialize Transpose.++ Args:+ shape: TODO.+ """super(Transpose, self).__init__()self.shape = shapedef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """return x.transpose(*self.shape)@@ -66,6 +106,16 @@ class DepthwiseConv1d(nn.Module):padding: int = 0,bias: bool = False,) -> None:+ """Initialize DepthwiseConv1d.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ padding: TODO.+ bias: TODO.+ """super(DepthwiseConv1d, self).__init__()assert (out_channels % in_channels == 0@@ -81,6 +131,11 @@ class DepthwiseConv1d(nn.Module):)def forward(self, inputs):+ """Forward pass for training.++ Args:+ inputs: TODO.+ """return self.conv(inputs)@@ -106,6 +161,14 @@ class ConvModule(nn.Module):expansion_factor: int = 2,dropout_p: float = 0.1,) -> None:+ """Initialize ConvModule.++ Args:+ in_channels: TODO.+ kernel_size: Size/dimension parameter.+ expansion_factor: TODO.+ dropout_p: TODO.+ """super(ConvModule, self).__init__()assert (kernel_size - 1) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding"assert expansion_factor == 2, "Currently, Only Supports expansion_factor 2"@@ -118,23 +181,47 @@ class ConvModule(nn.Module):)def forward(self, inputs):+ """Forward pass for training.++ Args:+ inputs: TODO.+ """return inputs + self.sequential(inputs).transpose(1, 2)class OffsetScale(nn.Module):def __init__(self, dim, heads=1):+ """Initialize OffsetScale.++ Args:+ dim: TODO.+ heads: TODO.+ """super().__init__()self.gamma = nn.Parameter(torch.ones(heads, dim))self.beta = nn.Parameter(torch.zeros(heads, dim))nn.init.normal_(self.gamma, std=0.02)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """out = einsum("... d, h d -> ... h d", x, self.gamma) + self.betareturn out.unbind(dim=-2)class FFConvM(nn.Module):def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1):+ """Initialize FFConvM.++ Args:+ dim_in: TODO.+ dim_out: TODO.+ norm_klass: TODO.+ dropout: TODO.+ """super().__init__()self.mdl = nn.Sequential(norm_klass(dim_in),@@ -148,6 +235,11 @@ class FFConvM(nn.Module):self,x,):+ """Forward pass for training.++ Args:+ x: TODO.+ """output = self.mdl(x)return output@@ -166,6 +258,7 @@ class FLASH_ShareA_FFConvM(nn.Module):norm_klass=nn.LayerNorm,shift_tokens=True):+ """Initialize FLASH_ShareA_FFConvM."""super().__init__()hidden_dim = int(dim * expansion_factor)self.group_size = group_size@@ -236,6 +329,18 @@ class FLASH_ShareA_FFConvM(nn.Module):return xdef cal_attention(self, x, quad_q, lin_q, quad_k, lin_k, v, u, mask=None):+ """Cal attention.++ Args:+ x: TODO.+ quad_q: TODO.+ lin_q: TODO.+ quad_k: TODO.+ lin_k: TODO.+ v: TODO.+ u: TODO.+ mask: TODO.+ """b, n, device, g = x.shape[0], x.shape[-2], x.device, self.group_sizeif exists(mask):diff --git a/funasr/models/mossformer/mossformer_decoder.py b/funasr/models/mossformer/mossformer_decoder.pyindex 56ad2ecb3..6b447c782 100644--- a/funasr/models/mossformer/mossformer_decoder.py+++ b/funasr/models/mossformer/mossformer_decoder.py@@ -25,6 +25,12 @@ class MossFormerDecoder(nn.ConvTranspose1d):"""def __init__(self, *args, **kwargs):+ """Initialize MossFormerDecoder.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super(MossFormerDecoder, self).__init__(*args, **kwargs)def forward(self, x):diff --git a/funasr/models/mossformer/mossformer_encoder.py b/funasr/models/mossformer/mossformer_encoder.pyindex a691d8c62..2642ace67 100644--- a/funasr/models/mossformer/mossformer_encoder.py+++ b/funasr/models/mossformer/mossformer_encoder.py@@ -41,6 +41,7 @@ class MossformerBlock(nn.Module):norm_type="scalenorm",shift_tokens=True):+ """Initialize MossformerBlock."""super().__init__()assert norm_type in ("scalenorm",@@ -74,6 +75,11 @@ class MossformerBlock(nn.Module):)def forward(self, x, *, mask=None):+ """Forward pass for training.++ Args:+ x: TODO.+ """ii = 0for flash in self.layers:x = flash(x, mask=mask)@@ -124,6 +130,18 @@ class MossFormer_MaskNet(nn.Module):use_global_pos_enc=True,max_length=20000,):+ """Initialize MossFormer_MaskNet.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ num_blocks: TODO.+ norm: TODO.+ num_spks: TODO.+ skip_around_intra: TODO.+ use_global_pos_enc: TODO.+ max_length: TODO.+ """super(MossFormer_MaskNet, self).__init__()self.num_spks = num_spksself.num_blocks = num_blocks@@ -236,6 +254,13 @@ class MossFormerEncoder(nn.Module):"""def __init__(self, kernel_size=2, out_channels=64, in_channels=1):+ """Initialize MossFormerEncoder.++ Args:+ kernel_size: Size/dimension parameter.+ out_channels: TODO.+ in_channels: TODO.+ """super(MossFormerEncoder, self).__init__()self.conv1d = nn.Conv1d(in_channels=in_channels,@@ -314,6 +339,17 @@ class MossFormerM(nn.Module):expansion_factor=4.0,attn_dropout=0.1,):+ """Initialize MossFormerM.++ Args:+ num_blocks: TODO.+ d_model: D Model instance.+ causal: TODO.+ group_size: Size/dimension parameter.+ query_key_dim: Size/dimension parameter.+ expansion_factor: TODO.+ attn_dropout: TODO.+ """super().__init__()self.mossformerM = MossformerBlock(@@ -379,6 +415,14 @@ class Computation_Block(nn.Module):norm="ln",skip_around_intra=True,):+ """Initialize Computation_Block.++ Args:+ num_blocks: TODO.+ out_channels: TODO.+ norm: TODO.+ skip_around_intra: TODO.+ """super(Computation_Block, self).__init__()##MossFormer2M: MossFormer with recurrencediff --git a/funasr/models/normalize/global_mvn.py b/funasr/models/normalize/global_mvn.pyindex 55a728706..c12da5110 100644--- a/funasr/models/normalize/global_mvn.py+++ b/funasr/models/normalize/global_mvn.py@@ -27,6 +27,14 @@ class GlobalMVN(torch.nn.Module):norm_vars: bool = True,eps: float = 1.0e-20,):+ """Initialize GlobalMVN.++ Args:+ stats_file: TODO.+ norm_means: TODO.+ norm_vars: TODO.+ eps: TODO.+ """super().__init__()self.norm_means = norm_meansself.norm_vars = norm_vars@@ -53,6 +61,7 @@ class GlobalMVN(torch.nn.Module):self.register_buffer("std", torch.from_numpy(std))def extra_repr(self):+ """Extra repr."""return (f"stats_file={self.stats_file}, "f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"@@ -93,6 +102,12 @@ class GlobalMVN(torch.nn.Module):def inverse(self, x: torch.Tensor, ilens: torch.Tensor = None) -> Tuple[torch.Tensor, torch.Tensor]:+ """Inverse.++ Args:+ x: TODO.+ ilens: TODO.+ """if ilens is None:ilens = x.new_full([x.size(0)], x.size(1))norm_means = self.norm_meansdiff --git a/funasr/models/normalize/utterance_mvn.py b/funasr/models/normalize/utterance_mvn.pyindex 8216328c8..9128c6d04 100644--- a/funasr/models/normalize/utterance_mvn.py+++ b/funasr/models/normalize/utterance_mvn.py@@ -14,12 +14,20 @@ class UtteranceMVN(torch.nn.Module):norm_vars: bool = False,eps: float = 1.0e-20,):+ """Initialize UtteranceMVN.++ Args:+ norm_means: TODO.+ norm_vars: TODO.+ eps: TODO.+ """super().__init__()self.norm_means = norm_meansself.norm_vars = norm_varsself.eps = epsdef extra_repr(self):+ """Extra repr."""return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"def forward(diff --git a/funasr/models/paraformer/cif_predictor.py b/funasr/models/paraformer/cif_predictor.pyindex d59705032..d72bd0f7b 100644--- a/funasr/models/paraformer/cif_predictor.py+++ b/funasr/models/paraformer/cif_predictor.py@@ -1,760 +1,920 @@-#!/usr/bin/env python3-# -*- encoding: utf-8 -*--# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.-# MIT License (https://opensource.org/licenses/MIT)--import torch-import logging-import numpy as np--from funasr.register import tables-from funasr.train_utils.device_funcs import to_device-from funasr.models.transformer.utils.nets_utils import make_pad_mask-from torch.cuda.amp import autocast--[email protected]("predictor_classes", "CifPredictor")-class CifPredictor(torch.nn.Module):- def __init__(- self,- idim,- l_order,- r_order,- threshold=1.0,- dropout=0.1,- smooth_factor=1.0,- noise_threshold=0,- tail_threshold=0.45,- ):- super().__init__()-- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)- self.cif_output = torch.nn.Linear(idim, 1)- self.dropout = torch.nn.Dropout(p=dropout)- self.threshold = threshold- self.smooth_factor = smooth_factor- self.noise_threshold = noise_threshold- self.tail_threshold = tail_threshold-- def forward(- self,- hidden,- target_label=None,- mask=None,- ignore_id=-1,- mask_chunk_predictor=None,- target_label_length=None,- ):-- with autocast(False):- h = hidden- context = h.transpose(1, 2)- queries = self.pad(context)- memory = self.cif_conv1d(queries)- output = memory + context- output = self.dropout(output)- output = output.transpose(1, 2)- output = torch.relu(output)- output = self.cif_output(output)- alphas = torch.sigmoid(output)- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)- if mask is not None:- mask = mask.transpose(-1, -2).float()- alphas = alphas * mask- if mask_chunk_predictor is not None:- alphas = alphas * mask_chunk_predictor- alphas = alphas.squeeze(-1)- mask = mask.squeeze(-1)- if target_label_length is not None:- target_length = target_label_length- elif target_label is not None:- target_length = (target_label != ignore_id).float().sum(-1)- else:- target_length = None- token_num = alphas.sum(-1)- if target_length is not None:- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))- elif self.tail_threshold > 0.0:- hidden, alphas, token_num = self.tail_process_fn(- hidden, alphas, token_num, mask=mask- )-- acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)-- if target_length is None and self.tail_threshold > 0.0:- token_num_int = torch.max(token_num).type(torch.int32).item()- acoustic_embeds = acoustic_embeds[:, :token_num_int, :]-- return acoustic_embeds, token_num, alphas, cif_peak-- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):- b, t, d = hidden.size()- tail_threshold = self.tail_threshold- if mask is not None:- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)- ones_t = torch.ones_like(zeros_t)- mask_1 = torch.cat([mask, zeros_t], dim=1)- mask_2 = torch.cat([ones_t, mask], dim=1)- mask = mask_2 - mask_1- tail_threshold = mask * tail_threshold- alphas = torch.cat([alphas, zeros_t], dim=1)- alphas = torch.add(alphas, tail_threshold)- else:- tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)- tail_threshold = torch.reshape(tail_threshold, (1, 1))- alphas = torch.cat([alphas, tail_threshold], dim=1)- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)- hidden = torch.cat([hidden, zeros], dim=1)- token_num = alphas.sum(dim=-1)- token_num_floor = torch.floor(token_num)-- return hidden, alphas, token_num_floor-- def gen_frame_alignments(- self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None- ):- batch_size, maximum_length = alphas.size()- int_type = torch.int32-- is_training = self.training- if is_training:- token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)- else:- token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)-- max_token_num = torch.max(token_num).item()-- alphas_cumsum = torch.cumsum(alphas, dim=1)- alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)- alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)-- index = torch.ones([batch_size, max_token_num], dtype=int_type)- index = torch.cumsum(index, dim=1)- index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)-- index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)- index_div_bool_zeros = index_div.eq(0)- index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1- index_div_bool_zeros_count = torch.clamp(- index_div_bool_zeros_count, 0, encoder_sequence_length.max()- )- token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)- index_div_bool_zeros_count *= token_num_mask-- index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(- 1, 1, maximum_length- )- ones = torch.ones_like(index_div_bool_zeros_count_tile)- zeros = torch.zeros_like(index_div_bool_zeros_count_tile)- ones = torch.cumsum(ones, dim=2)- cond = index_div_bool_zeros_count_tile == ones- index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)-- index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)- index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)- index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)- predictor_mask = (- (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))- .type(int_type)- .to(encoder_sequence_length.device)- )- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask-- predictor_alignments = index_div_bool_zeros_count_tile_out- predictor_alignments_length = predictor_alignments.sum(-1).type(- encoder_sequence_length.dtype- )- return predictor_alignments.detach(), predictor_alignments_length.detach()--[email protected]("predictor_classes", "CifPredictorV2")-class CifPredictorV2(torch.nn.Module):- def __init__(- self,- idim,- l_order,- r_order,- threshold=1.0,- dropout=0.1,- smooth_factor=1.0,- noise_threshold=0,- tail_threshold=0.0,- tf2torch_tensor_name_prefix_torch="predictor",- tf2torch_tensor_name_prefix_tf="seq2seq/cif",- tail_mask=True,- ):- super().__init__()-- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)- self.cif_output = torch.nn.Linear(idim, 1)- self.dropout = torch.nn.Dropout(p=dropout)- self.threshold = threshold- self.smooth_factor = smooth_factor- self.noise_threshold = noise_threshold- self.tail_threshold = tail_threshold- self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch- self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf- self.tail_mask = tail_mask-- def forward(- self,- hidden,- target_label=None,- mask=None,- ignore_id=-1,- mask_chunk_predictor=None,- target_label_length=None,- ):-- with autocast(False):- h = hidden- context = h.transpose(1, 2)- queries = self.pad(context)- output = torch.relu(self.cif_conv1d(queries))- output = output.transpose(1, 2)-- output = self.cif_output(output)- alphas = torch.sigmoid(output)- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)- if mask is not None:- mask = mask.transpose(-1, -2).float()- alphas = alphas * mask- if mask_chunk_predictor is not None:- alphas = alphas * mask_chunk_predictor- alphas = alphas.squeeze(-1)- mask = mask.squeeze(-1)- if target_label_length is not None:- target_length = target_label_length.squeeze(-1)- elif target_label is not None:- target_length = (target_label != ignore_id).float().sum(-1)- else:- target_length = None- token_num = alphas.sum(-1)- if target_length is not None:- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))- elif self.tail_threshold > 0.0:- if self.tail_mask:- hidden, alphas, token_num = self.tail_process_fn(- hidden, alphas, token_num, mask=mask- )- else:- hidden, alphas, token_num = self.tail_process_fn(- hidden, alphas, token_num, mask=None- )-- acoustic_embeds, cif_peak = cif_v1(hidden, alphas, self.threshold)- if target_length is None and self.tail_threshold > 0.0:- token_num_int = torch.max(token_num).type(torch.int32).item()- acoustic_embeds = acoustic_embeds[:, :token_num_int, :]-- return acoustic_embeds, token_num, alphas, cif_peak-- def forward_chunk(self, hidden, cache=None, **kwargs):- is_final = kwargs.get("is_final", False)- batch_size, len_time, hidden_size = hidden.shape- h = hidden- context = h.transpose(1, 2)- queries = self.pad(context)- output = torch.relu(self.cif_conv1d(queries))- output = output.transpose(1, 2)- output = self.cif_output(output)- alphas = torch.sigmoid(output)- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)-- alphas = alphas.squeeze(-1)-- token_length = []- list_fires = []- list_frames = []- cache_alphas = []- cache_hiddens = []-- if cache is not None and "chunk_size" in cache:- alphas[:, : cache["chunk_size"][0]] = 0.0- if not is_final:- alphas[:, sum(cache["chunk_size"][:2]) :] = 0.0- if cache is not None and "cif_alphas" in cache and "cif_hidden" in cache:- cache["cif_hidden"] = to_device(cache["cif_hidden"], device=hidden.device)- cache["cif_alphas"] = to_device(cache["cif_alphas"], device=alphas.device)- hidden = torch.cat((cache["cif_hidden"], hidden), dim=1)- alphas = torch.cat((cache["cif_alphas"], alphas), dim=1)- if cache is not None and is_final:- tail_hidden = torch.zeros((batch_size, 1, hidden_size), device=hidden.device)- tail_alphas = torch.tensor([[self.tail_threshold]], device=alphas.device)- tail_alphas = torch.tile(tail_alphas, (batch_size, 1))- hidden = torch.cat((hidden, tail_hidden), dim=1)- alphas = torch.cat((alphas, tail_alphas), dim=1)-- len_time = alphas.shape[1]- for b in range(batch_size):- integrate = 0.0- frames = torch.zeros((hidden_size), device=hidden.device)- list_frame = []- list_fire = []- for t in range(len_time):- alpha = alphas[b][t]- if alpha + integrate < self.threshold:- integrate += alpha- list_fire.append(integrate)- frames += alpha * hidden[b][t]- else:- frames += (self.threshold - integrate) * hidden[b][t]- list_frame.append(frames)- integrate += alpha- list_fire.append(integrate)- integrate -= self.threshold- frames = integrate * hidden[b][t]-- cache_alphas.append(integrate)- if integrate > 0.0:- cache_hiddens.append(frames / integrate)- else:- cache_hiddens.append(frames)-- token_length.append(torch.tensor(len(list_frame), device=alphas.device))- list_fires.append(list_fire)- list_frames.append(list_frame)-- cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)- cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)- cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)- cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)-- max_token_len = max(token_length)- if max_token_len == 0:- return hidden, torch.stack(token_length, 0), None, None- list_ls = []- for b in range(batch_size):- pad_frames = torch.zeros(- (max_token_len - token_length[b], hidden_size), device=alphas.device- )- if token_length[b] == 0:- list_ls.append(pad_frames)- else:- list_frames[b] = torch.stack(list_frames[b])- list_ls.append(torch.cat((list_frames[b], pad_frames), dim=0))-- cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)- cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)- cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)- cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)- return torch.stack(list_ls, 0), torch.stack(token_length, 0), None, None-- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):- b, t, d = hidden.size()- tail_threshold = self.tail_threshold- if mask is not None:- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)- ones_t = torch.ones_like(zeros_t)- mask_1 = torch.cat([mask, zeros_t], dim=1)- mask_2 = torch.cat([ones_t, mask], dim=1)- mask = mask_2 - mask_1- tail_threshold = mask * tail_threshold- alphas = torch.cat([alphas, zeros_t], dim=1)- alphas = torch.add(alphas, tail_threshold)- else:- tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)- tail_threshold = torch.reshape(tail_threshold, (1, 1))- if b > 1:- alphas = torch.cat([alphas, tail_threshold.repeat(b, 1)], dim=1)- else:- alphas = torch.cat([alphas, tail_threshold], dim=1)- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)- hidden = torch.cat([hidden, zeros], dim=1)- token_num = alphas.sum(dim=-1)- token_num_floor = torch.floor(token_num)-- return hidden, alphas, token_num_floor-- def gen_frame_alignments(- self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None- ):- batch_size, maximum_length = alphas.size()- int_type = torch.int32-- is_training = self.training- if is_training:- token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)- else:- token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)-- max_token_num = torch.max(token_num).item()-- alphas_cumsum = torch.cumsum(alphas, dim=1)- alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)- alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)-- index = torch.ones([batch_size, max_token_num], dtype=int_type)- index = torch.cumsum(index, dim=1)- index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)-- index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)- index_div_bool_zeros = index_div.eq(0)- index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1- index_div_bool_zeros_count = torch.clamp(- index_div_bool_zeros_count, 0, encoder_sequence_length.max()- )- token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)- index_div_bool_zeros_count *= token_num_mask-- index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(- 1, 1, maximum_length- )- ones = torch.ones_like(index_div_bool_zeros_count_tile)- zeros = torch.zeros_like(index_div_bool_zeros_count_tile)- ones = torch.cumsum(ones, dim=2)- cond = index_div_bool_zeros_count_tile == ones- index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)-- index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)- index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)- index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)- predictor_mask = (- (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))- .type(int_type)- .to(encoder_sequence_length.device)- )- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask-- predictor_alignments = index_div_bool_zeros_count_tile_out- predictor_alignments_length = predictor_alignments.sum(-1).type(- encoder_sequence_length.dtype- )- return predictor_alignments.detach(), predictor_alignments_length.detach()--[email protected]("predictor_classes", "CifPredictorV2Export")-class CifPredictorV2Export(torch.nn.Module):- def __init__(self, model, **kwargs):- super().__init__()-- self.pad = model.pad- self.cif_conv1d = model.cif_conv1d- self.cif_output = model.cif_output- self.threshold = model.threshold- self.smooth_factor = model.smooth_factor- self.noise_threshold = model.noise_threshold- self.tail_threshold = model.tail_threshold-- def forward(- self,- hidden: torch.Tensor,- mask: torch.Tensor,- ):- alphas, token_num = self.forward_cnn(hidden, mask)- mask = mask.transpose(-1, -2).float()- mask = mask.squeeze(-1)- hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)- acoustic_embeds, cif_peak = cif_v1_export(hidden, alphas, self.threshold)-- return acoustic_embeds, token_num, alphas, cif_peak-- def forward_cnn(- self,- hidden: torch.Tensor,- mask: torch.Tensor,- ):- h = hidden- context = h.transpose(1, 2)- queries = self.pad(context)- output = torch.relu(self.cif_conv1d(queries))- output = output.transpose(1, 2)-- output = self.cif_output(output)- alphas = torch.sigmoid(output)- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)- mask = mask.transpose(-1, -2).float()- alphas = alphas * mask- alphas = alphas.squeeze(-1)- token_num = alphas.sum(-1)-- return alphas, token_num-- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):- b, t, d = hidden.size()- tail_threshold = self.tail_threshold-- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)- ones_t = torch.ones_like(zeros_t)-- mask_1 = torch.cat([mask, zeros_t], dim=1)- mask_2 = torch.cat([ones_t, mask], dim=1)- mask = mask_2 - mask_1- tail_threshold = mask * tail_threshold- alphas = torch.cat([alphas, zeros_t], dim=1)- alphas = torch.add(alphas, tail_threshold)-- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)- hidden = torch.cat([hidden, zeros], dim=1)- token_num = alphas.sum(dim=-1)- token_num_floor = torch.floor(token_num)-- return hidden, alphas, token_num_floor---def cif_v1_export(hidden, alphas, threshold: float):- device = hidden.device- dtype = hidden.dtype- batch_size, len_time, hidden_size = hidden.size()- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)-- frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)- fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)-- # prefix_sum = torch.cumsum(alphas, dim=1)- prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(- torch.float32- ) # cumsum precision degradation cause wrong result in extreme- prefix_sum_floor = torch.floor(prefix_sum)- dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)- dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)-- dislocation_prefix_sum_floor[:, 0] = 0- dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor-- fire_idxs = dislocation_diff > 0- fires[fire_idxs] = 1- fires = fires + prefix_sum - prefix_sum_floor-- # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)- prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)- frames = prefix_sum_hidden[fire_idxs]- shift_frames = torch.roll(frames, 1, dims=0)-- batch_len = fire_idxs.sum(1)- batch_idxs = torch.cumsum(batch_len, dim=0)- shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)- shift_batch_idxs[0] = 0- shift_frames[shift_batch_idxs] = 0-- remains = fires - torch.floor(fires)- # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]- remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]-- shift_remain_frames = torch.roll(remain_frames, 1, dims=0)- shift_remain_frames[shift_batch_idxs] = 0-- frames = frames - shift_frames + shift_remain_frames - remain_frames-- # max_label_len = batch_len.max()- max_label_len = alphas.sum(dim=-1)- max_label_len = torch.floor(max_label_len).max().to(dtype=torch.int64)-- # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)- frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)- indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)- frame_fires_idxs = indices < batch_len.unsqueeze(1)- frame_fires[frame_fires_idxs] = frames- return frame_fires, fires---def cif_export(hidden, alphas, threshold: float):- batch_size, len_time, hidden_size = hidden.size()- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)-- # loop varss- integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)- frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)- # intermediate vars along time- list_fires = []- list_frames = []-- for t in range(len_time):- alpha = alphas[:, t]- distribution_completion = (- torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate- )-- integrate += alpha- list_fires.append(integrate)-- fire_place = integrate >= threshold- integrate = torch.where(- fire_place,- integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),- integrate,- )- cur = torch.where(fire_place, distribution_completion, alpha)- remainds = alpha - cur-- frame += cur[:, None] * hidden[:, t, :]- list_frames.append(frame)- frame = torch.where(- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame- )-- fires = torch.stack(list_fires, 1)- frames = torch.stack(list_frames, 1)-- fire_idxs = fires >= threshold- frame_fires = torch.zeros_like(hidden)- max_label_len = frames[0, fire_idxs[0]].size(0)- for b in range(batch_size):- frame_fire = frames[b, fire_idxs[b]]- frame_len = frame_fire.size(0)- frame_fires[b, :frame_len, :] = frame_fire-- if frame_len >= max_label_len:- max_label_len = frame_len- frame_fires = frame_fires[:, :max_label_len, :]- return frame_fires, fires---class mae_loss(torch.nn.Module):-- def __init__(self, normalize_length=False):- super(mae_loss, self).__init__()- self.normalize_length = normalize_length- self.criterion = torch.nn.L1Loss(reduction="sum")-- def forward(self, token_length, pre_token_length):- loss_token_normalizer = token_length.size(0)- if self.normalize_length:- loss_token_normalizer = token_length.sum().type(torch.float32)- loss = self.criterion(token_length, pre_token_length)- loss = loss / loss_token_normalizer- return loss---def cif(hidden, alphas, threshold):- batch_size, len_time, hidden_size = hidden.size()-- # loop varss- integrate = torch.zeros([batch_size], device=hidden.device)- frame = torch.zeros([batch_size, hidden_size], device=hidden.device)- # intermediate vars along time- list_fires = []- list_frames = []-- for t in range(len_time):- alpha = alphas[:, t]- distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate-- integrate += alpha- list_fires.append(integrate)-- fire_place = integrate >= threshold- integrate = torch.where(- fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate- )- cur = torch.where(fire_place, distribution_completion, alpha)- remainds = alpha - cur-- frame += cur[:, None] * hidden[:, t, :]- list_frames.append(frame)- frame = torch.where(- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame- )-- fires = torch.stack(list_fires, 1)- frames = torch.stack(list_frames, 1)- list_ls = []- len_labels = torch.round(alphas.sum(-1)).int()- max_label_len = len_labels.max()- for b in range(batch_size):- fire = fires[b, :]- l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze())- pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)- list_ls.append(torch.cat([l, pad_l], 0))- return torch.stack(list_ls, 0), fires---def cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=False):- batch_size, len_time = alphas.size()- device = alphas.device- dtype = alphas.dtype-- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)-- fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)-- # prefix_sum = torch.cumsum(alphas, dim=1)- prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(- torch.float32- ) # cumsum precision degradation cause wrong result in extreme- prefix_sum_floor = torch.floor(prefix_sum)- dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)- dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)-- dislocation_prefix_sum_floor[:, 0] = 0- dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor-- fire_idxs = dislocation_diff > 0- fires[fire_idxs] = 1- fires = fires + prefix_sum - prefix_sum_floor- if return_fire_idxs:- return fires, fire_idxs- return fires---def cif_v1(hidden, alphas, threshold):- fires, fire_idxs = cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=True)-- device = hidden.device- dtype = hidden.dtype- batch_size, len_time, hidden_size = hidden.size()- # frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)- # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)- frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)- prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)-- frames = prefix_sum_hidden[fire_idxs]- shift_frames = torch.roll(frames, 1, dims=0)-- batch_len = fire_idxs.sum(1)- batch_idxs = torch.cumsum(batch_len, dim=0)- shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)- shift_batch_idxs[0] = 0- shift_frames[shift_batch_idxs] = 0-- remains = fires - torch.floor(fires)- # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]- remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]-- shift_remain_frames = torch.roll(remain_frames, 1, dims=0)- shift_remain_frames[shift_batch_idxs] = 0-- frames = frames - shift_frames + shift_remain_frames - remain_frames-- # max_label_len = batch_len.max()- max_label_len = (- torch.round(alphas.sum(-1)).int().max()- ) # torch.round to calculate the max length-- # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)- frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)- indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)- frame_fires_idxs = indices < batch_len.unsqueeze(1)- frame_fires[frame_fires_idxs] = frames- return frame_fires, fires---def cif_wo_hidden(alphas, threshold):- batch_size, len_time = alphas.size()-- # loop varss- integrate = torch.zeros([batch_size], device=alphas.device)- # intermediate vars along time- list_fires = []-- for t in range(len_time):- alpha = alphas[:, t]-- integrate += alpha- list_fires.append(integrate)-- fire_place = integrate >= threshold- integrate = torch.where(- fire_place,- integrate - torch.ones([batch_size], device=alphas.device) * threshold,- integrate,- )-- fires = torch.stack(list_fires, 1)- return fires+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++import torch+import logging+import numpy as np++from funasr.register import tables+from funasr.train_utils.device_funcs import to_device+from funasr.models.transformer.utils.nets_utils import make_pad_mask+from torch.cuda.amp import autocast++[email protected]("predictor_classes", "CifPredictor")+class CifPredictor(torch.nn.Module):+ def __init__(+ self,+ idim,+ l_order,+ r_order,+ threshold=1.0,+ dropout=0.1,+ smooth_factor=1.0,+ noise_threshold=0,+ tail_threshold=0.45,+ ):+ """Initialize CifPredictor.++ Args:+ idim: TODO.+ l_order: TODO.+ r_order: TODO.+ threshold: TODO.+ dropout: TODO.+ smooth_factor: TODO.+ noise_threshold: TODO.+ tail_threshold: TODO.+ """+ super().__init__()++ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)+ self.cif_output = torch.nn.Linear(idim, 1)+ self.dropout = torch.nn.Dropout(p=dropout)+ self.threshold = threshold+ self.smooth_factor = smooth_factor+ self.noise_threshold = noise_threshold+ self.tail_threshold = tail_threshold++ def forward(+ self,+ hidden,+ target_label=None,+ mask=None,+ ignore_id=-1,+ mask_chunk_predictor=None,+ target_label_length=None,+ ):++ """Forward pass for training.++ Args:+ hidden: TODO.+ target_label: TODO.+ mask: TODO.+ ignore_id: TODO.+ mask_chunk_predictor: TODO.+ target_label_length: TODO.+ """+ with autocast(False):+ h = hidden+ context = h.transpose(1, 2)+ queries = self.pad(context)+ memory = self.cif_conv1d(queries)+ output = memory + context+ output = self.dropout(output)+ output = output.transpose(1, 2)+ output = torch.relu(output)+ output = self.cif_output(output)+ alphas = torch.sigmoid(output)+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)+ if mask is not None:+ mask = mask.transpose(-1, -2).float()+ alphas = alphas * mask+ if mask_chunk_predictor is not None:+ alphas = alphas * mask_chunk_predictor+ alphas = alphas.squeeze(-1)+ mask = mask.squeeze(-1)+ if target_label_length is not None:+ target_length = target_label_length+ elif target_label is not None:+ target_length = (target_label != ignore_id).float().sum(-1)+ else:+ target_length = None+ token_num = alphas.sum(-1)+ if target_length is not None:+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))+ elif self.tail_threshold > 0.0:+ hidden, alphas, token_num = self.tail_process_fn(+ hidden, alphas, token_num, mask=mask+ )++ acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)++ if target_length is None and self.tail_threshold > 0.0:+ token_num_int = torch.max(token_num).type(torch.int32).item()+ acoustic_embeds = acoustic_embeds[:, :token_num_int, :]++ return acoustic_embeds, token_num, alphas, cif_peak++ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):+ """Tail process fn.++ Args:+ hidden: TODO.+ alphas: TODO.+ token_num: TODO.+ mask: TODO.+ """+ b, t, d = hidden.size()+ tail_threshold = self.tail_threshold+ if mask is not None:+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)+ ones_t = torch.ones_like(zeros_t)+ mask_1 = torch.cat([mask, zeros_t], dim=1)+ mask_2 = torch.cat([ones_t, mask], dim=1)+ mask = mask_2 - mask_1+ tail_threshold = mask * tail_threshold+ alphas = torch.cat([alphas, zeros_t], dim=1)+ alphas = torch.add(alphas, tail_threshold)+ else:+ tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)+ tail_threshold = torch.reshape(tail_threshold, (1, 1))+ alphas = torch.cat([alphas, tail_threshold], dim=1)+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)+ hidden = torch.cat([hidden, zeros], dim=1)+ token_num = alphas.sum(dim=-1)+ token_num_floor = torch.floor(token_num)++ return hidden, alphas, token_num_floor++ def gen_frame_alignments(+ self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None+ ):+ """Gen frame alignments.++ Args:+ alphas: TODO.+ encoder_sequence_length: TODO.+ """+ batch_size, maximum_length = alphas.size()+ int_type = torch.int32++ is_training = self.training+ if is_training:+ token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)+ else:+ token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)++ max_token_num = torch.max(token_num).item()++ alphas_cumsum = torch.cumsum(alphas, dim=1)+ alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)+ alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)++ index = torch.ones([batch_size, max_token_num], dtype=int_type)+ index = torch.cumsum(index, dim=1)+ index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)++ index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)+ index_div_bool_zeros = index_div.eq(0)+ index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1+ index_div_bool_zeros_count = torch.clamp(+ index_div_bool_zeros_count, 0, encoder_sequence_length.max()+ )+ token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)+ index_div_bool_zeros_count *= token_num_mask++ index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(+ 1, 1, maximum_length+ )+ ones = torch.ones_like(index_div_bool_zeros_count_tile)+ zeros = torch.zeros_like(index_div_bool_zeros_count_tile)+ ones = torch.cumsum(ones, dim=2)+ cond = index_div_bool_zeros_count_tile == ones+ index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)++ index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)+ index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)+ index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)+ predictor_mask = (+ (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))+ .type(int_type)+ .to(encoder_sequence_length.device)+ )+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask++ predictor_alignments = index_div_bool_zeros_count_tile_out+ predictor_alignments_length = predictor_alignments.sum(-1).type(+ encoder_sequence_length.dtype+ )+ return predictor_alignments.detach(), predictor_alignments_length.detach()++[email protected]("predictor_classes", "CifPredictorV2")+class CifPredictorV2(torch.nn.Module):+ def __init__(+ self,+ idim,+ l_order,+ r_order,+ threshold=1.0,+ dropout=0.1,+ smooth_factor=1.0,+ noise_threshold=0,+ tail_threshold=0.0,+ tf2torch_tensor_name_prefix_torch="predictor",+ tf2torch_tensor_name_prefix_tf="seq2seq/cif",+ tail_mask=True,+ ):+ """Initialize CifPredictorV2.++ Args:+ idim: TODO.+ l_order: TODO.+ r_order: TODO.+ threshold: TODO.+ dropout: TODO.+ smooth_factor: TODO.+ noise_threshold: TODO.+ tail_threshold: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ tail_mask: TODO.+ """+ super().__init__()++ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)+ self.cif_output = torch.nn.Linear(idim, 1)+ self.dropout = torch.nn.Dropout(p=dropout)+ self.threshold = threshold+ self.smooth_factor = smooth_factor+ self.noise_threshold = noise_threshold+ self.tail_threshold = tail_threshold+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf+ self.tail_mask = tail_mask++ def forward(+ self,+ hidden,+ target_label=None,+ mask=None,+ ignore_id=-1,+ mask_chunk_predictor=None,+ target_label_length=None,+ ):++ """Forward pass for training.++ Args:+ hidden: TODO.+ target_label: TODO.+ mask: TODO.+ ignore_id: TODO.+ mask_chunk_predictor: TODO.+ target_label_length: TODO.+ """+ with autocast(False):+ h = hidden+ context = h.transpose(1, 2)+ queries = self.pad(context)+ output = torch.relu(self.cif_conv1d(queries))+ output = output.transpose(1, 2)++ output = self.cif_output(output)+ alphas = torch.sigmoid(output)+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)+ if mask is not None:+ mask = mask.transpose(-1, -2).float()+ alphas = alphas * mask+ if mask_chunk_predictor is not None:+ alphas = alphas * mask_chunk_predictor+ alphas = alphas.squeeze(-1)+ mask = mask.squeeze(-1)+ if target_label_length is not None:+ target_length = target_label_length.squeeze(-1)+ elif target_label is not None:+ target_length = (target_label != ignore_id).float().sum(-1)+ else:+ target_length = None+ token_num = alphas.sum(-1)+ if target_length is not None:+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))+ elif self.tail_threshold > 0.0:+ if self.tail_mask:+ hidden, alphas, token_num = self.tail_process_fn(+ hidden, alphas, token_num, mask=mask+ )+ else:+ hidden, alphas, token_num = self.tail_process_fn(+ hidden, alphas, token_num, mask=None+ )++ acoustic_embeds, cif_peak = cif_v1(hidden, alphas, self.threshold)+ if target_length is None and self.tail_threshold > 0.0:+ token_num_int = torch.max(token_num).type(torch.int32).item()+ acoustic_embeds = acoustic_embeds[:, :token_num_int, :]++ return acoustic_embeds, token_num, alphas, cif_peak++ def forward_chunk(self, hidden, cache=None, **kwargs):+ """Forward chunk.++ Args:+ hidden: TODO.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """+ is_final = kwargs.get("is_final", False)+ batch_size, len_time, hidden_size = hidden.shape+ h = hidden+ context = h.transpose(1, 2)+ queries = self.pad(context)+ output = torch.relu(self.cif_conv1d(queries))+ output = output.transpose(1, 2)+ output = self.cif_output(output)+ alphas = torch.sigmoid(output)+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)++ alphas = alphas.squeeze(-1)++ token_length = []+ list_fires = []+ list_frames = []+ cache_alphas = []+ cache_hiddens = []++ if cache is not None and "chunk_size" in cache:+ alphas[:, : cache["chunk_size"][0]] = 0.0+ if not is_final:+ alphas[:, sum(cache["chunk_size"][:2]) :] = 0.0+ if cache is not None and "cif_alphas" in cache and "cif_hidden" in cache:+ cache["cif_hidden"] = to_device(cache["cif_hidden"], device=hidden.device)+ cache["cif_alphas"] = to_device(cache["cif_alphas"], device=alphas.device)+ hidden = torch.cat((cache["cif_hidden"], hidden), dim=1)+ alphas = torch.cat((cache["cif_alphas"], alphas), dim=1)+ if cache is not None and is_final:+ tail_hidden = torch.zeros((batch_size, 1, hidden_size), device=hidden.device)+ tail_alphas = torch.tensor([[self.tail_threshold]], device=alphas.device)+ tail_alphas = torch.tile(tail_alphas, (batch_size, 1))+ hidden = torch.cat((hidden, tail_hidden), dim=1)+ alphas = torch.cat((alphas, tail_alphas), dim=1)++ len_time = alphas.shape[1]+ for b in range(batch_size):+ integrate = 0.0+ frames = torch.zeros((hidden_size), device=hidden.device)+ list_frame = []+ list_fire = []+ for t in range(len_time):+ alpha = alphas[b][t]+ if alpha + integrate < self.threshold:+ integrate += alpha+ list_fire.append(integrate)+ frames += alpha * hidden[b][t]+ else:+ frames += (self.threshold - integrate) * hidden[b][t]+ list_frame.append(frames)+ integrate += alpha+ list_fire.append(integrate)+ integrate -= self.threshold+ frames = integrate * hidden[b][t]++ cache_alphas.append(integrate)+ if integrate > 0.0:+ cache_hiddens.append(frames / integrate)+ else:+ cache_hiddens.append(frames)++ token_length.append(torch.tensor(len(list_frame), device=alphas.device))+ list_fires.append(list_fire)+ list_frames.append(list_frame)++ cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)+ cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)+ cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)+ cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)++ max_token_len = max(token_length)+ if max_token_len == 0:+ return hidden, torch.stack(token_length, 0), None, None+ list_ls = []+ for b in range(batch_size):+ pad_frames = torch.zeros(+ (max_token_len - token_length[b], hidden_size), device=alphas.device+ )+ if token_length[b] == 0:+ list_ls.append(pad_frames)+ else:+ list_frames[b] = torch.stack(list_frames[b])+ list_ls.append(torch.cat((list_frames[b], pad_frames), dim=0))++ cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)+ cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)+ cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)+ cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)+ return torch.stack(list_ls, 0), torch.stack(token_length, 0), None, None++ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):+ """Tail process fn.++ Args:+ hidden: TODO.+ alphas: TODO.+ token_num: TODO.+ mask: TODO.+ """+ b, t, d = hidden.size()+ tail_threshold = self.tail_threshold+ if mask is not None:+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)+ ones_t = torch.ones_like(zeros_t)+ mask_1 = torch.cat([mask, zeros_t], dim=1)+ mask_2 = torch.cat([ones_t, mask], dim=1)+ mask = mask_2 - mask_1+ tail_threshold = mask * tail_threshold+ alphas = torch.cat([alphas, zeros_t], dim=1)+ alphas = torch.add(alphas, tail_threshold)+ else:+ tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)+ tail_threshold = torch.reshape(tail_threshold, (1, 1))+ if b > 1:+ alphas = torch.cat([alphas, tail_threshold.repeat(b, 1)], dim=1)+ else:+ alphas = torch.cat([alphas, tail_threshold], dim=1)+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)+ hidden = torch.cat([hidden, zeros], dim=1)+ token_num = alphas.sum(dim=-1)+ token_num_floor = torch.floor(token_num)++ return hidden, alphas, token_num_floor++ def gen_frame_alignments(+ self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None+ ):+ """Gen frame alignments.++ Args:+ alphas: TODO.+ encoder_sequence_length: TODO.+ """+ batch_size, maximum_length = alphas.size()+ int_type = torch.int32++ is_training = self.training+ if is_training:+ token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)+ else:+ token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)++ max_token_num = torch.max(token_num).item()++ alphas_cumsum = torch.cumsum(alphas, dim=1)+ alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)+ alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)++ index = torch.ones([batch_size, max_token_num], dtype=int_type)+ index = torch.cumsum(index, dim=1)+ index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)++ index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)+ index_div_bool_zeros = index_div.eq(0)+ index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1+ index_div_bool_zeros_count = torch.clamp(+ index_div_bool_zeros_count, 0, encoder_sequence_length.max()+ )+ token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)+ index_div_bool_zeros_count *= token_num_mask++ index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(+ 1, 1, maximum_length+ )+ ones = torch.ones_like(index_div_bool_zeros_count_tile)+ zeros = torch.zeros_like(index_div_bool_zeros_count_tile)+ ones = torch.cumsum(ones, dim=2)+ cond = index_div_bool_zeros_count_tile == ones+ index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)++ index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)+ index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)+ index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)+ predictor_mask = (+ (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))+ .type(int_type)+ .to(encoder_sequence_length.device)+ )+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask++ predictor_alignments = index_div_bool_zeros_count_tile_out+ predictor_alignments_length = predictor_alignments.sum(-1).type(+ encoder_sequence_length.dtype+ )+ return predictor_alignments.detach(), predictor_alignments_length.detach()++[email protected]("predictor_classes", "CifPredictorV2Export")+class CifPredictorV2Export(torch.nn.Module):+ def __init__(self, model, **kwargs):+ """Initialize CifPredictorV2Export.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """+ super().__init__()++ self.pad = model.pad+ self.cif_conv1d = model.cif_conv1d+ self.cif_output = model.cif_output+ self.threshold = model.threshold+ self.smooth_factor = model.smooth_factor+ self.noise_threshold = model.noise_threshold+ self.tail_threshold = model.tail_threshold++ def forward(+ self,+ hidden: torch.Tensor,+ mask: torch.Tensor,+ ):+ """Forward pass for training.++ Args:+ hidden: TODO.+ mask: TODO.+ """+ alphas, token_num = self.forward_cnn(hidden, mask)+ mask = mask.transpose(-1, -2).float()+ mask = mask.squeeze(-1)+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)+ acoustic_embeds, cif_peak = cif_v1_export(hidden, alphas, self.threshold)++ return acoustic_embeds, token_num, alphas, cif_peak++ def forward_cnn(+ self,+ hidden: torch.Tensor,+ mask: torch.Tensor,+ ):+ """Forward cnn.++ Args:+ hidden: TODO.+ mask: TODO.+ """+ h = hidden+ context = h.transpose(1, 2)+ queries = self.pad(context)+ output = torch.relu(self.cif_conv1d(queries))+ output = output.transpose(1, 2)++ output = self.cif_output(output)+ alphas = torch.sigmoid(output)+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)+ mask = mask.transpose(-1, -2).float()+ alphas = alphas * mask+ alphas = alphas.squeeze(-1)+ token_num = alphas.sum(-1)++ return alphas, token_num++ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):+ """Tail process fn.++ Args:+ hidden: TODO.+ alphas: TODO.+ token_num: TODO.+ mask: TODO.+ """+ b, t, d = hidden.size()+ tail_threshold = self.tail_threshold++ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)+ ones_t = torch.ones_like(zeros_t)++ mask_1 = torch.cat([mask, zeros_t], dim=1)+ mask_2 = torch.cat([ones_t, mask], dim=1)+ mask = mask_2 - mask_1+ tail_threshold = mask * tail_threshold+ alphas = torch.cat([alphas, zeros_t], dim=1)+ alphas = torch.add(alphas, tail_threshold)++ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)+ hidden = torch.cat([hidden, zeros], dim=1)+ token_num = alphas.sum(dim=-1)+ token_num_floor = torch.floor(token_num)++ return hidden, alphas, token_num_floor+++def cif_v1_export(hidden, alphas, threshold: float):+ """Cif v1 export.++ Args:+ hidden: TODO.+ alphas: TODO.+ threshold: TODO.+ """+ device = hidden.device+ dtype = hidden.dtype+ batch_size, len_time, hidden_size = hidden.size()+ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)++ frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)+ fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)++ # prefix_sum = torch.cumsum(alphas, dim=1)+ prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(+ torch.float32+ ) # cumsum precision degradation cause wrong result in extreme+ prefix_sum_floor = torch.floor(prefix_sum)+ dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)+ dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)++ dislocation_prefix_sum_floor[:, 0] = 0+ dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor++ fire_idxs = dislocation_diff > 0+ fires[fire_idxs] = 1+ fires = fires + prefix_sum - prefix_sum_floor++ # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)+ prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)+ frames = prefix_sum_hidden[fire_idxs]+ shift_frames = torch.roll(frames, 1, dims=0)++ batch_len = fire_idxs.sum(1)+ batch_idxs = torch.cumsum(batch_len, dim=0)+ shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)+ shift_batch_idxs[0] = 0+ shift_frames[shift_batch_idxs] = 0++ remains = fires - torch.floor(fires)+ # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]+ remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]++ shift_remain_frames = torch.roll(remain_frames, 1, dims=0)+ shift_remain_frames[shift_batch_idxs] = 0++ frames = frames - shift_frames + shift_remain_frames - remain_frames++ # max_label_len = batch_len.max()+ max_label_len = alphas.sum(dim=-1)+ max_label_len = torch.floor(max_label_len).max().to(dtype=torch.int64)++ # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)+ frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)+ indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)+ frame_fires_idxs = indices < batch_len.unsqueeze(1)+ frame_fires[frame_fires_idxs] = frames+ return frame_fires, fires+++def cif_export(hidden, alphas, threshold: float):+ """Cif export.++ Args:+ hidden: TODO.+ alphas: TODO.+ threshold: TODO.+ """+ batch_size, len_time, hidden_size = hidden.size()+ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)++ # loop varss+ integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)+ frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)+ # intermediate vars along time+ list_fires = []+ list_frames = []++ for t in range(len_time):+ alpha = alphas[:, t]+ distribution_completion = (+ torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate+ )++ integrate += alpha+ list_fires.append(integrate)++ fire_place = integrate >= threshold+ integrate = torch.where(+ fire_place,+ integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),+ integrate,+ )+ cur = torch.where(fire_place, distribution_completion, alpha)+ remainds = alpha - cur++ frame += cur[:, None] * hidden[:, t, :]+ list_frames.append(frame)+ frame = torch.where(+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame+ )++ fires = torch.stack(list_fires, 1)+ frames = torch.stack(list_frames, 1)++ fire_idxs = fires >= threshold+ frame_fires = torch.zeros_like(hidden)+ max_label_len = frames[0, fire_idxs[0]].size(0)+ for b in range(batch_size):+ frame_fire = frames[b, fire_idxs[b]]+ frame_len = frame_fire.size(0)+ frame_fires[b, :frame_len, :] = frame_fire++ if frame_len >= max_label_len:+ max_label_len = frame_len+ frame_fires = frame_fires[:, :max_label_len, :]+ return frame_fires, fires+++class mae_loss(torch.nn.Module):++ def __init__(self, normalize_length=False):+ """Initialize mae_loss.++ Args:+ normalize_length: TODO.+ """+ super(mae_loss, self).__init__()+ self.normalize_length = normalize_length+ self.criterion = torch.nn.L1Loss(reduction="sum")++ def forward(self, token_length, pre_token_length):+ """Forward pass for training.++ Args:+ token_length: TODO.+ pre_token_length: TODO.+ """+ loss_token_normalizer = token_length.size(0)+ if self.normalize_length:+ loss_token_normalizer = token_length.sum().type(torch.float32)+ loss = self.criterion(token_length, pre_token_length)+ loss = loss / loss_token_normalizer+ return loss+++def cif(hidden, alphas, threshold):+ """Cif.++ Args:+ hidden: TODO.+ alphas: TODO.+ threshold: TODO.+ """+ batch_size, len_time, hidden_size = hidden.size()++ # loop varss+ integrate = torch.zeros([batch_size], device=hidden.device)+ frame = torch.zeros([batch_size, hidden_size], device=hidden.device)+ # intermediate vars along time+ list_fires = []+ list_frames = []++ for t in range(len_time):+ alpha = alphas[:, t]+ distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate++ integrate += alpha+ list_fires.append(integrate)++ fire_place = integrate >= threshold+ integrate = torch.where(+ fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate+ )+ cur = torch.where(fire_place, distribution_completion, alpha)+ remainds = alpha - cur++ frame += cur[:, None] * hidden[:, t, :]+ list_frames.append(frame)+ frame = torch.where(+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame+ )++ fires = torch.stack(list_fires, 1)+ frames = torch.stack(list_frames, 1)+ list_ls = []+ len_labels = torch.round(alphas.sum(-1)).int()+ max_label_len = len_labels.max()+ for b in range(batch_size):+ fire = fires[b, :]+ l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze())+ pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)+ list_ls.append(torch.cat([l, pad_l], 0))+ return torch.stack(list_ls, 0), fires+++def cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=False):+ """Cif wo hidden v1.++ Args:+ alphas: TODO.+ threshold: TODO.+ return_fire_idxs: TODO.+ """+ batch_size, len_time = alphas.size()+ device = alphas.device+ dtype = alphas.dtype++ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)++ fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)++ # prefix_sum = torch.cumsum(alphas, dim=1)+ prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(+ torch.float32+ ) # cumsum precision degradation cause wrong result in extreme+ prefix_sum_floor = torch.floor(prefix_sum)+ dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)+ dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)++ dislocation_prefix_sum_floor[:, 0] = 0+ dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor++ fire_idxs = dislocation_diff > 0+ fires[fire_idxs] = 1+ fires = fires + prefix_sum - prefix_sum_floor+ if return_fire_idxs:+ return fires, fire_idxs+ return fires+++def cif_v1(hidden, alphas, threshold):+ """Cif v1.++ Args:+ hidden: TODO.+ alphas: TODO.+ threshold: TODO.+ """+ fires, fire_idxs = cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=True)++ device = hidden.device+ dtype = hidden.dtype+ batch_size, len_time, hidden_size = hidden.size()+ # frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)+ # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)+ frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)+ prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)++ frames = prefix_sum_hidden[fire_idxs]+ shift_frames = torch.roll(frames, 1, dims=0)++ batch_len = fire_idxs.sum(1)+ batch_idxs = torch.cumsum(batch_len, dim=0)+ shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)+ shift_batch_idxs[0] = 0+ shift_frames[shift_batch_idxs] = 0++ remains = fires - torch.floor(fires)+ # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]+ remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]++ shift_remain_frames = torch.roll(remain_frames, 1, dims=0)+ shift_remain_frames[shift_batch_idxs] = 0++ frames = frames - shift_frames + shift_remain_frames - remain_frames++ # max_label_len = batch_len.max()+ max_label_len = (+ torch.round(alphas.sum(-1)).int().max()+ ) # torch.round to calculate the max length++ # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)+ frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)+ indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)+ frame_fires_idxs = indices < batch_len.unsqueeze(1)+ frame_fires[frame_fires_idxs] = frames+ return frame_fires, fires+++def cif_wo_hidden(alphas, threshold):+ """Cif wo hidden.++ Args:+ alphas: TODO.+ threshold: TODO.+ """+ batch_size, len_time = alphas.size()++ # loop varss+ integrate = torch.zeros([batch_size], device=alphas.device)+ # intermediate vars along time+ list_fires = []++ for t in range(len_time):+ alpha = alphas[:, t]++ integrate += alpha+ list_fires.append(integrate)++ fire_place = integrate >= threshold+ integrate = torch.where(+ fire_place,+ integrate - torch.ones([batch_size], device=alphas.device) * threshold,+ integrate,+ )++ fires = torch.stack(list_fires, 1)+ return firesdiff --git a/funasr/models/paraformer/decoder.py b/funasr/models/paraformer/decoder.pyindex 7edd91a2a..f784e0914 100644--- a/funasr/models/paraformer/decoder.py+++ b/funasr/models/paraformer/decoder.py@@ -121,6 +121,15 @@ class DecoderLayerSANM(torch.nn.Module):return x, tgt_mask, memory, memory_mask, cachedef get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Get attn mat.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt = self.feed_forward(tgt)@@ -257,6 +266,35 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):tf2torch_tensor_name_prefix_torch: str = "decoder",tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",):+ """Initialize ParaformerSANMDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ wo_input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ att_layer_num: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ lora_list: TODO.+ lora_rank: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ chunk_multiply_factor: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -426,6 +464,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):ys_in_lens: torch.Tensor,):+ """Forward asf2.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -444,6 +490,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):ys_in_lens: torch.Tensor,):+ """Forward asf6.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -592,6 +646,11 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):class DecoderLayerSANMExport(torch.nn.Module):def __init__(self, model):+ """Initialize DecoderLayerSANMExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.self_attn = model.self_attnself.src_attn = model.src_attn@@ -603,6 +662,15 @@ class DecoderLayerSANMExport(torch.nn.Module):def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Forward pass for training.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt = self.feed_forward(tgt)@@ -621,6 +689,15 @@ class DecoderLayerSANMExport(torch.nn.Module):return x, tgt_mask, memory, memory_mask, cachedef get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Get attn mat.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt = self.feed_forward(tgt)@@ -640,6 +717,15 @@ class DecoderLayerSANMExport(torch.nn.Module):@tables.register("decoder_classes", "ParaformerSANMDecoderExport")class ParaformerSANMDecoderExport(torch.nn.Module):def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):+ """Initialize ParaformerSANMDecoderExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ model_name: TODO.+ onnx: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# self.embed = model.embed #Embedding(model.embed, max_seq_len)@@ -673,6 +759,11 @@ class ParaformerSANMDecoderExport(torch.nn.Module):self.model_name = model_namedef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -692,6 +783,16 @@ class ParaformerSANMDecoderExport(torch.nn.Module):return_both: bool = False,):+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ return_hidden: TODO.+ return_both: TODO.+ """tgt = ys_in_padtgt_mask = self.make_pad_mask(ys_in_lens)tgt_mask, _ = self.prepare_mask(tgt_mask)@@ -728,6 +829,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):ys_in_lens: torch.Tensor,):+ """Forward asf2.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -749,6 +858,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):ys_in_lens: torch.Tensor,):+ """Forward asf6.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -829,6 +946,15 @@ class ParaformerSANMDecoderExport(torch.nn.Module):@tables.register("decoder_classes", "ParaformerSANMDecoderOnlineExport")class ParaformerSANMDecoderOnlineExport(torch.nn.Module):def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):+ """Initialize ParaformerSANMDecoderOnlineExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ model_name: TODO.+ onnx: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()# self.embed = model.embed #Embedding(model.embed, max_seq_len)self.model = model@@ -863,6 +989,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):self.model_name = model_namedef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -881,6 +1012,15 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):*args,):+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ *args: Variable positional arguments.+ """tgt = ys_in_padtgt_mask = self.make_pad_mask(ys_in_lens)tgt_mask, _ = self.prepare_mask(tgt_mask)@@ -913,6 +1053,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):return x, out_cachesdef get_dummy_inputs(self, enc_size):+ """Get dummy inputs.++ Args:+ enc_size: Size/dimension parameter.+ """enc = torch.randn(2, 100, enc_size).type(torch.float32)enc_len = torch.tensor([30, 100], dtype=torch.int32)acoustic_embeds = torch.randn(2, 10, enc_size).type(torch.float32)@@ -930,6 +1075,7 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):return (enc, enc_len, acoustic_embeds, acoustic_embeds_len, *cache)def get_input_names(self):+ """Get input names."""cache_num = len(self.model.decoders)if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:cache_num += len(self.model.decoders2)@@ -938,12 +1084,14 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):]def get_output_names(self):+ """Get output names."""cache_num = len(self.model.decoders)if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:cache_num += len(self.model.decoders2)return ["logits", "sample_ids"] + ["out_cache_%d" % i for i in range(cache_num)]def get_dynamic_axes(self):+ """Get dynamic axes."""ret = {"enc": {0: "batch_size", 1: "enc_length"},"acoustic_embeds": {0: "batch_size", 1: "token_length"},@@ -1004,6 +1152,25 @@ class ParaformerSANDecoder(BaseTransformerDecoder):concat_after: bool = False,embeds_id: int = -1,):+ """Initialize ParaformerSANDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ embeds_id: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -1093,6 +1260,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):model_name="decoder",onnx: bool = True,):+ """Initialize ParaformerDecoderSANExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ model_name: TODO.+ onnx: TODO.+ """super().__init__()# self.embed = model.embed #Embedding(model.embed, max_seq_len)self.model = model@@ -1116,6 +1291,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):self.model_name = model_namedef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -1133,6 +1313,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):ys_in_lens: torch.Tensor,):+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = self.make_pad_mask(ys_in_lens)tgt_mask, _ = self.prepare_mask(tgt_mask)@@ -1151,6 +1339,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):return x, ys_in_lensdef get_dummy_inputs(self, enc_size):+ """Get dummy inputs.++ Args:+ enc_size: Size/dimension parameter.+ """tgt = torch.LongTensor([0]).unsqueeze(0)memory = torch.randn(1, 100, enc_size)pre_acoustic_embeds = torch.randn(1, 1, enc_size)@@ -1164,17 +1357,21 @@ class ParaformerDecoderSANExport(torch.nn.Module):return (tgt, memory, pre_acoustic_embeds, cache)def is_optimizable(self):+ """Is optimizable."""return Truedef get_input_names(self):+ """Get input names."""cache_num = len(self.model.decoders) + len(self.model.decoders2)return ["tgt", "memory", "pre_acoustic_embeds"] + ["cache_%d" % i for i in range(cache_num)]def get_output_names(self):+ """Get output names."""cache_num = len(self.model.decoders) + len(self.model.decoders2)return ["y"] + ["out_cache_%d" % i for i in range(cache_num)]def get_dynamic_axes(self):+ """Get dynamic axes."""ret = {"tgt": {0: "tgt_batch", 1: "tgt_length"},"memory": {0: "memory_batch", 1: "memory_length"},diff --git a/funasr/models/paraformer/export_meta.py b/funasr/models/paraformer/export_meta.pyindex 8e086a212..0e8d194af 100644--- a/funasr/models/paraformer/export_meta.py+++ b/funasr/models/paraformer/export_meta.py@@ -9,6 +9,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """model.device = kwargs.get("device")is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")@@ -41,6 +47,12 @@ def export_forward(speech_lengths: torch.Tensor,):# a. To device+ """Export forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """batch = {"speech": speech, "speech_lengths": speech_lengths}# batch = to_device(batch, device=self.device)@@ -57,20 +69,24 @@ def export_forward(def export_dummy_inputs(self):+ """Export dummy inputs."""speech = torch.randn(2, 30, 560)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)return (speech, speech_lengths)def export_input_names(self):+ """Export input names."""return ["speech", "speech_lengths"]def export_output_names(self):+ """Export output names."""return ["logits", "token_num"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"},"speech_lengths": {@@ -84,4 +100,5 @@ def export_dynamic_axes(self):def export_name(self,):+ """Export name."""return "model.onnx"diff --git a/funasr/models/paraformer/model.py b/funasr/models/paraformer/model.pyindex fc20c13de..5060d8f20 100644--- a/funasr/models/paraformer/model.py+++ b/funasr/models/paraformer/model.py@@ -89,6 +89,37 @@ class Paraformer(torch.nn.Module):**kwargs,):+ """Initialize Paraformer.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ predictor: TODO.+ predictor_conf: Configuration dict for predictor.+ ctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ predictor_weight: TODO.+ predictor_bias: TODO.+ sampling_ratio: TODO.+ share_embedding: TODO.+ use_1st_decoder_loss: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -283,6 +314,12 @@ class Paraformer(torch.nn.Module):def calc_predictor(self, encoder_out, encoder_out_lens):+ """Calc predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -295,6 +332,14 @@ class Paraformer(torch.nn.Module):self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens):+ """Cal decoder with predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ sematic_embeds: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """decoder_outs = self.decoder(encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens)decoder_out = decoder_outs[0]decoder_out = torch.log_softmax(decoder_out, dim=-1)@@ -307,6 +352,14 @@ class Paraformer(torch.nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -354,6 +407,15 @@ class Paraformer(torch.nn.Module):def sampler(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):+ """Sampler.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ pre_acoustic_embeds: TODO.+ """tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)@@ -400,6 +462,14 @@ class Paraformer(torch.nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -413,6 +483,11 @@ class Paraformer(torch.nn.Module):self,**kwargs,):+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.paraformer.search import BeamSearchParafrom funasr.models.transformer.scorers.ctc import CTCPrefixScorerfrom funasr.models.transformer.scorers.length_bonus import LengthBonus@@ -466,6 +541,16 @@ class Paraformer(torch.nn.Module):**kwargs,):# init beamsearch+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != Noneis_use_lm = (kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None@@ -612,6 +697,11 @@ class Paraformer(torch.nn.Module):return results, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelif "max_seq_len" not in kwargs:diff --git a/funasr/models/paraformer_streaming/export_meta.py b/funasr/models/paraformer_streaming/export_meta.pyindex 681568849..448a8266d 100644--- a/funasr/models/paraformer_streaming/export_meta.py+++ b/funasr/models/paraformer_streaming/export_meta.py@@ -10,6 +10,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):# self.device = kwargs.get("device")+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")model.encoder = encoder_class(model.encoder, onnx=is_onnx)@@ -57,6 +63,12 @@ def export_encoder_forward(speech_lengths: torch.Tensor,):# a. To device+ """Export encoder forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """batch = {"speech": speech, "speech_lengths": speech_lengths, "online": True}# batch = to_device(batch, device=self.device)@@ -68,20 +80,24 @@ def export_encoder_forward(def export_encoder_dummy_inputs(self):+ """Export encoder dummy inputs."""speech = torch.randn(2, 30, 560)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)return (speech, speech_lengths)def export_encoder_input_names(self):+ """Export encoder input names."""return ["speech", "speech_lengths"]def export_encoder_output_names(self):+ """Export encoder output names."""return ["enc", "enc_len", "alphas"]def export_encoder_dynamic_axes(self):+ """Export encoder dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"},"speech_lengths": {@@ -96,6 +112,7 @@ def export_encoder_dynamic_axes(self):def export_encoder_name(self):+ """Export encoder name."""return "model.onnx"@@ -107,6 +124,15 @@ def export_decoder_forward(acoustic_embeds_len: torch.Tensor,*args,):+ """Export decoder forward.++ Args:+ enc: TODO.+ enc_len: TODO.+ acoustic_embeds: TODO.+ acoustic_embeds_len: TODO.+ *args: Variable positional arguments.+ """decoder_out, out_caches = self.decoder(enc, enc_len, acoustic_embeds, acoustic_embeds_len, *args)@@ -116,21 +142,26 @@ def export_decoder_forward(def export_decoder_dummy_inputs(self):+ """Export decoder dummy inputs."""dummy_inputs = self.decoder.get_dummy_inputs(enc_size=self.encoder._output_size)return dummy_inputsdef export_decoder_input_names(self):+ """Export decoder input names."""return self.decoder.get_input_names()def export_decoder_output_names(self):+ """Export decoder output names."""return self.decoder.get_output_names()def export_decoder_dynamic_axes(self):+ """Export decoder dynamic axes."""return self.decoder.get_dynamic_axes()def export_decoder_name(self):+ """Export decoder name."""return "decoder.onnx"diff --git a/funasr/models/paraformer_streaming/model.py b/funasr/models/paraformer_streaming/model.pyindex 9b9e609ab..d463aa269 100644--- a/funasr/models/paraformer_streaming/model.py+++ b/funasr/models/paraformer_streaming/model.py@@ -31,6 +31,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -48,6 +53,12 @@ class ParaformerStreaming(Paraformer):**kwargs,):+ """Initialize ParaformerStreaming.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)self.sampling_ratio = kwargs.get("sampling_ratio", 0.2)@@ -155,6 +166,14 @@ class ParaformerStreaming(Paraformer):cache: dict = None,**kwargs,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Encode chunk.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}"""Frontend + Encoder. Note that this method is used by asr_inference.py@@ -189,6 +208,14 @@ class ParaformerStreaming(Paraformer):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att predictor loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -313,6 +340,16 @@ class ParaformerStreaming(Paraformer):chunk_mask=None,):+ """Sampler.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ pre_acoustic_embeds: TODO.+ chunk_mask: TODO.+ """tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(ys_pad.device)@@ -351,6 +388,12 @@ class ParaformerStreaming(Paraformer):def calc_predictor(self, encoder_out, encoder_out_lens):+ """Calc predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -412,6 +455,14 @@ class ParaformerStreaming(Paraformer):return pre_acoustic_embeds, pre_token_length, pre_alphas, pre_peak_indexdef calc_predictor_chunk(self, encoder_out, encoder_out_lens, cache=None, **kwargs):+ """Calc predictor chunk.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """is_final = kwargs.get("is_final", False)return self.predictor.forward_chunk(encoder_out, cache["encoder"], is_final=is_final)@@ -419,6 +470,14 @@ class ParaformerStreaming(Paraformer):def cal_decoder_with_predictor(self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens):+ """Cal decoder with predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ sematic_embeds: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """decoder_outs = self.decoder(encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, self.scama_mask)@@ -429,12 +488,27 @@ class ParaformerStreaming(Paraformer):def cal_decoder_with_predictor_chunk(self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, cache=None):+ """Cal decoder with predictor chunk.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ sematic_embeds: TODO.+ ys_pad_lens: Lengths of ys_pad.+ cache: State cache dict for streaming inference.+ """decoder_outs = self.decoder.forward_chunk(encoder_out, sematic_embeds, cache["decoder"])decoder_out = decoder_outsdecoder_out = torch.log_softmax(decoder_out, dim=-1)return decoder_out, ys_pad_lensdef init_cache(self, cache: dict = None, **kwargs):+ """Init cache.++ Args:+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}chunk_size = kwargs.get("chunk_size", [0, 10, 5])@@ -478,6 +552,16 @@ class ParaformerStreaming(Paraformer):frontend=None,**kwargs,):+ """Generate chunk.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """cache = kwargs.get("cache", {})speech = speech.to(device=kwargs["device"])speech_lengths = speech_lengths.to(device=kwargs["device"])@@ -567,6 +651,17 @@ class ParaformerStreaming(Paraformer):cache: dict = None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}@@ -662,6 +757,11 @@ class ParaformerStreaming(Paraformer):return result, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelmodels = export_rebuild_model(model=self, **kwargs)diff --git a/funasr/models/paraformer_v2_community/decoder.py b/funasr/models/paraformer_v2_community/decoder.pyindex 556873db5..1dcbaff92 100644--- a/funasr/models/paraformer_v2_community/decoder.py+++ b/funasr/models/paraformer_v2_community/decoder.py@@ -121,6 +121,15 @@ class DecoderLayerSANM(torch.nn.Module):return x, tgt_mask, memory, memory_mask, cachedef get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):+ """Get attn mat.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt = self.feed_forward(tgt)@@ -257,6 +266,35 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):tf2torch_tensor_name_prefix_torch: str = "decoder",tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",):+ """Initialize ParaformerSANMDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ wo_input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ att_layer_num: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ lora_list: TODO.+ lora_rank: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ chunk_multiply_factor: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -426,6 +464,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):ys_in_lens: torch.Tensor,):+ """Forward asf2.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]@@ -444,6 +490,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):ys_in_lens: torch.Tensor,):+ """Forward asf6.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ """tgt = ys_in_padtgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]diff --git a/funasr/models/paraformer_v2_community/model.py b/funasr/models/paraformer_v2_community/model.pyindex 2d3ac9070..eb11ede5b 100644--- a/funasr/models/paraformer_v2_community/model.py+++ b/funasr/models/paraformer_v2_community/model.py@@ -69,6 +69,32 @@ class Paraformer(torch.nn.Module):**kwargs,):+ """Initialize Paraformer.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ share_embedding: TODO.+ use_1st_decoder_loss: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -259,6 +285,14 @@ class Paraformer(torch.nn.Module):):# 0. sampler+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """decoder_out_1st = Nonebatch_size = encoder_out.size(0)@@ -312,6 +346,14 @@ class Paraformer(torch.nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -449,6 +491,16 @@ class Paraformer(torch.nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}if (isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"@@ -538,6 +590,11 @@ class Paraformer(torch.nn.Module):return results, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelif "max_seq_len" not in kwargs:diff --git a/funasr/models/qwen3_asr/model.py b/funasr/models/qwen3_asr/model.pyindex 0054ebc42..4a494c13e 100644--- a/funasr/models/qwen3_asr/model.py+++ b/funasr/models/qwen3_asr/model.py@@ -29,6 +29,11 @@ class Qwen3ASR(nn.Module):"""def __init__(self, **kwargs):+ """Initialize Qwen3ASR.++ Args:+ **kwargs: Additional keyword arguments.+ """super().__init__()model_path = kwargs.get("model_path", kwargs.get("model", "Qwen/Qwen3-ASR-1.7B"))device = kwargs.get("device", "cuda:0")@@ -98,6 +103,11 @@ class Qwen3ASR(nn.Module):return model_pathdef forward(self, **kwargs):+ """Forward pass for training.++ Args:+ **kwargs: Additional keyword arguments.+ """raise NotImplementedError("Qwen3ASR only supports inference mode")def inference(diff --git a/funasr/models/qwen_audio/audio.py b/funasr/models/qwen_audio/audio.pyindex a1aad79ea..133ad86de 100644--- a/funasr/models/qwen_audio/audio.py+++ b/funasr/models/qwen_audio/audio.py@@ -16,6 +16,12 @@ from funasr.register import tablesdef exact_div(x, y):+ """Exact div.++ Args:+ x: TODO.+ y: TODO.+ """assert x % y == 0return x // y@@ -35,6 +41,12 @@ TOKENS_PER_SECOND = exact_div(SAMPLE_RATE, N_SAMPLES_PER_TOKEN) # 20ms per audidef get_T_after_cnn(L_in, dilation=1):+ """Get t after cnn.++ Args:+ L_in: TODO.+ dilation: TODO.+ """for padding, kernel_size, stride in eval("[(1,3,1)] + [(1,3,2)] "):L_out = L_in + 2 * padding - dilation * (kernel_size - 1) - 1L_out = 1 + L_out // stride@@ -43,6 +55,12 @@ def get_T_after_cnn(L_in, dilation=1):def load_bytesio_audio(content, sr: int = SAMPLE_RATE):+ """Load bytesio audio.++ Args:+ content: TODO.+ sr: TODO.+ """cmd = ["ffmpeg","-nostdin",@@ -228,11 +246,21 @@ class ModelDimensions:class LayerNorm(nn.LayerNorm):def forward(self, x: Tensor) -> Tensor:# return super().forward(x.float()).type(x.dtype)+ """Forward pass for training.++ Args:+ x: TODO.+ """return super().forward(x).type(x.dtype)class Linear(nn.Linear):def forward(self, x: Tensor) -> Tensor:+ """Forward pass for training.++ Args:+ x: TODO.+ """return F.linear(x,self.weight.to(x.dtype),@@ -242,6 +270,13 @@ class Linear(nn.Linear):class Conv1d(nn.Conv1d):def _conv_forward(self, x: Tensor, weight: Tensor, bias: Optional[Tensor]) -> Tensor:+ """Internal: conv forward.++ Args:+ x: TODO.+ weight: TODO.+ bias: TODO.+ """return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))@@ -258,6 +293,12 @@ def sinusoids(length, channels, max_timescale=10000):class MultiHeadAttention(nn.Module):def __init__(self, n_state: int, n_head: int):+ """Initialize MultiHeadAttention.++ Args:+ n_state: TODO.+ n_head: TODO.+ """super().__init__()self.n_head = n_headself.query = Linear(n_state, n_state)@@ -272,6 +313,14 @@ class MultiHeadAttention(nn.Module):mask: Optional[Tensor] = None,kv_cache: Optional[dict] = None,):+ """Forward pass for training.++ Args:+ x: TODO.+ xa: TODO.+ mask: TODO.+ kv_cache: TODO.+ """q = self.query(x)if kv_cache is None or xa is None or self.key not in kv_cache:@@ -288,6 +337,14 @@ class MultiHeadAttention(nn.Module):return self.out(wv), qkdef qkv_attention(self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None):+ """Qkv attention.++ Args:+ q: TODO.+ k: TODO.+ v: TODO.+ mask: TODO.+ """n_batch, n_ctx, n_state = q.shapescale = (n_state // self.n_head) ** -0.25q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale@@ -304,6 +361,13 @@ class MultiHeadAttention(nn.Module):class ResidualAttentionBlock(nn.Module):def __init__(self, n_state: int, n_head: int, cross_attention: bool = False):+ """Initialize ResidualAttentionBlock.++ Args:+ n_state: TODO.+ n_head: TODO.+ cross_attention: TODO.+ """super().__init__()self.attn = MultiHeadAttention(n_state, n_head)@@ -323,6 +387,14 @@ class ResidualAttentionBlock(nn.Module):mask: Optional[Tensor] = None,kv_cache: Optional[dict] = None,):+ """Forward pass for training.++ Args:+ x: TODO.+ xa: TODO.+ mask: TODO.+ kv_cache: TODO.+ """x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0]if self.cross_attn:x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0]@@ -344,6 +416,19 @@ class QwenAudioEncoder(nn.Module):add_audio_bos_eos_token: bool = True,**kwargs,):+ """Initialize QwenAudioEncoder.++ Args:+ n_mels: TODO.+ n_ctx: TODO.+ n_state: TODO.+ n_head: TODO.+ n_layer: TODO.+ output_dim: Size/dimension parameter.+ avg_pool: TODO.+ add_audio_bos_eos_token: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)@@ -423,6 +508,13 @@ class QwenAudioEncoder(nn.Module):return x, bos, eosdef encode(self, input_audios: Tensor, input_audio_lengths: Tensor, audio_span_tokens: List):+ """Encode.++ Args:+ input_audios: TODO.+ input_audio_lengths: Lengths of input_audio.+ audio_span_tokens: TODO.+ """real_input_audio_lens = input_audio_lengths[:, 0].tolist()max_len_in_batch = max(real_input_audio_lens)padding_mask = torch.ones([input_audios.size(0), max_len_in_batch]).to(diff --git a/funasr/models/qwen_audio/model.py b/funasr/models/qwen_audio/model.pyindex b0af4563c..f78d8914b 100644--- a/funasr/models/qwen_audio/model.py+++ b/funasr/models/qwen_audio/model.py@@ -27,6 +27,12 @@ class QwenAudioWarp(nn.Module):"""def __init__(self, *args, **kwargs):+ """Initialize QwenAudioWarp.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__()from transformers import AutoModelForCausalLM, AutoTokenizerfrom transformers.generation import GenerationConfig@@ -43,6 +49,7 @@ class QwenAudioWarp(nn.Module):def forward(self,):+ """Forward pass for training."""passdef inference(@@ -54,6 +61,16 @@ class QwenAudioWarp(nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")@@ -109,6 +126,7 @@ class QwenAudioChatWarp(nn.Module):def forward(self,):+ """Forward pass for training."""passdef inference(@@ -120,6 +138,16 @@ class QwenAudioChatWarp(nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")diff --git a/funasr/models/rwkv_bat/rwkv_encoder.py b/funasr/models/rwkv_bat/rwkv_encoder.pyindex a27088d69..6b33f2823 100644--- a/funasr/models/rwkv_bat/rwkv_encoder.py+++ b/funasr/models/rwkv_bat/rwkv_encoder.py@@ -92,6 +92,7 @@ class RWKVEncoder(torch.nn.Module):self.time_reduction_factor = time_reduction_factordef output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(self, x: torch.Tensor, x_len) -> torch.Tensor:@@ -133,6 +134,11 @@ class RWKVEncoder(torch.nn.Module):def rwkv_infer(self, xs_pad):+ """Rwkv infer.++ Args:+ xs_pad: TODO.+ """batch_size = xs_pad.shape[0]hidden_sizes = [self._output_size for i in range(5)]diff --git a/funasr/models/sa_asr/attention.py b/funasr/models/sa_asr/attention.pyindex 74e687a52..109a98e4f 100644--- a/funasr/models/sa_asr/attention.py+++ b/funasr/models/sa_asr/attention.py@@ -23,6 +23,7 @@ class CosineDistanceAttention(nn.Module):"""def __init__(self):+ """Initialize CosineDistanceAttention."""super().__init__()self.softmax = nn.Softmax(dim=-1)diff --git a/funasr/models/sa_asr/e2e_sa_asr.py b/funasr/models/sa_asr/e2e_sa_asr.pyindex f4827b257..6d9422d2f 100644--- a/funasr/models/sa_asr/e2e_sa_asr.py+++ b/funasr/models/sa_asr/e2e_sa_asr.py@@ -34,6 +34,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -64,6 +69,31 @@ class SAASRModel(FunASRModel):sym_blank: str = "<blank>",extract_feats_in_collect_stats: bool = True,):+ """Initialize SAASRModel.++ Args:+ vocab_size: Size/dimension parameter.+ max_spk_num: TODO.+ token_list: TODO.+ frontend: Audio frontend for feature extraction.+ specaug: TODO.+ normalize: TODO.+ asr_encoder: TODO.+ spk_encoder: TODO.+ decoder: TODO.+ ctc: TODO.+ spk_weight: TODO.+ ctc_weight: TODO.+ interctc_weight: TODO.+ ignore_id: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ extract_feats_in_collect_stats: TODO.+ """assert 0.0 <= ctc_weight <= 1.0, ctc_weightassert 0.0 <= interctc_weight < 1.0, interctc_weight@@ -250,6 +280,14 @@ class SAASRModel(FunASRModel):text: torch.Tensor,text_lengths: torch.Tensor,) -> Dict[str, torch.Tensor]:+ """Collect feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ text: Text tensor or string input.+ text_lengths: Length of each text sample.+ """if self.extract_feats_in_collect_stats:feats, feats_lengths = self._extract_feats(speech, speech_lengths)else:@@ -326,6 +364,12 @@ class SAASRModel(FunASRModel):def _extract_feats(self, speech: torch.Tensor, speech_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Internal: extract feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """assert speech_lengths.dim() == 1, speech_lengths.shape# for data-parallel@@ -439,6 +483,19 @@ class SAASRModel(FunASRModel):text_id: torch.Tensor,text_id_lengths: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ asr_encoder_out: TODO.+ spk_encoder_out: TODO.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ profile: TODO.+ profile_lens: Lengths of profile.+ text_id: TODO.+ text_id_lengths: Lengths of text_id.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -498,6 +555,14 @@ class SAASRModel(FunASRModel):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTCdiff --git a/funasr/models/sa_asr/transformer_decoder.py b/funasr/models/sa_asr/transformer_decoder.pyindex be33fa527..32dffbb50 100644--- a/funasr/models/sa_asr/transformer_decoder.py+++ b/funasr/models/sa_asr/transformer_decoder.py@@ -181,6 +181,18 @@ class BaseTransformerDecoder(nn.Module, BatchScorerInterface):pos_enc_class=PositionalEncoding,normalize_before: bool = True,):+ """Initialize BaseTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ """super().__init__()attention_dim = encoder_output_size@@ -359,6 +371,24 @@ class TransformerDecoder(BaseTransformerDecoder):normalize_before: bool = True,concat_after: bool = False,):+ """Initialize TransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -411,6 +441,25 @@ class ParaformerDecoderSAN(BaseTransformerDecoder):concat_after: bool = False,embeds_id: int = -1,):+ """Initialize ParaformerDecoderSAN.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ embeds_id: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -513,6 +562,27 @@ class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize LightweightConvolutionTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "@@ -573,6 +643,27 @@ class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize LightweightConvolution2DTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "@@ -633,6 +724,27 @@ class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize DynamicConvolutionTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "@@ -693,6 +805,27 @@ class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize DynamicConvolution2DTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "diff --git a/funasr/models/sanm/attention.py b/funasr/models/sanm/attention.pyindex 47d60cb67..34dfbbae2 100644--- a/funasr/models/sanm/attention.py+++ b/funasr/models/sanm/attention.py@@ -19,6 +19,15 @@ import funasr.models.lora.layers as loradef preprocess_for_attn(x, mask, cache, pad_fn, kernel_size):+ """Preprocess for attn.++ Args:+ x: TODO.+ mask: TODO.+ cache: State cache dict for streaming inference.+ pad_fn: TODO.+ kernel_size: Size/dimension parameter.+ """x = x * maskx = x.transpose(1, 2)if cache is None:@@ -205,6 +214,13 @@ class MultiHeadedAttentionSANM(nn.Module):self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None):+ """Forward fsmn.++ Args:+ inputs: TODO.+ mask: TODO.+ mask_shfit_chunk: TODO.+ """b, t, d = inputs.size()if mask is not None:mask = torch.reshape(mask, (b, -1, 1))@@ -352,6 +368,11 @@ class MultiHeadedAttentionSANM(nn.Module):class MultiHeadedAttentionSANMExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionSANMExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.d_k = model.d_kself.h = model.h@@ -364,6 +385,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):self.all_head_size = self.h * self.d_kdef forward(self, x, mask):+ """Forward pass for training.++ Args:+ x: TODO.+ mask: TODO.+ """mask_3d_btd, mask_4d_bhlt = maskq_h, k_h, v_h, v = self.forward_qkv(x)fsmn_memory = self.forward_fsmn(v, mask_3d_btd)@@ -373,11 +400,21 @@ class MultiHeadedAttentionSANMExport(nn.Module):return att_outs + fsmn_memorydef transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:+ """Transpose for scores.++ Args:+ x: TODO.+ """new_x_shape = x.size()[:-1] + (self.h, self.d_k)x = x.view(new_x_shape)return x.permute(0, 2, 1, 3)def forward_qkv(self, x):+ """Forward qkv.++ Args:+ x: TODO.+ """q_k_v = self.linear_q_k_v(x)q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)q_h = self.transpose_for_scores(q)@@ -388,6 +425,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):def forward_fsmn(self, inputs, mask):# b, t, d = inputs.size()# mask = torch.reshape(mask, (b, -1, 1))+ """Forward fsmn.++ Args:+ inputs: TODO.+ mask: TODO.+ """inputs = inputs * maskx = inputs.transpose(1, 2)x = self.pad_fn(x)@@ -398,6 +441,13 @@ class MultiHeadedAttentionSANMExport(nn.Module):return xdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)@@ -411,6 +461,11 @@ class MultiHeadedAttentionSANMExport(nn.Module):class MultiHeadedAttentionSANMExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionSANMExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.d_k = model.d_kself.h = model.h@@ -423,6 +478,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):self.all_head_size = self.h * self.d_kdef forward(self, x, mask):+ """Forward pass for training.++ Args:+ x: TODO.+ mask: TODO.+ """mask_3d_btd, mask_4d_bhlt = maskq_h, k_h, v_h, v = self.forward_qkv(x)fsmn_memory = self.forward_fsmn(v, mask_3d_btd)@@ -432,11 +493,21 @@ class MultiHeadedAttentionSANMExport(nn.Module):return att_outs + fsmn_memorydef transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:+ """Transpose for scores.++ Args:+ x: TODO.+ """new_x_shape = x.size()[:-1] + (self.h, self.d_k)x = x.view(new_x_shape)return x.permute(0, 2, 1, 3)def forward_qkv(self, x):+ """Forward qkv.++ Args:+ x: TODO.+ """q_k_v = self.linear_q_k_v(x)q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)q_h = self.transpose_for_scores(q)@@ -447,6 +518,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):def forward_fsmn(self, inputs, mask):# b, t, d = inputs.size()# mask = torch.reshape(mask, (b, -1, 1))+ """Forward fsmn.++ Args:+ inputs: TODO.+ mask: TODO.+ """inputs = inputs * maskx = inputs.transpose(1, 2)x = self.pad_fn(x)@@ -457,6 +534,13 @@ class MultiHeadedAttentionSANMExport(nn.Module):return xdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)@@ -549,6 +633,11 @@ class MultiHeadedAttentionSANMDecoder(nn.Module):class MultiHeadedAttentionSANMDecoderExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionSANMDecoderExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.fsmn_block = model.fsmn_blockself.pad_fn = model.pad_fn@@ -556,6 +645,13 @@ class MultiHeadedAttentionSANMDecoderExport(nn.Module):self.attn = Nonedef forward(self, inputs, mask, cache=None):+ """Forward pass for training.++ Args:+ inputs: TODO.+ mask: TODO.+ cache: State cache dict for streaming inference.+ """x, cache = preprocess_for_attn(inputs, mask, cache, self.pad_fn, self.kernel_size)x = self.fsmn_block(x)x = x.transpose(1, 2)@@ -750,6 +846,11 @@ class MultiHeadedAttentionCrossAtt(nn.Module):class MultiHeadedAttentionCrossAttExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionCrossAttExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.d_k = model.d_kself.h = model.h@@ -760,16 +861,35 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):self.all_head_size = self.h * self.d_kdef forward(self, x, memory, memory_mask, ret_attn=False):+ """Forward pass for training.++ Args:+ x: TODO.+ memory: TODO.+ memory_mask: TODO.+ ret_attn: TODO.+ """q, k, v = self.forward_qkv(x, memory)scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)return self.forward_attention(v, scores, memory_mask, ret_attn)def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:+ """Transpose for scores.++ Args:+ x: TODO.+ """new_x_shape = x.size()[:-1] + (self.h, self.d_k)x = x.view(new_x_shape)return x.permute(0, 2, 1, 3)def forward_qkv(self, x, memory):+ """Forward qkv.++ Args:+ x: TODO.+ memory: TODO.+ """q = self.linear_q(x)k_v = self.linear_k_v(memory)@@ -780,6 +900,14 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):return q, k, vdef forward_attention(self, value, scores, mask, ret_attn):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ ret_attn: TODO.+ """scores = scores + mask.to(scores.device)attn = torch.softmax(scores, dim=-1)diff --git a/funasr/models/sanm/decoder.py b/funasr/models/sanm/decoder.pyindex 1a4fb26e2..1a0de72d5 100644--- a/funasr/models/sanm/decoder.py+++ b/funasr/models/sanm/decoder.py@@ -233,6 +233,32 @@ class FsmnDecoder(BaseTransformerDecoder):tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",embed_tensor_name_prefix_tf: str = None,):+ """Initialize FsmnDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ att_layer_num: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ concat_embeds: TODO.+ attention_dim: Size/dimension parameter.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ embed_tensor_name_prefix_tf: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,diff --git a/funasr/models/sanm/encoder.py b/funasr/models/sanm/encoder.pyindex 735595971..d8a720b7a 100644--- a/funasr/models/sanm/encoder.py+++ b/funasr/models/sanm/encoder.py@@ -221,6 +221,36 @@ class SANMEncoder(nn.Module):tf2torch_tensor_name_prefix_torch: str = "encoder",tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",):+ """Initialize SANMEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ padding_idx: TODO.+ interctc_layer_idx: TODO.+ interctc_use_conditioning: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ lora_list: TODO.+ lora_rank: TODO.+ lora_alpha: TODO.+ lora_dropout: TODO.+ selfattention_layer_type: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ """super().__init__()self._output_size = output_size@@ -356,6 +386,7 @@ class SANMEncoder(nn.Module):self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tfdef output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(@@ -430,6 +461,12 @@ class SANMEncoder(nn.Module):return xs_pad, olens, Nonedef _add_overlap_chunk(self, feats: np.ndarray, cache: dict = None):+ """Internal: add overlap chunk.++ Args:+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}if len(cache) == 0:@@ -446,6 +483,14 @@ class SANMEncoder(nn.Module):cache: dict = None,ctc: CTC = None,):+ """Forward chunk.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ cache: State cache dict for streaming inference.+ ctc: TODO.+ """if cache is None:cache = {}xs_pad *= self.output_size() ** 0.5@@ -504,6 +549,12 @@ class EncoderLayerSANMExport(nn.Module):def forward(self, x, mask):+ """Forward pass for training.++ Args:+ x: TODO.+ mask: TODO.+ """residual = xx = self.norm1(x)x = self.self_attn(x, mask)@@ -529,6 +580,16 @@ class SANMEncoderExport(nn.Module):onnx: bool = True,ctc_linear: nn.Module = None,):+ """Initialize SANMEncoderExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ feats_dim: Size/dimension parameter.+ model_name: TODO.+ onnx: TODO.+ ctc_linear: TODO.+ """super().__init__()self.embed = model.embedif isinstance(self.embed, StreamSinusoidalPositionEncoder):@@ -561,6 +622,11 @@ class SANMEncoderExport(nn.Module):self.ctc_linear = ctc_lineardef prepare_mask(self, mask):+ """Prepare mask.++ Args:+ mask: TODO.+ """mask_3d_btd = mask[:, :, None]if len(mask.shape) == 2:mask_4d_bhlt = 1 - mask[:, None, None, :]@@ -571,6 +637,13 @@ class SANMEncoderExport(nn.Module):return mask_3d_btd, mask_4d_bhltdef forward(self, speech: torch.Tensor, speech_lengths: torch.Tensor, online: bool = False):+ """Forward pass for training.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ online: TODO.+ """if not online:speech = speech * self._output_size**0.5@@ -596,19 +669,24 @@ class SANMEncoderExport(nn.Module):return xs_pad, speech_lengthsdef get_output_size(self):+ """Get output size."""return self.model.encoders[0].sizedef get_dummy_inputs(self):+ """Get dummy inputs."""feats = torch.randn(1, 100, self.feats_dim)return featsdef get_input_names(self):+ """Get input names."""return ["feats"]def get_output_names(self):+ """Get output names."""return ["encoder_out", "encoder_out_lens", "predictor_weight"]def get_dynamic_axes(self):+ """Get dynamic axes."""return {"feats": {1: "feats_length"},"encoder_out": {1: "enc_out_length"},diff --git a/funasr/models/sanm/model.py b/funasr/models/sanm/model.pyindex b3f84b2b9..f0d25b6d9 100644--- a/funasr/models/sanm/model.py+++ b/funasr/models/sanm/model.py@@ -25,4 +25,10 @@ class SANM(Transformer):**kwargs,):+ """Initialize SANM.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)diff --git a/funasr/models/sanm/multihead_att.py b/funasr/models/sanm/multihead_att.pyindex 671d46011..3e9d18d82 100644--- a/funasr/models/sanm/multihead_att.py+++ b/funasr/models/sanm/multihead_att.py@@ -7,6 +7,11 @@ import torch.nn as nnclass MultiHeadedAttentionSANMExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionSANMExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.d_k = model.d_kself.h = model.h@@ -19,6 +24,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):self.all_head_size = self.h * self.d_kdef forward(self, x, mask):+ """Forward pass for training.++ Args:+ x: TODO.+ mask: TODO.+ """mask_3d_btd, mask_4d_bhlt = maskq_h, k_h, v_h, v = self.forward_qkv(x)fsmn_memory = self.forward_fsmn(v, mask_3d_btd)@@ -28,11 +39,21 @@ class MultiHeadedAttentionSANMExport(nn.Module):return att_outs + fsmn_memorydef transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:+ """Transpose for scores.++ Args:+ x: TODO.+ """new_x_shape = x.size()[:-1] + (self.h, self.d_k)x = x.view(new_x_shape)return x.permute(0, 2, 1, 3)def forward_qkv(self, x):+ """Forward qkv.++ Args:+ x: TODO.+ """q_k_v = self.linear_q_k_v(x)q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)q_h = self.transpose_for_scores(q)@@ -43,6 +64,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):def forward_fsmn(self, inputs, mask):# b, t, d = inputs.size()# mask = torch.reshape(mask, (b, -1, 1))+ """Forward fsmn.++ Args:+ inputs: TODO.+ mask: TODO.+ """inputs = inputs * maskx = inputs.transpose(1, 2)x = self.pad_fn(x)@@ -53,6 +80,13 @@ class MultiHeadedAttentionSANMExport(nn.Module):return xdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)@@ -65,6 +99,15 @@ class MultiHeadedAttentionSANMExport(nn.Module):def preprocess_for_attn(x, mask, cache, pad_fn, kernel_size):+ """Preprocess for attn.++ Args:+ x: TODO.+ mask: TODO.+ cache: State cache dict for streaming inference.+ pad_fn: TODO.+ kernel_size: Size/dimension parameter.+ """x = x * maskx = x.transpose(1, 2)if cache is None:@@ -84,6 +127,11 @@ if torch_version >= (1, 8):class MultiHeadedAttentionSANMDecoderExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionSANMDecoderExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.fsmn_block = model.fsmn_blockself.pad_fn = model.pad_fn@@ -91,6 +139,13 @@ class MultiHeadedAttentionSANMDecoderExport(nn.Module):self.attn = Nonedef forward(self, inputs, mask, cache=None):+ """Forward pass for training.++ Args:+ inputs: TODO.+ mask: TODO.+ cache: State cache dict for streaming inference.+ """x, cache = preprocess_for_attn(inputs, mask, cache, self.pad_fn, self.kernel_size)x = self.fsmn_block(x)x = x.transpose(1, 2)@@ -102,6 +157,11 @@ class MultiHeadedAttentionSANMDecoderExport(nn.Module):class MultiHeadedAttentionCrossAttExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionCrossAttExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.d_k = model.d_kself.h = model.h@@ -112,16 +172,34 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):self.all_head_size = self.h * self.d_kdef forward(self, x, memory, memory_mask):+ """Forward pass for training.++ Args:+ x: TODO.+ memory: TODO.+ memory_mask: TODO.+ """q, k, v = self.forward_qkv(x, memory)scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)return self.forward_attention(v, scores, memory_mask)def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:+ """Transpose for scores.++ Args:+ x: TODO.+ """new_x_shape = x.size()[:-1] + (self.h, self.d_k)x = x.view(new_x_shape)return x.permute(0, 2, 1, 3)def forward_qkv(self, x, memory):+ """Forward qkv.++ Args:+ x: TODO.+ memory: TODO.+ """q = self.linear_q(x)k_v = self.linear_k_v(memory)@@ -132,6 +210,13 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):return q, k, vdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)@@ -145,6 +230,11 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):class OnnxMultiHeadedAttention(nn.Module):def __init__(self, model):+ """Initialize OnnxMultiHeadedAttention.++ Args:+ model: Model instance or model name.+ """super().__init__()self.d_k = model.d_kself.h = model.h@@ -156,16 +246,36 @@ class OnnxMultiHeadedAttention(nn.Module):self.all_head_size = self.h * self.d_kdef forward(self, query, key, value, mask):+ """Forward pass for training.++ Args:+ query: TODO.+ key: Sample identifiers.+ value: TODO.+ mask: TODO.+ """q, k, v = self.forward_qkv(query, key, value)scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)return self.forward_attention(v, scores, mask)def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:+ """Transpose for scores.++ Args:+ x: TODO.+ """new_x_shape = x.size()[:-1] + (self.h, self.d_k)x = x.view(new_x_shape)return x.permute(0, 2, 1, 3)def forward_qkv(self, query, key, value):+ """Forward qkv.++ Args:+ query: TODO.+ key: Sample identifiers.+ value: TODO.+ """q = self.linear_q(query)k = self.linear_k(key)v = self.linear_v(value)@@ -175,6 +285,13 @@ class OnnxMultiHeadedAttention(nn.Module):return q, k, vdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)@@ -188,12 +305,26 @@ class OnnxMultiHeadedAttention(nn.Module):class OnnxRelPosMultiHeadedAttention(OnnxMultiHeadedAttention):def __init__(self, model):+ """Initialize OnnxRelPosMultiHeadedAttention.++ Args:+ model: Model instance or model name.+ """super().__init__(model)self.linear_pos = model.linear_posself.pos_bias_u = model.pos_bias_uself.pos_bias_v = model.pos_bias_vdef forward(self, query, key, value, pos_emb, mask):+ """Forward pass for training.++ Args:+ query: TODO.+ key: Sample identifiers.+ value: TODO.+ pos_emb: TODO.+ mask: TODO.+ """q, k, v = self.forward_qkv(query, key, value)q = q.transpose(1, 2) # (batch, time1, head, d_k)@@ -220,6 +351,11 @@ class OnnxRelPosMultiHeadedAttention(OnnxMultiHeadedAttention):return self.forward_attention(v, scores, mask)def rel_shift(self, x):+ """Rel shift.++ Args:+ x: TODO.+ """zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype)x_padded = torch.cat([zero_pad, x], dim=-1)@@ -230,6 +366,13 @@ class OnnxRelPosMultiHeadedAttention(OnnxMultiHeadedAttention):return xdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)diff --git a/funasr/models/sanm_kws/export_meta.py b/funasr/models/sanm_kws/export_meta.pyindex a91a22cb0..2d55a2bf7 100644--- a/funasr/models/sanm_kws/export_meta.py+++ b/funasr/models/sanm_kws/export_meta.py@@ -11,6 +11,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):# self.device = kwargs.get("device")+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")@@ -47,6 +53,12 @@ def export_encoder_forward(speech_lengths: torch.Tensor,):# a. To device+ """Export encoder forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """batch = {"speech": speech,"speech_lengths": speech_lengths,@@ -63,21 +75,25 @@ def export_encoder_forward(def export_encoder_dummy_inputs(self):+ """Export encoder dummy inputs."""speech = torch.randn(2, 30, 280)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)return (speech, speech_lengths)def export_encoder_input_names(self):+ """Export encoder input names."""return ["speech", "speech_lengths"]def export_encoder_output_names(self):# return ["encoder_out", "encoder_out_len", "alphas"]+ """Export encoder output names."""return ["encoder_out", "encoder_out_len"]def export_encoder_dynamic_axes(self):+ """Export encoder dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"@@ -95,4 +111,5 @@ def export_encoder_dynamic_axes(self):def export_encoder_name(self):+ """Export encoder name."""return "encoder.onnx"diff --git a/funasr/models/sanm_kws/model.py b/funasr/models/sanm_kws/model.pyindex d8d8f955c..8577cf425 100644--- a/funasr/models/sanm_kws/model.py+++ b/funasr/models/sanm_kws/model.py@@ -52,6 +52,26 @@ class SanmKWS(torch.nn.Module):**kwargs,):+ """Initialize SanmKWS.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -167,6 +187,14 @@ class SanmKWS(torch.nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -185,6 +213,16 @@ class SanmKWS(torch.nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """keywords = kwargs.get("keywords")from funasr.utils.kws_utils import KwsCtcPrefixDecoderself.kws_decoder = KwsCtcPrefixDecoder(@@ -258,6 +296,11 @@ class SanmKWS(torch.nn.Module):return results, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelif "max_seq_len" not in kwargs:diff --git a/funasr/models/sanm_kws_streaming/export_meta.py b/funasr/models/sanm_kws_streaming/export_meta.pyindex a91a22cb0..2d55a2bf7 100644--- a/funasr/models/sanm_kws_streaming/export_meta.py+++ b/funasr/models/sanm_kws_streaming/export_meta.py@@ -11,6 +11,12 @@ from funasr.register import tablesdef export_rebuild_model(model, **kwargs):# self.device = kwargs.get("device")+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")@@ -47,6 +53,12 @@ def export_encoder_forward(speech_lengths: torch.Tensor,):# a. To device+ """Export encoder forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """batch = {"speech": speech,"speech_lengths": speech_lengths,@@ -63,21 +75,25 @@ def export_encoder_forward(def export_encoder_dummy_inputs(self):+ """Export encoder dummy inputs."""speech = torch.randn(2, 30, 280)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)return (speech, speech_lengths)def export_encoder_input_names(self):+ """Export encoder input names."""return ["speech", "speech_lengths"]def export_encoder_output_names(self):# return ["encoder_out", "encoder_out_len", "alphas"]+ """Export encoder output names."""return ["encoder_out", "encoder_out_len"]def export_encoder_dynamic_axes(self):+ """Export encoder dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"@@ -95,4 +111,5 @@ def export_encoder_dynamic_axes(self):def export_encoder_name(self):+ """Export encoder name."""return "encoder.onnx"diff --git a/funasr/models/sanm_kws_streaming/model.py b/funasr/models/sanm_kws_streaming/model.pyindex 184965955..3228a0643 100644--- a/funasr/models/sanm_kws_streaming/model.py+++ b/funasr/models/sanm_kws_streaming/model.py@@ -31,6 +31,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -47,6 +52,12 @@ class SanmKWSStreaming(SanmKWS):*args,**kwargs,):+ """Initialize SanmKWSStreaming.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(@@ -112,6 +123,14 @@ class SanmKWSStreaming(SanmKWS):cache: dict = None,**kwargs,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Encode chunk.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}"""Frontend + Encoder. Note that this method is used by asr_inference.py@@ -140,6 +159,12 @@ class SanmKWSStreaming(SanmKWS):return encoder_out, torch.tensor([encoder_out.size(1)])def init_cache(self, cache: dict = None, **kwargs):+ """Init cache.++ Args:+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}chunk_size = kwargs.get("chunk_size", [0, 10, 5])@@ -185,6 +210,16 @@ class SanmKWSStreaming(SanmKWS):frontend=None,**kwargs,):+ """Generate chunk.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """cache = kwargs.get("cache", {})speech = speech.to(device=kwargs["device"])speech_lengths = speech_lengths.to(device=kwargs["device"])@@ -259,6 +294,17 @@ class SanmKWSStreaming(SanmKWS):cache: dict = None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}keywords = kwargs.get("keywords")@@ -442,6 +488,11 @@ class SanmKWSStreaming(SanmKWS):return result, meta_datadef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelmodels = export_rebuild_model(model=self, **kwargs)diff --git a/funasr/models/scama/beam_search.py b/funasr/models/scama/beam_search.pyindex bf0bb5cf7..e114fb773 100644--- a/funasr/models/scama/beam_search.py+++ b/funasr/models/scama/beam_search.py@@ -623,6 +623,15 @@ class BeamSearchScamaStreaming(torch.nn.Module):pre_acoustic_embeds: torch.Tensor = None,cache: dict = None,) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]:+ """Score full.++ Args:+ hyp: TODO.+ x: TODO.+ x_mask: TODO.+ pre_acoustic_embeds: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}"""Score new hypothesis by `self.full_scorers`.@@ -766,6 +775,15 @@ class BeamSearchScamaStreaming(torch.nn.Module):pre_acoustic_embeds: torch.Tensor = None,cache: dict = None,) -> List[Hypothesis]:+ """Search.++ Args:+ running_hyps: TODO.+ x: TODO.+ x_mask: TODO.+ pre_acoustic_embeds: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}"""Search new tokens for running hypotheses and encoded speech x.@@ -831,6 +849,18 @@ class BeamSearchScamaStreaming(torch.nn.Module):minlen: int = 0,cache: dict = None,) -> List[Hypothesis]:+ """Forward pass for training.++ Args:+ x: TODO.+ scama_mask: TODO.+ pre_acoustic_embeds: TODO.+ maxlenratio: TODO.+ minlenratio: TODO.+ maxlen: TODO.+ minlen: TODO.+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}"""Perform beam search.diff --git a/funasr/models/scama/chunk_utilis.py b/funasr/models/scama/chunk_utilis.pyindex 2fe3fa4e3..7d92ce5d9 100644--- a/funasr/models/scama/chunk_utilis.py+++ b/funasr/models/scama/chunk_utilis.py@@ -25,6 +25,16 @@ class overlap_chunk:decoder_att_look_back_factor: tuple = (1,),):+ """Initialize overlap_chunk.++ Args:+ chunk_size: Size/dimension parameter.+ stride: TODO.+ pad_left: TODO.+ encoder_att_look_back_factor: TODO.+ shfit_fsmn: TODO.+ decoder_att_look_back_factor: TODO.+ """pad_left = self.check_chunk_size_args(chunk_size, pad_left)encoder_att_look_back_factor = self.check_chunk_size_args(chunk_size, encoder_att_look_back_factor@@ -63,12 +73,23 @@ class overlap_chunk:) = (None, None, None, None, None)def check_chunk_size_args(self, chunk_size, x):+ """Check chunk size args.++ Args:+ chunk_size: Size/dimension parameter.+ x: TODO.+ """if len(x) < len(chunk_size):x = [x[0] for i in chunk_size]return xdef get_chunk_size(self, ind: int = 0):# with torch.no_grad:+ """Get chunk size.++ Args:+ ind: TODO.+ """chunk_size, stride, pad_left, encoder_att_look_back_factor, decoder_att_look_back_factor = (self.chunk_size[ind],self.stride[ind],@@ -100,6 +121,12 @@ class overlap_chunk:)def random_choice(self, training=True, decoding_ind=None):+ """Random choice.++ Args:+ training: TODO.+ decoding_ind: TODO.+ """chunk_num = len(self.chunk_size)ind = 0if training and chunk_num > 1:@@ -111,6 +138,14 @@ class overlap_chunk:def gen_chunk_mask(self, x_len, ind=0, num_units=1, num_units_predictor=1):+ """Gen chunk mask.++ Args:+ x_len: TODO.+ ind: TODO.+ num_units: TODO.+ num_units_predictor: TODO.+ """with torch.no_grad():x_len = x_len.cpu().numpy()x_len_max = x_len.max()@@ -271,6 +306,13 @@ class overlap_chunk:return x_chunk, x_len_chunkdef remove_chunk(self, x_chunk, x_len_chunk, chunk_outs):+ """Remove chunk.++ Args:+ x_chunk: TODO.+ x_len_chunk: TODO.+ chunk_outs: TODO.+ """x_chunk = x_chunk[:, : x_len_chunk.max(), :]b, t, d = x_chunk.size()x_len_chunk_mask = (~make_pad_mask(x_len_chunk, maxlen=t)).to(x_chunk.device)@@ -286,24 +328,56 @@ class overlap_chunk:return x, x_lendef get_x_add_mask(self, chunk_outs=None, device="cpu", idx=0, dtype=torch.float32):+ """Get x add mask.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = torch.from_numpy(x).type(dtype).to(device)return xdef get_x_len_chunk(self, chunk_outs=None, device="cpu", idx=1, dtype=torch.float32):+ """Get x len chunk.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = torch.from_numpy(x).type(dtype).to(device)return xdef get_x_rm_mask(self, chunk_outs=None, device="cpu", idx=2, dtype=torch.float32):+ """Get x rm mask.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = torch.from_numpy(x).type(dtype).to(device)return xdef get_x_len(self, chunk_outs=None, device="cpu", idx=3, dtype=torch.float32):+ """Get x len.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = torch.from_numpy(x).type(dtype).to(device)@@ -312,6 +386,16 @@ class overlap_chunk:def get_mask_shfit_chunk(self, chunk_outs=None, device="cpu", batch_size=1, num_units=1, idx=4, dtype=torch.float32):+ """Get mask shfit chunk.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ batch_size: Number of samples per batch.+ num_units: TODO.+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = np.tile(@@ -328,6 +412,16 @@ class overlap_chunk:def get_mask_chunk_predictor(self, chunk_outs=None, device="cpu", batch_size=1, num_units=1, idx=5, dtype=torch.float32):+ """Get mask chunk predictor.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ batch_size: Number of samples per batch.+ num_units: TODO.+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = np.tile(@@ -344,6 +438,15 @@ class overlap_chunk:def get_mask_att_chunk_encoder(self, chunk_outs=None, device="cpu", batch_size=1, idx=6, dtype=torch.float32):+ """Get mask att chunk encoder.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ batch_size: Number of samples per batch.+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = np.tile(@@ -360,6 +463,15 @@ class overlap_chunk:def get_mask_shift_att_chunk_decoder(self, chunk_outs=None, device="cpu", batch_size=1, idx=7, dtype=torch.float32):+ """Get mask shift att chunk decoder.++ Args:+ chunk_outs: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ batch_size: Number of samples per batch.+ idx: TODO.+ dtype: TODO.+ """with torch.no_grad():x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]x = np.tile(x[None, None, :, 0], [batch_size, 1, 1])@@ -383,6 +495,24 @@ def build_scama_mask_for_cross_attention_decoder(is_training=True,dtype: torch.dtype = torch.float32,):+ """Build scama mask for cross attention decoder.++ Args:+ predictor_alignments: TODO.+ encoder_sequence_length: TODO.+ chunk_size: Size/dimension parameter.+ encoder_chunk_size: Size/dimension parameter.+ attention_chunk_center_bias: TODO.+ attention_chunk_size: Size/dimension parameter.+ attention_chunk_type: TODO.+ step: TODO.+ predictor_mask_chunk_hopping: TODO.+ decoder_att_look_back_factor: TODO.+ mask_shift_att_chunk_decoder: TODO.+ target_length: TODO.+ is_training: Boolean flag for training.+ dtype: TODO.+ """with torch.no_grad():device = predictor_alignments.devicebatch_size, chunk_num = predictor_alignments.size()@@ -454,6 +584,7 @@ def build_scama_mask_for_cross_attention_decoder(mask_mask_flip = mask_flip * maskdef _fn():+ """Internal: fn."""mask_sliced = mask[:b, :k, encoder_chunk_size:t]zero_pad_right = torch.zeros([b, k, encoder_chunk_size], dtype=mask_sliced.dtypediff --git a/funasr/models/scama/decoder.py b/funasr/models/scama/decoder.pyindex 31b235778..e8495967d 100644--- a/funasr/models/scama/decoder.py+++ b/funasr/models/scama/decoder.py@@ -233,6 +233,32 @@ class FsmnDecoderSCAMAOpt(BaseTransformerDecoder):tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",embed_tensor_name_prefix_tf: str = None,):+ """Initialize FsmnDecoderSCAMAOpt.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ att_layer_num: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ concat_embeds: TODO.+ attention_dim: Size/dimension parameter.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ embed_tensor_name_prefix_tf: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,diff --git a/funasr/models/scama/encoder.py b/funasr/models/scama/encoder.pyindex 0cf48f526..88d3536d8 100644--- a/funasr/models/scama/encoder.py+++ b/funasr/models/scama/encoder.py@@ -222,6 +222,37 @@ class SANMEncoderChunkOpt(nn.Module):tf2torch_tensor_name_prefix_torch: str = "encoder",tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",):+ """Initialize SANMEncoderChunkOpt.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ padding_idx: TODO.+ interctc_layer_idx: TODO.+ interctc_use_conditioning: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ selfattention_layer_type: TODO.+ chunk_size: Size/dimension parameter.+ stride: TODO.+ pad_left: TODO.+ encoder_att_look_back_factor: TODO.+ decoder_att_look_back_factor: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ """super().__init__()self._output_size = output_size@@ -356,6 +387,7 @@ class SANMEncoderChunkOpt(nn.Module):self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tfdef output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(@@ -446,6 +478,12 @@ class SANMEncoderChunkOpt(nn.Module):return xs_pad, olens, Nonedef _add_overlap_chunk(self, feats: np.ndarray, cache: dict = None):+ """Internal: add overlap chunk.++ Args:+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).+ cache: State cache dict for streaming inference.+ """if cache is None:cache = {}if len(cache) == 0:@@ -462,6 +500,14 @@ class SANMEncoderChunkOpt(nn.Module):cache: dict = None,**kwargs,):+ """Forward chunk.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}is_final = kwargs.get("is_final", False)diff --git a/funasr/models/scama/model.py b/funasr/models/scama/model.pyindex bb207c88e..28be561ba 100644--- a/funasr/models/scama/model.py+++ b/funasr/models/scama/model.py@@ -33,6 +33,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -73,6 +78,35 @@ class SCAMA(nn.Module):**kwargs,):+ """Initialize SCAMA.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ predictor: TODO.+ predictor_conf: Configuration dict for predictor.+ predictor_bias: TODO.+ predictor_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -270,6 +304,14 @@ class SCAMA(nn.Module):cache: dict = None,**kwargs,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Encode chunk.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}"""Frontend + Encoder. Note that this method is used by asr_inference.py@@ -298,6 +340,14 @@ class SCAMA(nn.Module):return encoder_out, torch.tensor([encoder_out.size(1)])def calc_predictor_chunk(self, encoder_out, encoder_out_lens, cache=None, **kwargs):+ """Calc predictor chunk.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """is_final = kwargs.get("is_final", False)return self.predictor.forward_chunk(encoder_out, cache["encoder"], is_final=is_final)@@ -309,6 +359,14 @@ class SCAMA(nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att predictor loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -404,6 +462,14 @@ class SCAMA(nn.Module):):# ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)# ys_in_lens = ys_pad_lens + 1+ """Calc predictor mask.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_out_pad, ys_in_lens = None, Noneencoder_out_mask = sequence_mask(@@ -473,6 +539,11 @@ class SCAMA(nn.Module):**kwargs,):+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.scama.beam_search import BeamSearchScamaStreamingfrom funasr.models.transformer.scorers.ctc import CTCPrefixScorer@@ -528,6 +599,16 @@ class SCAMA(nn.Module):frontend=None,**kwargs,):+ """Generate chunk.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """cache = kwargs.get("cache", {})speech = speech.to(device=kwargs["device"])speech_lengths = speech_lengths.to(device=kwargs["device"])@@ -605,6 +686,12 @@ class SCAMA(nn.Module):return resultsdef init_cache(self, cache: dict = None, **kwargs):+ """Init cache.++ Args:+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}device = kwargs.get("device", "cuda")@@ -655,6 +742,17 @@ class SCAMA(nn.Module):cache: dict = None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ cache: State cache dict for streaming inference.+ **kwargs: Additional keyword arguments.+ """if cache is None:cache = {}diff --git a/funasr/models/scama/utils.py b/funasr/models/scama/utils.pyindex c3f7bc37c..7c7c59683 100644--- a/funasr/models/scama/utils.py+++ b/funasr/models/scama/utils.py@@ -6,6 +6,14 @@ from torch.nn import functional as Fdef sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):+ """Sequence mask.++ Args:+ lengths: TODO.+ maxlen: TODO.+ dtype: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """if maxlen is None:maxlen = lengths.max()row_vector = torch.arange(0, maxlen, 1).to(lengths.device)@@ -17,6 +25,12 @@ def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):def apply_cmvn(inputs, mvn):+ """Apply cmvn.++ Args:+ inputs: TODO.+ mvn: TODO.+ """device = inputs.devicedtype = inputs.dtypeframe, dim = inputs.shape@@ -36,6 +50,15 @@ def drop_and_add(stoch_layer_coeff: float = 1.0,):+ """Drop and add.++ Args:+ inputs: TODO.+ outputs: TODO.+ training: TODO.+ dropout_rate: TODO.+ stoch_layer_coeff: TODO.+ """outputs = F.dropout(outputs, p=dropout_rate, training=training, inplace=True)outputs *= stoch_layer_coeff@@ -48,6 +71,11 @@ def drop_and_add(def proc_tf_vocab(vocab_path):+ """Proc tf vocab.++ Args:+ vocab_path: TODO.+ """with open(vocab_path, encoding="utf-8") as f:token_list = [line.rstrip() for line in f]if "<unk>" not in token_list:@@ -56,6 +84,13 @@ def proc_tf_vocab(vocab_path):def gen_config_for_tfmodel(config_path, vocab_path, output_dir):+ """Gen config for tfmodel.++ Args:+ config_path: TODO.+ vocab_path: TODO.+ output_dir: Directory for saving output files.+ """token_list = proc_tf_vocab(vocab_path)with open(config_path, encoding="utf-8") as f:config = yaml.safe_load(f)@@ -72,6 +107,11 @@ def gen_config_for_tfmodel(config_path, vocab_path, output_dir):class NoAliasSafeDumper(yaml.SafeDumper):# Disable anchor/alias in yaml because looks uglydef ignore_aliases(self, data):+ """Ignore aliases.++ Args:+ data: TODO.+ """return Truediff --git a/funasr/models/seaco_paraformer/export_meta.py b/funasr/models/seaco_paraformer/export_meta.pyindex 947f4d956..980e062ab 100644--- a/funasr/models/seaco_paraformer/export_meta.py+++ b/funasr/models/seaco_paraformer/export_meta.py@@ -16,17 +16,31 @@ class ContextualEmbedderExport(torch.nn.Module):feats_dim=560,**kwargs,):+ """Initialize ContextualEmbedderExport.++ Args:+ model: Model instance or model name.+ max_seq_len: TODO.+ feats_dim: Size/dimension parameter.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.embedding = model.decoder.embed # model.bias_embedmodel.bias_encoder.batch_first = Falseself.bias_encoder = model.bias_encoderdef forward(self, hotword):+ """Forward pass for training.++ Args:+ hotword: Hotword/keyword for boosting recognition.+ """hotword = self.embedding(hotword).transpose(0, 1) # batch secondhw_embed, (_, _) = self.bias_encoder(hotword)return hw_embeddef export_dummy_inputs(self):+ """Export dummy inputs."""hotword = torch.tensor([[10, 11, 12, 13, 14, 10, 11, 12, 13, 14],@@ -42,12 +56,15 @@ class ContextualEmbedderExport(torch.nn.Module):return hotworddef export_input_names(self):+ """Export input names."""return ["hotword"]def export_output_names(self):+ """Export output names."""return ["hw_embed"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"hotword": {0: "num_hotwords",@@ -58,10 +75,17 @@ class ContextualEmbedderExport(torch.nn.Module):}def export_name(self):+ """Export name."""return "model_eb.onnx"def export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """model.device = kwargs.get("device")is_onnx = kwargs.get("type", "onnx") == "onnx"encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")@@ -124,6 +148,13 @@ def export_backbone_forward(# lmbd: float,):# a. To device+ """Export backbone forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ bias_embed: TODO.+ """batch = {"speech": speech, "speech_lengths": speech_lengths}enc, enc_len = self.encoder(**batch)@@ -173,6 +204,7 @@ def export_backbone_forward(def export_backbone_dummy_inputs(self):+ """Export backbone dummy inputs."""speech = torch.randn(2, 30, self.feats_dim)speech_lengths = torch.tensor([15, 30], dtype=torch.int32)bias_embed = torch.randn(2, 1, 512)@@ -180,14 +212,17 @@ def export_backbone_dummy_inputs(self):def export_backbone_input_names(self):+ """Export backbone input names."""return ["speech", "speech_lengths", "bias_embed"]def export_backbone_output_names(self):+ """Export backbone output names."""return ["logits", "token_num", "us_alphas", "us_cif_peak"]def export_backbone_dynamic_axes(self):+ """Export backbone dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"},"speech_lengths": {diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.pyindex b30c5cb98..d6ae4bee9 100644--- a/funasr/models/seaco_paraformer/model.py+++ b/funasr/models/seaco_paraformer/model.py@@ -38,6 +38,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -54,6 +59,12 @@ class SeacoParaformer(BiCifParaformer, Paraformer):*args,**kwargs,):+ """Initialize SeacoParaformer.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)self.inner_dim = kwargs.get("inner_dim", 256)@@ -177,9 +188,21 @@ class SeacoParaformer(BiCifParaformer, Paraformer):return loss, stats, weightdef _merge(self, cif_attended, dec_attended):+ """Internal: merge.++ Args:+ cif_attended: TODO.+ dec_attended: TODO.+ """return cif_attended + dec_attendeddef calc_predictor(self, encoder_out, encoder_out_lens):+ """Calc predictor.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -199,6 +222,17 @@ class SeacoParaformer(BiCifParaformer, Paraformer):seaco_label_pad: torch.Tensor,):# predictor forward+ """Internal: calc seaco loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_lengths: Lengths of ys.+ hotword_pad: TODO.+ hotword_lengths: Lengths of hotword.+ seaco_label_pad: TODO.+ """encoder_out_mask = (~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]).to(encoder_out.device)@@ -243,6 +277,17 @@ class SeacoParaformer(BiCifParaformer, Paraformer):):# decoder forward+ """Internal: seaco decode with ASF.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ sematic_embeds: TODO.+ ys_pad_lens: Lengths of ys_pad.+ hw_list: TODO.+ nfilter: TODO.+ seaco_weight: TODO.+ """decoder_out, decoder_hidden, _ = self.decoder(encoder_out,encoder_out_lens,@@ -311,6 +356,12 @@ class SeacoParaformer(BiCifParaformer, Paraformer):dha_pred = torch.log_softmax(dha_output, dim=-1)def _merge_res(dec_output, dha_output):+ """Internal: merge res.++ Args:+ dec_output: TODO.+ dha_output: TODO.+ """lmbd = torch.Tensor([seaco_weight] * dha_output.shape[0])dha_ids = dha_output.max(-1)[-1] # [0]dha_mask = (dha_ids == self.NO_BIAS).int().unsqueeze(-1)@@ -328,6 +379,12 @@ class SeacoParaformer(BiCifParaformer, Paraformer):return decoder_preddef _hotword_representation(self, hotword_pad, hotword_lengths):+ """Internal: hotword representation.++ Args:+ hotword_pad: TODO.+ hotword_lengths: Lengths of hotword.+ """if self.bias_encoder_type != "lstm":logging.error("Unsupported bias encoder type")@@ -370,6 +427,16 @@ class SeacoParaformer(BiCifParaformer, Paraformer):):# init beamsearch+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != Noneis_use_lm = (kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None@@ -512,7 +579,20 @@ class SeacoParaformer(BiCifParaformer, Paraformer):def generate_hotwords_list(self, hotword_list_or_file, tokenizer=None, frontend=None):+ """Generate hotwords list.++ Args:+ hotword_list_or_file: TODO.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ """def seg_tokenize(txt, seg_dict):+ """Seg tokenize.++ Args:+ txt: TODO.+ seg_dict: TODO.+ """pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$")out_txt = ""for word in txt:@@ -610,6 +690,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):self,**kwargs,):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """if "max_seq_len" not in kwargs:kwargs["max_seq_len"] = 512from .export_meta import export_rebuild_model@@ -620,6 +705,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):@lru_cache(maxsize=1)def load_seg_dict(seg_dict_file):+ """Load seg dict.++ Args:+ seg_dict_file: TODO.+ """seg_dict = {}assert isinstance(seg_dict_file, str)with open(seg_dict_file, "r", encoding="utf8") as f:diff --git a/funasr/models/sense_voice/export_meta.py b/funasr/models/sense_voice/export_meta.pyindex 449388ef7..58bccd3ba 100644--- a/funasr/models/sense_voice/export_meta.py+++ b/funasr/models/sense_voice/export_meta.py@@ -9,6 +9,12 @@ from funasr.utils.torch_function import sequence_maskdef export_rebuild_model(model, **kwargs):+ """Export rebuild model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """model.device = kwargs.get("device")model.make_pad_mask = sequence_mask(kwargs["max_seq_len"], flip=False)model.forward = types.MethodType(export_forward, model)@@ -29,6 +35,15 @@ def export_forward():# speech = speech.to(device="cuda")# speech_lengths = speech_lengths.to(device="cuda")+ """Export forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ language: Language identifier.+ textnorm: TODO.+ **kwargs: Additional keyword arguments.+ """language_query = self.embed(language.to(speech.device)).unsqueeze(1)textnorm_query = self.embed(textnorm.to(speech.device)).unsqueeze(1)@@ -51,6 +66,7 @@ def export_forward(return ctc_logits, encoder_out_lensdef export_dummy_inputs(self):+ """Export dummy inputs."""speech = torch.randn(2, 30, 560)speech_lengths = torch.tensor([6, 30], dtype=torch.int32)language = torch.tensor([0, 0], dtype=torch.int32)@@ -58,12 +74,15 @@ def export_dummy_inputs(self):return (speech, speech_lengths, language, textnorm)def export_input_names(self):+ """Export input names."""return ["speech", "speech_lengths", "language", "textnorm"]def export_output_names(self):+ """Export output names."""return ["ctc_logits", "encoder_out_lens"]def export_dynamic_axes(self):+ """Export dynamic axes."""return {"speech": {0: "batch_size", 1: "feats_length"},"speech_lengths": {0: "batch_size"},@@ -74,4 +93,5 @@ def export_dynamic_axes(self):}def export_name(self):+ """Export name."""return "model.onnx"diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.pyindex 30a9e42b9..3ec936c54 100644--- a/funasr/models/sense_voice/model.py+++ b/funasr/models/sense_voice/model.py@@ -26,11 +26,24 @@ class SinusoidalPositionEncoder(torch.nn.Module):""" """def __init__(self, d_model=80, dropout_rate=0.1):+ """Initialize SinusoidalPositionEncoder.++ Args:+ d_model: D Model instance.+ dropout_rate: TODO.+ """super().__init__()def encode(self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32):+ """Encode.++ Args:+ positions: TODO.+ depth: TODO.+ dtype: TODO.+ """batch_size = positions.size(0)positions = positions.type(dtype)device = positions.device@@ -48,6 +61,11 @@ class SinusoidalPositionEncoder(torch.nn.Module):return encoding.type(dtype)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """batch_size, timesteps, input_dim = x.size()positions = torch.arange(1, timesteps + 1, device=x.device)[None, :]position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device)@@ -127,6 +145,13 @@ class MultiHeadedAttentionSANM(nn.Module):self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None):+ """Forward fsmn.++ Args:+ inputs: TODO.+ mask: TODO.+ mask_shfit_chunk: TODO.+ """b, t, d = inputs.size()if mask is not None:mask = torch.reshape(mask, (b, -1, 1))@@ -274,9 +299,20 @@ class MultiHeadedAttentionSANM(nn.Module):class LayerNorm(nn.LayerNorm):def __init__(self, *args, **kwargs):+ """Initialize LayerNorm.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = F.layer_norm(input.float(),self.normalized_shape,@@ -288,6 +324,14 @@ class LayerNorm(nn.LayerNorm):def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):+ """Sequence mask.++ Args:+ lengths: TODO.+ maxlen: TODO.+ dtype: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """if maxlen is None:maxlen = lengths.max()row_vector = torch.arange(0, maxlen, 1).to(lengths.device)@@ -473,6 +517,31 @@ class SenseVoiceEncoderSmall(nn.Module):selfattention_layer_type: str = "sanm",**kwargs,):+ """Initialize SenseVoiceEncoderSmall.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ tp_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ stochastic_depth_rate: TODO.+ input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ padding_idx: TODO.+ kernel_size: Size/dimension parameter.+ sanm_shfit: TODO.+ selfattention_layer_type: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self._output_size = output_size@@ -548,6 +617,7 @@ class SenseVoiceEncoderSmall(nn.Module):self.tp_norm = LayerNorm(output_size)def output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(@@ -608,6 +678,25 @@ class SenseVoiceSmall(nn.Module):**kwargs,):+ """Initialize SenseVoiceSmall.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ ctc_conf: Configuration dict for ctc.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ length_normalized_loss: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -663,6 +752,12 @@ class SenseVoiceSmall(nn.Module):@staticmethoddef from_pretrained(model: str = None, **kwargs):+ """From pretrained.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """from funasr import AutoModelmodel, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)@@ -781,6 +876,14 @@ class SenseVoiceSmall(nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -795,6 +898,12 @@ class SenseVoiceSmall(nn.Module):encoder_out: torch.Tensor,ys_pad: torch.Tensor,):+ """Internal: calc rich ce loss.++ Args:+ encoder_out: Encoder output tensor.+ ys_pad: TODO.+ """decoder_out = self.ctc.ctc_lo(encoder_out)# 2. Compute attention lossloss_rich = self.criterion_att(decoder_out, ys_pad.contiguous())@@ -816,6 +925,16 @@ class SenseVoiceSmall(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """meta_data = {}if (isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"@@ -961,6 +1080,11 @@ class SenseVoiceSmall(nn.Module):return results, meta_datadef post(self, timestamp):+ """Post.++ Args:+ timestamp: TODO.+ """timestamp_new = []words_new = []prev_word = None@@ -990,6 +1114,11 @@ class SenseVoiceSmall(nn.Module):return timestamp_new, words_newdef export(self, **kwargs):+ """Export.++ Args:+ **kwargs: Additional keyword arguments.+ """from .export_meta import export_rebuild_modelif "max_seq_len" not in kwargs:diff --git a/funasr/models/sense_voice/whisper_lib/__init__.py b/funasr/models/sense_voice/whisper_lib/__init__.pyindex 855b8cf26..2cd10869b 100644--- a/funasr/models/sense_voice/whisper_lib/__init__.py+++ b/funasr/models/sense_voice/whisper_lib/__init__.py@@ -48,6 +48,13 @@ _ALIGNMENT_HEADS = {def _download(url: str, root: str, in_memory: bool) -> Union[bytes, str]:+ """Internal: download.++ Args:+ url: TODO.+ root: TODO.+ in_memory: TODO.+ """os.makedirs(root, exist_ok=True)expected_sha256 = url.split("/")[-2]diff --git a/funasr/models/sense_voice/whisper_lib/decoding.py b/funasr/models/sense_voice/whisper_lib/decoding.pyindex 644755f5f..73bb0a71c 100644--- a/funasr/models/sense_voice/whisper_lib/decoding.py+++ b/funasr/models/sense_voice/whisper_lib/decoding.py@@ -175,6 +175,12 @@ class Inference:class PyTorchInference(Inference):def __init__(self, model: "Whisper", initial_token_length: int):+ """Initialize PyTorchInference.++ Args:+ model: Model instance or model name.+ initial_token_length: TODO.+ """self.model: "Whisper" = modelself.initial_token_length = initial_token_lengthself.kv_cache = {}@@ -185,6 +191,12 @@ class PyTorchInference(Inference):self.kv_modules = key_modules + value_modulesdef logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:+ """Logits.++ Args:+ tokens: TODO.+ audio_features: TODO.+ """if not self.kv_cache:self.kv_cache, self.hooks = self.model.install_kv_cache_hooks()@@ -195,6 +207,7 @@ class PyTorchInference(Inference):return self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache)def cleanup_caching(self):+ """Cleanup caching."""for hook in self.hooks:hook.remove()@@ -202,6 +215,11 @@ class PyTorchInference(Inference):self.hooks = []def rearrange_kv_cache(self, source_indices):+ """Rearrange kv cache.++ Args:+ source_indices: TODO.+ """if source_indices != list(range(len(source_indices))):for module in self.kv_modules:# update the key/value cache to contain the selected sequences@@ -224,10 +242,27 @@ class MaximumLikelihoodRanker(SequenceRanker):"""def __init__(self, length_penalty: Optional[float]):+ """Initialize MaximumLikelihoodRanker.++ Args:+ length_penalty: TODO.+ """self.length_penalty = length_penaltydef rank(self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]):+ """Rank.++ Args:+ tokens: TODO.+ sum_logprobs: TODO.+ """def scores(logprobs, lengths):+ """Scores.++ Args:+ logprobs: TODO.+ lengths: TODO.+ """result = []for logprob, length in zip(logprobs, lengths):if self.length_penalty is None:@@ -299,10 +334,23 @@ class TokenDecoder:class GreedyDecoder(TokenDecoder):def __init__(self, temperature: float, eot: int):+ """Initialize GreedyDecoder.++ Args:+ temperature: TODO.+ eot: TODO.+ """self.temperature = temperatureself.eot = eotdef update(self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor) -> Tuple[Tensor, bool]:+ """Update.++ Args:+ tokens: TODO.+ logits: TODO.+ sum_logprobs: TODO.+ """if self.temperature == 0:next_tokens = logits.argmax(dim=-1)else:@@ -320,6 +368,12 @@ class GreedyDecoder(TokenDecoder):def finalize(self, tokens: Tensor, sum_logprobs: Tensor):# make sure each sequence has at least one EOT token at the end+ """Finalize.++ Args:+ tokens: TODO.+ sum_logprobs: TODO.+ """tokens = F.pad(tokens, (0, 1), value=self.eot)return tokens, sum_logprobs.tolist()@@ -332,6 +386,14 @@ class BeamSearchDecoder(TokenDecoder):inference: Inference,patience: Optional[float] = None,):+ """Initialize BeamSearchDecoder.++ Args:+ beam_size: Size/dimension parameter.+ eot: TODO.+ inference: TODO.+ patience: TODO.+ """self.beam_size = beam_sizeself.eot = eotself.inference = inference@@ -342,9 +404,17 @@ class BeamSearchDecoder(TokenDecoder):assert self.max_candidates > 0, f"Invalid beam size ({beam_size}) or patience ({patience})"def reset(self):+ """Reset."""self.finished_sequences = Nonedef update(self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor) -> Tuple[Tensor, bool]:+ """Update.++ Args:+ tokens: TODO.+ logits: TODO.+ sum_logprobs: TODO.+ """if tokens.shape[0] % self.beam_size != 0:raise ValueError(f"{tokens.shape}[0] % {self.beam_size} != 0")@@ -402,6 +472,12 @@ class BeamSearchDecoder(TokenDecoder):def finalize(self, preceding_tokens: Tensor, sum_logprobs: Tensor):# collect all finished sequences, including patience, and add unfinished ones if not enough+ """Finalize.++ Args:+ preceding_tokens: TODO.+ sum_logprobs: TODO.+ """sum_logprobs = sum_logprobs.cpu()for i, sequences in enumerate(self.finished_sequences):if len(sequences) < self.beam_size: # when not enough sequences are finished@@ -438,19 +514,42 @@ class LogitFilter:class SuppressBlank(LogitFilter):def __init__(self, tokenizer: Tokenizer, sample_begin: int):+ """Initialize SuppressBlank.++ Args:+ tokenizer: Tokenizer instance for text encoding/decoding.+ sample_begin: TODO.+ """self.tokenizer = tokenizerself.sample_begin = sample_begindef apply(self, logits: Tensor, tokens: Tensor):+ """Apply.++ Args:+ logits: TODO.+ tokens: TODO.+ """if tokens.shape[1] == self.sample_begin:logits[:, self.tokenizer.encode(" ") + [self.tokenizer.eot]] = -np.infclass SuppressTokens(LogitFilter):def __init__(self, suppress_tokens: Sequence[int]):+ """Initialize SuppressTokens.++ Args:+ suppress_tokens: TODO.+ """self.suppress_tokens = list(suppress_tokens)def apply(self, logits: Tensor, tokens: Tensor):+ """Apply.++ Args:+ logits: TODO.+ tokens: TODO.+ """logits[:, self.suppress_tokens] = -np.inf@@ -458,6 +557,13 @@ class GainEventToken(LogitFilter):def __init__(self, bg_tokens: Sequence[int], ed_tokens: Sequence[int], gain_values: Sequence[float]):+ """Initialize GainEventToken.++ Args:+ bg_tokens: TODO.+ ed_tokens: TODO.+ gain_values: TODO.+ """self.bg_tokens = list(bg_tokens)self.ed_tokens = list(ed_tokens)self.gain_value = [np.log(max(ga, 1e-9)) for ga in gain_values]@@ -465,6 +571,12 @@ class GainEventToken(LogitFilter):assert len(self.bg_tokens) == len(self.gain_value)def apply(self, logits: Tensor, tokens: Tensor):+ """Apply.++ Args:+ logits: TODO.+ tokens: TODO.+ """for i in range(len(tokens)):for bg, ed, ga in zip(self.bg_tokens, self.ed_tokens, self.gain_value):sum_bg = sum([1 if x == bg else 0 for x in tokens[i]])@@ -480,12 +592,25 @@ class ThresholdEmoToken(LogitFilter):def __init__(self, unk_tokens: Sequence[int], emo_tokens: Sequence[int], th_values: Sequence[float]):+ """Initialize ThresholdEmoToken.++ Args:+ unk_tokens: TODO.+ emo_tokens: TODO.+ th_values: TODO.+ """self.unk_token = list(unk_tokens)[0]self.emo_tokens = list(emo_tokens)self.th_values = list(th_values)assert len(self.emo_tokens) == len(self.th_values)def apply(self, logits: Tensor, tokens: Tensor):+ """Apply.++ Args:+ logits: TODO.+ tokens: TODO.+ """for i in range(len(tokens)):for emo, th in zip(self.emo_tokens, self.th_values):if logits[i].argmax() == emo and logits[i].softmax(dim=-1)[emo] < th:@@ -509,12 +634,25 @@ class ApplyTimestampRules(LogitFilter):sample_begin: int,max_initial_timestamp_index: Optional[int],):+ """Initialize ApplyTimestampRules.++ Args:+ tokenizer: Tokenizer instance for text encoding/decoding.+ sample_begin: TODO.+ max_initial_timestamp_index: TODO.+ """self.tokenizer = tokenizerself.sample_begin = sample_beginself.max_initial_timestamp_index = max_initial_timestamp_indexdef apply(self, logits: Tensor, tokens: Tensor):# suppress <|notimestamps|> which is handled by without_timestamps+ """Apply.++ Args:+ logits: TODO.+ tokens: TODO.+ """if self.tokenizer.no_timestamps is not None:logits[:, self.tokenizer.no_timestamps] = -np.inf@@ -566,6 +704,12 @@ class DecodingTask:logit_filters: List[LogitFilter]def __init__(self, model: "Whisper", options: DecodingOptions):+ """Initialize DecodingTask.++ Args:+ model: Model instance or model name.+ options: TODO.+ """self.model = modellanguage = options.language or "en"@@ -637,6 +781,11 @@ class DecodingTask:)def _verify_options(self, options: DecodingOptions) -> DecodingOptions:+ """Internal: verify options.++ Args:+ options: TODO.+ """if options.beam_size is not None and options.best_of is not None:raise ValueError("beam_size and best_of can't be given together")if options.temperature == 0:@@ -650,6 +799,7 @@ class DecodingTask:return optionsdef _get_initial_tokens(self) -> Tuple[int]:+ """Internal: get initial tokens."""tokens = list(self.sot_sequence)if prefix := self.options.prefix:@@ -678,6 +828,7 @@ class DecodingTask:return tuple(tokens)def _get_suppress_tokens(self) -> Tuple[int]:+ """Internal: get suppress tokens."""suppress_tokens = self.options.suppress_tokensif isinstance(suppress_tokens, str):@@ -707,6 +858,11 @@ class DecodingTask:return tuple(sorted(set(suppress_tokens)))def _get_audio_features(self, mel: Tensor):+ """Internal: get audio features.++ Args:+ mel: TODO.+ """if self.options.fp16:mel = mel.half()@@ -727,6 +883,12 @@ class DecodingTask:return audio_featuresdef _detect_language(self, audio_features: Tensor, tokens: Tensor):+ """Internal: detect language.++ Args:+ audio_features: TODO.+ tokens: TODO.+ """languages = [self.options.language] * audio_features.shape[0]lang_probs = None@@ -755,6 +917,12 @@ class DecodingTask:return languages, lang_probsdef _main_loop(self, audio_features: Tensor, tokens: Tensor):+ """Internal: main loop.++ Args:+ audio_features: TODO.+ tokens: TODO.+ """n_batch = tokens.shape[0]sum_logprobs: Tensor = torch.zeros(n_batch, device=audio_features.device)no_speech_probs = [np.nan] * n_batch@@ -786,6 +954,11 @@ class DecodingTask:@torch.no_grad()def run(self, mel: Tensor) -> List[DecodingResult]:+ """Run.++ Args:+ mel: TODO.+ """self.decoder.reset()tokenizer: Tokenizer = self.tokenizern_audio: int = mel.shape[0]diff --git a/funasr/models/sense_voice/whisper_lib/model.py b/funasr/models/sense_voice/whisper_lib/model.pyindex eb1380d11..d4d293a0e 100644--- a/funasr/models/sense_voice/whisper_lib/model.py+++ b/funasr/models/sense_voice/whisper_lib/model.py@@ -35,9 +35,20 @@ class ModelDimensions:class LayerNorm(nn.LayerNorm):def __init__(self, *args, **kwargs):+ """Initialize LayerNorm.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(*args, **kwargs)def forward(self, input):+ """Forward pass for training.++ Args:+ input: Input audio/text data.+ """output = F.layer_norm(input.float(),self.normalized_shape,@@ -50,6 +61,11 @@ class LayerNorm(nn.LayerNorm):class Linear(nn.Linear):def forward(self, x: Tensor) -> Tensor:+ """Forward pass for training.++ Args:+ x: TODO.+ """return F.linear(x,self.weight.to(x.dtype),@@ -59,6 +75,13 @@ class Linear(nn.Linear):class Conv1d(nn.Conv1d):def _conv_forward(self, x: Tensor, weight: Tensor, bias: Optional[Tensor]) -> Tensor:+ """Internal: conv forward.++ Args:+ x: TODO.+ weight: TODO.+ bias: TODO.+ """return super()._conv_forward(x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype))@@ -75,6 +98,12 @@ def sinusoids(length, channels, max_timescale=10000):class MultiHeadAttention(nn.Module):def __init__(self, n_state: int, n_head: int):+ """Initialize MultiHeadAttention.++ Args:+ n_state: TODO.+ n_head: TODO.+ """super().__init__()self.n_head = n_headself.query = Linear(n_state, n_state)@@ -90,6 +119,15 @@ class MultiHeadAttention(nn.Module):kv_cache: Optional[dict] = None,**kwargs,):+ """Forward pass for training.++ Args:+ x: TODO.+ xa: TODO.+ mask: TODO.+ kv_cache: TODO.+ **kwargs: Additional keyword arguments.+ """is_pad_mask = kwargs.get("is_pad_mask", False)q = self.query(x)@@ -115,6 +153,15 @@ class MultiHeadAttention(nn.Module):mask: Optional[Tensor] = None,**kwargs,):+ """Qkv attention.++ Args:+ q: TODO.+ k: TODO.+ v: TODO.+ mask: TODO.+ **kwargs: Additional keyword arguments.+ """is_pad_mask = kwargs.get("is_pad_mask", False)n_batch, n_ctx, n_state = q.shapescale = (n_state // self.n_head) ** -0.25@@ -143,6 +190,12 @@ class MultiHeadAttention(nn.Module):class MultiHeadAttentionSdpa(nn.Module):def __init__(self, n_state: int, n_head: int):+ """Initialize MultiHeadAttentionSdpa.++ Args:+ n_state: TODO.+ n_head: TODO.+ """super().__init__()self.n_head = n_headself.query = Linear(n_state, n_state)@@ -158,6 +211,15 @@ class MultiHeadAttentionSdpa(nn.Module):kv_cache: Optional[dict] = None,**kwargs,):+ """Forward pass for training.++ Args:+ x: TODO.+ xa: TODO.+ mask: TODO.+ kv_cache: TODO.+ **kwargs: Additional keyword arguments.+ """is_pad_mask = kwargs.get("is_pad_mask", False)q = self.query(x)@@ -183,6 +245,15 @@ class MultiHeadAttentionSdpa(nn.Module):mask: Optional[Tensor] = None,**kwargs,):+ """Qkv attention.++ Args:+ q: TODO.+ k: TODO.+ v: TODO.+ mask: TODO.+ **kwargs: Additional keyword arguments.+ """is_pad_mask = kwargs.get("is_pad_mask", False)is_causal = kwargs.get("is_causal", False)n_batch, n_ctx, n_state = q.shape@@ -222,6 +293,14 @@ att_type_dict = {class ResidualAttentionBlock(nn.Module):def __init__(self, n_state: int, n_head: int, cross_attention: bool = False, **kwargs):+ """Initialize ResidualAttentionBlock.++ Args:+ n_state: TODO.+ n_head: TODO.+ cross_attention: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()att_type = kwargs.get("att_type", "default")@@ -245,6 +324,15 @@ class ResidualAttentionBlock(nn.Module):kv_cache: Optional[dict] = None,**kwargs,):+ """Forward pass for training.++ Args:+ x: TODO.+ xa: TODO.+ mask: TODO.+ kv_cache: TODO.+ **kwargs: Additional keyword arguments.+ """is_pad_mask = kwargs.get("is_pad_mask", False)is_pad_memory_mask = kwargs.get("is_pad_memory_mask", False)memory_mask = kwargs.get("memory_mask", None)@@ -266,6 +354,16 @@ class ResidualAttentionBlock(nn.Module):class AudioEncoder(nn.Module):def __init__(self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int, **kwargs):+ """Initialize AudioEncoder.++ Args:+ n_mels: TODO.+ n_ctx: TODO.+ n_state: TODO.+ n_head: TODO.+ n_layer: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, stride=2, padding=1)self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)@@ -301,6 +399,16 @@ class AudioEncoder(nn.Module):class TextDecoder(nn.Module):def __init__(self, n_vocab: int, n_ctx: int, n_state: int, n_head: int, n_layer: int, **kwargs):+ """Initialize TextDecoder.++ Args:+ n_vocab: TODO.+ n_ctx: TODO.+ n_state: TODO.+ n_head: TODO.+ n_layer: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()self.token_embedding = nn.Embedding(n_vocab, n_state)@@ -342,6 +450,11 @@ class TextDecoder(nn.Module):return logitsdef init_state(self, x):+ """Init state.++ Args:+ x: TODO.+ """state = {}return state@@ -369,6 +482,11 @@ class TextDecoder(nn.Module):class Whisper(nn.Module):def __init__(self, dims: ModelDimensions):+ """Initialize Whisper.++ Args:+ dims: TODO.+ """super().__init__()self.dims = dimsself.encoder = AudioEncoder(@@ -396,29 +514,54 @@ class Whisper(nn.Module):# model.register_buffer("alignment_heads", alignment_heads_dense, persistent=False)def set_alignment_heads(self, dump: bytes):+ """Set alignment heads.++ Args:+ dump: TODO.+ """array = np.frombuffer(gzip.decompress(base64.b85decode(dump)), dtype=bool).copy()mask = torch.from_numpy(array).reshape(self.dims.n_text_layer, self.dims.n_text_head)self.register_buffer("alignment_heads", mask.to_sparse(), persistent=False)def embed_audio(self, mel: torch.Tensor):+ """Embed audio.++ Args:+ mel: TODO.+ """return self.encoder(mel)def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor):+ """Logits.++ Args:+ tokens: TODO.+ audio_features: TODO.+ """return self.decoder(tokens, audio_features)def forward(self, mel: torch.Tensor, tokens: torch.Tensor) -> Dict[str, torch.Tensor]:+ """Forward pass for training.++ Args:+ mel: TODO.+ tokens: TODO.+ """return self.decoder(tokens, self.encoder(mel))@propertydef device(self):+ """Device."""return next(self.parameters()).device@propertydef is_multilingual(self):+ """Is multilingual."""return self.dims.n_vocab >= 51865@propertydef num_languages(self):+ """Num languages."""return self.dims.n_vocab - 51765 - int(self.is_multilingual)def install_kv_cache_hooks(self, cache: Optional[dict] = None):@@ -439,6 +582,13 @@ class Whisper(nn.Module):hooks = []def save_to_cache(module, _, output):+ """Save to cache.++ Args:+ module: TODO.+ _: TODO.+ output: TODO.+ """if module not in cache or output.shape[1] > self.dims.n_text_ctx:# save as-is, for the first token or cross attentioncache[module] = output@@ -447,6 +597,11 @@ class Whisper(nn.Module):return cache[module]def install_hooks(layer: nn.Module):+ """Install hooks.++ Args:+ layer: TODO.+ """if isinstance(layer, MultiHeadAttention):hooks.append(layer.key.register_forward_hook(save_to_cache))hooks.append(layer.value.register_forward_hook(save_to_cache))diff --git a/funasr/models/sense_voice/whisper_lib/normalizers/basic.py b/funasr/models/sense_voice/whisper_lib/normalizers/basic.pyindex 76addc520..021fddc24 100644--- a/funasr/models/sense_voice/whisper_lib/normalizers/basic.py+++ b/funasr/models/sense_voice/whisper_lib/normalizers/basic.py@@ -58,10 +58,21 @@ def remove_symbols(s: str):class BasicTextNormalizer:def __init__(self, remove_diacritics: bool = False, split_letters: bool = False):+ """Initialize BasicTextNormalizer.++ Args:+ remove_diacritics: TODO.+ split_letters: TODO.+ """self.clean = remove_symbols_and_diacritics if remove_diacritics else remove_symbolsself.split_letters = split_lettersdef __call__(self, s: str):+ """Internal: call .++ Args:+ s: TODO.+ """s = s.lower()s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between bracketss = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesisdiff --git a/funasr/models/sense_voice/whisper_lib/normalizers/english.py b/funasr/models/sense_voice/whisper_lib/normalizers/english.pyindex cb4653098..651e53eb6 100644--- a/funasr/models/sense_voice/whisper_lib/normalizers/english.py+++ b/funasr/models/sense_voice/whisper_lib/normalizers/english.py@@ -21,6 +21,7 @@ class EnglishNumberNormalizer:"""def __init__(self):+ """Initialize EnglishNumberNormalizer."""super().__init__()self.zeros = {"o", "oh", "zero"}@@ -161,17 +162,32 @@ class EnglishNumberNormalizer:self.literal_words = {"one", "ones"}def process_words(self, words: List[str]) -> Iterator[str]:+ """Process words.++ Args:+ words: TODO.+ """prefix: Optional[str] = Nonevalue: Optional[Union[str, int]] = Noneskip = Falsedef to_fraction(s: str):+ """To fraction.++ Args:+ s: TODO.+ """try:return Fraction(s)except ValueError:return Nonedef output(result: Union[str, int]):+ """Output.++ Args:+ result: TODO.+ """nonlocal prefix, valueresult = str(result)if prefix is not None:@@ -383,6 +399,11 @@ class EnglishNumberNormalizer:def preprocess(self, s: str):# replace "<number> and a half" with "<number> point five"+ """Preprocess.++ Args:+ s: TODO.+ """results = []segments = re.split(r"\band\s+a\s+half\b", s)@@ -411,7 +432,17 @@ class EnglishNumberNormalizer:return sdef postprocess(self, s: str):+ """Postprocess.++ Args:+ s: TODO.+ """def combine_cents(m: Match):+ """Combine cents.++ Args:+ m: TODO.+ """try:currency = m.group(1)integer = m.group(2)@@ -421,6 +452,11 @@ class EnglishNumberNormalizer:return m.stringdef extract_cents(m: Match):+ """Extract cents.++ Args:+ m: TODO.+ """try:return f"¢{int(m.group(1))}"except ValueError:@@ -436,6 +472,11 @@ class EnglishNumberNormalizer:return sdef __call__(self, s: str):+ """Internal: call .++ Args:+ s: TODO.+ """s = self.preprocess(s)s = " ".join(word for word in self.process_words(s.split()) if word is not None)s = self.postprocess(s)@@ -451,15 +492,22 @@ class EnglishSpellingNormalizer:"""def __init__(self):+ """Initialize EnglishSpellingNormalizer."""mapping_path = os.path.join(os.path.dirname(__file__), "english.json")self.mapping = json.load(open(mapping_path))def __call__(self, s: str):+ """Internal: call .++ Args:+ s: TODO.+ """return " ".join(self.mapping.get(word, word) for word in s.split())class EnglishTextNormalizer:def __init__(self):+ """Initialize EnglishTextNormalizer."""self.ignore_patterns = r"\b(hmm|mm|mhm|mmm|uh|um)\b"self.replacers = {# common contractions@@ -520,6 +568,11 @@ class EnglishTextNormalizer:self.standardize_spellings = EnglishSpellingNormalizer()def __call__(self, s: str):+ """Internal: call .++ Args:+ s: TODO.+ """s = s.lower()s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between bracketsdiff --git a/funasr/models/sense_voice/whisper_lib/timing.py b/funasr/models/sense_voice/whisper_lib/timing.pyindex ba9cb1399..4e28fbc59 100644--- a/funasr/models/sense_voice/whisper_lib/timing.py+++ b/funasr/models/sense_voice/whisper_lib/timing.py@@ -54,6 +54,11 @@ def median_filter(x: torch.Tensor, filter_width: int):@numba.jit(nopython=True)def backtrace(trace: np.ndarray):+ """Backtrace.++ Args:+ trace: TODO.+ """i = trace.shape[0] - 1j = trace.shape[1] - 1trace[0, :] = 2@@ -79,6 +84,11 @@ def backtrace(trace: np.ndarray):@numba.jit(nopython=True, parallel=True)def dtw_cpu(x: np.ndarray):+ """Dtw cpu.++ Args:+ x: TODO.+ """N, M = x.shapecost = np.ones((N + 1, M + 1), dtype=np.float32) * np.inftrace = -np.ones((N + 1, M + 1), dtype=np.float32)@@ -104,6 +114,12 @@ def dtw_cpu(x: np.ndarray):def dtw_cuda(x, BLOCK_SIZE=1024):+ """Dtw cuda.++ Args:+ x: TODO.+ BLOCK_SIZE: TODO.+ """from .triton_ops import dtw_kernelM, N = x.shape@@ -133,6 +149,11 @@ def dtw_cuda(x, BLOCK_SIZE=1024):def dtw(x: torch.Tensor) -> np.ndarray:+ """Dtw.++ Args:+ x: TODO.+ """if x.is_cuda:try:return dtw_cuda(x)@@ -164,6 +185,15 @@ def find_alignment(medfilt_width: int = 7,qk_scale: float = 1.0,) -> List[WordTiming]:+ """Find alignment.++ Args:+ model: Model instance or model name.+ tokenizer: Tokenizer instance for text encoding/decoding.+ text_tokens: TODO.+ mel: TODO.+ num_frames: TODO.+ """if len(text_tokens) == 0:return []@@ -235,6 +265,13 @@ def find_alignment(def merge_punctuations(alignment: List[WordTiming], prepended: str, appended: str):# merge prepended punctuations+ """Merge punctuations.++ Args:+ alignment: TODO.+ prepended: TODO.+ appended: TODO.+ """i = len(alignment) - 2j = len(alignment) - 1while i >= 0:@@ -279,6 +316,11 @@ def add_word_timestamps(last_speech_timestamp: float,**kwargs,):+ """Add word timestamps.++ Args:+ **kwargs: Additional keyword arguments.+ """if len(segments) == 0:returndiff --git a/funasr/models/sense_voice/whisper_lib/tokenizer.py b/funasr/models/sense_voice/whisper_lib/tokenizer.pyindex 5b276c2a0..4a7520519 100644--- a/funasr/models/sense_voice/whisper_lib/tokenizer.py+++ b/funasr/models/sense_voice/whisper_lib/tokenizer.py@@ -168,6 +168,7 @@ class Tokenizer:special_tokens: Dict[str, int] = field(default_factory=dict)def __post_init__(self):+ """Internal: post init ."""for special in self.encoding.special_tokens_set:special_token = self.encoding.encode_single_token(special)self.special_tokens[special] = special_token@@ -192,9 +193,21 @@ class Tokenizer:self.sot_sequence = tuple(sot_sequence)def encode(self, text, **kwargs):+ """Encode.++ Args:+ text: Text tensor or string input.+ **kwargs: Additional keyword arguments.+ """return self.encoding.encode(text, **kwargs)def decode(self, token_ids: List[int], **kwargs) -> str:+ """Decode.++ Args:+ token_ids: TODO.+ **kwargs: Additional keyword arguments.+ """token_ids = [t for t in token_ids if t < self.timestamp_begin]return self.encoding.decode(token_ids, **kwargs)@@ -206,46 +219,57 @@ class Tokenizer:return self.encoding.decode(token_ids, **kwargs)def get_vocab_size(self) -> int:+ """Get vocab size."""return self.encoding.n_vocab@cached_propertydef eot(self) -> int:+ """Eot."""return self.encoding.eot_token@cached_propertydef transcribe(self) -> int:+ """Transcribe."""return self.special_tokens["<|transcribe|>"]@cached_propertydef translate(self) -> int:+ """Translate."""return self.special_tokens["<|translate|>"]@cached_propertydef sot(self) -> int:+ """Sot."""return self.special_tokens["<|startoftranscript|>"]@cached_propertydef sot_sense(self) -> int:+ """Sot sense."""return self.special_tokens["<|startoftranscript|>"]@cached_propertydef sot_lm(self) -> int:+ """Sot lm."""return self.special_tokens["<|startoflm|>"]@cached_propertydef sot_prev(self) -> int:+ """Sot prev."""return self.special_tokens["<|startofprev|>"]@cached_propertydef no_speech(self) -> int:+ """No speech."""return self.special_tokens["<|nospeech|>"]@cached_propertydef no_timestamps(self) -> int:+ """No timestamps."""return self.special_tokens["<|notimestamps|>"]@cached_propertydef timestamp_begin(self) -> int:+ """Timestamp begin."""return self.special_tokens["<|0.00|>"]@cached_property@@ -257,6 +281,11 @@ class Tokenizer:return self.to_language_token(self.language)def to_language_token(self, language):+ """To language token.++ Args:+ language: Language identifier.+ """if token := self.special_tokens.get(f"<|{language}|>", None):return token@@ -264,6 +293,7 @@ class Tokenizer:@cached_propertydef all_language_tokens(self) -> Tuple[int]:+ """All language tokens."""result = []for token, token_id in self.special_tokens.items():if token.strip("<|>") in LANGUAGES:@@ -272,10 +302,12 @@ class Tokenizer:@cached_propertydef all_language_codes(self) -> Tuple[str]:+ """All language codes."""return tuple(self.decode([_l]).strip("<|>") for _l in self.all_language_tokens)@cached_propertydef sot_sequence_including_notimestamps(self) -> Tuple[int]:+ """Sot sequence including notimestamps."""return tuple(list(self.sot_sequence) + [self.no_timestamps])@cached_property@@ -313,6 +345,11 @@ class Tokenizer:return tuple(sorted(result))def split_to_word_tokens(self, tokens: List[int]):+ """Split to word tokens.++ Args:+ tokens: TODO.+ """if self.language in {"zh", "ja", "th", "lo", "my", "yue"}:# These languages don't typically use spaces, so it is difficult to split words# without morpheme analysis. Here, we instead split words at any@@ -322,6 +359,11 @@ class Tokenizer:return self.split_tokens_on_spaces(tokens)def split_tokens_on_unicode(self, tokens: List[int]):+ """Split tokens on unicode.++ Args:+ tokens: TODO.+ """decoded_full = self.decode_with_timestamps(tokens)replacement_char = "\ufffd"@@ -347,6 +389,11 @@ class Tokenizer:return words, word_tokensdef split_tokens_on_spaces(self, tokens: List[int]):+ """Split tokens on spaces.++ Args:+ tokens: TODO.+ """subwords, subword_tokens_list = self.split_tokens_on_unicode(tokens)words = []word_tokens = []@@ -367,6 +414,13 @@ class Tokenizer:@lru_cache(maxsize=None)def get_encoding(name: str = "gpt2", num_languages: int = 99, vocab_path: str = None):+ """Get encoding.++ Args:+ name: TODO.+ num_languages: TODO.+ vocab_path: TODO.+ """if vocab_path is None:vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken")@@ -430,6 +484,11 @@ def get_tokenizer(encoding_path: Optional[str] = None,vocab_path: Optional[str] = None,) -> Tokenizer:+ """Get tokenizer.++ Args:+ multilingual: TODO.+ """if language is not None:language = language.lower()if language not in LANGUAGES:diff --git a/funasr/models/sense_voice/whisper_lib/transcribe.py b/funasr/models/sense_voice/whisper_lib/transcribe.pyindex 5c5f49d80..3124f8245 100644--- a/funasr/models/sense_voice/whisper_lib/transcribe.py+++ b/funasr/models/sense_voice/whisper_lib/transcribe.py@@ -174,6 +174,11 @@ def transcribe(warnings.warn("Word-level timestamps on translations may not be reliable.")def decode_with_fallback(segment: torch.Tensor) -> DecodingResult:+ """Decode with fallback.++ Args:+ segment: TODO.+ """temperatures = [temperature] if isinstance(temperature, (int, float)) else temperaturedecode_result = None@@ -225,6 +230,7 @@ def transcribe(initial_prompt_tokens = []def new_segment(*, start: float, end: float, tokens: torch.Tensor, result: DecodingResult):+ """New segment."""tokens = tokens.tolist()text_tokens = [token for token in tokens if token < tokenizer.eot]return {@@ -282,6 +288,11 @@ def transcribe(# anomalous words are very long/short/improbabledef word_anomaly_score(word: dict) -> float:+ """Word anomaly score.++ Args:+ word: TODO.+ """probability = word.get("probability", 0.0)duration = word["end"] - word["start"]score = 0.0@@ -294,6 +305,11 @@ def transcribe(return scoredef is_segment_anomaly(segment: Optional[dict]) -> bool:+ """Is segment anomaly.++ Args:+ segment: TODO.+ """if segment is None or not segment["words"]:return Falsewords = [w for w in segment["words"] if w["word"] not in punctuation]@@ -302,6 +318,11 @@ def transcribe(return score >= 3 or score + 0.01 >= len(words)def next_words_segment(segments: List[dict]) -> Optional[dict]:+ """Next words segment.++ Args:+ segments: TODO.+ """return next((s for s in segments if s["words"]), None)timestamp_tokens: torch.Tensor = tokens.ge(tokenizer.timestamp_begin)@@ -467,9 +488,15 @@ def transcribe(def cli():+ """Cli."""from . import available_modelsdef valid_model_name(name):+ """Valid model name.++ Args:+ name: TODO.+ """if name in available_models() or os.path.exists(name):return nameraise ValueError(diff --git a/funasr/models/sense_voice/whisper_lib/triton_ops.py b/funasr/models/sense_voice/whisper_lib/triton_ops.pyindex 9919595ce..ecb27b6ac 100644--- a/funasr/models/sense_voice/whisper_lib/triton_ops.py+++ b/funasr/models/sense_voice/whisper_lib/triton_ops.py@@ -12,6 +12,19 @@ except ImportError:@triton.jitdef dtw_kernel(cost, trace, x, x_stride, cost_stride, trace_stride, N, M, BLOCK_SIZE: tl.constexpr):+ """Dtw kernel.++ Args:+ cost: TODO.+ trace: TODO.+ x: TODO.+ x_stride: TODO.+ cost_stride: TODO.+ trace_stride: TODO.+ N: TODO.+ M: TODO.+ BLOCK_SIZE: TODO.+ """offsets = tl.arange(0, BLOCK_SIZE)mask = offsets < M@@ -41,7 +54,21 @@ def dtw_kernel(cost, trace, x, x_stride, cost_stride, trace_stride, N, M, BLOCK_@lru_cache(maxsize=None)def median_kernel(filter_width: int):@triton.jit+ """Median kernel.++ Args:+ filter_width: TODO.+ """def kernel(y, x, x_stride, y_stride, BLOCK_SIZE: tl.constexpr): # x.shape[-1] == filter_width+ """Kernel.++ Args:+ y: TODO.+ x: TODO.+ x_stride: TODO.+ y_stride: TODO.+ BLOCK_SIZE: TODO.+ """row_idx = tl.program_id(0)offsets = tl.arange(0, BLOCK_SIZE)mask = offsets < y_stridediff --git a/funasr/models/sense_voice/whisper_lib/utils.py b/funasr/models/sense_voice/whisper_lib/utils.pyindex 5fc61256f..bbd3f6cff 100644--- a/funasr/models/sense_voice/whisper_lib/utils.py+++ b/funasr/models/sense_voice/whisper_lib/utils.py@@ -12,21 +12,42 @@ if system_encoding != "utf-8":def make_safe(string):# replaces any character not representable using the system default encoding with an '?',# avoiding UnicodeEncodeError (https://github.com/openai/whisper/discussions/729).+ """Make safe.++ Args:+ string: TODO.+ """return string.encode(system_encoding, errors="replace").decode(system_encoding)else:def make_safe(string):# utf-8 can encode any Unicode code point, so no need to do the round-trip encoding+ """Make safe.++ Args:+ string: TODO.+ """return stringdef exact_div(x, y):+ """Exact div.++ Args:+ x: TODO.+ y: TODO.+ """assert x % y == 0return x // ydef str2bool(string):+ """Str2bool.++ Args:+ string: TODO.+ """str2val = {"True": True, "False": False}if string in str2val:return str2val[string]@@ -35,19 +56,41 @@ def str2bool(string):def optional_int(string):+ """Optional int.++ Args:+ string: TODO.+ """return None if string == "None" else int(string)def optional_float(string):+ """Optional float.++ Args:+ string: TODO.+ """return None if string == "None" else float(string)def compression_ratio(text) -> float:+ """Compression ratio.++ Args:+ text: Text tensor or string input.+ """text_bytes = text.encode("utf-8")return len(text_bytes) / len(zlib.compress(text_bytes))def format_timestamp(seconds: float, always_include_hours: bool = False, decimal_marker: str = "."):+ """Format timestamp.++ Args:+ seconds: TODO.+ always_include_hours: TODO.+ decimal_marker: TODO.+ """assert seconds >= 0, "non-negative timestamp expected"milliseconds = round(seconds * 1000.0)@@ -65,6 +108,11 @@ def format_timestamp(seconds: float, always_include_hours: bool = False, decimaldef get_start(segments: List[dict]) -> Optional[float]:+ """Get start.++ Args:+ segments: TODO.+ """return next((w["start"] for s in segments for w in s["words"]),segments[0]["start"] if segments else None,@@ -72,6 +120,11 @@ def get_start(segments: List[dict]) -> Optional[float]:def get_end(segments: List[dict]) -> Optional[float]:+ """Get end.++ Args:+ segments: TODO.+ """return next((w["end"] for s in reversed(segments) for w in reversed(s["words"])),segments[-1]["end"] if segments else None,@@ -82,9 +135,22 @@ class ResultWriter:extension: strdef __init__(self, output_dir: str):+ """Initialize ResultWriter.++ Args:+ output_dir: Directory for saving output files.+ """self.output_dir = output_dirdef __call__(self, result: dict, audio_path: str, options: Optional[dict] = None, **kwargs):+ """Internal: call .++ Args:+ result: TODO.+ audio_path: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """audio_basename = os.path.basename(audio_path)audio_basename = os.path.splitext(audio_basename)[0]output_path = os.path.join(self.output_dir, audio_basename + "." + self.extension)@@ -93,6 +159,14 @@ class ResultWriter:self.write_result(result, file=f, options=options, **kwargs)def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ """Write result.++ Args:+ result: TODO.+ file: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """raise NotImplementedError@@ -100,6 +174,14 @@ class WriteTXT(ResultWriter):extension: str = "txt"def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ """Write result.++ Args:+ result: TODO.+ file: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """for segment in result["segments"]:print(segment["text"].strip(), file=file, flush=True)@@ -118,6 +200,12 @@ class SubtitlesWriter(ResultWriter):highlight_words: bool = False,max_words_per_line: Optional[int] = None,):+ """Iterate result.++ Args:+ result: TODO.+ options: TODO.+ """options = options or {}max_line_width = max_line_width or options.get("max_line_width")max_line_count = max_line_count or options.get("max_line_count")@@ -128,6 +216,7 @@ class SubtitlesWriter(ResultWriter):max_words_per_line = max_words_per_line or 1000def iterate_subtitles():+ """Iterate subtitles."""line_len = 0line_count = 1# the next subtitle to yield (a list of word timings with whitespace)@@ -205,6 +294,11 @@ class SubtitlesWriter(ResultWriter):yield segment_start, segment_end, segment_textdef format_timestamp(self, seconds: float):+ """Format timestamp.++ Args:+ seconds: TODO.+ """return format_timestamp(seconds=seconds,always_include_hours=self.always_include_hours,@@ -218,6 +312,14 @@ class WriteVTT(SubtitlesWriter):decimal_marker: str = "."def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ """Write result.++ Args:+ result: TODO.+ file: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """print("WEBVTT\n", file=file)for start, end, text in self.iterate_result(result, options, **kwargs):print(f"{start} --> {end}\n{text}\n", file=file, flush=True)@@ -229,6 +331,14 @@ class WriteSRT(SubtitlesWriter):decimal_marker: str = ","def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ """Write result.++ Args:+ result: TODO.+ file: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """for i, (start, end, text) in enumerate(self.iterate_result(result, options, **kwargs), start=1):@@ -248,6 +358,14 @@ class WriteTSV(ResultWriter):extension: str = "tsv"def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ """Write result.++ Args:+ result: TODO.+ file: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """print("start", "end", "text", sep="\t", file=file)for segment in result["segments"]:print(round(1000 * segment["start"]), file=file, end="\t")@@ -259,10 +377,24 @@ class WriteJSON(ResultWriter):extension: str = "json"def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ """Write result.++ Args:+ result: TODO.+ file: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """json.dump(result, file)def get_writer(output_format: str, output_dir: str) -> Callable[[dict, TextIO, dict], None]:+ """Get writer.++ Args:+ output_format: TODO.+ output_dir: Directory for saving output files.+ """writers = {"txt": WriteTXT,"vtt": WriteVTT,@@ -275,6 +407,14 @@ def get_writer(output_format: str, output_dir: str) -> Callable[[dict, TextIO, dall_writers = [writer(output_dir) for writer in writers.values()]def write_all(result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ """Write all.++ Args:+ result: TODO.+ file: TODO.+ options: TODO.+ **kwargs: Additional keyword arguments.+ """for writer in all_writers:writer(result, file, options, **kwargs)diff --git a/funasr/models/sond/e2e_diar_sond.py b/funasr/models/sond/e2e_diar_sond.pyindex 0f5a7c27c..44cf6d361 100644--- a/funasr/models/sond/e2e_diar_sond.py+++ b/funasr/models/sond/e2e_diar_sond.py@@ -34,6 +34,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -69,6 +74,33 @@ class DiarSondModel(FunASRModel):onfly_shuffle_speaker: bool = True,):+ """Initialize DiarSondModel.++ Args:+ vocab_size: Size/dimension parameter.+ frontend: Audio frontend for feature extraction.+ specaug: TODO.+ profileaug: TODO.+ normalize: TODO.+ encoder: TODO.+ speaker_encoder: TODO.+ ci_scorer: TODO.+ cd_scorer: TODO.+ decoder: TODO.+ token_list: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ max_spk_num: TODO.+ label_aggregator: TODO.+ normalize_speech_speaker: TODO.+ ignore_id: TODO.+ speaker_discrimination_loss_weight: TODO.+ inter_score_loss_weight: TODO.+ inputs_type: TODO.+ model_regularizer_weight: TODO.+ freeze_encoder: TODO.+ onfly_shuffle_speaker: TODO.+ """super().__init__()self.encoder = encoder@@ -109,6 +141,7 @@ class DiarSondModel(FunASRModel):self.to_regularize_parameters = Nonedef get_regularize_parameters(self):+ """Get regularize parameters."""to_regularize_parameters, normal_parameters = [], []for name, param in self.named_parameters():if (@@ -124,6 +157,7 @@ class DiarSondModel(FunASRModel):return to_regularize_parameters, normal_parametersdef generate_pse_embedding(self):+ """Generate pse embedding."""embedding = np.zeros((len(self.token_list), self.max_spk_num), dtype=np.float32)for idx, pse_label in enumerate(self.token_list):emb = int2vec(int(pse_label), vec_dim=self.max_spk_num, dtype=np.float32)@@ -299,6 +333,7 @@ class DiarSondModel(FunASRModel):return loss, stats, weightdef calculate_regularizer_loss(self):+ """Calculate regularizer loss."""regularizer_loss = 0.0for name, param in self.to_regularize_parameters:regularizer_loss = regularizer_loss + torch.norm(param, p=2)@@ -307,6 +342,13 @@ class DiarSondModel(FunASRModel):def classification_loss(self, predictions: torch.Tensor, labels: torch.Tensor, prediction_lengths: torch.Tensor) -> torch.Tensor:+ """Classification loss.++ Args:+ predictions: TODO.+ labels: TODO.+ prediction_lengths: Lengths of prediction.+ """mask = make_pad_mask(prediction_lengths, maxlen=labels.shape[1])pad_labels = labels.masked_fill(mask.to(predictions.device), value=self.ignore_id)loss = self.criterion_diar(predictions.contiguous(), pad_labels)@@ -316,6 +358,12 @@ class DiarSondModel(FunASRModel):def speaker_discrimination_loss(self, profile: torch.Tensor, profile_lengths: torch.Tensor) -> torch.Tensor:+ """Speaker discrimination loss.++ Args:+ profile: TODO.+ profile_lengths: Lengths of profile.+ """profile_mask = (torch.linalg.norm(profile, ord=2, dim=2, keepdim=True) > 0).float() # (B, N, 1)@@ -339,6 +387,12 @@ class DiarSondModel(FunASRModel):return lossdef calculate_multi_labels(self, pse_labels, pse_labels_lengths):+ """Calculate multi labels.++ Args:+ pse_labels: TODO.+ pse_labels_lengths: Lengths of pse_labels.+ """mask = make_pad_mask(pse_labels_lengths, maxlen=pse_labels.shape[1])padding_labels = pse_labels.masked_fill(mask.to(pse_labels.device), value=0).to(pse_labels)multi_labels = F.embedding(padding_labels, self.pse_embedding)@@ -352,6 +406,14 @@ class DiarSondModel(FunASRModel):pse_labels: torch.Tensor,pse_labels_lengths: torch.Tensor,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Internal score loss.++ Args:+ cd_score: TODO.+ ci_score: TODO.+ pse_labels: TODO.+ pse_labels_lengths: Lengths of pse_labels.+ """multi_labels = self.calculate_multi_labels(pse_labels, pse_labels_lengths)ci_loss = self.criterion_bce(ci_score, multi_labels, pse_labels_lengths)cd_loss = self.criterion_bce(cd_score, multi_labels, pse_labels_lengths)@@ -366,6 +428,16 @@ class DiarSondModel(FunASRModel):binary_labels: torch.Tensor = None,binary_labels_lengths: torch.Tensor = None,) -> Dict[str, torch.Tensor]:+ """Collect feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ profile: TODO.+ profile_lengths: Lengths of profile.+ binary_labels: TODO.+ binary_labels_lengths: Lengths of binary_labels.+ """feats, feats_lengths = self._extract_feats(speech, speech_lengths)return {"feats": feats, "feats_lengths": feats_lengths}@@ -374,6 +446,12 @@ class DiarSondModel(FunASRModel):profile: torch.Tensor,profile_lengths: torch.Tensor,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Encode speaker.++ Args:+ profile: TODO.+ profile_lengths: Lengths of profile.+ """with autocast(False):if profile.shape[1] < self.max_spk_num:profile = F.pad(@@ -392,6 +470,12 @@ class DiarSondModel(FunASRModel):speech: torch.Tensor,speech_lengths: torch.Tensor,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Encode speech.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """if self.encoder is not None and self.inputs_type == "raw":speech, speech_lengths = self.encode(speech, speech_lengths)speech_mask = ~make_pad_mask(speech_lengths, maxlen=speech.shape[1])@@ -402,6 +486,12 @@ class DiarSondModel(FunASRModel):@staticmethoddef concate_speech_ivc(speech: torch.Tensor, ivc: torch.Tensor) -> torch.Tensor:+ """Concate speech ivc.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ ivc: TODO.+ """nn, tt = ivc.shape[1], speech.shape[1]speech = speech.unsqueeze(dim=1) # B x 1 x T x Dspeech = speech.expand(-1, nn, -1, -1) # B x N x T x D@@ -417,6 +507,14 @@ class DiarSondModel(FunASRModel):seq_len: torch.Tensor = None,spk_len: torch.Tensor = None,) -> Tuple[torch.Tensor, torch.Tensor]:+ """Calc similarity.++ Args:+ speech_encoder_outputs: TODO.+ speaker_encoder_outputs: TODO.+ seq_len: TODO.+ spk_len: TODO.+ """bb, tt = speech_encoder_outputs.shape[0], speech_encoder_outputs.shape[1]d_sph, d_spk = speech_encoder_outputs.shape[2], speaker_encoder_outputs.shape[2]if self.normalize_speech_speaker:@@ -439,6 +537,12 @@ class DiarSondModel(FunASRModel):return ci_simi, cd_simidef post_net_forward(self, simi, seq_len):+ """Post net forward.++ Args:+ simi: TODO.+ seq_len: TODO.+ """logits = self.decoder(simi, seq_len)[0]return logits@@ -452,6 +556,15 @@ class DiarSondModel(FunASRModel):return_inter_outputs: bool = False,) -> [torch.Tensor, Optional[list]]:# speech encoding+ """Prediction forward.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ profile: TODO.+ profile_lengths: Lengths of profile.+ return_inter_outputs: TODO.+ """speech, speech_lengths = self.encode_speech(speech, speech_lengths)# speaker encodingprofile, profile_lengths = self.encode_speaker(profile, profile_lengths)@@ -510,6 +623,12 @@ class DiarSondModel(FunASRModel):def _extract_feats(self, speech: torch.Tensor, speech_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Internal: extract feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """batch_size = speech.shape[0]speech_lengths = (speech_lengths@@ -537,6 +656,13 @@ class DiarSondModel(FunASRModel):def calc_diarization_error(pred, label, length):# Note (jiatong): Credit to https://github.com/hitachi-speech/EEND+ """Calc diarization error.++ Args:+ pred: TODO.+ label: TODO.+ length: TODO.+ """(batch_size, max_len, num_output) = label.size()# mask the padding partmask = ~make_pad_mask(length, maxlen=label.shape[1]).unsqueeze(-1).numpy()diff --git a/funasr/models/sond/encoder/ci_scorers.py b/funasr/models/sond/encoder/ci_scorers.pyindex a60560b7c..043fa30d2 100644--- a/funasr/models/sond/encoder/ci_scorers.py+++ b/funasr/models/sond/encoder/ci_scorers.py@@ -4,6 +4,7 @@ from torch.nn import functional as Fclass DotScorer(torch.nn.Module):def __init__(self):+ """Initialize DotScorer."""super().__init__()def forward(@@ -13,12 +14,19 @@ class DotScorer(torch.nn.Module):):# xs_pad: B, T, D# spk_emb: B, N, D+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ spk_emb: TODO.+ """scores = torch.matmul(xs_pad, spk_emb.transpose(1, 2))return scoresclass CosScorer(torch.nn.Module):def __init__(self):+ """Initialize CosScorer."""super().__init__()def forward(@@ -28,5 +36,11 @@ class CosScorer(torch.nn.Module):):# xs_pad: B, T, D# spk_emb: B, N, D+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ spk_emb: TODO.+ """scores = F.cosine_similarity(xs_pad.unsqueeze(2), spk_emb.unsqueeze(1), dim=-1)return scoresdiff --git a/funasr/models/sond/encoder/conv_encoder.py b/funasr/models/sond/encoder/conv_encoder.pyindex c4f7098c7..44023ef56 100644--- a/funasr/models/sond/encoder/conv_encoder.py+++ b/funasr/models/sond/encoder/conv_encoder.py@@ -26,6 +26,17 @@ class EncoderLayer(nn.Module):include_batch_norm=False,residual=False,):+ """Initialize EncoderLayer.++ Args:+ input_units: TODO.+ num_units: TODO.+ kernel_size: Size/dimension parameter.+ activation: TODO.+ stride: TODO.+ include_batch_norm: TODO.+ residual: TODO.+ """super().__init__()left_padding = math.ceil((kernel_size - stride) / 2)right_padding = kernel_size - stride - left_padding@@ -47,12 +58,23 @@ class EncoderLayer(nn.Module):@staticmethoddef get_activation(activation):+ """Get activation.++ Args:+ activation: TODO.+ """if activation == "tanh":return nn.Tanh()else:return nn.ReLU()def forward(self, xs_pad, ilens=None):+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ """outputs = self.conv1d(self.conv_padding(xs_pad))if self.residual and self.stride == 1 and self.input_units == self.num_units:outputs = outputs + xs_pad@@ -89,6 +111,26 @@ class ConvEncoder(AbsEncoder):tf2torch_tensor_name_prefix_torch: str = "speaker_encoder",tf2torch_tensor_name_prefix_tf: str = "EAND/speaker_encoder",):+ """Initialize ConvEncoder.++ Args:+ num_layers: TODO.+ input_units: TODO.+ num_units: TODO.+ kernel_size: Size/dimension parameter.+ dropout_rate: TODO.+ position_encoder: TODO.+ activation: TODO.+ auxiliary_states: TODO.+ out_units: TODO.+ out_norm: TODO.+ out_residual: TODO.+ include_batchnorm: TODO.+ regularization_weight: TODO.+ stride: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ """super().__init__()self._output_size = num_units@@ -143,6 +185,7 @@ class ConvEncoder(AbsEncoder):self.after_norm = LayerNorm(out_units)def output_size(self) -> int:+ """Output size."""return self.num_unitsdef forward(@@ -152,6 +195,13 @@ class ConvEncoder(AbsEncoder):prev_states: torch.Tensor = None,) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ prev_states: TODO.+ """inputs = xs_padif self.position_encoder is not None:inputs = self.position_encoder(inputs)diff --git a/funasr/models/sond/encoder/ecapa_tdnn_encoder.py b/funasr/models/sond/encoder/ecapa_tdnn_encoder.pyindex 1af8b7033..a66b5db47 100644--- a/funasr/models/sond/encoder/ecapa_tdnn_encoder.py+++ b/funasr/models/sond/encoder/ecapa_tdnn_encoder.py@@ -16,6 +16,18 @@ class _BatchNorm1d(nn.Module):combine_batch_time=False,skip_transpose=False,):+ """Initialize _BatchNorm1d.++ Args:+ input_shape: TODO.+ input_size: Size/dimension parameter.+ eps: TODO.+ momentum: TODO.+ affine: TODO.+ track_running_stats: TODO.+ combine_batch_time: TODO.+ skip_transpose: TODO.+ """super().__init__()self.combine_batch_time = combine_batch_timeself.skip_transpose = skip_transpose@@ -34,6 +46,11 @@ class _BatchNorm1d(nn.Module):)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """shape_or = x.shapeif self.combine_batch_time:if x.ndim == 3:@@ -69,6 +86,21 @@ class _Conv1d(nn.Module):padding_mode="reflect",skip_transpose=False,):+ """Initialize _Conv1d.++ Args:+ out_channels: TODO.+ kernel_size: Size/dimension parameter.+ input_shape: TODO.+ in_channels: TODO.+ stride: TODO.+ dilation: TODO.+ padding: TODO.+ groups: TODO.+ bias: TODO.+ padding_mode: TODO.+ skip_transpose: TODO.+ """super().__init__()self.kernel_size = kernel_sizeself.stride = stride@@ -96,6 +128,11 @@ class _Conv1d(nn.Module):)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """if not self.skip_transpose:x = x.transpose(1, -1)@@ -133,6 +170,14 @@ class _Conv1d(nn.Module):stride: int,):# Detecting input shape+ """Internal: manage padding.++ Args:+ x: TODO.+ kernel_size: Size/dimension parameter.+ dilation: TODO.+ stride: TODO.+ """L_in = x.shape[-1]# Time padding@@ -165,6 +210,14 @@ class _Conv1d(nn.Module):def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int):+ """Get padding elem.++ Args:+ L_in: TODO.+ stride: TODO.+ kernel_size: Size/dimension parameter.+ dilation: TODO.+ """if stride > 1:n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1)L_out = stride * (n_steps - 1) + kernel_size * dilation@@ -180,15 +233,35 @@ def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int):# Skip transpose as much as possible for efficiencyclass Conv1d(_Conv1d):def __init__(self, *args, **kwargs):+ """Initialize Conv1d.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(skip_transpose=True, *args, **kwargs)class BatchNorm1d(_BatchNorm1d):def __init__(self, *args, **kwargs):+ """Initialize BatchNorm1d.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__(skip_transpose=True, *args, **kwargs)def length_to_mask(length, max_len=None, dtype=None, device=None):+ """Length to mask.++ Args:+ length: TODO.+ max_len: TODO.+ dtype: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """assert len(length.shape) == 1if max_len is None:@@ -217,6 +290,16 @@ class TDNNBlock(nn.Module):activation=nn.ReLU,groups=1,):+ """Initialize TDNNBlock.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ kernel_size: Size/dimension parameter.+ dilation: TODO.+ activation: TODO.+ groups: TODO.+ """super(TDNNBlock, self).__init__()self.conv = Conv1d(in_channels=in_channels,@@ -229,6 +312,11 @@ class TDNNBlock(nn.Module):self.norm = BatchNorm1d(input_size=out_channels)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """return self.norm(self.activation(self.conv(x)))@@ -258,6 +346,15 @@ class Res2NetBlock(torch.nn.Module):"""def __init__(self, in_channels, out_channels, scale=8, kernel_size=3, dilation=1):+ """Initialize Res2NetBlock.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ scale: TODO.+ kernel_size: Size/dimension parameter.+ dilation: TODO.+ """super(Res2NetBlock, self).__init__()assert in_channels % scale == 0assert out_channels % scale == 0@@ -279,6 +376,11 @@ class Res2NetBlock(torch.nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """y = []for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)):if i == 0:@@ -315,6 +417,13 @@ class SEBlock(nn.Module):"""def __init__(self, in_channels, se_channels, out_channels):+ """Initialize SEBlock.++ Args:+ in_channels: TODO.+ se_channels: TODO.+ out_channels: TODO.+ """super(SEBlock, self).__init__()self.conv1 = Conv1d(in_channels=in_channels, out_channels=se_channels, kernel_size=1)@@ -323,6 +432,12 @@ class SEBlock(nn.Module):self.sigmoid = torch.nn.Sigmoid()def forward(self, x, lengths=None):+ """Forward pass for training.++ Args:+ x: TODO.+ lengths: TODO.+ """L = x.shape[-1]if lengths is not None:mask = length_to_mask(lengths * L, max_len=L, device=x.device)@@ -360,6 +475,13 @@ class AttentiveStatisticsPooling(nn.Module):"""def __init__(self, channels, attention_channels=128, global_context=True):+ """Initialize AttentiveStatisticsPooling.++ Args:+ channels: TODO.+ attention_channels: TODO.+ global_context: TODO.+ """super().__init__()self.eps = 1e-12@@ -382,6 +504,14 @@ class AttentiveStatisticsPooling(nn.Module):L = x.shape[-1]def _compute_statistics(x, m, dim=2, eps=self.eps):+ """Internal: compute statistics.++ Args:+ x: TODO.+ m: TODO.+ dim: TODO.+ eps: TODO.+ """mean = (m * x).sum(dim)std = torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps))return mean, std@@ -460,6 +590,18 @@ class SERes2NetBlock(nn.Module):activation=torch.nn.ReLU,groups=1,):+ """Initialize SERes2NetBlock.++ Args:+ in_channels: TODO.+ out_channels: TODO.+ res2net_scale: TODO.+ se_channels: TODO.+ kernel_size: Size/dimension parameter.+ dilation: TODO.+ activation: TODO.+ groups: TODO.+ """super().__init__()self.out_channels = out_channelsself.tdnn1 = TDNNBlock(@@ -492,6 +634,12 @@ class SERes2NetBlock(nn.Module):)def forward(self, x, lengths=None):+ """Forward pass for training.++ Args:+ x: TODO.+ lengths: TODO.+ """residual = xif self.shortcut:residual = self.shortcut(x)@@ -550,6 +698,23 @@ class ECAPA_TDNN(torch.nn.Module):window_shift=1,):+ """Initialize ECAPA_TDNN.++ Args:+ input_size: Size/dimension parameter.+ lin_neurons: TODO.+ activation: TODO.+ channels: TODO.+ kernel_sizes: TODO.+ dilations: TODO.+ attention_channels: TODO.+ res2net_scale: TODO.+ se_channels: TODO.+ global_context: TODO.+ groups: TODO.+ window_size: Size/dimension parameter.+ window_shift: TODO.+ """super().__init__()assert len(channels) == len(kernel_sizes)assert len(channels) == len(dilations)@@ -612,6 +777,12 @@ class ECAPA_TDNN(torch.nn.Module):def windowed_pooling(self, x, lengths=None):# x: Batch, Channel, Time+ """Windowed pooling.++ Args:+ x: TODO.+ lengths: TODO.+ """tt = x.shape[2]num_chunk = int(math.ceil(tt / self.window_shift))pad = self.window_size // 2diff --git a/funasr/models/sond/encoder/fsmn_encoder.py b/funasr/models/sond/encoder/fsmn_encoder.pyindex 9ec9912a4..b1c785198 100644--- a/funasr/models/sond/encoder/fsmn_encoder.py+++ b/funasr/models/sond/encoder/fsmn_encoder.py@@ -24,6 +24,14 @@ class FsmnBlock(torch.nn.Module):kernel_size,fsmn_shift=0,):+ """Initialize FsmnBlock.++ Args:+ n_feat: TODO.+ dropout_rate: TODO.+ kernel_size: Size/dimension parameter.+ fsmn_shift: TODO.+ """super().__init__()self.dropout = nn.Dropout(p=dropout_rate)self.fsmn_block = nn.Conv1d(@@ -37,6 +45,13 @@ class FsmnBlock(torch.nn.Module):self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)def forward(self, inputs, mask, mask_shfit_chunk=None):+ """Forward pass for training.++ Args:+ inputs: TODO.+ mask: TODO.+ mask_shfit_chunk: TODO.+ """b, t, d = inputs.size()if mask is not None:mask = torch.reshape(mask, (b, -1, 1))@@ -55,6 +70,15 @@ class FsmnBlock(torch.nn.Module):class EncoderLayer(torch.nn.Module):def __init__(self, in_size, size, feed_forward, fsmn_block, dropout_rate=0.0):+ """Initialize EncoderLayer.++ Args:+ in_size: Size/dimension parameter.+ size: TODO.+ feed_forward: TODO.+ fsmn_block: TODO.+ dropout_rate: TODO.+ """super().__init__()self.in_size = in_sizeself.size = size@@ -67,6 +91,12 @@ class EncoderLayer(torch.nn.Module):) -> Tuple[torch.Tensor, torch.Tensor]:# xs_pad in Batch, Time, Dim+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ mask: TODO.+ """context = self.ffn(xs_pad)[0]memory = self.memory(context, mask)@@ -160,11 +190,19 @@ class FsmnEncoder(AbsEncoder):self.conv1d = nn.Conv1d(num_memory_units, out_units, 1, 1)def output_size(self) -> int:+ """Output size."""return self.num_memory_unitsdef forward(self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ prev_states: TODO.+ """inputs = xs_padif self.position_encoder is not None:inputs = self.position_encoder(inputs)diff --git a/funasr/models/sond/encoder/resnet34_encoder.py b/funasr/models/sond/encoder/resnet34_encoder.pyindex ba577a3a4..ddbdc3571 100644--- a/funasr/models/sond/encoder/resnet34_encoder.py+++ b/funasr/models/sond/encoder/resnet34_encoder.py@@ -12,6 +12,14 @@ class BasicLayer(torch.nn.Module):def __init__(self, in_filters: int, filters: int, stride: int, bn_momentum: float = 0.5):+ """Initialize BasicLayer.++ Args:+ in_filters: TODO.+ filters: TODO.+ stride: TODO.+ bn_momentum: TODO.+ """super().__init__()self.stride = strideself.in_filters = in_filters@@ -31,6 +39,12 @@ class BasicLayer(torch.nn.Module):def proper_padding(self, x, stride):# align padding mode to tf.layers.conv2d with padding_mod="same"+ """Proper padding.++ Args:+ x: TODO.+ stride: TODO.+ """if stride == 1:return F.pad(x, (1, 1, 1, 1), "constant", 0)elif stride == 2:@@ -39,6 +53,12 @@ class BasicLayer(torch.nn.Module):return F.pad(x, (w % 2, 1, h % 2, 1), "constant", 0)def forward(self, xs_pad, ilens):+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ """identity = xs_padif self.in_filters != self.filters or self.stride > 1:identity = self.conv_sc(identity)@@ -60,6 +80,15 @@ class BasicLayer(torch.nn.Module):class BasicBlock(torch.nn.Module):def __init__(self, in_filters, filters, num_layer, stride, bn_momentum=0.5):+ """Initialize BasicBlock.++ Args:+ in_filters: TODO.+ filters: TODO.+ num_layer: TODO.+ stride: TODO.+ bn_momentum: TODO.+ """super().__init__()self.num_layer = num_layer@@ -71,6 +100,12 @@ class BasicBlock(torch.nn.Module):def forward(self, xs_pad, ilens):+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ """for i in range(self.num_layer):xs_pad, ilens = self._modules["layer_{}".format(i)](xs_pad, ilens)@@ -88,6 +123,17 @@ class ResNet34(AbsEncoder):layers_in_block=(3, 4, 6, 3),filters_in_block=(32, 64, 128, 256),):+ """Initialize ResNet34.++ Args:+ input_size: Size/dimension parameter.+ use_head_conv: TODO.+ batchnorm_momentum: TODO.+ use_head_maxpool: TODO.+ num_nodes_pooling_layer: TODO.+ layers_in_block: TODO.+ filters_in_block: TODO.+ """super(ResNet34, self).__init__()self.use_head_conv = use_head_conv@@ -132,12 +178,20 @@ class ResNet34(AbsEncoder):self.time_ds_ratio = 8def output_size(self) -> int:+ """Output size."""return self.num_nodes_pooling_layerdef forward(self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ prev_states: TODO.+ """features = xs_padassert (features.size(-1) == self.input_size@@ -181,6 +235,20 @@ class ResNet34_SP_L2Reg(AbsEncoder):tf2torch_tensor_name_prefix_tf="EAND/speech_encoder",tf_train_steps=720000,):+ """Initialize ResNet34_SP_L2Reg.++ Args:+ input_size: Size/dimension parameter.+ use_head_conv: TODO.+ batchnorm_momentum: TODO.+ use_head_maxpool: TODO.+ num_nodes_pooling_layer: TODO.+ layers_in_block: TODO.+ filters_in_block: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ tf_train_steps: TODO.+ """super(ResNet34_SP_L2Reg, self).__init__()self.use_head_conv = use_head_conv@@ -230,12 +298,20 @@ class ResNet34_SP_L2Reg(AbsEncoder):self.time_ds_ratio = 8def output_size(self) -> int:+ """Output size."""return self.num_nodes_pooling_layerdef forward(self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None) -> Tuple[torch.Tensor, torch.Tensor]:+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ prev_states: TODO.+ """features = xs_padassert (features.size(-1) == self.input_size@@ -321,6 +397,7 @@ class ResNet34Diar(ResNet34):)def output_size(self) -> int:+ """Output size."""if self.embedding_node.startswith("resnet1"):return self.num_nodes_resnet1elif self.embedding_node.startswith("resnet2"):@@ -335,6 +412,13 @@ class ResNet34Diar(ResNet34):prev_states: torch.Tensor = None,) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ prev_states: TODO.+ """endpoints = OrderedDict()res_out, ilens = super().forward(xs_pad, ilens)endpoints["resnet0_bn"] = res_out@@ -419,6 +503,7 @@ class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg):)def output_size(self) -> int:+ """Output size."""if self.embedding_node.startswith("resnet1"):return self.num_nodes_resnet1elif self.embedding_node.startswith("resnet2"):@@ -433,6 +518,13 @@ class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg):prev_states: torch.Tensor = None,) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ prev_states: TODO.+ """endpoints = OrderedDict()res_out, ilens = super().forward(xs_pad, ilens)endpoints["resnet0_bn"] = res_outdiff --git a/funasr/models/sond/encoder/self_attention_encoder.py b/funasr/models/sond/encoder/self_attention_encoder.pyindex 2e979b1be..5796649f9 100644--- a/funasr/models/sond/encoder/self_attention_encoder.py+++ b/funasr/models/sond/encoder/self_attention_encoder.py@@ -145,6 +145,30 @@ class SelfAttentionEncoder(AbsEncoder):tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",out_units=None,):+ """Initialize SelfAttentionEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ padding_idx: TODO.+ interctc_layer_idx: TODO.+ interctc_use_conditioning: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ out_units: TODO.+ """super().__init__()self._output_size = output_size@@ -257,6 +281,7 @@ class SelfAttentionEncoder(AbsEncoder):self.output_linear = nn.Linear(output_size, out_units)def output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(diff --git a/funasr/models/sond/label_aggregation.py b/funasr/models/sond/label_aggregation.pyindex dddc4c4bf..f3fdb7d0a 100644--- a/funasr/models/sond/label_aggregation.py+++ b/funasr/models/sond/label_aggregation.py@@ -12,6 +12,13 @@ class LabelAggregate(torch.nn.Module):hop_length: int = 128,center: bool = True,):+ """Initialize LabelAggregate.++ Args:+ win_length: TODO.+ hop_length: TODO.+ center: TODO.+ """super().__init__()self.win_length = win_length@@ -19,6 +26,7 @@ class LabelAggregate(torch.nn.Module):self.center = centerdef extra_repr(self):+ """Extra repr."""return (f"win_length={self.win_length}, "f"hop_length={self.hop_length}, "@@ -85,11 +93,17 @@ class LabelAggregateMaxPooling(torch.nn.Module):self,hop_length: int = 8,):+ """Initialize LabelAggregateMaxPooling.++ Args:+ hop_length: TODO.+ """super().__init__()self.hop_length = hop_lengthdef extra_repr(self):+ """Extra repr."""return f"hop_length={self.hop_length}, "def forward(diff --git a/funasr/models/sond/pooling/pooling_layers.py b/funasr/models/sond/pooling/pooling_layers.pyindex 0a426a94f..4b8d69d64 100644--- a/funasr/models/sond/pooling/pooling_layers.py+++ b/funasr/models/sond/pooling/pooling_layers.py@@ -13,9 +13,19 @@ class TAP(nn.Module):"""def __init__(self, **kwargs):+ """Initialize TAP.++ Args:+ **kwargs: Additional keyword arguments.+ """super(TAP, self).__init__()def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """pooling_mean = x.mean(dim=-1)# To be compatable with 2D inputpooling_mean = pooling_mean.flatten(start_dim=1)@@ -28,10 +38,20 @@ class TSDP(nn.Module):"""def __init__(self, **kwargs):+ """Initialize TSDP.++ Args:+ **kwargs: Additional keyword arguments.+ """super(TSDP, self).__init__()def forward(self, x):# The last dimension is the temporal axis+ """Forward pass for training.++ Args:+ x: TODO.+ """pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8)pooling_std = pooling_std.flatten(start_dim=1)return pooling_std@@ -45,10 +65,20 @@ class TSTP(nn.Module):"""def __init__(self, **kwargs):+ """Initialize TSTP.++ Args:+ **kwargs: Additional keyword arguments.+ """super(TSTP, self).__init__()def forward(self, x):# The last dimension is the temporal axis+ """Forward pass for training.++ Args:+ x: TODO.+ """pooling_mean = x.mean(dim=-1)pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8)pooling_mean = pooling_mean.flatten(start_dim=1)@@ -64,6 +94,13 @@ class ASTP(nn.Module):"""def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False):+ """Initialize ASTP.++ Args:+ in_dim: Size/dimension parameter.+ bottleneck_dim: Size/dimension parameter.+ global_context_att: TODO.+ """super(ASTP, self).__init__()self.global_context_att = global_context_attdiff --git a/funasr/models/sond/pooling/statistic_pooling.py b/funasr/models/sond/pooling/statistic_pooling.pyindex 91db2393e..e35b74551 100644--- a/funasr/models/sond/pooling/statistic_pooling.py+++ b/funasr/models/sond/pooling/statistic_pooling.py@@ -10,6 +10,12 @@ VAR2STD_EPSILON = 1e-12class StatisticPooling(torch.nn.Module):def __init__(self, pooling_dim: Union[int, Tuple] = 2, eps=1e-12):+ """Initialize StatisticPooling.++ Args:+ pooling_dim: Size/dimension parameter.+ eps: TODO.+ """super(StatisticPooling, self).__init__()if isinstance(pooling_dim, int):pooling_dim = (pooling_dim,)@@ -19,6 +25,12 @@ class StatisticPooling(torch.nn.Module):def forward(self, xs_pad, ilens=None):# xs_pad in (Batch, Channel, Time, Frequency)+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ """if ilens is None:masks = torch.ones_like(xs_pad).to(xs_pad)else:@@ -47,6 +59,13 @@ def statistic_pooling() -> torch.Tensor:# xs_pad in (Batch, Channel, Time, Frequency)+ """Statistic pooling.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ pooling_dim: Size/dimension parameter.+ """if ilens is None:seq_mask = torch.ones_like(xs_pad).to(xs_pad)else:@@ -79,6 +98,15 @@ def windowed_statistic_pooling() -> Tuple[torch.Tensor, int]:# xs_pad in (Batch, Channel, Time, Frequency)+ """Windowed statistic pooling.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ pooling_dim: Size/dimension parameter.+ pooling_size: Size/dimension parameter.+ pooling_stride: TODO.+ """tt = xs_pad.shape[2]num_chunk = int(math.ceil(tt / pooling_stride))pad = pooling_size // 2diff --git a/funasr/models/sond/sv_decoder.py b/funasr/models/sond/sv_decoder.pyindex 7d8484db9..2051c530c 100644--- a/funasr/models/sond/sv_decoder.py+++ b/funasr/models/sond/sv_decoder.py@@ -12,6 +12,15 @@ class DenseDecoder(AbsDecoder):num_nodes_last_layer: int = 256,batchnorm_momentum: float = 0.5,):+ """Initialize DenseDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ num_nodes_resnet1: TODO.+ num_nodes_last_layer: TODO.+ batchnorm_momentum: TODO.+ """super(DenseDecoder, self).__init__()self.resnet1_dense = torch.nn.Linear(encoder_output_size, num_nodes_resnet1)self.resnet1_bn = torch.nn.BatchNorm1d(@@ -26,6 +35,11 @@ class DenseDecoder(AbsDecoder):self.output_dense = torch.nn.Linear(num_nodes_last_layer, vocab_size, bias=False)def forward(self, features):+ """Forward pass for training.++ Args:+ features: TODO.+ """embeddings = {}features = self.resnet1_dense(features)embeddings["resnet1_dense"] = featuresdiff --git a/funasr/models/specaug/mask_along_axis.py b/funasr/models/specaug/mask_along_axis.pyindex d4f237b3d..72e131a8f 100644--- a/funasr/models/specaug/mask_along_axis.py+++ b/funasr/models/specaug/mask_along_axis.py@@ -147,6 +147,14 @@ class MaskAlongAxis(torch.nn.Module):dim: Union[int, str] = "time",replace_with_zero: bool = True,):+ """Initialize MaskAlongAxis.++ Args:+ mask_width_range: TODO.+ num_mask: TODO.+ dim: TODO.+ replace_with_zero: TODO.+ """if isinstance(mask_width_range, int):mask_width_range = (0, mask_width_range)if len(mask_width_range) != 2:@@ -176,6 +184,7 @@ class MaskAlongAxis(torch.nn.Module):self.replace_with_zero = replace_with_zerodef extra_repr(self):+ """Extra repr."""return (f"mask_width_range={self.mask_width_range}, "f"num_mask={self.num_mask}, axis={self.mask_axis}"@@ -212,6 +221,14 @@ class MaskAlongAxisVariableMaxWidth(torch.nn.Module):dim: Union[int, str] = "time",replace_with_zero: bool = True,):+ """Initialize MaskAlongAxisVariableMaxWidth.++ Args:+ mask_width_ratio_range: TODO.+ num_mask: TODO.+ dim: TODO.+ replace_with_zero: TODO.+ """if isinstance(mask_width_ratio_range, float):mask_width_ratio_range = (0.0, mask_width_ratio_range)if len(mask_width_ratio_range) != 2:@@ -242,6 +259,7 @@ class MaskAlongAxisVariableMaxWidth(torch.nn.Module):self.replace_with_zero = replace_with_zerodef extra_repr(self):+ """Extra repr."""return (f"mask_width_ratio_range={self.mask_width_ratio_range}, "f"num_mask={self.num_mask}, axis={self.mask_axis}"@@ -281,6 +299,15 @@ class MaskAlongAxisLFR(torch.nn.Module):replace_with_zero: bool = True,lfr_rate: int = 1,):+ """Initialize MaskAlongAxisLFR.++ Args:+ mask_width_range: TODO.+ num_mask: TODO.+ dim: TODO.+ replace_with_zero: TODO.+ lfr_rate: TODO.+ """if isinstance(mask_width_range, int):mask_width_range = (0, mask_width_range)if len(mask_width_range) != 2:@@ -313,6 +340,7 @@ class MaskAlongAxisLFR(torch.nn.Module):self.lfr_rate = lfr_ratedef extra_repr(self):+ """Extra repr."""return (f"mask_width_range={self.mask_width_range}, "f"num_mask={self.num_mask}, axis={self.mask_axis}"diff --git a/funasr/models/specaug/profileaug.py b/funasr/models/specaug/profileaug.pyindex 669d323f8..05a5c63b5 100644--- a/funasr/models/specaug/profileaug.py+++ b/funasr/models/specaug/profileaug.py@@ -23,6 +23,17 @@ class ProfileAug(nn.Module):disturb_aug_prob: float = 0.4,disturb_alpha: float = 0.2,) -> None:+ """Initialize ProfileAug.++ Args:+ apply_split_aug: TODO.+ split_aug_prob: TODO.+ apply_merge_aug: TODO.+ merge_aug_prob: TODO.+ apply_disturb_aug: TODO.+ disturb_aug_prob: TODO.+ disturb_alpha: TODO.+ """super().__init__()self.apply_split_aug = apply_split_augself.split_aug_prob = split_aug_prob@@ -34,6 +45,13 @@ class ProfileAug(nn.Module):def split_aug(self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor):# B, N+ """Split aug.++ Args:+ profile: TODO.+ binary_labels: TODO.+ mask: TODO.+ """bsz, dim = profile.shape[0], profile.shape[-1]profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False)spk_count = binary_labels.sum(dim=1)@@ -56,6 +74,13 @@ class ProfileAug(nn.Module):return profile, binary_labels, maskdef merge_aug(self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor):+ """Merge aug.++ Args:+ profile: TODO.+ binary_labels: TODO.+ mask: TODO.+ """bsz, dim = profile.shape[0], profile.shape[-1]profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False)spk_count = binary_labels.sum(dim=1)@@ -86,6 +111,13 @@ class ProfileAug(nn.Module):return profile, binary_labels, maskdef disturb_aug(self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor):+ """Disturb aug.++ Args:+ profile: TODO.+ binary_labels: TODO.+ mask: TODO.+ """bsz, dim = profile.shape[0], profile.shape[-1]profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False)spk_count = binary_labels.sum(dim=1)@@ -118,6 +150,16 @@ class ProfileAug(nn.Module):) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:# copy inputs to avoid inplace-operation+ """Forward pass for training.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ profile: TODO.+ profile_lengths: Lengths of profile.+ binary_labels: TODO.+ labels_length: TODO.+ """speech, profile, binary_labels = (torch.clone(speech),torch.clone(profile),diff --git a/funasr/models/specaug/specaug.py b/funasr/models/specaug/specaug.pyindex 75a4b0b28..5a23502d7 100644--- a/funasr/models/specaug/specaug.py+++ b/funasr/models/specaug/specaug.py@@ -41,6 +41,20 @@ class SpecAug(nn.Module):time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None,num_time_mask: int = 2,):+ """Initialize SpecAug.++ Args:+ apply_time_warp: TODO.+ time_warp_window: TODO.+ time_warp_mode: TODO.+ apply_freq_mask: TODO.+ freq_mask_width_range: TODO.+ num_freq_mask: TODO.+ apply_time_mask: TODO.+ time_mask_width_range: TODO.+ time_mask_width_ratio_range: TODO.+ num_time_mask: TODO.+ """if not apply_time_warp and not apply_time_mask and not apply_freq_mask:raise ValueError("Either one of time_warp, time_mask, or freq_mask should be applied")if (@@ -93,6 +107,12 @@ class SpecAug(nn.Module):self.time_mask = Nonedef forward(self, x, x_lengths=None):+ """Forward pass for training.++ Args:+ x: TODO.+ x_lengths: Lengths of x.+ """if self.time_warp is not None:x, x_lengths = self.time_warp(x, x_lengths)if self.freq_mask is not None:@@ -122,6 +142,21 @@ class SpecAugLFR(nn.Module):time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None,num_time_mask: int = 2,):+ """Initialize SpecAugLFR.++ Args:+ apply_time_warp: TODO.+ time_warp_window: TODO.+ time_warp_mode: TODO.+ apply_freq_mask: TODO.+ freq_mask_width_range: TODO.+ num_freq_mask: TODO.+ lfr_rate: TODO.+ apply_time_mask: TODO.+ time_mask_width_range: TODO.+ time_mask_width_ratio_range: TODO.+ num_time_mask: TODO.+ """if not apply_time_warp and not apply_time_mask and not apply_freq_mask:raise ValueError("Either one of time_warp, time_mask, or freq_mask should be applied")if (@@ -177,6 +212,12 @@ class SpecAugLFR(nn.Module):self.time_mask = Nonedef forward(self, x, x_lengths=None):+ """Forward pass for training.++ Args:+ x: TODO.+ x_lengths: Lengths of x.+ """if self.time_warp is not None:x, x_lengths = self.time_warp(x, x_lengths)if self.freq_mask is not None:diff --git a/funasr/models/specaug/time_warp.py b/funasr/models/specaug/time_warp.pyindex ef3532f6a..afdffbffc 100644--- a/funasr/models/specaug/time_warp.py+++ b/funasr/models/specaug/time_warp.py@@ -56,11 +56,18 @@ class TimeWarp(torch.nn.Module):"""def __init__(self, window: int = 80, mode: str = DEFAULT_TIME_WARP_MODE):+ """Initialize TimeWarp.++ Args:+ window: TODO.+ mode: TODO.+ """super().__init__()self.window = windowself.mode = modedef extra_repr(self):+ """Extra repr."""return f"window={self.window}, mode={self.mode}"def forward(self, x: torch.Tensor, x_lengths: torch.Tensor = None):diff --git a/funasr/models/transducer/model.py b/funasr/models/transducer/model.pyindex 37925fcf6..e72200083 100644--- a/funasr/models/transducer/model.py+++ b/funasr/models/transducer/model.py@@ -28,6 +28,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -72,6 +77,38 @@ class Transducer(torch.nn.Module):**kwargs,):+ """Initialize Transducer.++ Args:+ frontend: Audio frontend for feature extraction.+ frontend_conf: Configuration dict for frontend.+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ joint_network: TODO.+ joint_network_conf: Configuration dict for joint_network.+ transducer_weight: TODO.+ fastemit_lambda: TODO.+ auxiliary_ctc_weight: TODO.+ auxiliary_ctc_dropout_rate: TODO.+ auxiliary_lm_loss_weight: TODO.+ auxiliary_lm_loss_smoothing: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -413,6 +450,11 @@ class Transducer(torch.nn.Module):):# 1. Build ASR model+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """scorers = {}if self.ctc != None:@@ -449,6 +491,15 @@ class Transducer(torch.nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")diff --git a/funasr/models/transducer/rnn_decoder.py b/funasr/models/transducer/rnn_decoder.pyindex 4a0ffa3c3..10a56b295 100644--- a/funasr/models/transducer/rnn_decoder.py+++ b/funasr/models/transducer/rnn_decoder.py@@ -35,6 +35,27 @@ def build_attention_list(han_win: int = 5,):+ """Build attention list.++ Args:+ eprojs: TODO.+ dunits: TODO.+ atype: TODO.+ num_att: TODO.+ num_encs: TODO.+ aheads: TODO.+ adim: TODO.+ awin: TODO.+ aconv_chans: TODO.+ aconv_filts: TODO.+ han_mode: TODO.+ han_type: TODO.+ han_heads: TODO.+ han_dim: Size/dimension parameter.+ han_conv_chans: TODO.+ han_conv_filts: TODO.+ han_win: TODO.+ """att_list = torch.nn.ModuleList()if num_encs == 1:for i in range(num_att):@@ -99,6 +120,21 @@ class RNNDecoder(nn.Module):att_conf: dict = None,):# FIXME(kamo): The parts of num_spk should be refactored more more more+ """Initialize RNNDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ rnn_type: TODO.+ num_layers: TODO.+ hidden_size: Size/dimension parameter.+ sampling_probability: TODO.+ dropout: TODO.+ context_residual: TODO.+ replace_sos: TODO.+ num_encs: TODO.+ att_conf: Configuration dict for att.+ """if rnn_type not in {"lstm", "gru"}:raise ValueError(f"Not supported: rnn_type={rnn_type}")@@ -151,9 +187,23 @@ class RNNDecoder(nn.Module):self.att_list = build_attention_list(eprojs=eprojs, dunits=hidden_size, **att_conf)def zero_state(self, hs_pad):+ """Zero state.++ Args:+ hs_pad: TODO.+ """return hs_pad.new_zeros(hs_pad.size(0), self.dunits)def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev):+ """Rnn forward.++ Args:+ ey: TODO.+ z_list: TODO.+ c_list: TODO.+ z_prev: TODO.+ c_prev: TODO.+ """if self.dtype == "lstm":z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0]))for i in range(1, self.dlayers):@@ -170,6 +220,15 @@ class RNNDecoder(nn.Module):def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0):# to support mutiple encoder asr mode, in single encoder mode,# convert torch.Tensor to List of torch.Tensor+ """Forward pass for training.++ Args:+ hs_pad: TODO.+ hlens: TODO.+ ys_in_pad: TODO.+ ys_in_lens: Lengths of ys_in.+ strm_idx: TODO.+ """if self.num_encs == 1:hs_pad = [hs_pad]hlens = [hlens]@@ -254,6 +313,11 @@ class RNNDecoder(nn.Module):def init_state(self, x):# to support mutiple encoder asr mode, in single encoder mode,# convert torch.Tensor to List of torch.Tensor+ """Init state.++ Args:+ x: TODO.+ """if self.num_encs == 1:x = [x]@@ -283,6 +347,13 @@ class RNNDecoder(nn.Module):def score(self, yseq, state, x):# to support mutiple encoder asr mode, in single encoder mode,# convert torch.Tensor to List of torch.Tensor+ """Score.++ Args:+ yseq: TODO.+ state: TODO.+ x: TODO.+ """if self.num_encs == 1:x = [x]diff --git a/funasr/models/transformer/attention.py b/funasr/models/transformer/attention.pyindex 2844333a1..f99eaff16 100644--- a/funasr/models/transformer/attention.py+++ b/funasr/models/transformer/attention.py@@ -122,6 +122,11 @@ class MultiHeadedAttention(nn.Module):class MultiHeadedAttentionExport(nn.Module):def __init__(self, model):+ """Initialize MultiHeadedAttentionExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.d_k = model.d_kself.h = model.h@@ -133,16 +138,36 @@ class MultiHeadedAttentionExport(nn.Module):self.all_head_size = self.h * self.d_kdef forward(self, query, key, value, mask):+ """Forward pass for training.++ Args:+ query: TODO.+ key: Sample identifiers.+ value: TODO.+ mask: TODO.+ """q, k, v = self.forward_qkv(query, key, value)scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)return self.forward_attention(v, scores, mask)def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:+ """Transpose for scores.++ Args:+ x: TODO.+ """new_x_shape = x.size()[:-1] + (self.h, self.d_k)x = x.view(new_x_shape)return x.permute(0, 2, 1, 3)def forward_qkv(self, query, key, value):+ """Forward qkv.++ Args:+ query: TODO.+ key: Sample identifiers.+ value: TODO.+ """q = self.linear_q(query)k = self.linear_k(key)v = self.linear_v(value)@@ -152,6 +177,13 @@ class MultiHeadedAttentionExport(nn.Module):return q, k, vdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)@@ -165,12 +197,26 @@ class MultiHeadedAttentionExport(nn.Module):class RelPosMultiHeadedAttentionExport(MultiHeadedAttentionExport):def __init__(self, model):+ """Initialize RelPosMultiHeadedAttentionExport.++ Args:+ model: Model instance or model name.+ """super().__init__(model)self.linear_pos = model.linear_posself.pos_bias_u = model.pos_bias_uself.pos_bias_v = model.pos_bias_vdef forward(self, query, key, value, pos_emb, mask):+ """Forward pass for training.++ Args:+ query: TODO.+ key: Sample identifiers.+ value: TODO.+ pos_emb: TODO.+ mask: TODO.+ """q, k, v = self.forward_qkv(query, key, value)q = q.transpose(1, 2) # (batch, time1, head, d_k)@@ -197,6 +243,11 @@ class RelPosMultiHeadedAttentionExport(MultiHeadedAttentionExport):return self.forward_attention(v, scores, mask)def rel_shift(self, x):+ """Rel shift.++ Args:+ x: TODO.+ """zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype)x_padded = torch.cat([zero_pad, x], dim=-1)@@ -207,6 +258,13 @@ class RelPosMultiHeadedAttentionExport(MultiHeadedAttentionExport):return xdef forward_attention(self, value, scores, mask):+ """Forward attention.++ Args:+ value: TODO.+ scores: TODO.+ mask: TODO.+ """scores = scores + maskattn = torch.softmax(scores, dim=-1)diff --git a/funasr/models/transformer/decoder.py b/funasr/models/transformer/decoder.pyindex 7d849f349..307d5365b 100644--- a/funasr/models/transformer/decoder.py+++ b/funasr/models/transformer/decoder.py@@ -149,6 +149,11 @@ class DecoderLayer(nn.Module):class DecoderLayerExport(nn.Module):def __init__(self, model):+ """Initialize DecoderLayerExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.self_attn = model.self_attnself.src_attn = model.src_attn@@ -158,6 +163,15 @@ class DecoderLayerExport(nn.Module):self.norm3 = model.norm3def forward(self, tgt, tgt_mask, memory, memory_mask, cache=None):+ """Forward pass for training.++ Args:+ tgt: TODO.+ tgt_mask: TODO.+ memory: TODO.+ memory_mask: TODO.+ cache: State cache dict for streaming inference.+ """residual = tgttgt = self.norm1(tgt)tgt_q = tgt@@ -209,6 +223,18 @@ class BaseTransformerDecoder(nn.Module, BatchScorerInterface):pos_enc_class=PositionalEncoding,normalize_before: bool = True,):+ """Initialize BaseTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ """super().__init__()attention_dim = encoder_output_size@@ -387,6 +413,24 @@ class TransformerDecoder(BaseTransformerDecoder):normalize_before: bool = True,concat_after: bool = False,):+ """Initialize TransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ """super().__init__(vocab_size=vocab_size,encoder_output_size=encoder_output_size,@@ -435,6 +479,27 @@ class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize LightweightConvolutionTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "@@ -495,6 +560,27 @@ class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize LightweightConvolution2DTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "@@ -555,6 +641,27 @@ class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize DynamicConvolutionTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "@@ -615,6 +722,27 @@ class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder):conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),conv_usebias: int = False,):+ """Initialize DynamicConvolution2DTransformerDecoder.++ Args:+ vocab_size: Size/dimension parameter.+ encoder_output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ self_attention_dropout_rate: TODO.+ src_attention_dropout_rate: TODO.+ input_layer: TODO.+ use_output_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ conv_wshare: TODO.+ conv_kernel_length: TODO.+ conv_usebias: TODO.+ """if len(conv_kernel_length) != num_blocks:raise ValueError("conv_kernel_length must have equal number of values to num_blocks: "diff --git a/funasr/models/transformer/embedding.py b/funasr/models/transformer/embedding.pyindex 9ef2f4b0e..93863e623 100644--- a/funasr/models/transformer/embedding.py+++ b/funasr/models/transformer/embedding.py@@ -182,6 +182,7 @@ class LearnableFourierPosEnc(torch.nn.Module):)def _reset(self):+ """Internal: reset."""self.w_r.data = torch.normal(0, (1 / math.sqrt(self.gamma)), (1, self.d_model // 2))def extend_pe(self, x):@@ -384,11 +385,24 @@ class SinusoidalPositionEncoder(torch.nn.Module):""" """def __init__(self, d_model=80, dropout_rate=0.1):+ """Initialize SinusoidalPositionEncoder.++ Args:+ d_model: D Model instance.+ dropout_rate: TODO.+ """super().__init__()def encode(self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32):+ """Encode.++ Args:+ positions: TODO.+ depth: TODO.+ dtype: TODO.+ """batch_size = positions.size(0)positions = positions.type(dtype)device = positions.device@@ -406,6 +420,11 @@ class SinusoidalPositionEncoder(torch.nn.Module):return encoding.type(dtype)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """batch_size, timesteps, input_dim = x.size()positions = torch.arange(1, timesteps + 1, device=x.device)[None, :]position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device)@@ -417,11 +436,24 @@ class StreamSinusoidalPositionEncoder(torch.nn.Module):""" """def __init__(self, d_model=80, dropout_rate=0.1):+ """Initialize StreamSinusoidalPositionEncoder.++ Args:+ d_model: D Model instance.+ dropout_rate: TODO.+ """super().__init__()def encode(self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32):+ """Encode.++ Args:+ positions: TODO.+ depth: TODO.+ dtype: TODO.+ """batch_size = positions.size(0)positions = positions.type(dtype)log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype)) / (depth / 2 - 1)@@ -434,6 +466,12 @@ class StreamSinusoidalPositionEncoder(torch.nn.Module):return encoding.type(dtype)def forward(self, x, cache=None):+ """Forward pass for training.++ Args:+ x: TODO.+ cache: State cache dict for streaming inference.+ """batch_size, timesteps, input_dim = x.size()start_idx = 0if cache is not None:@@ -516,6 +554,11 @@ class StreamingRelPositionalEncoding(torch.nn.Module):class ScaledSinuEmbedding(torch.nn.Module):def __init__(self, dim):+ """Initialize ScaledSinuEmbedding.++ Args:+ dim: TODO.+ """super().__init__()self.scale = torch.nn.Parameter(torch.ones(@@ -526,6 +569,11 @@ class ScaledSinuEmbedding(torch.nn.Module):self.register_buffer("inv_freq", inv_freq)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """n, device = x.shape[1], x.devicet = torch.arange(n, device=device).type_as(self.inv_freq)sinu = einsum("i , j -> i j", t, self.inv_freq)diff --git a/funasr/models/transformer/encoder.py b/funasr/models/transformer/encoder.pyindex 987924f0c..78c6b8c7b 100644--- a/funasr/models/transformer/encoder.py+++ b/funasr/models/transformer/encoder.py@@ -182,6 +182,27 @@ class TransformerEncoder(nn.Module):interctc_layer_idx: List[int] = [],interctc_use_conditioning: bool = False,):+ """Initialize TransformerEncoder.++ Args:+ input_size: Size/dimension parameter.+ output_size: Size/dimension parameter.+ attention_heads: TODO.+ linear_units: TODO.+ num_blocks: TODO.+ dropout_rate: TODO.+ positional_dropout_rate: TODO.+ attention_dropout_rate: TODO.+ input_layer: TODO.+ pos_enc_class: TODO.+ normalize_before: TODO.+ concat_after: TODO.+ positionwise_layer_type: TODO.+ positionwise_conv_kernel_size: Size/dimension parameter.+ padding_idx: TODO.+ interctc_layer_idx: TODO.+ interctc_use_conditioning: TODO.+ """super().__init__()self._output_size = output_size@@ -260,6 +281,7 @@ class TransformerEncoder(nn.Module):self.conditioning_layer = Nonedef output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(diff --git a/funasr/models/transformer/layer_norm.py b/funasr/models/transformer/layer_norm.pyindex ce4f8877f..d73552409 100644--- a/funasr/models/transformer/layer_norm.py+++ b/funasr/models/transformer/layer_norm.py@@ -61,6 +61,14 @@ class GlobalLayerNorm(nn.Module):"""def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True):+ """Initialize GlobalLayerNorm.++ Args:+ dim: TODO.+ shape: TODO.+ eps: TODO.+ elementwise_affine: TODO.+ """super(GlobalLayerNorm, self).__init__()self.dim = dimself.eps = eps@@ -125,6 +133,12 @@ class CumulativeLayerNorm(nn.LayerNorm):"""def __init__(self, dim, elementwise_affine=True):+ """Initialize CumulativeLayerNorm.++ Args:+ dim: TODO.+ elementwise_affine: TODO.+ """super(CumulativeLayerNorm, self).__init__(dim, elementwise_affine=elementwise_affine, eps=1e-8)@@ -156,11 +170,22 @@ class CumulativeLayerNorm(nn.LayerNorm):class ScaleNorm(nn.Module):def __init__(self, dim, eps=1e-5):+ """Initialize ScaleNorm.++ Args:+ dim: TODO.+ eps: TODO.+ """super().__init__()self.scale = dim**-0.5self.eps = epsself.g = nn.Parameter(torch.ones(1))def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """norm = torch.norm(x, dim=-1, keepdim=True) * self.scalereturn x / norm.clamp(min=self.eps) * self.gdiff --git a/funasr/models/transformer/model.py b/funasr/models/transformer/model.pyindex adfd52520..a6c1c7ade 100644--- a/funasr/models/transformer/model.py+++ b/funasr/models/transformer/model.py@@ -56,6 +56,36 @@ class Transformer(nn.Module):**kwargs,):+ """Initialize Transformer.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ interctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -269,6 +299,14 @@ class Transformer(nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -300,6 +338,14 @@ class Transformer(nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -313,6 +359,11 @@ class Transformer(nn.Module):self,**kwargs,):+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.transformer.search import BeamSearchfrom funasr.models.transformer.scorers.ctc import CTCPrefixScorerfrom funasr.models.transformer.scorers.length_bonus import LengthBonus@@ -364,6 +415,16 @@ class Transformer(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")diff --git a/funasr/models/transformer/positionwise_feed_forward.py b/funasr/models/transformer/positionwise_feed_forward.pyindex 7cfa5f9d5..3cf7887a2 100644--- a/funasr/models/transformer/positionwise_feed_forward.py+++ b/funasr/models/transformer/positionwise_feed_forward.py@@ -36,6 +36,11 @@ class PositionwiseFeedForward(torch.nn.Module):class PositionwiseFeedForwardDecoderSANMExport(torch.nn.Module):def __init__(self, model):+ """Initialize PositionwiseFeedForwardDecoderSANMExport.++ Args:+ model: Model instance or model name.+ """super().__init__()self.w_1 = model.w_1self.w_2 = model.w_2@@ -43,6 +48,11 @@ class PositionwiseFeedForwardDecoderSANMExport(torch.nn.Module):self.norm = model.normdef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = self.activation(self.w_1(x))x = self.w_2(self.norm(x))return xdiff --git a/funasr/models/transformer/scorers/ctc_prefix_score.py b/funasr/models/transformer/scorers/ctc_prefix_score.pyindex ce0703e73..df5bcfd55 100644--- a/funasr/models/transformer/scorers/ctc_prefix_score.py+++ b/funasr/models/transformer/scorers/ctc_prefix_score.py@@ -262,6 +262,14 @@ class CTCPrefixScore(object):"""def __init__(self, x, blank, eos, xp):+ """Initialize CTCPrefixScore.++ Args:+ x: TODO.+ blank: TODO.+ eos: TODO.+ xp: TODO.+ """self.xp = xpself.logzero = -10000000000.0self.blank = blankdiff --git a/funasr/models/transformer/utils/nets_utils.py b/funasr/models/transformer/utils/nets_utils.pyindex 29d23ee59..856788828 100644--- a/funasr/models/transformer/utils/nets_utils.py+++ b/funasr/models/transformer/utils/nets_utils.py@@ -511,6 +511,12 @@ class Swish(torch.nn.Module):"""def __init__(self, beta: float = 1.0, use_builtin: bool = False) -> None:+ """Initialize Swish.++ Args:+ beta: TODO.+ use_builtin: TODO.+ """super().__init__()self.beta = betadiff --git a/funasr/models/transformer/utils/subsampling.py b/funasr/models/transformer/utils/subsampling.pyindex b18bd3132..12eff29ca 100644--- a/funasr/models/transformer/utils/subsampling.py+++ b/funasr/models/transformer/utils/subsampling.py@@ -350,6 +350,17 @@ class Conv1dSubsampling(torch.nn.Module):tf2torch_tensor_name_prefix_torch: str = "stride_conv",tf2torch_tensor_name_prefix_tf: str = "seq2seq/proj_encoder/downsampling",):+ """Initialize Conv1dSubsampling.++ Args:+ idim: TODO.+ odim: TODO.+ kernel_size: Size/dimension parameter.+ stride: TODO.+ pad: TODO.+ tf2torch_tensor_name_prefix_torch: TODO.+ tf2torch_tensor_name_prefix_tf: TODO.+ """super(Conv1dSubsampling, self).__init__()self.conv = torch.nn.Conv1d(idim, odim, kernel_size, stride)self.pad_fn = torch.nn.ConstantPad1d(pad, 0.0)@@ -359,6 +370,7 @@ class Conv1dSubsampling(torch.nn.Module):self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tfdef output_size(self) -> int:+ """Output size."""return self.odimdef forward(self, x, x_len):diff --git a/funasr/models/uniasr/model.py b/funasr/models/uniasr/model.pyindex bde637777..efe1f18c0 100644--- a/funasr/models/uniasr/model.py+++ b/funasr/models/uniasr/model.py@@ -73,6 +73,51 @@ class UniASR(torch.nn.Module):share_embedding: bool = False,**kwargs,):+ """Initialize UniASR.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ encoder2: TODO.+ encoder2_conf: Configuration dict for encoder2.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ decoder2: TODO.+ decoder2_conf: Configuration dict for decoder2.+ predictor: TODO.+ predictor_conf: Configuration dict for predictor.+ predictor_bias: TODO.+ predictor_weight: TODO.+ predictor2: TODO.+ predictor2_conf: Configuration dict for predictor2.+ predictor2_bias: TODO.+ predictor2_weight: TODO.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ ctc2: TODO.+ ctc2_conf: Configuration dict for ctc2.+ ctc2_weight: TODO.+ decoder_attention_chunk_type: TODO.+ decoder_attention_chunk_type2: TODO.+ stride_conv: TODO.+ stride_conv_conf: Configuration dict for stride_conv.+ loss_weight_model1: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -308,6 +353,14 @@ class UniASR(torch.nn.Module):text: torch.Tensor,text_lengths: torch.Tensor,) -> Dict[str, torch.Tensor]:+ """Collect feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ text: Text tensor or string input.+ text_lengths: Length of each text sample.+ """if self.extract_feats_in_collect_stats:feats, feats_lengths = self._extract_feats(speech, speech_lengths)else:@@ -477,6 +530,14 @@ class UniASR(torch.nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -507,6 +568,14 @@ class UniASR(torch.nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att predictor loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -608,6 +677,14 @@ class UniASR(torch.nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att predictor loss2.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -711,6 +788,14 @@ class UniASR(torch.nn.Module):):# ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)# ys_in_lens = ys_pad_lens + 1+ """Calc predictor mask.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_out_pad, ys_in_lens = None, Noneencoder_out_mask = sequence_mask(@@ -793,6 +878,14 @@ class UniASR(torch.nn.Module):):# ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)# ys_in_lens = ys_pad_lens + 1+ """Calc predictor mask2.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_out_pad, ys_in_lens = None, Noneencoder_out_mask = sequence_mask(@@ -870,6 +963,11 @@ class UniASR(torch.nn.Module):self,**kwargs,):+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.uniasr.beam_search import BeamSearchScamafrom funasr.models.transformer.scorers.ctc import CTCPrefixScorerfrom funasr.models.transformer.scorers.length_bonus import LengthBonus@@ -926,6 +1024,16 @@ class UniASR(torch.nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """decoding_model = kwargs.get("decoding_model", "normal")token_num_relax = kwargs.get("token_num_relax", 5)if decoding_model == "fast":diff --git a/funasr/models/whisper/model.py b/funasr/models/whisper/model.pyindex 398eea3fd..d2656dcce 100644--- a/funasr/models/whisper/model.py+++ b/funasr/models/whisper/model.py@@ -32,6 +32,12 @@ from funasr.register import tables@tables.register("model_classes", "WhisperWarp")class WhisperWarp(nn.Module):def __init__(self, *args, **kwargs):+ """Initialize WhisperWarp.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """super().__init__()hub = kwargs.get("hub", "funasr")if hub == "openai":@@ -51,6 +57,7 @@ class WhisperWarp(nn.Module):def forward(self,):+ """Forward pass for training."""passdef inference(@@ -62,6 +69,16 @@ class WhisperWarp(nn.Module):frontend=None,**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")diff --git a/funasr/models/whisper_lid/decoder.py b/funasr/models/whisper_lid/decoder.pyindex 1c8ab47f8..a355e508e 100644--- a/funasr/models/whisper_lid/decoder.py+++ b/funasr/models/whisper_lid/decoder.py@@ -26,6 +26,14 @@ class OpenAIWhisperDecoderWarp(nn.Module):download_dir: str = None,use_padmask: bool = False,):+ """Initialize OpenAIWhisperDecoderWarp.++ Args:+ dropout_rate: TODO.+ whisper_model: Whisper Model instance.+ download_dir: TODO.+ use_padmask: TODO.+ """super().__init__()assert whisper_model in whisper.available_models()diff --git a/funasr/models/whisper_lid/encoder.py b/funasr/models/whisper_lid/encoder.pyindex b6fb28b50..73de72e77 100644--- a/funasr/models/whisper_lid/encoder.py+++ b/funasr/models/whisper_lid/encoder.py@@ -30,6 +30,16 @@ class OpenAIWhisperEncoderWarp(nn.Module):use_padmask: bool = False,specaug_conf: Union[dict, None] = None,):+ """Initialize OpenAIWhisperEncoderWarp.++ Args:+ dropout_rate: TODO.+ whisper_model: Whisper Model instance.+ download_dir: TODO.+ use_specaug: TODO.+ use_padmask: TODO.+ specaug_conf: Configuration dict for specaug.+ """super().__init__()# note that originally Whisper doesn't use dropouts@@ -53,6 +63,12 @@ class OpenAIWhisperEncoderWarp(nn.Module):input: torch.Tensor,ilens: torch.Tensor = None,) -> torch.Tensor:+ """Whisper encode.++ Args:+ input: Input audio/text data.+ ilens: TODO.+ """x = F.gelu(self.encoders.conv1(input))x = F.gelu(self.encoders.conv2(x))x = x.permute(0, 2, 1)@@ -93,6 +109,7 @@ class OpenAIWhisperEncoderWarp(nn.Module):def output_size(self) -> int:# dummy output size+ """Output size."""return self.encoders.conv2.weight.shape[0]def forward(@@ -101,6 +118,13 @@ class OpenAIWhisperEncoderWarp(nn.Module):ilens: torch.Tensor,prev_states: torch.Tensor = None,) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:+ """Forward pass for training.++ Args:+ xs_pad: TODO.+ ilens: TODO.+ prev_states: TODO.+ """feats, feats_lens = xs_pad, ilensif self.specaug is not None and self.encoders.training:diff --git a/funasr/models/whisper_lid/eres2net/ResNet.py b/funasr/models/whisper_lid/eres2net/ResNet.pyindex df3b96d24..554ca8ec2 100644--- a/funasr/models/whisper_lid/eres2net/ResNet.py+++ b/funasr/models/whisper_lid/eres2net/ResNet.py@@ -20,9 +20,15 @@ from funasr.models.whisper_lid.eres2net.fusion import AFFclass ReLU(nn.Hardtanh):def __init__(self, inplace=False):+ """Initialize ReLU.++ Args:+ inplace: TODO.+ """super(ReLU, self).__init__(0, 20, inplace)def __repr__(self):+ """Internal: repr ."""inplace_str = "inplace" if self.inplace else ""return self.__class__.__name__ + " (" + inplace_str + ")"@@ -41,6 +47,15 @@ class BasicBlockERes2Net(nn.Module):expansion = 2def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):+ """Initialize BasicBlockERes2Net.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockERes2Net, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -71,6 +86,11 @@ class BasicBlockERes2Net(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -103,6 +123,15 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):expansion = 2def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):+ """Initialize BasicBlockERes2Net_diff_AFF.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockERes2Net_diff_AFF, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -138,6 +167,11 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -179,6 +213,18 @@ class ERes2Net(nn.Module):pooling_func="TSTP",two_emb_layer=False,):+ """Initialize ERes2Net.++ Args:+ block: TODO.+ block_fuse: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ pooling_func: TODO.+ two_emb_layer: TODO.+ """super(ERes2Net, self).__init__()self.in_planes = m_channelsself.feat_dim = feat_dim@@ -221,6 +267,14 @@ class ERes2Net(nn.Module):self.seg_2 = nn.Identity()def _make_layer(self, block, planes, num_blocks, stride):+ """Internal: make layer.++ Args:+ block: TODO.+ planes: TODO.+ num_blocks: TODO.+ stride: TODO.+ """strides = [stride] + [1] * (num_blocks - 1)layers = []for stride in strides:@@ -229,10 +283,17 @@ class ERes2Net(nn.Module):return nn.Sequential(*layers)def output_size(self) -> int:+ """Output size."""return self._output_sizedef forward(self, x, ilens):# assert x.shape[1] == ilens.max()+ """Forward pass for training.++ Args:+ x: TODO.+ ilens: TODO.+ """x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)x = x.unsqueeze_(1)out = F.relu(self.bn1(self.conv1(x)))@@ -263,6 +324,15 @@ class BasicBlockRes2Net(nn.Module):expansion = 2def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):+ """Initialize BasicBlockRes2Net.++ Args:+ in_planes: TODO.+ planes: TODO.+ stride: TODO.+ baseWidth: TODO.+ scale: TODO.+ """super(BasicBlockRes2Net, self).__init__()width = int(math.floor(planes * (baseWidth / 64.0)))self.conv1 = conv1x1(in_planes, width * scale, stride)@@ -292,6 +362,11 @@ class BasicBlockRes2Net(nn.Module):self.scale = scaledef forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """residual = xout = self.conv1(x)@@ -333,6 +408,17 @@ class Res2Net(nn.Module):pooling_func="TSTP",two_emb_layer=False,):+ """Initialize Res2Net.++ Args:+ block: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ pooling_func: TODO.+ two_emb_layer: TODO.+ """super(Res2Net, self).__init__()self.in_planes = m_channelsself.feat_dim = feat_dim@@ -358,6 +444,14 @@ class Res2Net(nn.Module):self.seg_2 = nn.Identity()def _make_layer(self, block, planes, num_blocks, stride):+ """Internal: make layer.++ Args:+ block: TODO.+ planes: TODO.+ num_blocks: TODO.+ stride: TODO.+ """strides = [stride] + [1] * (num_blocks - 1)layers = []for stride in strides:@@ -366,6 +460,11 @@ class Res2Net(nn.Module):return nn.Sequential(*layers)def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)x = x.unsqueeze_(1)diff --git a/funasr/models/whisper_lid/eres2net/fusion.py b/funasr/models/whisper_lid/eres2net/fusion.pyindex fbe699efb..74886e293 100644--- a/funasr/models/whisper_lid/eres2net/fusion.py+++ b/funasr/models/whisper_lid/eres2net/fusion.py@@ -8,6 +8,12 @@ import torch.nn as nnclass AFF(nn.Module):def __init__(self, channels=64, r=4):+ """Initialize AFF.++ Args:+ channels: TODO.+ r: TODO.+ """super(AFF, self).__init__()inter_channels = int(channels // r)@@ -20,6 +26,12 @@ class AFF(nn.Module):)def forward(self, x, ds_y):+ """Forward pass for training.++ Args:+ x: TODO.+ ds_y: TODO.+ """xa = torch.cat((x, ds_y), dim=1)x_att = self.local_att(xa)x_att = 1.0 + torch.tanh(x_att)diff --git a/funasr/models/whisper_lid/eres2net/pooling_layers.py b/funasr/models/whisper_lid/eres2net/pooling_layers.pyindex da23d7b4c..6128e1c5f 100644--- a/funasr/models/whisper_lid/eres2net/pooling_layers.py+++ b/funasr/models/whisper_lid/eres2net/pooling_layers.py@@ -15,9 +15,19 @@ class TAP(nn.Module):"""def __init__(self, **kwargs):+ """Initialize TAP.++ Args:+ **kwargs: Additional keyword arguments.+ """super(TAP, self).__init__()def forward(self, x):+ """Forward pass for training.++ Args:+ x: TODO.+ """pooling_mean = x.mean(dim=-1)# To be compatable with 2D inputpooling_mean = pooling_mean.flatten(start_dim=1)@@ -30,10 +40,20 @@ class TSDP(nn.Module):"""def __init__(self, **kwargs):+ """Initialize TSDP.++ Args:+ **kwargs: Additional keyword arguments.+ """super(TSDP, self).__init__()def forward(self, x):# The last dimension is the temporal axis+ """Forward pass for training.++ Args:+ x: TODO.+ """pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8)pooling_std = pooling_std.flatten(start_dim=1)return pooling_std@@ -47,10 +67,21 @@ class TSTP(nn.Module):"""def __init__(self, **kwargs):+ """Initialize TSTP.++ Args:+ **kwargs: Additional keyword arguments.+ """super(TSTP, self).__init__()def forward(self, x, olens):# The last dimension is the temporal axis+ """Forward pass for training.++ Args:+ x: TODO.+ olens: TODO.+ """masks = (~make_pad_mask(olens, maxlen=x.shape[-1])[:, None, None, :]).to(x.device)x_masked = x * maskssum_without_padding = torch.sum(x_masked, axis=-1)@@ -76,6 +107,13 @@ class ASTP(nn.Module):"""def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False):+ """Initialize ASTP.++ Args:+ in_dim: Size/dimension parameter.+ bottleneck_dim: Size/dimension parameter.+ global_context_att: TODO.+ """super(ASTP, self).__init__()self.global_context_att = global_context_attdiff --git a/funasr/models/whisper_lid/eres2net/simple_avg.py b/funasr/models/whisper_lid/eres2net/simple_avg.pyindex 9dbe7b398..fcf7c6ff3 100644--- a/funasr/models/whisper_lid/eres2net/simple_avg.py+++ b/funasr/models/whisper_lid/eres2net/simple_avg.py@@ -6,13 +6,25 @@ from funasr.modules.nets_utils import make_pad_maskclass SimpleAvg(AbsEncoder):def __init__(self, feat_dim):+ """Initialize SimpleAvg.++ Args:+ feat_dim: Size/dimension parameter.+ """super(SimpleAvg, self).__init__()self.feat_dim = feat_dimdef forward(self, x, ilens):+ """Forward pass for training.++ Args:+ x: TODO.+ ilens: TODO.+ """mask = ~make_pad_mask(ilens, maxlen=x.shape[1]).to(x.device)avg_x = (x * mask[:, :, None]).sum(1) / mask.sum(-1)[:, None]return avg_xdef output_size(self) -> int:+ """Output size."""return self.feat_dimdiff --git a/funasr/models/whisper_lid/lid_predictor.py b/funasr/models/whisper_lid/lid_predictor.pyindex d5b0d37ac..b3ccf11a5 100644--- a/funasr/models/whisper_lid/lid_predictor.py+++ b/funasr/models/whisper_lid/lid_predictor.py@@ -19,6 +19,18 @@ class LidPredictor(ERes2Net):pooling_func="TSTP",two_emb_layer=False,):+ """Initialize LidPredictor.++ Args:+ block: TODO.+ block_fuse: TODO.+ num_blocks: TODO.+ m_channels: TODO.+ feat_dim: Size/dimension parameter.+ embedding_size: Size/dimension parameter.+ pooling_func: TODO.+ two_emb_layer: TODO.+ """super(LidPredictor, self).__init__(block=block,block_fuse=block_fuse,diff --git a/funasr/models/whisper_lid/model.py b/funasr/models/whisper_lid/model.pyindex 02cd3730a..baca008b0 100644--- a/funasr/models/whisper_lid/model.py+++ b/funasr/models/whisper_lid/model.py@@ -55,6 +55,36 @@ class OpenAIWhisperModel(nn.Module):**kwargs,):+ """Initialize OpenAIWhisperModel.++ Args:+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ normalize: TODO.+ normalize_conf: Configuration dict for normalize.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ decoder: TODO.+ decoder_conf: Configuration dict for decoder.+ ctc: TODO.+ ctc_conf: Configuration dict for ctc.+ ctc_weight: TODO.+ interctc_weight: TODO.+ input_size: Size/dimension parameter.+ vocab_size: Size/dimension parameter.+ ignore_id: TODO.+ blank_id: TODO.+ sos: TODO.+ eos: TODO.+ lsm_weight: TODO.+ length_normalized_loss: TODO.+ report_cer: TODO.+ report_wer: TODO.+ sym_space: TODO.+ sym_blank: TODO.+ share_embedding: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:@@ -264,6 +294,14 @@ class OpenAIWhisperModel(nn.Module):ys_pad: torch.Tensor,ys_pad_lens: torch.Tensor,):+ """Internal: calc att loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)ys_in_lens = ys_pad_lens + 1@@ -295,6 +333,14 @@ class OpenAIWhisperModel(nn.Module):ys_pad_lens: torch.Tensor,):# Calc CTC loss+ """Internal: calc ctc loss.++ Args:+ encoder_out: Encoder output tensor.+ encoder_out_lens: Encoder output lengths.+ ys_pad: TODO.+ ys_pad_lens: Lengths of ys_pad.+ """loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)# Calc CER using CTC@@ -308,6 +354,11 @@ class OpenAIWhisperModel(nn.Module):self,**kwargs,):+ """Init beam search.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.transformer.search import BeamSearchfrom funasr.models.transformer.scorers.ctc import CTCPrefixScorerfrom funasr.models.transformer.scorers.length_bonus import LengthBonus@@ -359,6 +410,16 @@ class OpenAIWhisperModel(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")@@ -472,6 +533,21 @@ class OpenAIWhisperLIDModel(nn.Module):random_clip: bool = False,**kwargs,):+ """Initialize OpenAIWhisperLIDModel.++ Args:+ vocab_size: Size/dimension parameter.+ specaug: TODO.+ specaug_conf: Configuration dict for specaug.+ encoder: TODO.+ encoder_conf: Configuration dict for encoder.+ lid_predictor: TODO.+ lid_predictor_conf: Configuration dict for lid_predictor.+ proj_dim: Size/dimension parameter.+ clip_frames: TODO.+ random_clip: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__()if specaug is not None:specaug_class = tables.specaug_classes.get(specaug)@@ -509,6 +585,14 @@ class OpenAIWhisperLIDModel(nn.Module):lid: torch.Tensor, # lid label, (batch_size, 1)lid_lengths: torch.Tensor,):+ """Forward pass for training.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ lid: TODO.+ lid_lengths: Lengths of lid.+ """assert lid.shape[1] == 1batch_size = speech.shape[0]encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)@@ -611,6 +695,16 @@ class OpenAIWhisperLIDModel(nn.Module):**kwargs,):+ """Run inference on input data.++ Args:+ data_in: Input data (audio samples, file paths, or text).+ data_lengths: Lengths of each input sample in the batch.+ key: Sample identifiers.+ tokenizer: Tokenizer instance for text encoding/decoding.+ frontend: Audio frontend for feature extraction.+ **kwargs: Additional keyword arguments.+ """if kwargs.get("batch_size", 1) > 1:raise NotImplementedError("batch decoding is not implemented")diff --git a/funasr/models/xvector/e2e_sv.py b/funasr/models/xvector/e2e_sv.pyindex 56c9e3b10..c6062b59c 100644--- a/funasr/models/xvector/e2e_sv.py+++ b/funasr/models/xvector/e2e_sv.py@@ -36,6 +36,11 @@ else:# Nothing to do if torch<1.6.0@contextmanagerdef autocast(enabled=True):+ """Autocast.++ Args:+ enabled: TODO.+ """yield@@ -56,6 +61,20 @@ class ESPnetSVModel(FunASRModel):decoder: AbsDecoder,):+ """Initialize ESPnetSVModel.++ Args:+ vocab_size: Size/dimension parameter.+ token_list: TODO.+ frontend: Audio frontend for feature extraction.+ specaug: TODO.+ normalize: TODO.+ preencoder: TODO.+ encoder: TODO.+ postencoder: TODO.+ pooling_layer: TODO.+ decoder: TODO.+ """super().__init__()# note that eos is the same as sos (equivalent ID)self.vocab_size = vocab_size@@ -197,6 +216,14 @@ class ESPnetSVModel(FunASRModel):text: torch.Tensor,text_lengths: torch.Tensor,) -> Dict[str, torch.Tensor]:+ """Collect feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ text: Text tensor or string input.+ text_lengths: Length of each text sample.+ """if self.extract_feats_in_collect_stats:feats, feats_lengths = self._extract_feats(speech, speech_lengths)else:@@ -246,6 +273,12 @@ class ESPnetSVModel(FunASRModel):def _extract_feats(self, speech: torch.Tensor, speech_lengths: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ """Internal: extract feats.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ """assert speech_lengths.dim() == 1, speech_lengths.shape# for data-paralleldiff --git a/funasr/optimizers/fairseq_adam.py b/funasr/optimizers/fairseq_adam.pyindex 32fae6a25..c140c62f0 100644--- a/funasr/optimizers/fairseq_adam.py+++ b/funasr/optimizers/fairseq_adam.py@@ -46,6 +46,16 @@ class FairseqAdam(torch.optim.Optimizer):weight_decay=0,amsgrad=False,):+ """Initialize FairseqAdam.++ Args:+ params: TODO.+ lr: TODO.+ adam_betas: TODO.+ adam_eps: TODO.+ weight_decay: TODO.+ amsgrad: TODO.+ """defaults = dict(lr=lr, betas=adam_betas, eps=adam_eps, weight_decay=weight_decay, amsgrad=amsgrad)@@ -54,10 +64,12 @@ class FairseqAdam(torch.optim.Optimizer):@propertydef supports_memory_efficient_fp16(self):+ """Supports memory efficient fp16."""return True@propertydef supports_flat_params(self):+ """Supports flat params."""return Truedef step(self, closure=None):diff --git a/funasr/optimizers/sgd.py b/funasr/optimizers/sgd.pyindex fb7a3df01..04993916f 100644--- a/funasr/optimizers/sgd.py+++ b/funasr/optimizers/sgd.py@@ -20,6 +20,16 @@ class SGD(torch.optim.SGD):weight_decay: float = 0.0,nesterov: bool = False,):+ """Initialize SGD.++ Args:+ params: TODO.+ lr: TODO.+ momentum: TODO.+ dampening: TODO.+ weight_decay: TODO.+ nesterov: TODO.+ """super().__init__(params,lr=lr,diff --git a/funasr/register.py b/funasr/register.pyindex a8b1fd333..b2ef75a3d 100644--- a/funasr/register.py+++ b/funasr/register.py@@ -50,6 +50,11 @@ class RegisterTables:"""Decorator to register a class."""def decorator(target_class):+ """Decorator.++ Args:+ target_class: TODO.+ """if not hasattr(self, register_tables_key):setattr(self, register_tables_key, {})logging.debug(f"New registry table added: {register_tables_key}")diff --git a/funasr/schedulers/abs_scheduler.py b/funasr/schedulers/abs_scheduler.pyindex 7395f259c..7181a8756 100644--- a/funasr/schedulers/abs_scheduler.py+++ b/funasr/schedulers/abs_scheduler.py@@ -7,14 +7,25 @@ import torch.optim.lr_scheduler as Lclass AbsScheduler(ABC):@abstractmethoddef step(self, epoch: int = None):+ """Step.++ Args:+ epoch: TODO.+ """pass@abstractmethoddef state_dict(self):+ """State dict."""pass@abstractmethoddef load_state_dict(self, state):+ """Load state dict.++ Args:+ state: TODO.+ """pass@@ -22,42 +33,76 @@ class AbsScheduler(ABC):class AbsBatchStepScheduler(AbsScheduler):@abstractmethoddef step(self, epoch: int = None):+ """Step.++ Args:+ epoch: TODO.+ """pass@abstractmethoddef state_dict(self):+ """State dict."""pass@abstractmethoddef load_state_dict(self, state):+ """Load state dict.++ Args:+ state: TODO.+ """passclass AbsEpochStepScheduler(AbsScheduler):@abstractmethoddef step(self, epoch: int = None):+ """Step.++ Args:+ epoch: TODO.+ """pass@abstractmethoddef state_dict(self):+ """State dict."""pass@abstractmethoddef load_state_dict(self, state):+ """Load state dict.++ Args:+ state: TODO.+ """passclass AbsValEpochStepScheduler(AbsEpochStepScheduler):@abstractmethoddef step(self, val, epoch: int = None):+ """Step.++ Args:+ val: TODO.+ epoch: TODO.+ """pass@abstractmethoddef state_dict(self):+ """State dict."""pass@abstractmethoddef load_state_dict(self, state):+ """Load state dict.++ Args:+ state: TODO.+ """passdiff --git a/funasr/schedulers/lambdalr_cus.py b/funasr/schedulers/lambdalr_cus.pyindex e3bb1fb47..899cceb3c 100644--- a/funasr/schedulers/lambdalr_cus.py+++ b/funasr/schedulers/lambdalr_cus.py@@ -25,12 +25,22 @@ class CustomLambdaLR(_LRScheduler):last_epoch=-1,verbose=False,):+ """Initialize CustomLambdaLR.++ Args:+ optimizer: TODO.+ warmup_steps: TODO.+ total_steps: TODO.+ last_epoch: TODO.+ verbose: TODO.+ """self.warmup_steps = warmup_stepsself.total_steps = total_stepssuper().__init__(optimizer, last_epoch, verbose)def get_lr(self):+ """Get lr."""step = self.last_epoch + 1if step < self.warmup_steps:lr_scale = step / self.warmup_stepsdiff --git a/funasr/schedulers/noam_lr.py b/funasr/schedulers/noam_lr.pyindex b33b4bde6..8cec25ddf 100644--- a/funasr/schedulers/noam_lr.py+++ b/funasr/schedulers/noam_lr.py@@ -31,6 +31,14 @@ class NoamLR(_LRScheduler, AbsBatchStepScheduler):warmup_steps: Union[int, float] = 25000,last_epoch: int = -1,):+ """Initialize NoamLR.++ Args:+ optimizer: TODO.+ model_size: Size/dimension parameter.+ warmup_steps: TODO.+ last_epoch: TODO.+ """self.model_size = model_sizeself.warmup_steps = warmup_steps@@ -46,15 +54,22 @@ class NoamLR(_LRScheduler, AbsBatchStepScheduler):super().__init__(optimizer, last_epoch)def lr_for_WarmupLR(self, lr: float) -> float:+ """Lr for warmuplr.++ Args:+ lr: TODO.+ """return lr / self.model_size**0.5 / self.warmup_steps**0.5def __repr__(self):+ """Internal: repr ."""return (f"{self.__class__.__name__}(model_size={self.model_size}, "f"warmup_steps={self.warmup_steps})")def get_lr(self):+ """Get lr."""step_num = self.last_epoch + 1return [lr * self.model_size**-0.5 * min(step_num**-0.5, step_num * self.warmup_steps**-1.5)diff --git a/funasr/schedulers/tri_stage_scheduler.py b/funasr/schedulers/tri_stage_scheduler.pyindex 7fc13b2ff..110477803 100644--- a/funasr/schedulers/tri_stage_scheduler.py+++ b/funasr/schedulers/tri_stage_scheduler.py@@ -21,6 +21,15 @@ class TriStageLR(_LRScheduler, AbsBatchStepScheduler):init_lr_scale: float = 0.01,final_lr_scale: float = 0.01,):+ """Initialize TriStageLR.++ Args:+ optimizer: TODO.+ last_epoch: TODO.+ phase_ratio: TODO.+ init_lr_scale: TODO.+ final_lr_scale: TODO.+ """self.optimizer = optimizerself.last_epoch = last_epochself.phase_ratio = phase_ratio@@ -29,6 +38,11 @@ class TriStageLR(_LRScheduler, AbsBatchStepScheduler):self.optimizer_lr = self.optimizer.defaults["lr"]def init_tri_stage_scheudler(self, max_update):+ """Init tri stage scheudler.++ Args:+ max_update: TODO.+ """self.max_update = max_updateself.peak_lr = self.optimizer_lrself.init_lr = self.init_lr_scale * self.optimizer_lr@@ -95,10 +109,16 @@ class TriStageLR(_LRScheduler, AbsBatchStepScheduler):self.set_optimizer_lr(self.lr)def set_optimizer_lr(self, lr):+ """Set optimizer lr.++ Args:+ lr: TODO.+ """for param_group in self.optimizer.param_groups:param_group["lr"] = lrdef get_lr(self):+ """Get lr."""step_num = self.last_epoch + 1self.step_update(step_num)return [self.lr]diff --git a/funasr/schedulers/warmup_lr.py b/funasr/schedulers/warmup_lr.pyindex 847b53ad7..13af0226f 100644--- a/funasr/schedulers/warmup_lr.py+++ b/funasr/schedulers/warmup_lr.py@@ -30,6 +30,13 @@ class WarmupLR(_LRScheduler, AbsBatchStepScheduler):warmup_steps: Union[int, float] = 25000,last_epoch: int = -1,):+ """Initialize WarmupLR.++ Args:+ optimizer: TODO.+ warmup_steps: TODO.+ last_epoch: TODO.+ """self.warmup_steps = warmup_steps# __init__() must be invoked before setting field@@ -37,9 +44,11 @@ class WarmupLR(_LRScheduler, AbsBatchStepScheduler):super().__init__(optimizer, last_epoch)def __repr__(self):+ """Internal: repr ."""return f"{self.__class__.__name__}(warmup_steps={self.warmup_steps})"def get_lr(self):+ """Get lr."""step_num = self.last_epoch + 1return [lr * self.warmup_steps**0.5 * min(step_num**-0.5, step_num * self.warmup_steps**-1.5)diff --git a/funasr/tokenizer/abs_tokenizer.py b/funasr/tokenizer/abs_tokenizer.pyindex e125d292b..a1d51b228 100644--- a/funasr/tokenizer/abs_tokenizer.py+++ b/funasr/tokenizer/abs_tokenizer.py@@ -9,10 +9,20 @@ from typing import Union, Iterable, List, Dictclass AbsTokenizer(ABC):@abstractmethoddef text2tokens(self, line: str) -> List[str]:+ """Text2tokens.++ Args:+ line: TODO.+ """raise NotImplementedError@abstractmethoddef tokens2text(self, tokens: Iterable[str]) -> str:+ """Tokens2text.++ Args:+ tokens: TODO.+ """raise NotImplementedError@@ -24,6 +34,13 @@ class BaseTokenizer(ABC):**kwargs,):+ """Initialize BaseTokenizer.++ Args:+ token_list: TODO.+ unk_symbol: TODO.+ **kwargs: Additional keyword arguments.+ """if token_list is not None:if isinstance(token_list, (Path, str)) and token_list.endswith(".txt"):token_list = Path(token_list)@@ -63,31 +80,63 @@ class BaseTokenizer(ABC):self.unk_id = self.token2id[self.unk_symbol]def encode(self, text, **kwargs):+ """Encode.++ Args:+ text: Text tensor or string input.+ **kwargs: Additional keyword arguments.+ """tokens = self.text2tokens(text)text_ints = self.tokens2ids(tokens)return text_intsdef decode(self, text_ints):+ """Decode.++ Args:+ text_ints: TODO.+ """token = self.ids2tokens(text_ints)text = self.tokens2text(token)return textdef get_num_vocabulary_size(self) -> int:+ """Get num vocabulary size."""return len(self.token_list)def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]:+ """Ids2tokens.++ Args:+ integers: TODO.+ """if isinstance(integers, np.ndarray) and integers.ndim != 1:raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}")return [self.token_list[i] for i in integers]def tokens2ids(self, tokens: Iterable[str]) -> List[int]:+ """Tokens2ids.++ Args:+ tokens: TODO.+ """return [self.token2id.get(i, self.unk_id) for i in tokens]@abstractmethoddef text2tokens(self, line: str) -> List[str]:+ """Text2tokens.++ Args:+ line: TODO.+ """raise NotImplementedError@abstractmethoddef tokens2text(self, tokens: Iterable[str]) -> str:+ """Tokens2text.++ Args:+ tokens: TODO.+ """raise NotImplementedErrordiff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.pyindex 7b517dacb..d25c98926 100644--- a/funasr/tokenizer/char_tokenizer.py+++ b/funasr/tokenizer/char_tokenizer.py@@ -20,6 +20,16 @@ class CharTokenizer(BaseTokenizer):seg_dict: str = None,**kwargs,):+ """Initialize CharTokenizer.++ Args:+ non_linguistic_symbols: TODO.+ space_symbol: TODO.+ remove_non_linguistic_symbols: TODO.+ split_with_space: TODO.+ seg_dict: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__(**kwargs)self.space_symbol = space_symbolif non_linguistic_symbols is None:@@ -42,6 +52,7 @@ class CharTokenizer(BaseTokenizer):self.seg_dict = load_seg_dict(seg_dict)def __repr__(self):+ """Internal: repr ."""return (f"{self.__class__.__name__}("f'space_symbol="{self.space_symbol}"'@@ -51,6 +62,11 @@ class CharTokenizer(BaseTokenizer):def text2tokens(self, line: Union[str, list]) -> List[str]:# if self.split_with_space:+ """Text2tokens.++ Args:+ line: TODO.+ """if self.seg_dict is not None:tokens = line.strip().split(" ")tokens = seg_tokenize(tokens, self.seg_dict)@@ -74,11 +90,21 @@ class CharTokenizer(BaseTokenizer):return tokensdef tokens2text(self, tokens: Iterable[str]) -> str:+ """Tokens2text.++ Args:+ tokens: TODO.+ """tokens = [t if t != self.space_symbol else " " for t in tokens]return "".join(tokens)def load_seg_dict(seg_dict_file):+ """Load seg dict.++ Args:+ seg_dict_file: TODO.+ """seg_dict = {}assert isinstance(seg_dict_file, str)with open(seg_dict_file, "r", encoding="utf8") as f:@@ -93,6 +119,12 @@ def load_seg_dict(seg_dict_file):def seg_tokenize(txt, seg_dict):# pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')+ """Seg tokenize.++ Args:+ txt: TODO.+ seg_dict: TODO.+ """pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")out_txt = ""for word in txt:diff --git a/funasr/tokenizer/cleaner.py b/funasr/tokenizer/cleaner.pyindex 882db5c97..1c609caf0 100644--- a/funasr/tokenizer/cleaner.py+++ b/funasr/tokenizer/cleaner.py@@ -22,6 +22,11 @@ class TextCleaner:def __init__(self, cleaner_types: Collection[str] = None):+ """Initialize TextCleaner.++ Args:+ cleaner_types: TODO.+ """if cleaner_types is None:self.cleaner_types = []elif isinstance(cleaner_types, str):@@ -30,6 +35,11 @@ class TextCleaner:self.cleaner_types = list(cleaner_types)def __call__(self, text: str) -> str:+ """Internal: call .++ Args:+ text: Text tensor or string input.+ """for t in self.cleaner_types:if t == "tacotron":# text = tacotron_cleaner.cleaners.custom_english_cleaners(text)diff --git a/funasr/tokenizer/hf_tokenizer.py b/funasr/tokenizer/hf_tokenizer.pyindex 2b782179c..57c23cb53 100644--- a/funasr/tokenizer/hf_tokenizer.py+++ b/funasr/tokenizer/hf_tokenizer.py@@ -3,6 +3,12 @@ from funasr.register import tables@tables.register("tokenizer_classes", "HuggingfaceTokenizer")def HuggingfaceTokenizer(init_param_path, **kwargs):+ """Huggingfacetokenizer.++ Args:+ init_param_path: TODO.+ **kwargs: Additional keyword arguments.+ """try:from transformers import AutoTokenizerexcept Exception as e:diff --git a/funasr/tokenizer/korean_cleaner.py b/funasr/tokenizer/korean_cleaner.pyindex eceebe216..ff0beeb60 100644--- a/funasr/tokenizer/korean_cleaner.py+++ b/funasr/tokenizer/korean_cleaner.py@@ -6,6 +6,11 @@ import reclass KoreanCleaner:@classmethoddef _normalize_numbers(cls, text):+ """Internal: normalize numbers.++ Args:+ text: Text tensor or string input.+ """number_to_kor = {"0": "영","1": "일",@@ -25,6 +30,11 @@ class KoreanCleaner:@classmethoddef _normalize_english_text(cls, text):+ """Internal: normalize english text.++ Args:+ text: Text tensor or string input.+ """upper_alphabet_to_kor = {"A": "에이","B": "비",@@ -64,6 +74,11 @@ class KoreanCleaner:@classmethoddef normalize_text(cls, text):# stage 0 : text strip+ """Normalize text.++ Args:+ text: Text tensor or string input.+ """text = text.strip()# stage 1 : normalize numbersdiff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.pyindex 12b29ec22..ab479242a 100644--- a/funasr/tokenizer/phoneme_tokenizer.py+++ b/funasr/tokenizer/phoneme_tokenizer.py@@ -43,6 +43,11 @@ g2p_classes = [def split_by_space(text) -> List[str]:+ """Split by space.++ Args:+ text: Text tensor or string input.+ """if " " in text:text = text.replace(" ", " <space> ")return [c.replace("<space>", " ") for c in text.split(" ")]@@ -51,6 +56,11 @@ def split_by_space(text) -> List[str]:def pyopenjtalk_g2p(text) -> List[str]:+ """Pyopenjtalk g2p.++ Args:+ text: Text tensor or string input.+ """import pyopenjtalk# phones is a str object separated by space@@ -60,6 +70,11 @@ def pyopenjtalk_g2p(text) -> List[str]:def pyopenjtalk_g2p_accent(text) -> List[str]:+ """Pyopenjtalk g2p accent.++ Args:+ text: Text tensor or string input.+ """import pyopenjtalkimport re@@ -72,6 +87,11 @@ def pyopenjtalk_g2p_accent(text) -> List[str]:def pyopenjtalk_g2p_accent_with_pause(text) -> List[str]:+ """Pyopenjtalk g2p accent with pause.++ Args:+ text: Text tensor or string input.+ """import pyopenjtalkimport re@@ -87,6 +107,11 @@ def pyopenjtalk_g2p_accent_with_pause(text) -> List[str]:def pyopenjtalk_g2p_kana(text) -> List[str]:+ """Pyopenjtalk g2p kana.++ Args:+ text: Text tensor or string input.+ """import pyopenjtalkkanas = pyopenjtalk.g2p(text, kana=True)@@ -173,6 +198,12 @@ def pyopenjtalk_g2p_prosody(text: str, drop_unvoiced_vowels: bool = True) -> Lisdef _numeric_feature_by_regex(regex, s):+ """Internal: numeric feature by regex.++ Args:+ regex: TODO.+ s: TODO.+ """match = re.search(regex, s)if match is None:return -50@@ -180,6 +211,11 @@ def _numeric_feature_by_regex(regex, s):def pypinyin_g2p(text) -> List[str]:+ """Pypinyin g2p.++ Args:+ text: Text tensor or string input.+ """from pypinyin import pinyinfrom pypinyin import Style@@ -188,6 +224,11 @@ def pypinyin_g2p(text) -> List[str]:def pypinyin_g2p_phone(text) -> List[str]:+ """Pypinyin g2p phone.++ Args:+ text: Text tensor or string input.+ """from pypinyin import pinyinfrom pypinyin import Stylefrom pypinyin.style._utils import get_finals@@ -215,10 +256,20 @@ class G2p_en:"""def __init__(self, no_space: bool = False):+ """Initialize G2p_en.++ Args:+ no_space: TODO.+ """self.no_space = no_spaceself.g2p = Nonedef __call__(self, text) -> List[str]:+ """Internal: call .++ Args:+ text: Text tensor or string input.+ """if self.g2p is None:self.g2p = g2p_en.G2p()@@ -239,6 +290,14 @@ class G2pk:"""def __init__(self, descritive=False, group_vowels=False, to_syl=False, no_space=False):+ """Initialize G2pk.++ Args:+ descritive: TODO.+ group_vowels: TODO.+ to_syl: TODO.+ no_space: TODO.+ """self.descritive = descritiveself.group_vowels = group_vowelsself.to_syl = to_syl@@ -246,6 +305,11 @@ class G2pk:self.g2p = Nonedef __call__(self, text) -> List[str]:+ """Internal: call .++ Args:+ text: Text tensor or string input.+ """if self.g2p is None:import g2pk@@ -276,18 +340,39 @@ class Jaso:VALID_CHARS = JAMO_LEADS + JAMO_VOWELS + JAMO_TAILS + PUNC + SPACEdef __init__(self, space_symbol=" ", no_space=False):+ """Initialize Jaso.++ Args:+ space_symbol: TODO.+ no_space: TODO.+ """self.space_symbol = space_symbolself.no_space = no_spacedef _text_to_jaso(self, line: str) -> List[str]:+ """Internal: text to jaso.++ Args:+ line: TODO.+ """jasos = list(jamo.hangul_to_jamo(line))return jasosdef _remove_non_korean_characters(self, tokens):+ """Internal: remove non korean characters.++ Args:+ tokens: TODO.+ """new_tokens = [token for token in tokens if token in self.VALID_CHARS]return new_tokensdef __call__(self, text) -> List[str]:+ """Internal: call .++ Args:+ text: Text tensor or string input.+ """graphemes = [x for x in self._text_to_jaso(text)]graphemes = self._remove_non_korean_characters(graphemes)@@ -320,6 +405,17 @@ class Phonemizer:**phonemizer_kwargs,):# delayed import+ """Initialize Phonemizer.++ Args:+ backend: TODO.+ word_separator: TODO.+ syllable_separator: TODO.+ phone_separator: TODO.+ strip: TODO.+ split_by_single_token: TODO.+ **phonemizer_kwargs: Additional keyword arguments.+ """from phonemizer.backend import BACKENDSfrom phonemizer.separator import Separator@@ -340,6 +436,11 @@ class Phonemizer:self.split_by_single_token = split_by_single_tokendef __call__(self, text) -> List[str]:+ """Internal: call .++ Args:+ text: Text tensor or string input.+ """tokens = self.phonemizer.phonemize([text],separator=self.separator,@@ -362,6 +463,14 @@ class PhonemeTokenizer(AbsTokenizer):space_symbol: str = "<space>",remove_non_linguistic_symbols: bool = False,):+ """Initialize PhonemeTokenizer.++ Args:+ g2p_type: TODO.+ non_linguistic_symbols: TODO.+ space_symbol: TODO.+ remove_non_linguistic_symbols: TODO.+ """if g2p_type is None:self.g2p = split_by_spaceelif g2p_type == "g2p_en":@@ -493,6 +602,7 @@ class PhonemeTokenizer(AbsTokenizer):self.remove_non_linguistic_symbols = remove_non_linguistic_symbolsdef __repr__(self):+ """Internal: repr ."""return (f"{self.__class__.__name__}("f'g2p_type="{self.g2p_type}", '@@ -502,6 +612,11 @@ class PhonemeTokenizer(AbsTokenizer):)def text2tokens(self, line: str) -> List[str]:+ """Text2tokens.++ Args:+ line: TODO.+ """tokens = []while len(line) != 0:for w in self.non_linguistic_symbols:@@ -521,4 +636,9 @@ class PhonemeTokenizer(AbsTokenizer):def tokens2text(self, tokens: Iterable[str]) -> str:# phoneme type is not invertible+ """Tokens2text.++ Args:+ tokens: TODO.+ """return "".join(tokens)diff --git a/funasr/tokenizer/sentencepiece_tokenizer.py b/funasr/tokenizer/sentencepiece_tokenizer.pyindex 0b47a9f6a..acba22c2a 100644--- a/funasr/tokenizer/sentencepiece_tokenizer.py+++ b/funasr/tokenizer/sentencepiece_tokenizer.py@@ -12,6 +12,12 @@ from funasr.register import tables@tables.register("tokenizer_classes", "SentencepiecesTokenizer")class SentencepiecesTokenizer(BaseTokenizer):def __init__(self, bpemodel: Union[Path, str], **kwargs):+ """Initialize SentencepiecesTokenizer.++ Args:+ bpemodel: TODO.+ **kwargs: Additional keyword arguments.+ """super().__init__(**kwargs)self.bpemodel = str(bpemodel)# NOTE(kamo):@@ -23,35 +29,72 @@ class SentencepiecesTokenizer(BaseTokenizer):self._build_sentence_piece_processor()def __repr__(self):+ """Internal: repr ."""return f'{self.__class__.__name__}(model="{self.bpemodel}")'def _build_sentence_piece_processor(self):# Build SentencePieceProcessor lazily.+ """Internal: build sentence piece processor."""if self.sp is None:self.sp = spm.SentencePieceProcessor()self.sp.load(self.bpemodel)def text2tokens(self, line: str) -> List[str]:+ """Text2tokens.++ Args:+ line: TODO.+ """self._build_sentence_piece_processor()return self.sp.EncodeAsPieces(line)def tokens2text(self, tokens: Iterable[str]) -> str:+ """Tokens2text.++ Args:+ tokens: TODO.+ """self._build_sentence_piece_processor()return self.sp.DecodePieces(list(tokens))def encode(self, line: str, **kwargs) -> List[int]:+ """Encode.++ Args:+ line: TODO.+ **kwargs: Additional keyword arguments.+ """self._build_sentence_piece_processor()return self.sp.EncodeAsIds(line)def decode(self, line: List[int], **kwargs):+ """Decode.++ Args:+ line: TODO.+ **kwargs: Additional keyword arguments.+ """self._build_sentence_piece_processor()return self.sp.DecodeIds(line)def get_vocab_size(self):+ """Get vocab size."""return self.sp.GetPieceSize()def ids2tokens(self, *args, **kwargs):+ """Ids2tokens.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """return self.decode(*args, **kwargs)def tokens2ids(self, *args, **kwargs):+ """Tokens2ids.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """return self.encode(*args, **kwargs)diff --git a/funasr/tokenizer/token_id_converter.py b/funasr/tokenizer/token_id_converter.pyindex c3fc71d5a..ff60d4992 100644--- a/funasr/tokenizer/token_id_converter.py+++ b/funasr/tokenizer/token_id_converter.py@@ -14,6 +14,12 @@ class TokenIDConverter:unk_symbol: str = "<unk>",):+ """Initialize TokenIDConverter.++ Args:+ token_list: TODO.+ unk_symbol: TODO.+ """if isinstance(token_list, (Path, str)):token_list = Path(token_list)self.token_list_repr = str(token_list)@@ -45,12 +51,23 @@ class TokenIDConverter:self.unk_id = self.token2id[self.unk_symbol]def get_num_vocabulary_size(self) -> int:+ """Get num vocabulary size."""return len(self.token_list)def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]:+ """Ids2tokens.++ Args:+ integers: TODO.+ """if isinstance(integers, np.ndarray) and integers.ndim != 1:raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}")return [self.token_list[i] for i in integers]def tokens2ids(self, tokens: Iterable[str]) -> List[int]:+ """Tokens2ids.++ Args:+ tokens: TODO.+ """return [self.token2id.get(i, self.unk_id) for i in tokens]diff --git a/funasr/tokenizer/whisper_tokenizer.py b/funasr/tokenizer/whisper_tokenizer.pyindex 3fd029c57..c085b4811 100644--- a/funasr/tokenizer/whisper_tokenizer.py+++ b/funasr/tokenizer/whisper_tokenizer.py@@ -3,6 +3,11 @@ from funasr.register import tables@tables.register("tokenizer_classes", "WhisperTokenizer")def WhisperTokenizer(**kwargs):+ """Whispertokenizer.++ Args:+ **kwargs: Additional keyword arguments.+ """try:from whisper.tokenizer import get_tokenizerexcept:@@ -24,6 +29,11 @@ def WhisperTokenizer(**kwargs):@tables.register("tokenizer_classes", "SenseVoiceTokenizer")def SenseVoiceTokenizer(**kwargs):+ """Sensevoicetokenizer.++ Args:+ **kwargs: Additional keyword arguments.+ """from funasr.models.sense_voice.whisper_lib.tokenizer import get_tokenizerlanguage = kwargs.get("language", None)diff --git a/funasr/tokenizer/word_tokenizer.py b/funasr/tokenizer/word_tokenizer.pyindex c1fd6e039..ded95f444 100644--- a/funasr/tokenizer/word_tokenizer.py+++ b/funasr/tokenizer/word_tokenizer.py@@ -15,6 +15,13 @@ class WordTokenizer(AbsTokenizer):non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,remove_non_linguistic_symbols: bool = False,):+ """Initialize WordTokenizer.++ Args:+ delimiter: TODO.+ non_linguistic_symbols: TODO.+ remove_non_linguistic_symbols: TODO.+ """self.delimiter = delimiterif not remove_non_linguistic_symbols and non_linguistic_symbols is not None:@@ -37,9 +44,15 @@ class WordTokenizer(AbsTokenizer):self.remove_non_linguistic_symbols = remove_non_linguistic_symbolsdef __repr__(self):+ """Internal: repr ."""return f'{self.__class__.__name__}(delimiter="{self.delimiter}")'def text2tokens(self, line: str) -> List[str]:+ """Text2tokens.++ Args:+ line: TODO.+ """tokens = []for t in line.split(self.delimiter):if self.remove_non_linguistic_symbols and t in self.non_linguistic_symbols:@@ -48,6 +61,11 @@ class WordTokenizer(AbsTokenizer):return tokensdef tokens2text(self, tokens: Iterable[str]) -> str:+ """Tokens2text.++ Args:+ tokens: TODO.+ """if self.delimiter is None:delimiter = " "else:diff --git a/funasr/train_utils/forward_adaptor.py b/funasr/train_utils/forward_adaptor.pyindex eb6da2bb1..b66c13fe2 100644--- a/funasr/train_utils/forward_adaptor.py+++ b/funasr/train_utils/forward_adaptor.py@@ -20,6 +20,12 @@ class ForwardAdaptor(torch.nn.Module):"""def __init__(self, module: torch.nn.Module, name: str):+ """Initialize ForwardAdaptor.++ Args:+ module: TODO.+ name: TODO.+ """super().__init__()self.module = moduleself.name = name@@ -27,5 +33,11 @@ class ForwardAdaptor(torch.nn.Module):raise ValueError(f"{module} doesn't have {name}")def forward(self, *args, **kwargs):+ """Forward pass for training.++ Args:+ *args: Variable positional arguments.+ **kwargs: Additional keyword arguments.+ """func = getattr(self.module, self.name)return func(*args, **kwargs)diff --git a/funasr/train_utils/model_summary.py b/funasr/train_utils/model_summary.pyindex 842cd21fa..c9215fffa 100644--- a/funasr/train_utils/model_summary.py+++ b/funasr/train_utils/model_summary.py@@ -41,10 +41,20 @@ def get_human_readable_count(number: int) -> str:def to_bytes(dtype) -> int:# torch.float16 -> 16+ """To bytes.++ Args:+ dtype: TODO.+ """return int(str(dtype)[-2:]) // 8def model_summary(model: torch.nn.Module) -> str:+ """Model summary.++ Args:+ model: Model instance or model name.+ """message = "Model structure:\n"message += str(model)diff --git a/funasr/train_utils/recursive_op.py b/funasr/train_utils/recursive_op.pyindex 1c2a92fd8..1e471012e 100644--- a/funasr/train_utils/recursive_op.py+++ b/funasr/train_utils/recursive_op.py@@ -7,6 +7,13 @@ if torch.distributed.is_available():def recursive_sum(obj, weight: torch.Tensor, distributed: bool = False):+ """Recursive sum.++ Args:+ obj: TODO.+ weight: TODO.+ distributed: TODO.+ """assert weight.dim() == 1, weight.size()if isinstance(obj, (tuple, list)):return type(obj)(recursive_sum(v, weight, distributed) for v in obj)@@ -25,6 +32,12 @@ def recursive_sum(obj, weight: torch.Tensor, distributed: bool = False):def recursive_divide(a, b: torch.Tensor):+ """Recursive divide.++ Args:+ a: TODO.+ b: TODO.+ """if isinstance(a, (tuple, list)):return type(a)(recursive_divide(v, b) for v in a)elif isinstance(a, dict):@@ -39,6 +52,13 @@ def recursive_divide(a, b: torch.Tensor):def recursive_average(obj, weight: torch.Tensor, distributed: bool = False):+ """Recursive average.++ Args:+ obj: TODO.+ weight: TODO.+ distributed: TODO.+ """obj = recursive_sum(obj, weight, distributed)weight = weight.sum()if distributed:diff --git a/funasr/train_utils/set_all_random_seed.py b/funasr/train_utils/set_all_random_seed.pyindex ebdca3f53..f23b0a89b 100644--- a/funasr/train_utils/set_all_random_seed.py+++ b/funasr/train_utils/set_all_random_seed.py@@ -5,6 +5,11 @@ import torchdef set_all_random_seed(seed: int):+ """Set all random seed.++ Args:+ seed: TODO.+ """random.seed(seed)np.random.seed(seed)torch.random.manual_seed(seed)diff --git a/funasr/train_utils/trainer.py b/funasr/train_utils/trainer.pyindex badf8fa2c..c95acd658 100644--- a/funasr/train_utils/trainer.py+++ b/funasr/train_utils/trainer.py@@ -23,6 +23,11 @@ except:@contextmanagerdef maybe_autocast(enabled):+ """Maybe autocast.++ Args:+ enabled: TODO.+ """if enabled:with autocast():yield@@ -661,6 +666,24 @@ class Trainer:**kwargs,):+ """Log.++ Args:+ epoch: TODO.+ batch_idx: TODO.+ step_in_epoch: TODO.+ batch_num_epoch: TODO.+ lr: TODO.+ loss: TODO.+ speed_stats: TODO.+ stats: TODO.+ writer: TODO.+ tag: TODO.+ data_split_i: TODO.+ data_split_num: TODO.+ log_step: TODO.+ **kwargs: Additional keyword arguments.+ """if (batch_idx + 1) % self.log_interval == 0:batch_idx = log_step if log_step is not None else batch_idxgpu_info = (@@ -720,6 +743,11 @@ class Trainer:def close(self, writer=None):+ """Close.++ Args:+ writer: TODO.+ """if self.use_ddp or self.use_fsdp:dist.barrier()diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.pyindex b1f2f6857..3e5f68eee 100644--- a/funasr/train_utils/trainer_ds.py+++ b/funasr/train_utils/trainer_ds.py@@ -25,6 +25,12 @@ except:@contextmanagerdef maybe_autocast(dtype=None, use_deepspeed=False):+ """Maybe autocast.++ Args:+ dtype: TODO.+ use_deepspeed: TODO.+ """if use_deepspeed:with torch.cuda.amp.autocast(enabled=True, dtype=dtype, cache_enabled=False):yield@@ -668,6 +674,13 @@ class Trainer:self.train_acc_avg = train_acc_avg.detach().cpu().item() / self.world_sizedef forward_step(self, model, batch, loss_dict={}):+ """Forward step.++ Args:+ model: Model instance or model name.+ batch: TODO.+ loss_dict: TODO.+ """with maybe_autocast(dtype=self.dtype, use_deepspeed=self.use_deepspeed):retval = model(**batch)@@ -679,6 +692,13 @@ class Trainer:loss_dict["weight"] = weightdef backward_step(self, model, scaler, loss_dict={}):+ """Backward step.++ Args:+ model: Model instance or model name.+ scaler: TODO.+ loss_dict: TODO.+ """loss = loss_dict["loss"]if self.use_deepspeed:@@ -691,6 +711,15 @@ class Trainer:loss.backward()def update_step(self, model, optim, scheduler, scaler, loss_dict=None):+ """Update step.++ Args:+ model: Model instance or model name.+ optim: TODO.+ scheduler: TODO.+ scaler: TODO.+ loss_dict: TODO.+ """batch_idx = loss_dict["batch_idx"]if self.use_deepspeed:model.step()@@ -822,6 +851,13 @@ class Trainer:tag="train",**kwargs,):+ """Log.++ Args:+ loss_dict: TODO.+ tag: TODO.+ **kwargs: Additional keyword arguments.+ """loss = loss_dict["loss"].detach().cpu().item()epoch = loss_dict["epoch"]batch_idx = loss_dict["batch_idx"]@@ -892,6 +928,11 @@ class Trainer:def close(self, writer=None):+ """Close.++ Args:+ writer: TODO.+ """if self.use_ddp or self.use_fsdp:dist.barrier()@@ -903,6 +944,12 @@ class Trainer:def warp_model(self, model, **kwargs):+ """Warp model.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """if self.use_deepspeed:from deepspeed.runtime.zero.stage_1_and_2 import (estimate_zero2_model_states_mem_needs_all_live,@@ -948,6 +995,12 @@ class Trainer:return modeldef warp_optim_scheduler(self, model, **kwargs):+ """Warp optim scheduler.++ Args:+ model: Model instance or model name.+ **kwargs: Additional keyword arguments.+ """from funasr.optimizers import optim_classesfrom funasr.schedulers import scheduler_classesfrom omegaconf import OmegaConf, DictConfig@@ -989,6 +1042,11 @@ class Trainer:else:def scheduler(opt):+ """Scheduler.++ Args:+ opt: TODO.+ """return scheduler_class(opt, **kwargs.get("scheduler_conf"))model, optimizer, _, scheduler = deepspeed.initialize(diff --git a/funasr/utils/compute_det_ctc.py b/funasr/utils/compute_det_ctc.pyindex 47c81821b..6bd8e5fef 100644--- a/funasr/utils/compute_det_ctc.py+++ b/funasr/utils/compute_det_ctc.py@@ -13,15 +13,23 @@ from funasr.utils.kws_utils import split_mixed_labelclass thread_wrapper(threading.Thread):def __init__(self, func, args=()):+ """Initialize thread_wrapper.++ Args:+ func: TODO.+ args: TODO.+ """super(thread_wrapper, self).__init__()self.func = funcself.args = argsself.result = []def run(self):+ """Run."""self.result = self.func(*self.args)def get_result(self):+ """Get result."""try:return self.resultexcept Exception:@@ -29,12 +37,22 @@ class thread_wrapper(threading.Thread):def space_mixed_label(input_str):+ """Space mixed label.++ Args:+ input_str: TODO.+ """splits = split_mixed_label(input_str)space_str = ''.join(f'{sub} ' for sub in splits)return space_str.strip()def read_lists(list_file):+ """Read lists.++ Args:+ list_file: TODO.+ """lists = []with open(list_file, 'r', encoding='utf8') as fin:for line in fin:@@ -44,6 +62,12 @@ def read_lists(list_file):def make_pair(wav_lists, trans_lists):+ """Make pair.++ Args:+ wav_lists: TODO.+ trans_lists: TODO.+ """logging.info('make pair for wav-trans list')trans_table = {}@@ -74,6 +98,12 @@ def make_pair(wav_lists, trans_lists):def count_duration(tid, data_lists):+ """Count duration.++ Args:+ tid: TODO.+ data_lists: TODO.+ """results = []for obj in data_lists:@@ -102,6 +132,14 @@ def count_duration(tid, data_lists):def load_data_and_score(keywords_list, data_file, trans_file, score_file):# score_table: {uttid: [keywordlist]}+ """Load data and score.++ Args:+ keywords_list: TODO.+ data_file: TODO.+ trans_file: TODO.+ score_file: TODO.+ """score_table = {}with open(score_file, 'r', encoding='utf8') as fin:# read score file and store in tablediff --git a/funasr/utils/datadir_writer.py b/funasr/utils/datadir_writer.pyindex 882db7708..628ee27ea 100644--- a/funasr/utils/datadir_writer.py+++ b/funasr/utils/datadir_writer.py@@ -17,6 +17,11 @@ class DatadirWriter:"""def __init__(self, p: Union[Path, str]):+ """Initialize DatadirWriter.++ Args:+ p: TODO.+ """self.path = Path(p)self.chilidren = {}self.fd = None@@ -24,9 +29,15 @@ class DatadirWriter:self.keys = set()def __enter__(self):+ """Internal: enter ."""return selfdef __getitem__(self, key: str) -> "DatadirWriter":+ """Internal: getitem .++ Args:+ key: Sample identifiers.+ """if self.fd is not None:raise RuntimeError("This writer points out a file")@@ -39,6 +50,12 @@ class DatadirWriter:return retvaldef __setitem__(self, key: str, value: str):+ """Internal: setitem .++ Args:+ key: Sample identifiers.+ value: TODO.+ """if self.has_children:raise RuntimeError("This writer points out a directory")if key in self.keys:@@ -53,9 +70,17 @@ class DatadirWriter:self.fd.flush()def __exit__(self, exc_type, exc_val, exc_tb):+ """Internal: exit .++ Args:+ exc_type: TODO.+ exc_val: TODO.+ exc_tb: TODO.+ """self.close()def close(self):+ """Close."""if self.has_children:prev_child = Nonefor child in self.chilidren.values():diff --git a/funasr/utils/dynamic_import.py b/funasr/utils/dynamic_import.pyindex e0a056bd3..cdd6b3106 100644--- a/funasr/utils/dynamic_import.py+++ b/funasr/utils/dynamic_import.py@@ -22,6 +22,11 @@ def load_module_from_path(file_path):def import_module_from_path(file_path: str):+ """Import module from path.++ Args:+ file_path: TODO.+ """if file_path.startswith("http"):from funasr.download.file import download_from_urldiff --git a/funasr/utils/export_utils.py b/funasr/utils/export_utils.pyindex b03b05272..b48417f39 100644--- a/funasr/utils/export_utils.py+++ b/funasr/utils/export_utils.py@@ -10,6 +10,16 @@ warnings.filterwarnings("ignore")def export(model, data_in=None, quantize: bool = False, opset_version: int = 14, type="onnx", **kwargs):+ """Export.++ Args:+ model: Model instance or model name.+ data_in: Input data (audio samples, file paths, or text).+ quantize: TODO.+ opset_version: TODO.+ type: TODO.+ **kwargs: Additional keyword arguments.+ """model_scripts = model.export(**kwargs)export_dir = kwargs.get("output_dir", os.path.dirname(kwargs.get("init_param")))os.makedirs(export_dir, exist_ok=True)@@ -62,6 +72,16 @@ def _onnx(**kwargs,):+ """Internal: onnx.++ Args:+ model: Model instance or model name.+ data_in: Input data (audio samples, file paths, or text).+ quantize: TODO.+ opset_version: TODO.+ export_dir: TODO.+ **kwargs: Additional keyword arguments.+ """device = kwargs.get("device", "cpu")dummy_input = model.export_dummy_inputs()@@ -117,6 +137,13 @@ def _onnx(def _torchscripts(model, path, device="cuda"):+ """Internal: torchscripts.++ Args:+ model: Model instance or model name.+ path: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """dummy_input = model.export_dummy_inputs()if device == "cuda":@@ -136,6 +163,13 @@ def _torchscripts(model, path, device="cuda"):def _bladedisc_opt(model, model_inputs, enable_fp16=True):+ """Internal: bladedisc opt.++ Args:+ model: Model instance or model name.+ model_inputs: TODO.+ enable_fp16: TODO.+ """model = model.eval()try:import torch_blade@@ -155,6 +189,13 @@ def _bladedisc_opt(model, model_inputs, enable_fp16=True):def _rescale_input_hook(m, x, scale):+ """Internal: rescale input hook.++ Args:+ m: TODO.+ x: TODO.+ scale: TODO.+ """if len(x) > 1:return (x[0] / scale, *x[1:])else:@@ -162,6 +203,14 @@ def _rescale_input_hook(m, x, scale):def _rescale_output_hook(m, x, y, scale):+ """Internal: rescale output hook.++ Args:+ m: TODO.+ x: TODO.+ y: TODO.+ scale: TODO.+ """if isinstance(y, tuple):return (y[0] / scale, *y[1:])else:@@ -170,9 +219,22 @@ def _rescale_output_hook(m, x, y, scale):def _rescale_encoder_model(model, input_data):# Calculate absmax+ """Internal: rescale encoder model.++ Args:+ model: Model instance or model name.+ input_data: TODO.+ """absmax = torch.tensor(0).cuda()def stat_input_hook(m, x, y):+ """Stat input hook.++ Args:+ m: TODO.+ x: TODO.+ y: TODO.+ """val = x[0] if isinstance(x, tuple) else xabsmax.copy_(torch.max(absmax, val.detach().abs().max()))@@ -200,6 +262,13 @@ def _rescale_encoder_model(model, input_data):def _bladedisc_opt_for_encdec(model, path, enable_fp16):# Get input data# TODO: better to use real data+ """Internal: bladedisc opt for encdec.++ Args:+ model: Model instance or model name.+ path: TODO.+ enable_fp16: TODO.+ """input_data = model.export_dummy_inputs()if isinstance(input_data, torch.Tensor):input_data = input_data.cuda()@@ -210,6 +279,12 @@ def _bladedisc_opt_for_encdec(model, path, enable_fp16):decoder_inputs = list()def get_input_hook(m, x):+ """Get input hook.++ Args:+ m: TODO.+ x: TODO.+ """decoder_inputs.extend(list(x))hook = model.decoder.register_forward_pre_hook(get_input_hook)@@ -232,6 +307,13 @@ def _onnx_opt_for_encdec(model, path, enable_fp16):# Get input data# TODO: better to use real data+ """Internal: onnx opt for encdec.++ Args:+ model: Model instance or model name.+ path: TODO.+ enable_fp16: TODO.+ """input_data = model.export_dummy_inputs()if isinstance(input_data, torch.Tensor):@@ -243,6 +325,12 @@ def _onnx_opt_for_encdec(model, path, enable_fp16):decoder_inputs = list()def get_input_hook(m, x):+ """Get input hook.++ Args:+ m: TODO.+ x: TODO.+ """decoder_inputs.extend(list(x))hook = model.decoder.register_forward_pre_hook(get_input_hook)diff --git a/funasr/utils/install_model_requirements.py b/funasr/utils/install_model_requirements.pyindex 73a2dcb39..56a1f5424 100644--- a/funasr/utils/install_model_requirements.py+++ b/funasr/utils/install_model_requirements.py@@ -3,6 +3,11 @@ import subprocessdef install_requirements(requirements_path):+ """Install requirements.++ Args:+ requirements_path: TODO.+ """try:result = pip_install_r(requirements_path)# check status@@ -26,6 +31,11 @@ def install_requirements(requirements_path):def pip_install_r(requirements_path):+ """Pip install r.++ Args:+ requirements_path: TODO.+ """cmd = []if shutil.which("pip") is not None:cmd = ["pip"]diff --git a/funasr/utils/kws_utils.py b/funasr/utils/kws_utils.pyindex f133fe659..1086050e4 100644--- a/funasr/utils/kws_utils.py+++ b/funasr/utils/kws_utils.py@@ -11,6 +11,11 @@ symbol_str = '[’!"#$%&\'()*+,-./:;<>=?@,。?★、…【】《》?“”def split_mixed_label(input_str):+ """Split mixed label.++ Args:+ input_str: TODO.+ """tokens = []s = input_str.lower()while len(s) > 0:@@ -25,6 +30,13 @@ def split_mixed_label(input_str):def query_token_set(txt, symbol_table, lexicon_table):+ """Query token set.++ Args:+ txt: TODO.+ symbol_table: TODO.+ lexicon_table: TODO.+ """tokens_str = tuple()tokens_idx = tuple()@@ -218,6 +230,12 @@ class KwsCtcPrefixDecoder():def is_sublist(self, main_list, check_list):+ """Is sublist.++ Args:+ main_list: TODO.+ check_list: TODO.+ """if len(main_list) < len(check_list):return -1@@ -240,6 +258,12 @@ class KwsCtcPrefixDecoder():logits: torch.Tensor,logits_lengths: torch.Tensor,):+ """Internal: decode inside.++ Args:+ logits: TODO.+ logits_lengths: Lengths of logits.+ """hyps = self.beam_search(logits, logits_lengths, self.keywords_idxset)hit_keyword = Nonediff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.pyindex e287d0c7a..ce8525b74 100644--- a/funasr/utils/load_utils.py+++ b/funasr/utils/load_utils.py@@ -25,6 +25,7 @@ except:def is_ffmpeg_installed():+ """Is ffmpeg installed."""try:output = subprocess.check_output(["ffmpeg", "-version"], stderr=subprocess.STDOUT)return "ffmpeg version" in output.decode("utf-8")@@ -222,6 +223,12 @@ def validate_frame_rate():# 将文件读取为字节流+ """Validate frame rate.++ Args:+ input: Input audio/text data.+ fs: TODO.+ """byte_data = BytesIO(input)# 使用 pydub 加载音频diff --git a/funasr/utils/misc.py b/funasr/utils/misc.pyindex ba16e05c1..f7ff937d2 100644--- a/funasr/utils/misc.py+++ b/funasr/utils/misc.py@@ -8,6 +8,12 @@ from omegaconf import DictConfig, OmegaConfdef statistic_model_parameters(model, prefix=None):+ """Statistic model parameters.++ Args:+ model: Model instance or model name.+ prefix: TODO.+ """var_dict = model.state_dict()numel = 0for i, key in enumerate(@@ -19,16 +25,36 @@ def statistic_model_parameters(model, prefix=None):def int2vec(x, vec_dim=8, dtype=np.int32):+ """Int2vec.++ Args:+ x: TODO.+ vec_dim: Size/dimension parameter.+ dtype: TODO.+ """b = ("{:0" + str(vec_dim) + "b}").format(x)# little-endian order: lower bit firstreturn (np.array(list(b)[::-1]) == "1").astype(dtype)def seq2arr(seq, vec_dim=8):+ """Seq2arr.++ Args:+ seq: TODO.+ vec_dim: Size/dimension parameter.+ """return np.row_stack([int2vec(int(x), vec_dim) for x in seq])def load_scp_as_dict(scp_path, value_type="str", kv_sep=" "):+ """Load scp as dict.++ Args:+ scp_path: TODO.+ value_type: TODO.+ kv_sep: TODO.+ """with io.open(scp_path, "r", encoding="utf-8") as f:ret_dict = OrderedDict()for one_line in f.readlines():@@ -42,6 +68,13 @@ def load_scp_as_dict(scp_path, value_type="str", kv_sep=" "):def load_scp_as_list(scp_path, value_type="str", kv_sep=" "):+ """Load scp as list.++ Args:+ scp_path: TODO.+ value_type: TODO.+ kv_sep: TODO.+ """with io.open(scp_path, "r", encoding="utf8") as f:ret_dict = []for one_line in f.readlines():@@ -75,6 +108,11 @@ def deep_update(original, update):def prepare_model_dir(**kwargs):+ """Prepare model dir.++ Args:+ **kwargs: Additional keyword arguments.+ """os.makedirs(kwargs.get("output_dir", "./"), exist_ok=True)yaml_file = os.path.join(kwargs.get("output_dir", "./"), "config.yaml")diff --git a/funasr/utils/postprocess_utils.py b/funasr/utils/postprocess_utils.pyindex 39a8a10dd..beaa87cb4 100644--- a/funasr/utils/postprocess_utils.py+++ b/funasr/utils/postprocess_utils.py@@ -6,12 +6,22 @@ from typing import Any, List, Uniondef isChinese(ch: str):+ """Ischinese.++ Args:+ ch: TODO.+ """if "\u4e00" <= ch <= "\u9fff" or "\u0030" <= ch <= "\u0039" or ch == "@":return Truereturn Falsedef isAllChinese(word: Union[List[Any], str]):+ """Isallchinese.++ Args:+ word: TODO.+ """word_lists = []for i in word:cur = i.replace(" ", "")@@ -31,6 +41,11 @@ def isAllChinese(word: Union[List[Any], str]):def isAllAlpha(word: Union[List[Any], str]):+ """Isallalpha.++ Args:+ word: TODO.+ """word_lists = []for i in word:cur = i.replace(" ", "")@@ -54,6 +69,12 @@ def isAllAlpha(word: Union[List[Any], str]):# def abbr_dispose(words: List[Any]) -> List[Any]:def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]:+ """Abbr dispose.++ Args:+ words: TODO.+ time_stamp: TODO.+ """words_size = len(words)word_lists = []abbr_begin = []@@ -142,6 +163,12 @@ def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]:def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):+ """Sentence postprocess.++ Args:+ words: TODO.+ time_stamp: TODO.+ """middle_lists = []word_lists = []word_item = ""@@ -252,6 +279,11 @@ def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):def sentence_postprocess_sentencepiece(words):+ """Sentence postprocess sentencepiece.++ Args:+ words: TODO.+ """middle_lists = []word_lists = []word_item = ""@@ -377,6 +409,11 @@ event_set = {def format_str_v2(s):+ """Format str v2.++ Args:+ s: TODO.+ """sptk_dict = {}for sptk in emoji_dict:sptk_dict[sptk] = s.count(sptk)@@ -397,10 +434,25 @@ def format_str_v2(s):def rich_transcription_postprocess(s):+ """Rich transcription postprocess.++ Args:+ s: TODO.+ """def get_emo(s):+ """Get emo.++ Args:+ s: TODO.+ """return s[-1] if s[-1] in emo_set else Nonedef get_event(s):+ """Get event.++ Args:+ s: TODO.+ """return s[0] if s[0] in event_set else Nones = s.replace("<|nospeech|><|Event_UNK|>", "❓")diff --git a/funasr/utils/speaker_utils.py b/funasr/utils/speaker_utils.pyindex e470849f8..2b811e948 100644--- a/funasr/utils/speaker_utils.py+++ b/funasr/utils/speaker_utils.py@@ -16,6 +16,11 @@ from funasr.utils.modelscope_file import Filedef check_audio_list(audio: list):+ """Check audio list.++ Args:+ audio: TODO.+ """audio_dur = 0for i in range(len(audio)):seg = audio[i]@@ -32,6 +37,11 @@ def check_audio_list(audio: list):def sv_preprocess(inputs: Union[np.ndarray, list]):+ """Sv preprocess.++ Args:+ inputs: TODO.+ """output = []for i in range(len(inputs)):if isinstance(inputs[i], str):@@ -58,12 +68,23 @@ def sv_preprocess(inputs: Union[np.ndarray, list]):def sv_chunk(vad_segments: list, fs=16000) -> list:+ """Sv chunk.++ Args:+ vad_segments: TODO.+ fs: TODO.+ """config = {"seg_dur": 1.5,"seg_shift": 0.75,}def seg_chunk(seg_data):+ """Seg chunk.++ Args:+ seg_data: TODO.+ """seg_st = seg_data[0]data = seg_data[2]chunk_len = int(config["seg_dur"] * fs)@@ -90,6 +111,11 @@ def sv_chunk(vad_segments: list, fs=16000) -> list:def extract_feature(audio):+ """Extract feature.++ Args:+ audio: TODO.+ """features = []for au in audio:feature = Kaldi.fbank(au.unsqueeze(0), num_mel_bins=80)@@ -102,6 +128,14 @@ def extract_feature(audio):def postprocess(segments: list, vad_segments: list, labels: np.ndarray, embeddings: np.ndarray) -> list:+ """Postprocess.++ Args:+ segments: TODO.+ vad_segments: TODO.+ labels: TODO.+ embeddings: TODO.+ """assert len(segments) == len(labels)labels = correct_labels(labels)distribute_res = []@@ -118,6 +152,12 @@ def postprocess(spk_embs = np.stack(spk_embs)def is_overlapped(t1, t2):+ """Is overlapped.++ Args:+ t1: TODO.+ t2: TODO.+ """if t1 > t2 + 1e-4:return Truereturn False@@ -136,6 +176,11 @@ def postprocess(def correct_labels(labels):+ """Correct labels.++ Args:+ labels: TODO.+ """labels_id = 0id2id = {}new_labels = []@@ -148,6 +193,11 @@ def correct_labels(labels):def merge_seque(distribute_res):+ """Merge seque.++ Args:+ distribute_res: TODO.+ """res = [distribute_res[0]]for i in range(1, len(distribute_res)):if distribute_res[i][2] != res[-1][2] or distribute_res[i][0] > res[-1][1]:@@ -159,6 +209,12 @@ def merge_seque(distribute_res):def smooth(res, mindur=1):# short segments are assigned to nearest speakers.+ """Smooth.++ Args:+ res: TODO.+ mindur: TODO.+ """for i in range(len(res)):res[i][0] = round(res[i][0], 2)res[i][1] = round(res[i][1], 2)@@ -178,6 +234,12 @@ def smooth(res, mindur=1):def distribute_spk(sentence_list, sd_time_list):+ """Distribute spk.++ Args:+ sentence_list: TODO.+ sd_time_list: TODO.+ """sd_sentence_list = []for d in sentence_list:sentence_start = d["ts_list"][0][0]diff --git a/funasr/utils/timestamp_tools.py b/funasr/utils/timestamp_tools.pyindex ccfc1a91d..ef5af8466 100644--- a/funasr/utils/timestamp_tools.py+++ b/funasr/utils/timestamp_tools.py@@ -9,6 +9,12 @@ from itertools import zip_longestdef cif_wo_hidden(alphas, threshold):+ """Cif wo hidden.++ Args:+ alphas: TODO.+ threshold: TODO.+ """batch_size, len_time = alphas.size()# loop varssintegrate = torch.zeros([batch_size], device=alphas.device)@@ -31,6 +37,17 @@ def cif_wo_hidden(alphas, threshold):def ts_prediction_lfr6_standard(us_alphas, us_peaks, char_list, vad_offset=0.0, force_time_shift=-1.5, sil_in_str=True, upsample_rate=3,):+ """Ts prediction lfr6 standard.++ Args:+ us_alphas: TODO.+ us_peaks: TODO.+ char_list: TODO.+ vad_offset: TODO.+ force_time_shift: TODO.+ sil_in_str: TODO.+ upsample_rate: TODO.+ """if not len(char_list):return "", []START_END_THRESHOLD = 5@@ -206,6 +223,14 @@ def timestamp_sentence(def timestamp_sentence_en(punc_id_list, timestamp_postprocessed, text_postprocessed, return_raw_text=False):+ """Timestamp sentence en.++ Args:+ punc_id_list: TODO.+ timestamp_postprocessed: TODO.+ text_postprocessed: TODO.+ return_raw_text: TODO.+ """punc_list = [",", ".", "?", ","]res = []if text_postprocessed is None:diff --git a/funasr/utils/torch_function.py b/funasr/utils/torch_function.pyindex f637bbf82..cc24e373d 100644--- a/funasr/utils/torch_function.py+++ b/funasr/utils/torch_function.py@@ -8,6 +8,12 @@ import numpy as npclass MakePadMask(nn.Module):def __init__(self, max_seq_len=512, flip=True):+ """Initialize MakePadMask.++ Args:+ max_seq_len: TODO.+ flip: TODO.+ """super().__init__()if flip:self.mask_pad = torch.Tensor(1 - np.tri(max_seq_len)).type(torch.bool)@@ -46,9 +52,23 @@ class MakePadMask(nn.Module):class sequence_mask(nn.Module):def __init__(self, max_seq_len=512, flip=True):+ """Initialize sequence_mask.++ Args:+ max_seq_len: TODO.+ flip: TODO.+ """super().__init__()def forward(self, lengths, max_seq_len=None, dtype=torch.float32, device=None):+ """Forward pass for training.++ Args:+ lengths: TODO.+ max_seq_len: TODO.+ dtype: TODO.+ device: Target device ("cuda:0", "cpu", etc.).+ """if max_seq_len is None:max_seq_len = lengths.max()row_vector = torch.arange(0, max_seq_len, 1).to(lengths.device)@@ -61,6 +81,14 @@ class sequence_mask(nn.Module):def normalize(input: torch.Tensor, p: float = 2.0, dim: int = 1, out: Optional[torch.Tensor] = None) -> torch.Tensor:+ """Normalize.++ Args:+ input: Input audio/text data.+ p: TODO.+ dim: TODO.+ out: TODO.+ """if out is None:denom = input.norm(p, dim, keepdim=True).expand_as(input)return input / denom@@ -70,10 +98,16 @@ def normalize(def subsequent_mask(size: torch.Tensor):+ """Subsequent mask.++ Args:+ size: TODO.+ """return torch.ones(size, size).tril()def MakePadMask_test():+ """Makepadmask test."""feats_length = torch.tensor([10]).type(torch.long)mask_fn = MakePadMask()mask = mask_fn(feats_length)diff --git a/funasr/utils/type_utils.py b/funasr/utils/type_utils.pyindex 6b36f9c4b..fe99013be 100644--- a/funasr/utils/type_utils.py+++ b/funasr/utils/type_utils.py@@ -7,10 +7,20 @@ import humanfriendlydef str2bool(value: str) -> bool:+ """Str2bool.++ Args:+ value: TODO.+ """return bool(strtobool(value))def remove_parenthesis(value: str):+ """Remove parenthesis.++ Args:+ value: TODO.+ """value = value.strip()if value.startswith("(") and value.endswith(")"):value = value[1:-1]@@ -20,6 +30,11 @@ def remove_parenthesis(value: str):def remove_quotes(value: str):+ """Remove quotes.++ Args:+ value: TODO.+ """value = value.strip()if value.startswith('"') and value.endswith('"'):value = value[1:-1]@@ -73,12 +88,22 @@ def float_or_none(value: str) -> Optional[float]:def humanfriendly_parse_size_or_none(value) -> Optional[float]:+ """Humanfriendly parse size or none.++ Args:+ value: TODO.+ """if value.strip().lower() in ("none", "null", "nil"):return Nonereturn humanfriendly.parse_size(value)def str_or_int(value: str) -> Union[str, int]:+ """Str or int.++ Args:+ value: TODO.+ """try:return int(value)except ValueError:diff --git a/funasr/utils/vad_utils.py b/funasr/utils/vad_utils.pyindex 8400e60b6..2622dd552 100644--- a/funasr/utils/vad_utils.py+++ b/funasr/utils/vad_utils.py@@ -3,6 +3,13 @@ from torch.nn.utils.rnn import pad_sequencedef slice_padding_fbank(speech, speech_lengths, vad_segments):+ """Slice padding fbank.++ Args:+ speech: Speech audio tensor, shape (batch, time).+ speech_lengths: Length of each speech sample.+ vad_segments: TODO.+ """speech_list = []speech_lengths_list = []for i, segment in enumerate(vad_segments):diff --git a/funasr/utils/version_checker.py b/funasr/utils/version_checker.pyindex 9bc11deb9..156881bab 100644--- a/funasr/utils/version_checker.py+++ b/funasr/utils/version_checker.py@@ -3,6 +3,11 @@ from funasr import __version__ # Ensure that __version__ is defined in your pacdef get_pypi_version(package_name):+ """Get pypi version.++ Args:+ package_name: TODO.+ """import requestsurl = f"https://pypi.org/pypi/{package_name}/json"@@ -15,6 +20,11 @@ def get_pypi_version(package_name):def check_for_update(disable=False):+ """Check for update.++ Args:+ disable: TODO.+ """current_version = version.parse(__version__)print(f"funasr version: {current_version}.")if disable: