Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

docs: add docstrings to 1950 functions across 276 files for API documentation

游雁78cc132a5157
diff --git a/funasr/__init__.py b/funasr/__init__.py
index e466840f9..37a5d8e2a 100644
--- a/funasr/__init__.py
+++ b/funasr/__init__.py
@@ -19,6 +19,12 @@ _STRICT_IMPORT = os.environ.get("FUNASR_STRICT_IMPORT") == "1"
def _record_import_error(name, error):
+ """Internal: record import error.
+
+ Args:
+ name: TODO.
+ error: TODO.
+ """
_IMPORT_ERRORS[name] = f"{error.__class__.__name__}: {error}"
_IMPORT_ERROR_TRACEBACKS[name] = traceback.format_exc()
if _IMPORT_DEBUG:
@@ -26,14 +32,22 @@ def _record_import_error(name, error):
def get_import_errors():
+ """Get import errors."""
return dict(_IMPORT_ERRORS)
def get_import_error_tracebacks():
+ """Get import error tracebacks."""
return dict(_IMPORT_ERROR_TRACEBACKS)
def import_submodules(package, recursive=True):
+ """Import submodules.
+
+ Args:
+ package: TODO.
+ recursive: TODO.
+ """
if isinstance(package, str):
try:
package = importlib.import_module(package)
diff --git a/funasr/auto/auto_frontend.py b/funasr/auto/auto_frontend.py
index 562d5f856..ed7ba0444 100644
--- a/funasr/auto/auto_frontend.py
+++ b/funasr/auto/auto_frontend.py
@@ -29,6 +29,11 @@ from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
class AutoFrontend:
def __init__(self, **kwargs):
+ """Initialize AutoFrontend.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
assert "model" in kwargs
if "model_conf" not in kwargs:
logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms")))
@@ -47,6 +52,14 @@ class AutoFrontend:
def __call__(self, input, input_len=None, kwargs=None, **cfg):
+ """Internal: call .
+
+ Args:
+ input: Input audio/text data.
+ input_len: TODO.
+ kwargs: Additional keyword arguments.
+ **cfg: Configuration overrides.
+ """
kwargs = self.kwargs if kwargs is None else kwargs
kwargs.update(cfg)
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index f3dbc5df5..05df315ca 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -52,6 +52,7 @@ def _resolve_ncpu(config, fallback=4):
def _get_import_errors():
+ """Internal: get import errors."""
try:
import funasr
except Exception:
@@ -63,6 +64,13 @@ def _get_import_errors():
def _format_unregistered_component_error(component_type, component_name, registry):
+ """Internal: format unregistered component error.
+
+ Args:
+ component_type: TODO.
+ component_name: TODO.
+ registry: TODO.
+ """
registered = sorted(registry.keys())
preview = ", ".join(registered[:80])
if len(registered) > 80:
@@ -414,6 +422,12 @@ class AutoModel:
return model, kwargs
def __call__(self, *args, **cfg):
+ """Internal: call .
+
+ Args:
+ *args: Variable positional arguments.
+ **cfg: Configuration overrides.
+ """
kwargs = self.kwargs
deep_update(kwargs, cfg)
res = self.model(*args, kwargs)
diff --git a/funasr/auto/auto_tokenizer.py b/funasr/auto/auto_tokenizer.py
index 59ff8893d..817dc768b 100644
--- a/funasr/auto/auto_tokenizer.py
+++ b/funasr/auto/auto_tokenizer.py
@@ -4,4 +4,5 @@ class AutoTokenizer:
"""
def __init__(self):
+ """Initialize AutoTokenizer."""
pass
diff --git a/funasr/bin/compute_audio_cmvn.py b/funasr/bin/compute_audio_cmvn.py
index 880a24745..1cdfa5bc0 100644
--- a/funasr/bin/compute_audio_cmvn.py
+++ b/funasr/bin/compute_audio_cmvn.py
@@ -13,6 +13,11 @@ from funasr.train_utils.set_all_random_seed import set_all_random_seed
@hydra.main(config_name=None, version_base=None)
def main_hydra(kwargs: DictConfig):
+ """Main hydra.
+
+ Args:
+ kwargs: Additional keyword arguments.
+ """
if kwargs.get("debug", False):
import pdb
@@ -27,6 +32,11 @@ def main_hydra(kwargs: DictConfig):
def main(**kwargs):
+ """Main.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
print(kwargs)
# set random seed
# tables.print()
diff --git a/funasr/bin/export.py b/funasr/bin/export.py
index 9d0140102..e151db1fe 100644
--- a/funasr/bin/export.py
+++ b/funasr/bin/export.py
@@ -8,7 +8,17 @@ from funasr.auto.auto_model import AutoModel
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
def to_plain_list(cfg_item):
+ """To plain list.
+
+ Args:
+ cfg_item: TODO.
+ """
if isinstance(cfg_item, ListConfig):
return OmegaConf.to_container(cfg_item, resolve=True)
elif isinstance(cfg_item, DictConfig):
diff --git a/funasr/bin/inference.py b/funasr/bin/inference.py
index 39ee5c067..b4c17d5fa 100644
--- a/funasr/bin/inference.py
+++ b/funasr/bin/inference.py
@@ -7,7 +7,17 @@ from funasr.auto.auto_model import AutoModel
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
def to_plain_list(cfg_item):
+ """To plain list.
+
+ Args:
+ cfg_item: TODO.
+ """
if isinstance(cfg_item, ListConfig):
return OmegaConf.to_container(cfg_item, resolve=True)
elif isinstance(cfg_item, DictConfig):
diff --git a/funasr/bin/tokenize_text.py b/funasr/bin/tokenize_text.py
index 8c2aae050..e127e1234 100755
--- a/funasr/bin/tokenize_text.py
+++ b/funasr/bin/tokenize_text.py
@@ -81,6 +81,26 @@ def tokenize(
g2p: Optional[str],
):
+ """Tokenize.
+
+ Args:
+ input: Input audio/text data.
+ output: TODO.
+ field: TODO.
+ delimiter: TODO.
+ token_type: TODO.
+ space_symbol: TODO.
+ non_linguistic_symbols: TODO.
+ bpemodel: TODO.
+ log_level: TODO.
+ write_vocabulary: TODO.
+ vocabulary_size: Size/dimension parameter.
+ remove_non_linguistic_symbols: TODO.
+ cutoff: TODO.
+ add_symbol: TODO.
+ cleaner: TODO.
+ g2p: TODO.
+ """
logging.basicConfig(
level=log_level,
format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s",
@@ -185,6 +205,7 @@ def tokenize(
def get_parser() -> argparse.ArgumentParser:
+ """Get parser."""
parser = argparse.ArgumentParser(
description="Tokenize texts",
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
@@ -266,6 +287,11 @@ def get_parser() -> argparse.ArgumentParser:
def main(cmd=None):
+ """Main.
+
+ Args:
+ cmd: TODO.
+ """
print(get_commandline_args(), file=sys.stderr)
parser = get_parser()
args = parser.parse_args(cmd)
diff --git a/funasr/bin/train.py b/funasr/bin/train.py
index 4e3144133..d54cd0bbe 100644
--- a/funasr/bin/train.py
+++ b/funasr/bin/train.py
@@ -40,6 +40,11 @@ from funasr import AutoModel
@hydra.main(config_name=None, version_base=None)
def main_hydra(kwargs: DictConfig):
+ """Main hydra.
+
+ Args:
+ kwargs: Additional keyword arguments.
+ """
if kwargs.get("debug", False):
import pdb
@@ -56,6 +61,11 @@ def main_hydra(kwargs: DictConfig):
def main(**kwargs):
# set random seed
+ """Main.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
set_all_random_seed(kwargs.get("seed", 0))
torch.backends.cudnn.enabled = kwargs.get("cudnn_enabled", torch.backends.cudnn.enabled)
torch.backends.cudnn.benchmark = kwargs.get("cudnn_benchmark", torch.backends.cudnn.benchmark)
@@ -137,6 +147,14 @@ def main(**kwargs):
min_num_params: int = int(1e8),
) -> bool:
# 根据自定义逻辑决定是否包装模块
+ """Custom auto wrap policy.
+
+ Args:
+ module: TODO.
+ recurse: TODO.
+ nonwrapped_numel: TODO.
+ min_num_params: TODO.
+ """
is_large = nonwrapped_numel >= min_num_params
requires_grad_uniform = len({p.requires_grad for p in module.parameters()}) == 1
return is_large and requires_grad_uniform
diff --git a/funasr/bin/train_ds.py b/funasr/bin/train_ds.py
index 4aa5f3bfd..b32881d99 100644
--- a/funasr/bin/train_ds.py
+++ b/funasr/bin/train_ds.py
@@ -43,6 +43,11 @@ except:
@hydra.main(config_name=None, version_base=None)
def main_hydra(kwargs: DictConfig):
+ """Main hydra.
+
+ Args:
+ kwargs: Additional keyword arguments.
+ """
if kwargs.get("debug", False):
import pdb
@@ -59,6 +64,11 @@ def main_hydra(kwargs: DictConfig):
def main(**kwargs):
# set random seed
+ """Main.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
set_all_random_seed(kwargs.get("seed", 0))
torch.backends.cudnn.enabled = kwargs.get("cudnn_enabled", torch.backends.cudnn.enabled)
torch.backends.cudnn.benchmark = kwargs.get("cudnn_benchmark", torch.backends.cudnn.benchmark)
diff --git a/funasr/datasets/audio_datasets/datasets.py b/funasr/datasets/audio_datasets/datasets.py
index 68b2d3cc7..35266b050 100644
--- a/funasr/datasets/audio_datasets/datasets.py
+++ b/funasr/datasets/audio_datasets/datasets.py
@@ -23,6 +23,18 @@ class AudioDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs,
):
+ """Initialize AudioDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ is_training: Boolean flag for training.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -53,17 +65,33 @@ class AudioDataset(torch.utils.data.Dataset):
self.float_pad_value = float_pad_value
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
# import pdb;
# pdb.set_trace()
@@ -97,6 +125,11 @@ class AudioDataset(torch.utils.data.Dataset):
}
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
for key in sample.keys():
@@ -127,10 +160,21 @@ class AudioDatasetHotword(AudioDataset):
seaco_id: bool = 0,
**kwargs,
):
+ """Initialize AudioDatasetHotword.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
self.seaco_id = seaco_id
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
# import pdb;
# pdb.set_trace()
@@ -164,6 +208,18 @@ class AudioDatasetHotword(AudioDataset):
pre_index=None,
pre_hwlist=None,
):
+ """Generate index.
+
+ Args:
+ length: TODO.
+ hotword_min_length: TODO.
+ hotword_max_length: TODO.
+ sample_rate: TODO.
+ double_rate: TODO.
+ pre_prob: TODO.
+ pre_index: TODO.
+ pre_hwlist: TODO.
+ """
if length < hotword_min_length:
return [-1]
if random.random() < sample_rate:
@@ -210,6 +266,11 @@ class AudioDatasetHotword(AudioDataset):
}
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
hotword_indxs = []
seaco_id = samples[0]["seaco_id"]
diff --git a/funasr/datasets/audio_datasets/espnet_samplers.py b/funasr/datasets/audio_datasets/espnet_samplers.py
index 004201e53..a411ba35f 100644
--- a/funasr/datasets/audio_datasets/espnet_samplers.py
+++ b/funasr/datasets/audio_datasets/espnet_samplers.py
@@ -12,6 +12,12 @@ from funasr.register import tables
@tables.register("batch_sampler_classes", "EspnetStyleBatchSampler")
def EspnetStyleBatchSampler_fn(dataset, **kwargs):
+ """Espnetstylebatchsampler fn.
+
+ Args:
+ dataset: TODO.
+ **kwargs: Additional keyword arguments.
+ """
dataloader_args = {}
batch_sampler = EspnetStyleBatchSampler(dataset, **kwargs)
@@ -45,6 +51,22 @@ class EspnetStyleBatchSampler(DistributedSampler):
**kwargs,
):
+ """Initialize EspnetStyleBatchSampler.
+
+ Args:
+ dataset: TODO.
+ batch_size: Number of samples per batch.
+ batch_type: TODO.
+ rank: TODO.
+ num_replicas: TODO.
+ rank_split: TODO.
+ shuffle: TODO.
+ drop_last: TODO.
+ is_training: Boolean flag for training.
+ sort_size: Size/dimension parameter.
+ start_step: TODO.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
num_replicas = dist.get_world_size()
@@ -79,6 +101,7 @@ class EspnetStyleBatchSampler(DistributedSampler):
# shuffle=shuffle, drop_last=drop_last)
def __iter__(self):
+ """Internal: iter ."""
if self.shuffle:
g = torch.Generator()
g.manual_seed(self.epoch)
@@ -158,8 +181,14 @@ class EspnetStyleBatchSampler(DistributedSampler):
def __len__(self):
# Calculate the number of batches per epoch for the current rank
+ """Internal: len ."""
return self.batch_num
def set_epoch(self, epoch):
# Set the epoch for shuffling
+ """Set epoch.
+
+ Args:
+ epoch: TODO.
+ """
self.epoch = epoch
diff --git a/funasr/datasets/audio_datasets/index_ds.py b/funasr/datasets/audio_datasets/index_ds.py
index dd7c52cd4..ff9af8533 100644
--- a/funasr/datasets/audio_datasets/index_ds.py
+++ b/funasr/datasets/audio_datasets/index_ds.py
@@ -16,6 +16,12 @@ from funasr.register import tables
class IndexDSJsonlRankFull(torch.utils.data.Dataset):
def __init__(self, path: str, **kwargs):
+ """Initialize IndexDSJsonlRankFull.
+
+ Args:
+ path: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.max_source_length = kwargs.get("max_source_length", 2048)
self.min_source_length = kwargs.get("min_source_length", 0)
@@ -135,17 +141,33 @@ class IndexDSJsonlRankFull(torch.utils.data.Dataset):
logging.info("total_num of samplers: {}, {}".format(len(self.contents), path))
def __len__(self):
+ """Internal: len ."""
return len(self.contents)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
data = self.contents[index]
return data
def get_source_len(self, data_dict):
+ """Get source len.
+
+ Args:
+ data_dict: TODO.
+ """
return data_dict.get("source_len", 1)
def get_target_len(self, data_dict):
+ """Get target len.
+
+ Args:
+ data_dict: TODO.
+ """
return data_dict.get("target_len", 0)
diff --git a/funasr/datasets/audio_datasets/jsonl2scp.py b/funasr/datasets/audio_datasets/jsonl2scp.py
index 3ce96ca78..cdd2f5e65 100644
--- a/funasr/datasets/audio_datasets/jsonl2scp.py
+++ b/funasr/datasets/audio_datasets/jsonl2scp.py
@@ -11,6 +11,14 @@ import torch.distributed as dist
def gen_scp_from_jsonl(jsonl_file, data_type_list, wav_scp_file, text_file):
+ """Gen scp from jsonl.
+
+ Args:
+ jsonl_file: TODO.
+ data_type_list: TODO.
+ wav_scp_file: TODO.
+ text_file: TODO.
+ """
wav_f = open(wav_scp_file, "w")
text_f = open(text_file, "w")
with open(jsonl_file, encoding="utf-8") as fin:
@@ -37,6 +45,11 @@ def gen_scp_from_jsonl(jsonl_file, data_type_list, wav_scp_file, text_file):
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
kwargs = OmegaConf.to_container(cfg, resolve=True)
print(kwargs)
diff --git a/funasr/datasets/audio_datasets/preprocessor.py b/funasr/datasets/audio_datasets/preprocessor.py
index b3f942156..9726ee9f5 100644
--- a/funasr/datasets/audio_datasets/preprocessor.py
+++ b/funasr/datasets/audio_datasets/preprocessor.py
@@ -18,10 +18,23 @@ from funasr.register import tables
@tables.register("preprocessor_classes", "SpeechPreprocessSpeedPerturb")
class SpeechPreprocessSpeedPerturb(nn.Module):
def __init__(self, speed_perturb: list = None, **kwargs):
+ """Initialize SpeechPreprocessSpeedPerturb.
+
+ Args:
+ speed_perturb: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.speed_perturb = speed_perturb
def forward(self, waveform, fs, **kwargs):
+ """Forward pass for training.
+
+ Args:
+ waveform: TODO.
+ fs: TODO.
+ **kwargs: Additional keyword arguments.
+ """
if self.speed_perturb is None:
return waveform
speed = random.choice(self.speed_perturb)
@@ -45,11 +58,25 @@ class TextPreprocessSegDict(nn.Module):
split_with_space: bool = False,
**kwargs
):
+ """Initialize TextPreprocessSegDict.
+
+ Args:
+ seg_dict: TODO.
+ text_cleaner: TODO.
+ split_with_space: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.text_cleaner = TextCleaner(text_cleaner)
def forward(self, text, **kwargs):
+ """Forward pass for training.
+
+ Args:
+ text: Text tensor or string input.
+ **kwargs: Additional keyword arguments.
+ """
text = self.text_cleaner(text)
return text
diff --git a/funasr/datasets/audio_datasets/samplers.py b/funasr/datasets/audio_datasets/samplers.py
index f7057de11..f478db002 100644
--- a/funasr/datasets/audio_datasets/samplers.py
+++ b/funasr/datasets/audio_datasets/samplers.py
@@ -18,6 +18,12 @@ from funasr.register import tables
@tables.register("batch_sampler_classes", "RankFullLocalShuffleBatchSampler")
@tables.register("batch_sampler_classes", "RankFullLocalShuffleDynamicBatchSampler")
def CustomDistributedBatchSampler_fn(dataset, **kwargs):
+ """Customdistributedbatchsampler fn.
+
+ Args:
+ dataset: TODO.
+ **kwargs: Additional keyword arguments.
+ """
dataloader_args = {}
batch_type = kwargs.get("batch_type", "example")
if batch_type == "example":
@@ -50,6 +56,18 @@ class CustomDistributedBatchSampler(Sampler):
**kwargs,
):
+ """Initialize CustomDistributedBatchSampler.
+
+ Args:
+ dataset: TODO.
+ batch_size: Number of samples per batch.
+ num_replicas: TODO.
+ rank: TODO.
+ shuffle: TODO.
+ drop_last: TODO.
+ is_training: Boolean flag for training.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
num_replicas = dist.get_world_size()
@@ -79,6 +97,7 @@ class CustomDistributedBatchSampler(Sampler):
def __iter__(self):
# Generate a list of indices
+ """Internal: iter ."""
if self.shuffle:
g = torch.Generator()
g.manual_seed(self.epoch)
@@ -123,9 +142,15 @@ class CustomDistributedBatchSampler(Sampler):
def __len__(self):
+ """Internal: len ."""
return self.num_samples // self.batch_size
def set_epoch(self, epoch):
+ """Set epoch.
+
+ Args:
+ epoch: TODO.
+ """
self.epoch = epoch
@@ -143,6 +168,19 @@ class CustomDistributedBufferBatchSampler(Sampler):
**kwargs,
):
+ """Initialize CustomDistributedBufferBatchSampler.
+
+ Args:
+ dataset: TODO.
+ batch_size: Number of samples per batch.
+ num_replicas: TODO.
+ rank: TODO.
+ shuffle: TODO.
+ drop_last: TODO.
+ is_training: Boolean flag for training.
+ sort_size: Size/dimension parameter.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
num_replicas = dist.get_world_size()
@@ -173,6 +211,7 @@ class CustomDistributedBufferBatchSampler(Sampler):
def __iter__(self):
# Generate a list of indices
+ """Internal: iter ."""
if self.shuffle:
g = torch.Generator()
g.manual_seed(self.epoch)
@@ -225,6 +264,11 @@ class CustomDistributedBufferBatchSampler(Sampler):
def _create_batches_from_buffer(self, buffer):
# Function to convert the sorted buffer into batches
+ """Internal: create batches from buffer.
+
+ Args:
+ buffer: TODO.
+ """
batched_buffer = [
buffer[i : i + self.batch_size] for i in range(0, len(buffer), self.batch_size)
]
@@ -234,9 +278,15 @@ class CustomDistributedBufferBatchSampler(Sampler):
def __len__(self):
+ """Internal: len ."""
return self.num_samples // self.batch_size
def set_epoch(self, epoch):
+ """Set epoch.
+
+ Args:
+ epoch: TODO.
+ """
self.epoch = epoch
@@ -253,6 +303,18 @@ class CustomDistributedDynamicBatchSampler(DistributedSampler):
**kwargs,
):
+ """Initialize CustomDistributedDynamicBatchSampler.
+
+ Args:
+ dataset: TODO.
+ batch_size: Number of samples per batch.
+ num_replicas: TODO.
+ rank: TODO.
+ shuffle: TODO.
+ drop_last: TODO.
+ is_training: Boolean flag for training.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
num_replicas = dist.get_world_size()
@@ -274,6 +336,7 @@ class CustomDistributedDynamicBatchSampler(DistributedSampler):
self.length_scale_source = kwargs.get("length_scale_source", 1.0)
def __iter__(self):
+ """Internal: iter ."""
if self.shuffle:
g = torch.Generator()
g.manual_seed(self.epoch)
@@ -315,9 +378,15 @@ class CustomDistributedDynamicBatchSampler(DistributedSampler):
def __len__(self):
+ """Internal: len ."""
return 1
def set_epoch(self, epoch):
+ """Set epoch.
+
+ Args:
+ epoch: TODO.
+ """
self.epoch = epoch
@@ -338,6 +407,22 @@ class CustomDistributedBufferDynamicBatchSampler(DistributedSampler):
**kwargs,
):
+ """Initialize CustomDistributedBufferDynamicBatchSampler.
+
+ Args:
+ dataset: TODO.
+ batch_size: Number of samples per batch.
+ batch_type: TODO.
+ num_replicas: TODO.
+ rank: TODO.
+ rank_split: TODO.
+ shuffle: TODO.
+ drop_last: TODO.
+ is_training: Boolean flag for training.
+ sort_size: Size/dimension parameter.
+ start_step: TODO.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
num_replicas = dist.get_world_size()
@@ -375,6 +460,7 @@ class CustomDistributedBufferDynamicBatchSampler(DistributedSampler):
# )
def __iter__(self):
+ """Internal: iter ."""
if self.shuffle:
g = torch.Generator()
g.manual_seed(self.epoch)
@@ -434,9 +520,15 @@ class CustomDistributedBufferDynamicBatchSampler(DistributedSampler):
def __len__(self):
# Calculate the number of batches per epoch for the current rank
+ """Internal: len ."""
return self.batch_num
def set_epoch(self, epoch):
+ """Set epoch.
+
+ Args:
+ epoch: TODO.
+ """
self.epoch = epoch
@@ -444,6 +536,16 @@ class DistributedSamplerWarp(BatchSampler):
def __init__(
self, dataset, batch_size, num_replicas=None, rank=None, shuffle=True, drop_last=False
):
+ """Initialize DistributedSamplerWarp.
+
+ Args:
+ dataset: TODO.
+ batch_size: Number of samples per batch.
+ num_replicas: TODO.
+ rank: TODO.
+ shuffle: TODO.
+ drop_last: TODO.
+ """
if num_replicas is None:
if not torch.distributed.is_available():
raise RuntimeError("Requires distributed package to be available")
@@ -470,6 +572,7 @@ class DistributedSamplerWarp(BatchSampler):
def __iter__(self):
# If we shuffle, we need to call the set_epoch method
+ """Internal: iter ."""
if self.shuffle:
self.sampler.set_epoch(self.epoch)
@@ -477,4 +580,9 @@ class DistributedSamplerWarp(BatchSampler):
return super().__iter__()
def set_epoch(self, epoch):
+ """Set epoch.
+
+ Args:
+ epoch: TODO.
+ """
self.epoch = epoch
diff --git a/funasr/datasets/audio_datasets/scp2jsonl.py b/funasr/datasets/audio_datasets/scp2jsonl.py
index 48c64d262..f1f2f710f 100644
--- a/funasr/datasets/audio_datasets/scp2jsonl.py
+++ b/funasr/datasets/audio_datasets/scp2jsonl.py
@@ -13,6 +13,14 @@ from tqdm import tqdm
def gen_jsonl_from_wav_text_list(
path, data_type_list=("source", "target"), jsonl_file_out: str = None, **kwargs
):
+ """Gen jsonl from wav text list.
+
+ Args:
+ path: TODO.
+ data_type_list: TODO.
+ jsonl_file_out: TODO.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
world_size = dist.get_world_size()
@@ -73,6 +81,13 @@ def gen_jsonl_from_wav_text_list(
def parse_context_length(data_list: list, data_type: str, id=0):
+ """Parse context length.
+
+ Args:
+ data_list: TODO.
+ data_type: TODO.
+ id: TODO.
+ """
pbar = tqdm(total=len(data_list), dynamic_ncols=True)
res = {}
for i, line in enumerate(data_list):
@@ -99,6 +114,11 @@ def parse_context_length(data_list: list, data_type: str, id=0):
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
kwargs = OmegaConf.to_container(cfg, resolve=True)
print(kwargs)
diff --git a/funasr/datasets/audio_datasets/scp2len.py b/funasr/datasets/audio_datasets/scp2len.py
index 5d742b17c..17819057e 100644
--- a/funasr/datasets/audio_datasets/scp2len.py
+++ b/funasr/datasets/audio_datasets/scp2len.py
@@ -13,6 +13,14 @@ from tqdm import tqdm
def gen_jsonl_from_wav_text_list(
path, data_type_list=("source",), jsonl_file_out: str = None, **kwargs
):
+ """Gen jsonl from wav text list.
+
+ Args:
+ path: TODO.
+ data_type_list: TODO.
+ jsonl_file_out: TODO.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
world_size = dist.get_world_size()
@@ -75,6 +83,13 @@ def gen_jsonl_from_wav_text_list(
def parse_context_length(data_list: list, data_type: str, id=0):
+ """Parse context length.
+
+ Args:
+ data_list: TODO.
+ data_type: TODO.
+ id: TODO.
+ """
pbar = tqdm(total=len(data_list), dynamic_ncols=True)
res = {}
for i, line in enumerate(data_list):
@@ -97,6 +112,11 @@ def parse_context_length(data_list: list, data_type: str, id=0):
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
kwargs = OmegaConf.to_container(cfg, resolve=True)
print(kwargs)
diff --git a/funasr/datasets/audio_datasets/sensevoice2jsonl.py b/funasr/datasets/audio_datasets/sensevoice2jsonl.py
index 102e82797..e4f72366b 100644
--- a/funasr/datasets/audio_datasets/sensevoice2jsonl.py
+++ b/funasr/datasets/audio_datasets/sensevoice2jsonl.py
@@ -15,6 +15,15 @@ from tqdm import tqdm
def gen_jsonl_from_wav_text_list(
path, data_type_list=("source", "target"), jsonl_file_out: str = None, model_dir: str = "iic/SenseVoiceSmall", **kwargs
):
+ """Gen jsonl from wav text list.
+
+ Args:
+ path: TODO.
+ data_type_list: TODO.
+ jsonl_file_out: TODO.
+ model_dir: TODO.
+ **kwargs: Additional keyword arguments.
+ """
try:
rank = dist.get_rank()
world_size = dist.get_world_size()
@@ -120,6 +129,11 @@ def gen_jsonl_from_wav_text_list(
dist.barrier()
def contains_punctuation(s):
+ """Contains punctuation.
+
+ Args:
+ s: TODO.
+ """
punctuations = (
string.punctuation +
',。、;:?!""''()【】《》〈〉「」『』〔〕[]{}~·…—–'
@@ -127,6 +141,13 @@ def contains_punctuation(s):
return any(char in punctuations for char in s)
def parse_context_length(data_list: list, data_type: str, id=0):
+ """Parse context length.
+
+ Args:
+ data_list: TODO.
+ data_type: TODO.
+ id: TODO.
+ """
pbar = tqdm(total=len(data_list), dynamic_ncols=True)
res = {}
for i, line in enumerate(data_list):
@@ -159,6 +180,11 @@ def parse_context_length(data_list: list, data_type: str, id=0):
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
kwargs = OmegaConf.to_container(cfg, resolve=True)
print(kwargs)
diff --git a/funasr/datasets/audio_datasets/update_jsonl.py b/funasr/datasets/audio_datasets/update_jsonl.py
index bc8470808..e21943736 100644
--- a/funasr/datasets/audio_datasets/update_jsonl.py
+++ b/funasr/datasets/audio_datasets/update_jsonl.py
@@ -13,6 +13,13 @@ from concurrent.futures import ThreadPoolExecutor
def gen_scp_from_jsonl(jsonl_file, jsonl_file_out, ncpu):
+ """Gen scp from jsonl.
+
+ Args:
+ jsonl_file: TODO.
+ jsonl_file_out: TODO.
+ ncpu: TODO.
+ """
jsonl_file_out_f = open(jsonl_file_out, "w")
with open(jsonl_file, encoding="utf-8") as fin:
lines = fin.readlines()
@@ -42,6 +49,12 @@ def gen_scp_from_jsonl(jsonl_file, jsonl_file_out, ncpu):
def update_data(lines, i):
+ """Update data.
+
+ Args:
+ lines: TODO.
+ i: TODO.
+ """
line = lines[i]
data = json.loads(line.strip())
@@ -61,6 +74,13 @@ def update_data(lines, i):
def update_wav_len(jsonl_file_list_in, jsonl_file_out_dir, ncpu=1):
+ """Update wav len.
+
+ Args:
+ jsonl_file_list_in: TODO.
+ jsonl_file_out_dir: TODO.
+ ncpu: TODO.
+ """
os.makedirs(jsonl_file_out_dir, exist_ok=True)
with open(jsonl_file_list_in, "r") as f:
data_file_lists = f.readlines()
@@ -76,6 +96,11 @@ def update_wav_len(jsonl_file_list_in, jsonl_file_out_dir, ncpu=1):
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
kwargs = OmegaConf.to_container(cfg, resolve=True)
logging.info(kwargs)
diff --git a/funasr/datasets/dataloader_entry.py b/funasr/datasets/dataloader_entry.py
index 055e4c8a7..c868c20d2 100644
--- a/funasr/datasets/dataloader_entry.py
+++ b/funasr/datasets/dataloader_entry.py
@@ -7,6 +7,13 @@ from funasr.register import tables
# @tables.register("dataloader_classes", "DataloaderMapStyle")
def DataloaderMapStyle(frontend=None, tokenizer=None, **kwargs):
# dataset
+ """Dataloadermapstyle.
+
+ Args:
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ **kwargs: Additional keyword arguments.
+ """
logging.info("Build dataloader")
dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset"))
dataset_tr = dataset_class(
@@ -48,6 +55,13 @@ def DataloaderMapStyle(frontend=None, tokenizer=None, **kwargs):
class DataloaderMapStyle:
def __init__(self, frontend=None, tokenizer=None, **kwargs):
# dataset
+ """Initialize DataloaderMapStyle.
+
+ Args:
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ **kwargs: Additional keyword arguments.
+ """
logging.info("Build dataloader")
dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "AudioDataset"))
@@ -82,6 +96,14 @@ class DataloaderMapStyle:
def build_iter(self, epoch=0, data_split_i=0, start_step=0, **kwargs):
# reload dataset slice
+ """Build iter.
+
+ Args:
+ epoch: TODO.
+ data_split_i: TODO.
+ start_step: TODO.
+ **kwargs: Additional keyword arguments.
+ """
if self.data_split_num > 1:
del self.dataset_tr
self.dataset_tr = self.dataset_class(
@@ -119,6 +141,13 @@ class DataloaderMapStyle:
@tables.register("dataloader_classes", "DataloaderIterable")
def DataloaderIterable(frontend=None, tokenizer=None, **kwargs):
+ """Dataloaderiterable.
+
+ Args:
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ **kwargs: Additional keyword arguments.
+ """
logging.info("Build dataloader")
dataset_class = tables.dataset_classes.get(kwargs.get("dataset", "LargeDataset"))
dataset_tr = dataset_class(
diff --git a/funasr/datasets/fun_asr_datasets/datasets.py b/funasr/datasets/fun_asr_datasets/datasets.py
index af8702a89..c70ff1a1b 100644
--- a/funasr/datasets/fun_asr_datasets/datasets.py
+++ b/funasr/datasets/fun_asr_datasets/datasets.py
@@ -26,6 +26,17 @@ class FunASR(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs,
):
+ """Initialize FunASR.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -82,14 +93,30 @@ class FunASR(torch.utils.data.Dataset):
self.min_output_non_mask_token_len = kwargs.get("min_non_mask_token_len", 6) # [eos]
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def get_random_user_prompt(self, item, user_prompt):
+ """Get random user prompt.
+
+ Args:
+ item: TODO.
+ user_prompt: TODO.
+ """
tasks = ["语音转写:", "Speech transcription:"]
language = item.get("language", None)
# LID in distill data is fake
@@ -112,9 +139,15 @@ class FunASR(torch.utils.data.Dataset):
return user_prompt
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
output = None
for idx in range(self.retry):
@@ -309,6 +342,11 @@ class FunASR(torch.utils.data.Dataset):
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
for idx in range(self.retry):
badcase_flag = False
@@ -354,6 +392,12 @@ class FunASR(torch.utils.data.Dataset):
class FunASR(torch.utils.data.Dataset): # torch.utils.data.Dataset
def __init__(self, path: str, **kwargs):
+ """Initialize FunASR.
+
+ Args:
+ path: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.max_source_length = kwargs.get("max_source_length", 8000)
@@ -490,15 +534,26 @@ class FunASR(torch.utils.data.Dataset): # torch.utils.data.Dataset
f"\n\ntotal_num of samplers: {len(self.contents)}, total_whrs: {total_whrs:.5f}, total_token_for_llm_B: {total_token_for_llm_B:.5g}, {path}, {file_list}\n\n")
def __len__(self):
+ """Internal: len ."""
return len(self.contents)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
data = self.contents[index]
return data
def get_source_len(self, data_dict):
+ """Get source len.
+
+ Args:
+ data_dict: TODO.
+ """
source_len = data_dict.get("source_len", -1)
if source_len < 0:
source_len = len(data_dict["system"]) + len(data_dict["user"])
@@ -506,4 +561,9 @@ class FunASR(torch.utils.data.Dataset): # torch.utils.data.Dataset
def get_target_len(self, data_dict):
+ """Get target len.
+
+ Args:
+ data_dict: TODO.
+ """
return 0
diff --git a/funasr/datasets/fun_asr_datasets/multicontext_prompt.py b/funasr/datasets/fun_asr_datasets/multicontext_prompt.py
index 6d46dceb2..1f8dd69dc 100644
--- a/funasr/datasets/fun_asr_datasets/multicontext_prompt.py
+++ b/funasr/datasets/fun_asr_datasets/multicontext_prompt.py
@@ -33,6 +33,16 @@ class MultiContextPrompt:
use_asr_hotwords=True,
use_multi_lingual_prompt=True,
**kwargs):
+ """Initialize MultiContextPrompt.
+
+ Args:
+ use_hist: TODO.
+ use_one_pass_result: TODO.
+ use_hotwords: TODO.
+ use_asr_hotwords: TODO.
+ use_multi_lingual_prompt: TODO.
+ **kwargs: Additional keyword arguments.
+ """
self.use_hist = use_hist
self.use_one_pass_result = use_one_pass_result
self.use_hotwords = use_hotwords
@@ -60,11 +70,21 @@ class MultiContextPrompt:
self.min_neg_hotwords_num = kwargs.get("min_neg_hotwords_num", 0)
def get_hotwords_list(self, hotwords_file):
+ """Get hotwords list.
+
+ Args:
+ hotwords_file: TODO.
+ """
with open(hotwords_file, "r") as f:
hotwords_list = f.read().strip().split("\n")
return hotwords_list, len(hotwords_list)
def detect_language(self, text):
+ """Detect language.
+
+ Args:
+ text: Text tensor or string input.
+ """
if isinstance(text, list):
text = " ".join(text)
@@ -95,6 +115,11 @@ class MultiContextPrompt:
def hotwords_sampling(self, hotwords):
# hotwords_list = hotwords.split(", ")
+ """Hotwords sampling.
+
+ Args:
+ hotwords: TODO.
+ """
hotwords_list = hotwords
selected_hotwords = []
if self.max_neg_hotwords_num > -1:
@@ -112,6 +137,12 @@ class MultiContextPrompt:
return selected_hotwords, selected_hotwords_num
def get_prompt(self, item, language):
+ """Get prompt.
+
+ Args:
+ item: TODO.
+ language: Language identifier.
+ """
template = self.CONTEXT_TEMPLATES[language]
prompt = template['header']
@@ -157,6 +188,12 @@ class MultiContextPrompt:
return prompt
def get_inference_prompt(self, item, language="zh"):
+ """Get inference prompt.
+
+ Args:
+ item: TODO.
+ language: Language identifier.
+ """
template = self.CONTEXT_TEMPLATES[language]
prompt = template['header']
@@ -233,6 +270,15 @@ class MultiContextPromptNew:
use_hotwords=True,
use_multi_lingual_prompt=True,
**kwargs):
+ """Initialize MultiContextPromptNew.
+
+ Args:
+ use_hist: TODO.
+ use_one_pass_result: TODO.
+ use_hotwords: TODO.
+ use_multi_lingual_prompt: TODO.
+ **kwargs: Additional keyword arguments.
+ """
self.use_hist = use_hist
self.use_one_pass_result = use_one_pass_result
self.use_hotwords = use_hotwords
@@ -244,6 +290,11 @@ class MultiContextPromptNew:
def hotwords_sampling(self, hotwords):
+ """Hotwords sampling.
+
+ Args:
+ hotwords: TODO.
+ """
hotwords_list = hotwords.split(", ")
if self.max_hotwords_num > 0:
max_hotwords_num = min(self.max_hotwords_num, len(hotwords_list))
@@ -261,6 +312,12 @@ class MultiContextPromptNew:
return hotwords_list, selected_hotwords_num
def get_prompt(self, item, language):
+ """Get prompt.
+
+ Args:
+ item: TODO.
+ language: Language identifier.
+ """
template = self.CONTEXT_TEMPLATES[language]
prompt = template['header']
@@ -287,6 +344,13 @@ class MultiContextPromptNew:
return prompt
def get_inference_prompt(self, hist_context="", one_pass_result="", hotwords=""):
+ """Get inference prompt.
+
+ Args:
+ hist_context: TODO.
+ one_pass_result: TODO.
+ hotwords: TODO.
+ """
language = 'zh' if self.use_multi_lingual_prompt and np.random.rand() < 0.5 else 'en'
template = self.CONTEXT_TEMPLATES[language]
diff --git a/funasr/datasets/kws_datasets/datasets.py b/funasr/datasets/kws_datasets/datasets.py
index 4679295cb..24a9ae75d 100644
--- a/funasr/datasets/kws_datasets/datasets.py
+++ b/funasr/datasets/kws_datasets/datasets.py
@@ -21,6 +21,18 @@ class KwsMTDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs,
):
+ """Initialize KwsMTDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ is_training: Boolean flag for training.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -53,17 +65,33 @@ class KwsMTDataset(torch.utils.data.Dataset):
self.float_pad_value = float_pad_value
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
# import pdb;
# pdb.set_trace()
@@ -111,6 +139,11 @@ class KwsMTDataset(torch.utils.data.Dataset):
def collator(self, samples: list=None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
for key in sample.keys():
diff --git a/funasr/datasets/llm_datasets/datasets.py b/funasr/datasets/llm_datasets/datasets.py
index 61cadedd1..f1ae4a524 100644
--- a/funasr/datasets/llm_datasets/datasets.py
+++ b/funasr/datasets/llm_datasets/datasets.py
@@ -21,6 +21,17 @@ class AudioLLMNARDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs
):
+ """Initialize AudioLLMNARDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -52,17 +63,33 @@ class AudioLLMNARDataset(torch.utils.data.Dataset):
self.int_pad_value = self.IGNORE_INDEX
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
source = item["source"]
data_src = load_audio_text_image_video(source, fs=self.fs)
@@ -143,6 +170,11 @@ class AudioLLMNARDataset(torch.utils.data.Dataset):
}
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
for key in sample.keys():
@@ -180,6 +212,17 @@ class AudioLLMDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs
):
+ """Initialize AudioLLMDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -211,17 +254,33 @@ class AudioLLMDataset(torch.utils.data.Dataset):
self.int_pad_value = self.IGNORE_INDEX
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
# import pdb;
# pdb.set_trace()
@@ -278,6 +337,11 @@ class AudioLLMDataset(torch.utils.data.Dataset):
}
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
for key in sample.keys():
@@ -315,6 +379,17 @@ class AudioLLMARDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs
):
+ """Initialize AudioLLMARDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -346,17 +421,33 @@ class AudioLLMARDataset(torch.utils.data.Dataset):
self.int_pad_value = self.IGNORE_INDEX
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
# import pdb;
# pdb.set_trace()
@@ -413,6 +504,11 @@ class AudioLLMARDataset(torch.utils.data.Dataset):
}
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
for key in sample.keys():
diff --git a/funasr/datasets/llm_datasets/preprocessor.py b/funasr/datasets/llm_datasets/preprocessor.py
index b99255e38..59edf67f0 100644
--- a/funasr/datasets/llm_datasets/preprocessor.py
+++ b/funasr/datasets/llm_datasets/preprocessor.py
@@ -19,10 +19,21 @@ from funasr.register import tables
@tables.register("preprocessor_classes", "TextPreprocessRemovePunctuation")
class TextPreprocessRemovePunctuation(nn.Module):
def __init__(self, **kwargs):
+ """Initialize TextPreprocessRemovePunctuation.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
def forward(self, text, **kwargs):
# 定义英文标点符号
+ """Forward pass for training.
+
+ Args:
+ text: Text tensor or string input.
+ **kwargs: Additional keyword arguments.
+ """
en_punct = string.punctuation
# 定义中文标点符号(部分常用的)
cn_punct = "。?!,、;:“”‘’()《》【】…—~·"
diff --git a/funasr/datasets/llm_datasets_qwenaudio/datasets.py b/funasr/datasets/llm_datasets_qwenaudio/datasets.py
index 569665a1d..ca9fbbceb 100644
--- a/funasr/datasets/llm_datasets_qwenaudio/datasets.py
+++ b/funasr/datasets/llm_datasets_qwenaudio/datasets.py
@@ -21,6 +21,17 @@ class AudioLLMQwenAudioDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs
):
+ """Initialize AudioLLMQwenAudioDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -54,17 +65,33 @@ class AudioLLMQwenAudioDataset(torch.utils.data.Dataset):
self.answer_template = "{}"
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
source = item["source"]
data_src = load_audio_text_image_video(source, fs=self.fs)
@@ -138,6 +165,11 @@ class AudioLLMQwenAudioDataset(torch.utils.data.Dataset):
}
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
for key in sample.keys():
diff --git a/funasr/datasets/llm_datasets_vicuna/datasets.py b/funasr/datasets/llm_datasets_vicuna/datasets.py
index cde29a94a..dd5ce96f9 100644
--- a/funasr/datasets/llm_datasets_vicuna/datasets.py
+++ b/funasr/datasets/llm_datasets_vicuna/datasets.py
@@ -21,6 +21,17 @@ class AudioLLMVicunaDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs
):
+ """Initialize AudioLLMVicunaDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -54,17 +65,33 @@ class AudioLLMVicunaDataset(torch.utils.data.Dataset):
self.answer_template = "{}"
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
source = item["source"]
data_src = load_audio_text_image_video(source, fs=self.fs)
@@ -138,6 +165,11 @@ class AudioLLMVicunaDataset(torch.utils.data.Dataset):
}
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
for key in sample.keys():
diff --git a/funasr/datasets/openai_datasets/datasets.py b/funasr/datasets/openai_datasets/datasets.py
index d670708a3..c373c424c 100644
--- a/funasr/datasets/openai_datasets/datasets.py
+++ b/funasr/datasets/openai_datasets/datasets.py
@@ -23,6 +23,17 @@ class OpenAIDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs,
):
+ """Initialize OpenAIDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -68,20 +79,36 @@ class OpenAIDataset(torch.utils.data.Dataset):
self.audio_encoder_downsample_rate = kwargs.get("audio_encoder_downsample_rate", 4)
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
# import pdb;
# pdb.set_trace()
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
output = None
for idx in range(self.retry):
@@ -191,6 +218,11 @@ class OpenAIDataset(torch.utils.data.Dataset):
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
for idx in range(self.retry):
badcase_flag = False
@@ -245,6 +277,17 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs,
):
+ """Initialize OpenAIDatasetMultiTurn.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -290,14 +333,25 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):
self.max_source_length = kwargs.get("max_source_length", 3000)
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
@@ -305,6 +359,11 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):
#
# pdb.set_trace()
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
output = None
for idx in range(self.retry):
@@ -440,6 +499,11 @@ class OpenAIDatasetMultiTurn(torch.utils.data.Dataset):
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
for idx in range(self.retry):
badcase_flag = False
diff --git a/funasr/datasets/openai_datasets/index_ds.py b/funasr/datasets/openai_datasets/index_ds.py
index 010d2d5d7..4b6e1091d 100644
--- a/funasr/datasets/openai_datasets/index_ds.py
+++ b/funasr/datasets/openai_datasets/index_ds.py
@@ -14,6 +14,12 @@ from funasr.register import tables
class OpenAIIndexDSJsonl(torch.utils.data.Dataset): # torch.utils.data.Dataset
def __init__(self, path: str, **kwargs):
+ """Initialize OpenAIIndexDSJsonl.
+
+ Args:
+ path: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.max_source_length = kwargs.get("max_source_length", 3000)
@@ -89,15 +95,26 @@ class OpenAIIndexDSJsonl(torch.utils.data.Dataset): # torch.utils.data.Dataset
logging.info("total_num of samplers: {}, {}".format(len(self.contents), path))
def __len__(self):
+ """Internal: len ."""
return len(self.contents)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
data = self.contents[index]
return data
def get_source_len(self, data_dict):
+ """Get source len.
+
+ Args:
+ data_dict: TODO.
+ """
source_len = data_dict.get("source_len", -1)
if source_len < 0:
source_len = len(data_dict["system"]) + len(data_dict["user"])
@@ -105,6 +122,11 @@ class OpenAIIndexDSJsonl(torch.utils.data.Dataset): # torch.utils.data.Dataset
def get_target_len(self, data_dict):
+ """Get target len.
+
+ Args:
+ data_dict: TODO.
+ """
return 0
diff --git a/funasr/datasets/sense_voice_datasets/datasets.py b/funasr/datasets/sense_voice_datasets/datasets.py
index 2e0266ef1..8e1f45961 100644
--- a/funasr/datasets/sense_voice_datasets/datasets.py
+++ b/funasr/datasets/sense_voice_datasets/datasets.py
@@ -24,6 +24,17 @@ class SenseVoiceDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs,
):
+ """Initialize SenseVoiceDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -61,18 +72,34 @@ class SenseVoiceDataset(torch.utils.data.Dataset):
self.permute = True
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
output = None
for idx in range(self.retry):
if idx == 0:
@@ -153,6 +180,11 @@ class SenseVoiceDataset(torch.utils.data.Dataset):
return output
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
if sample is None:
@@ -209,6 +241,12 @@ class SenseVoiceDataset(torch.utils.data.Dataset):
return outputs
def _filter_badcase(self, outputs, i=0):
+ """Internal: filter badcase.
+
+ Args:
+ outputs: TODO.
+ i: TODO.
+ """
b, t, _ = outputs["speech"].shape
if b * t > self.batch_size * 1.25:
@@ -247,6 +285,17 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):
float_pad_value: float = 0.0,
**kwargs,
):
+ """Initialize SenseVoiceCTCDataset.
+
+ Args:
+ path: TODO.
+ index_ds: TODO.
+ frontend: Audio frontend for feature extraction.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ int_pad_value: TODO.
+ float_pad_value: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
index_ds_class = tables.index_ds_classes.get(index_ds)
self.index_ds = index_ds_class(path, **kwargs)
@@ -284,18 +333,34 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):
self.permute = True
def get_source_len(self, index):
+ """Get source len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_source_len(item)
def get_target_len(self, index):
+ """Get target len.
+
+ Args:
+ index: TODO.
+ """
item = self.index_ds[index]
return self.index_ds.get_target_len(item)
def __len__(self):
+ """Internal: len ."""
return len(self.index_ds)
def __getitem__(self, index):
+ """Internal: getitem .
+
+ Args:
+ index: TODO.
+ """
output = None
for idx in range(self.retry):
if idx == 0:
@@ -357,6 +422,11 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):
return output
def collator(self, samples: list = None):
+ """Collator.
+
+ Args:
+ samples: TODO.
+ """
outputs = {}
for sample in samples:
if sample is None:
@@ -410,6 +480,12 @@ class SenseVoiceCTCDataset(torch.utils.data.Dataset):
return outputs
def _filter_badcase(self, outputs, i=0):
+ """Internal: filter badcase.
+
+ Args:
+ outputs: TODO.
+ i: TODO.
+ """
b, t, _ = outputs["speech"].shape
if b * t > self.batch_size * 1.25:
diff --git a/funasr/download/download_dataset_from_hub.py b/funasr/download/download_dataset_from_hub.py
index 2419ce869..26082e79f 100644
--- a/funasr/download/download_dataset_from_hub.py
+++ b/funasr/download/download_dataset_from_hub.py
@@ -1,8 +1,14 @@
def download_dataset():
+ """Download dataset."""
pass
def download_dataset_from_ms(**kwargs):
+ """Download dataset from ms.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from modelscope.msdatasets import MsDataset
dataset_name = kwargs.get("dataset_name", "speech_asr/speech_asr_aishell1_trainsets")
diff --git a/funasr/download/download_model_from_hub.py b/funasr/download/download_model_from_hub.py
index 0cdab34c2..322775519 100644
--- a/funasr/download/download_model_from_hub.py
+++ b/funasr/download/download_model_from_hub.py
@@ -42,6 +42,11 @@ def download_model(**kwargs):
def download_from_ms(**kwargs):
+ """Download from ms.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
model_or_path = kwargs.get("model")
if model_or_path in name_maps_ms:
model_or_path = name_maps_ms[model_or_path]
@@ -115,6 +120,11 @@ def download_from_ms(**kwargs):
def download_from_hf(**kwargs):
+ """Download from hf.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
model_or_path = kwargs.get("model")
if model_or_path in name_maps_hf:
model_or_path = name_maps_hf[model_or_path]
@@ -180,6 +190,13 @@ def download_from_hf(**kwargs):
def add_file_root_path(model_or_path: str, file_path_metas: dict, cfg={}):
+ """Add file root path.
+
+ Args:
+ model_or_path: TODO.
+ file_path_metas: TODO.
+ cfg: Configuration overrides.
+ """
if isinstance(file_path_metas, dict):
if isinstance(cfg, list):
cfg.append({})
diff --git a/funasr/download/file.py b/funasr/download/file.py
index a8a13f874..0375f0c87 100644
--- a/funasr/download/file.py
+++ b/funasr/download/file.py
@@ -12,6 +12,11 @@ from urllib.parse import urlparse
def download_from_url(url):
+ """Download from url.
+
+ Args:
+ url: TODO.
+ """
result = urlparse(url)
file_path = None
if result.scheme is not None and len(result.scheme) > 0:
@@ -38,18 +43,41 @@ class Storage(metaclass=ABCMeta):
@abstractmethod
def read(self, filepath: str):
+ """Read.
+
+ Args:
+ filepath: TODO.
+ """
pass
@abstractmethod
def read_text(self, filepath: str):
+ """Read text.
+
+ Args:
+ filepath: TODO.
+ """
pass
@abstractmethod
def write(self, obj: bytes, filepath: Union[str, Path]) -> None:
+ """Write.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ """
pass
@abstractmethod
def write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:
+ """Write text.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ encoding: TODO.
+ """
pass
@@ -133,11 +161,21 @@ class HTTPStorage(Storage):
def read(self, url):
# TODO @wenmeng.zwm add progress bar if file is too large
+ """Read.
+
+ Args:
+ url: TODO.
+ """
r = requests.get(url)
r.raise_for_status()
return r.content
def read_text(self, url):
+ """Read text.
+
+ Args:
+ url: TODO.
+ """
r = requests.get(url)
r.raise_for_status()
return r.text
@@ -169,9 +207,22 @@ class HTTPStorage(Storage):
os.remove(f.name)
def write(self, obj: bytes, url: Union[str, Path]) -> None:
+ """Write.
+
+ Args:
+ obj: TODO.
+ url: TODO.
+ """
raise NotImplementedError("write is not supported by HTTP Storage")
def write_text(self, obj: str, url: Union[str, Path], encoding: str = "utf-8") -> None:
+ """Write text.
+
+ Args:
+ obj: TODO.
+ url: TODO.
+ encoding: TODO.
+ """
raise NotImplementedError("write_text is not supported by HTTP Storage")
@@ -180,12 +231,28 @@ class OSSStorage(Storage):
def __init__(self, oss_config_file=None):
# read from config file or env var
+ """Initialize OSSStorage.
+
+ Args:
+ oss_config_file: TODO.
+ """
raise NotImplementedError("OSSStorage.__init__ to be implemented in the future")
def read(self, filepath):
+ """Read.
+
+ Args:
+ filepath: TODO.
+ """
raise NotImplementedError("OSSStorage.read to be implemented in the future")
def read_text(self, filepath, encoding="utf-8"):
+ """Read text.
+
+ Args:
+ filepath: TODO.
+ encoding: TODO.
+ """
raise NotImplementedError("OSSStorage.read_text to be implemented in the future")
@contextlib.contextmanager
@@ -215,9 +282,22 @@ class OSSStorage(Storage):
os.remove(f.name)
def write(self, obj: bytes, filepath: Union[str, Path]) -> None:
+ """Write.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ """
raise NotImplementedError("OSSStorage.write to be implemented in the future")
def write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:
+ """Write text.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ encoding: TODO.
+ """
raise NotImplementedError("OSSStorage.write_text to be implemented in the future")
@@ -234,6 +314,11 @@ class File(object):
@staticmethod
def _get_storage(uri):
+ """Internal: get storage.
+
+ Args:
+ uri: TODO.
+ """
assert isinstance(uri, str), f"uri should be str type, but got {type(uri)}"
if "://" not in uri:
diff --git a/funasr/download/runtime_sdk_download_tool.py b/funasr/download/runtime_sdk_download_tool.py
index fb0ff028d..98c5826df 100644
--- a/funasr/download/runtime_sdk_download_tool.py
+++ b/funasr/download/runtime_sdk_download_tool.py
@@ -6,6 +6,7 @@ from funasr.utils.type_utils import str2bool
def main():
+ """Main."""
parser = argparse.ArgumentParser()
parser.add_argument("--model-name", type=str, required=True)
parser.add_argument("--export-dir", type=str, required=True)
diff --git a/funasr/frontends/default.py b/funasr/frontends/default.py
index 6ea88d6f6..8ef283d40 100644
--- a/funasr/frontends/default.py
+++ b/funasr/frontends/default.py
@@ -45,6 +45,26 @@ class DefaultFrontend(nn.Module):
use_channel: int = None,
**kwargs,
):
+ """Initialize DefaultFrontend.
+
+ Args:
+ fs: TODO.
+ n_fft: TODO.
+ win_length: TODO.
+ hop_length: TODO.
+ window: TODO.
+ center: TODO.
+ normalized: TODO.
+ onesided: TODO.
+ n_mels: TODO.
+ fmin: TODO.
+ fmax: TODO.
+ htk: TODO.
+ frontend_conf: Configuration dict for frontend.
+ apply_stft: TODO.
+ use_channel: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# Deepcopy (In general, dict shouldn't be used as default arg)
@@ -84,11 +104,18 @@ class DefaultFrontend(nn.Module):
self.frontend_type = "default"
def output_size(self) -> int:
+ """Output size."""
return self.n_mels
def forward(
self, input: torch.Tensor, input_lengths: Union[torch.Tensor, list]
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
if isinstance(input_lengths, list):
input_lengths = torch.tensor(input_lengths)
if input.dtype == torch.float64:
@@ -131,6 +158,12 @@ class DefaultFrontend(nn.Module):
return input_feats, feats_lens
def _compute_stft(self, input: torch.Tensor, input_lengths: torch.Tensor) -> torch.Tensor:
+ """Internal: compute stft.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
input_stft, feats_lens = self.stft(input, input_lengths)
assert input_stft.dim() >= 4, input_stft.shape
@@ -172,6 +205,31 @@ class MultiChannelFrontend(nn.Module):
cmvn_file: str = None,
mc: bool = True,
):
+ """Initialize MultiChannelFrontend.
+
+ Args:
+ fs: TODO.
+ n_fft: TODO.
+ win_length: TODO.
+ hop_length: TODO.
+ frame_length: TODO.
+ frame_shift: TODO.
+ window: TODO.
+ center: TODO.
+ normalized: TODO.
+ onesided: TODO.
+ n_mels: TODO.
+ fmin: TODO.
+ fmax: TODO.
+ htk: TODO.
+ frontend_conf: Configuration dict for frontend.
+ apply_stft: TODO.
+ use_channel: TODO.
+ lfr_m: TODO.
+ lfr_n: TODO.
+ cmvn_file: TODO.
+ mc: TODO.
+ """
super().__init__()
# Deepcopy (In general, dict shouldn't be used as default arg)
frontend_conf = copy.deepcopy(frontend_conf)
@@ -230,12 +288,19 @@ class MultiChannelFrontend(nn.Module):
self.frontend_type = "multichannelfrontend"
def output_size(self) -> int:
+ """Output size."""
return self.n_mels
def forward(
self, input: torch.Tensor, input_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
# 1. Domain-conversion: e.g. Stft: time -> time-freq
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
if self.stft is not None:
input_stft, feats_lens = self._compute_stft(input, input_lengths)
else:
@@ -305,6 +370,12 @@ class MultiChannelFrontend(nn.Module):
return input_feats, feats_lens
def _compute_stft(self, input: torch.Tensor, input_lengths: torch.Tensor) -> torch.Tensor:
+ """Internal: compute stft.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
input_stft, feats_lens = self.stft(input, input_lengths)
assert input_stft.dim() >= 4, input_stft.shape
@@ -317,6 +388,11 @@ class MultiChannelFrontend(nn.Module):
return input_stft, feats_lens
def _load_cmvn(self, cmvn_file):
+ """Internal: load cmvn.
+
+ Args:
+ cmvn_file: TODO.
+ """
with open(cmvn_file, "r", encoding="utf-8") as f:
lines = f.readlines()
means_list = []
diff --git a/funasr/frontends/eend_ola_feature.py b/funasr/frontends/eend_ola_feature.py
index 0c79a3bb8..aead92daf 100644
--- a/funasr/frontends/eend_ola_feature.py
+++ b/funasr/frontends/eend_ola_feature.py
@@ -8,6 +8,12 @@ import numpy as np
def transform(Y, dtype=np.float32):
+ """Transform.
+
+ Args:
+ Y: TODO.
+ dtype: TODO.
+ """
Y = np.abs(Y)
n_fft = 2 * (Y.shape[1] - 1)
sr = 8000
@@ -21,12 +27,25 @@ def transform(Y, dtype=np.float32):
def subsample(Y, T, subsampling=1):
+ """Subsample.
+
+ Args:
+ Y: TODO.
+ T: TODO.
+ subsampling: TODO.
+ """
Y_ss = Y[::subsampling]
T_ss = T[::subsampling]
return Y_ss, T_ss
def splice(Y, context_size=0):
+ """Splice.
+
+ Args:
+ Y: TODO.
+ context_size: Size/dimension parameter.
+ """
Y_pad = np.pad(Y, [(context_size, context_size), (0, 0)], "constant")
Y_spliced = np.lib.stride_tricks.as_strided(
np.ascontiguousarray(Y_pad),
@@ -38,6 +57,13 @@ def splice(Y, context_size=0):
def stft(data, frame_size=1024, frame_shift=256):
+ """Stft.
+
+ Args:
+ data: TODO.
+ frame_size: Size/dimension parameter.
+ frame_shift: TODO.
+ """
fft_size = 1 << (frame_size - 1).bit_length()
if len(data) % frame_shift == 0:
return librosa.stft(data, n_fft=fft_size, win_length=frame_size, hop_length=frame_shift).T[
diff --git a/funasr/frontends/fused.py b/funasr/frontends/fused.py
index 1da1e9f73..8a82125f1 100644
--- a/funasr/frontends/fused.py
+++ b/funasr/frontends/fused.py
@@ -9,6 +9,14 @@ from typing import Tuple
class FusedFrontends(nn.Module):
def __init__(self, frontends=None, align_method="linear_projection", proj_dim=100, fs=16000):
+ """Initialize FusedFrontends.
+
+ Args:
+ frontends: TODO.
+ align_method: TODO.
+ proj_dim: Size/dimension parameter.
+ fs: TODO.
+ """
super().__init__()
self.align_method = align_method # fusing method : linear_projection only for now
self.proj_dim = proj_dim # dim of the projection done on each frontend
@@ -96,6 +104,7 @@ class FusedFrontends(nn.Module):
self.projection_layers = self.projection_layers.to(torch.device(dev))
def output_size(self) -> int:
+ """Output size."""
return len(self.frontends) * self.proj_dim
def forward(
@@ -103,6 +112,12 @@ class FusedFrontends(nn.Module):
) -> Tuple[torch.Tensor, torch.Tensor]:
# step 0 : get all frontends features
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
self.feats = []
for frontend in self.frontends:
with torch.no_grad():
diff --git a/funasr/frontends/s3prl.py b/funasr/frontends/s3prl.py
index 5c358cca6..36f1dc7a9 100644
--- a/funasr/frontends/s3prl.py
+++ b/funasr/frontends/s3prl.py
@@ -15,6 +15,11 @@ from funasr.models.transformer.utils.nets_utils import pad_list
def base_s3prl_setup(args):
+ """Base s3prl setup.
+
+ Args:
+ args: TODO.
+ """
args.upstream_feature_selection = getattr(args, "upstream_feature_selection", None)
args.upstream_model_config = getattr(args, "upstream_model_config", None)
args.upstream_refresh = getattr(args, "upstream_refresh", False)
@@ -35,6 +40,14 @@ class S3prlFrontend(nn.Module):
download_dir: str = None,
multilayer_feature: bool = False,
):
+ """Initialize S3prlFrontend.
+
+ Args:
+ fs: TODO.
+ frontend_conf: Configuration dict for frontend.
+ download_dir: TODO.
+ multilayer_feature: TODO.
+ """
super().__init__()
if isinstance(fs, str):
fs = humanfriendly.parse_size(fs)
@@ -112,11 +125,18 @@ class S3prlFrontend(nn.Module):
return tiled_feature
def output_size(self) -> int:
+ """Output size."""
return self.output_dim
def forward(
self, input: torch.Tensor, input_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
wavs = [wav[: input_lengths[i]] for i, wav in enumerate(input)]
self.upstream.eval()
with torch.no_grad():
@@ -135,5 +155,6 @@ class S3prlFrontend(nn.Module):
return input_feats, feats_lens
def reload_pretrained_parameters(self):
+ """Reload pretrained parameters."""
self.upstream.load_state_dict(self.pretrained_params)
logging.info("Pretrained S3PRL frontend model parameters reloaded!")
diff --git a/funasr/frontends/utils/beamformer.py b/funasr/frontends/utils/beamformer.py
index d19abe0ab..222847d61 100644
--- a/funasr/frontends/utils/beamformer.py
+++ b/funasr/frontends/utils/beamformer.py
@@ -78,5 +78,11 @@ def get_mvdr_vector(
def apply_beamforming_vector(beamform_vector: ComplexTensor, mix: ComplexTensor) -> ComplexTensor:
# (..., C) x (..., C, T) -> (..., T)
+ """Apply beamforming vector.
+
+ Args:
+ beamform_vector: TODO.
+ mix: TODO.
+ """
es = FC.einsum("...c,...ct->...t", [beamform_vector.conj(), mix])
return es
diff --git a/funasr/frontends/utils/complex_utils.py b/funasr/frontends/utils/complex_utils.py
index 06c69363b..d7e2714a9 100644
--- a/funasr/frontends/utils/complex_utils.py
+++ b/funasr/frontends/utils/complex_utils.py
@@ -23,6 +23,12 @@ def new_complex_like(
ref: Union[torch.Tensor, ComplexTensor],
real_imag: Tuple[torch.Tensor, torch.Tensor],
):
+ """New complex like.
+
+ Args:
+ ref: TODO.
+ real_imag: TODO.
+ """
if isinstance(ref, ComplexTensor):
return ComplexTensor(*real_imag)
elif is_torch_complex_tensor(ref):
@@ -32,14 +38,29 @@ def new_complex_like(
def is_torch_complex_tensor(c):
+ """Is torch complex tensor.
+
+ Args:
+ c: TODO.
+ """
return not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c)
def is_complex(c):
+ """Is complex.
+
+ Args:
+ c: TODO.
+ """
return isinstance(c, ComplexTensor) or is_torch_complex_tensor(c)
def to_double(c):
+ """To double.
+
+ Args:
+ c: TODO.
+ """
if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c):
return c.to(dtype=torch.complex128)
else:
@@ -47,6 +68,11 @@ def to_double(c):
def to_float(c):
+ """To float.
+
+ Args:
+ c: TODO.
+ """
if not isinstance(c, ComplexTensor) and is_torch_1_9_plus and torch.is_complex(c):
return c.to(dtype=torch.complex64)
else:
@@ -54,6 +80,13 @@ def to_float(c):
def cat(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs):
+ """Cat.
+
+ Args:
+ seq: TODO.
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
if not isinstance(seq, (list, tuple)):
raise TypeError(
"cat(): argument 'tensors' (position 1) must be tuple of Tensors, " "not Tensor"
@@ -65,6 +98,13 @@ def cat(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs):
def complex_norm(c: Union[torch.Tensor, ComplexTensor], dim=-1, keepdim=False) -> torch.Tensor:
+ """Complex norm.
+
+ Args:
+ c: TODO.
+ dim: TODO.
+ keepdim: TODO.
+ """
if not is_complex(c):
raise TypeError("Input is not a complex tensor.")
if is_torch_complex_tensor(c):
@@ -77,6 +117,12 @@ def einsum(equation, *operands):
# NOTE: Do not mix ComplexTensor and torch.complex in the input!
# NOTE (wangyou): Until PyTorch 1.9.0, torch.einsum does not support
# mixed input with complex and real tensors.
+ """Einsum.
+
+ Args:
+ equation: TODO.
+ *operands: Variable positional arguments.
+ """
if len(operands) == 1:
if isinstance(operands[0], (tuple, list)):
operands = operands[0]
@@ -109,6 +155,11 @@ def einsum(equation, *operands):
def inverse(c: Union[torch.Tensor, ComplexTensor]) -> Union[torch.Tensor, ComplexTensor]:
+ """Inverse.
+
+ Args:
+ c: TODO.
+ """
if isinstance(c, ComplexTensor):
return c.inverse2()
else:
@@ -121,6 +172,12 @@ def matmul(
# NOTE: Do not mix ComplexTensor and torch.complex in the input!
# NOTE (wangyou): Until PyTorch 1.9.0, torch.matmul does not support
# multiplication between complex and real tensors.
+ """Matmul.
+
+ Args:
+ a: TODO.
+ b: TODO.
+ """
if isinstance(a, ComplexTensor) or isinstance(b, ComplexTensor):
return FC.matmul(a, b)
elif is_torch_1_9_plus and (torch.is_complex(a) or torch.is_complex(b)):
@@ -141,10 +198,21 @@ def matmul(
def trace(a: Union[torch.Tensor, ComplexTensor]):
# NOTE (wangyou): until PyTorch 1.9.0, torch.trace does not
# support bacth processing. Use FC.trace() as fallback.
+ """Trace.
+
+ Args:
+ a: TODO.
+ """
return FC.trace(a)
def reverse(a: Union[torch.Tensor, ComplexTensor], dim=0):
+ """Reverse.
+
+ Args:
+ a: TODO.
+ dim: TODO.
+ """
if isinstance(a, ComplexTensor):
return FC.reverse(a, dim=dim)
else:
@@ -174,6 +242,13 @@ def solve(b: Union[torch.Tensor, ComplexTensor], a: Union[torch.Tensor, ComplexT
def stack(seq: Sequence[Union[ComplexTensor, torch.Tensor]], *args, **kwargs):
+ """Stack.
+
+ Args:
+ seq: TODO.
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
if not isinstance(seq, (list, tuple)):
raise TypeError(
"stack(): argument 'tensors' (position 1) must be tuple of Tensors, " "not Tensor"
diff --git a/funasr/frontends/utils/dnn_beamformer.py b/funasr/frontends/utils/dnn_beamformer.py
index 73b6bea42..2ba20d34e 100644
--- a/funasr/frontends/utils/dnn_beamformer.py
+++ b/funasr/frontends/utils/dnn_beamformer.py
@@ -36,6 +36,20 @@ class DNN_Beamformer(torch.nn.Module):
ref_channel: int = -1,
beamformer_type="mvdr",
):
+ """Initialize DNN_Beamformer.
+
+ Args:
+ bidim: TODO.
+ btype: TODO.
+ blayers: TODO.
+ bunits: TODO.
+ bprojs: TODO.
+ bnmask: TODO.
+ dropout_rate: TODO.
+ badim: TODO.
+ ref_channel: TODO.
+ beamformer_type: TODO.
+ """
super().__init__()
self.mask = MaskEstimator(btype, bidim, blayers, bunits, bprojs, dropout_rate, nmask=bnmask)
self.ref = AttentionReference(bidim, badim)
@@ -69,6 +83,14 @@ class DNN_Beamformer(torch.nn.Module):
def apply_beamforming(data, ilens, psd_speech, psd_noise):
# u: (B, C)
+ """Apply beamforming.
+
+ Args:
+ data: TODO.
+ ilens: TODO.
+ psd_speech: TODO.
+ psd_noise: TODO.
+ """
if self.ref_channel < 0:
u, _ = self.ref(psd_speech, ilens)
else:
@@ -126,6 +148,12 @@ class DNN_Beamformer(torch.nn.Module):
class AttentionReference(torch.nn.Module):
def __init__(self, bidim, att_dim):
+ """Initialize AttentionReference.
+
+ Args:
+ bidim: TODO.
+ att_dim: Size/dimension parameter.
+ """
super().__init__()
self.mlp_psd = torch.nn.Linear(bidim, att_dim)
self.gvec = torch.nn.Linear(att_dim, 1)
diff --git a/funasr/frontends/utils/dnn_wpe.py b/funasr/frontends/utils/dnn_wpe.py
index 917133953..a03d266a8 100644
--- a/funasr/frontends/utils/dnn_wpe.py
+++ b/funasr/frontends/utils/dnn_wpe.py
@@ -23,6 +23,21 @@ class DNN_WPE(torch.nn.Module):
iterations: int = 1,
normalization: bool = False,
):
+ """Initialize DNN_WPE.
+
+ Args:
+ wtype: TODO.
+ widim: TODO.
+ wlayers: TODO.
+ wunits: TODO.
+ wprojs: TODO.
+ dropout_rate: TODO.
+ taps: TODO.
+ delay: TODO.
+ use_dnn_mask: TODO.
+ iterations: TODO.
+ normalization: TODO.
+ """
super().__init__()
self.iterations = iterations
self.taps = taps
diff --git a/funasr/frontends/utils/feature_transform.py b/funasr/frontends/utils/feature_transform.py
index ab1486be9..49f9b32d4 100644
--- a/funasr/frontends/utils/feature_transform.py
+++ b/funasr/frontends/utils/feature_transform.py
@@ -25,6 +25,19 @@ class FeatureTransform(torch.nn.Module):
uttmvn_norm_means: bool = True,
uttmvn_norm_vars: bool = False,
):
+ """Initialize FeatureTransform.
+
+ Args:
+ fs: TODO.
+ n_fft: TODO.
+ n_mels: TODO.
+ fmin: TODO.
+ fmax: TODO.
+ stats_file: TODO.
+ apply_uttmvn: TODO.
+ uttmvn_norm_means: TODO.
+ uttmvn_norm_vars: TODO.
+ """
super().__init__()
self.apply_uttmvn = apply_uttmvn
@@ -44,6 +57,12 @@ class FeatureTransform(torch.nn.Module):
self, x: ComplexTensor, ilens: Union[torch.LongTensor, np.ndarray, List[int]]
) -> Tuple[torch.Tensor, torch.LongTensor]:
# (B, T, F) or (B, T, C, F)
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
if x.dim() not in (3, 4):
raise ValueError(f"Input dim must be 3 or 4: {x.dim()}")
if not torch.is_tensor(ilens):
@@ -103,6 +122,17 @@ class LogMel(torch.nn.Module):
htk: bool = False,
norm=1,
):
+ """Initialize LogMel.
+
+ Args:
+ fs: TODO.
+ n_fft: TODO.
+ n_mels: TODO.
+ fmin: TODO.
+ fmax: TODO.
+ htk: TODO.
+ norm: TODO.
+ """
super().__init__()
_mel_options = dict(
@@ -116,12 +146,19 @@ class LogMel(torch.nn.Module):
self.register_buffer("melmat", torch.from_numpy(melmat.T).float())
def extra_repr(self):
+ """Extra repr."""
return ", ".join(f"{k}={v}" for k, v in self.mel_options.items())
def forward(
self, feat: torch.Tensor, ilens: torch.LongTensor
) -> Tuple[torch.Tensor, torch.LongTensor]:
# feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2)
+ """Forward pass for training.
+
+ Args:
+ feat: TODO.
+ ilens: TODO.
+ """
mel_feat = torch.matmul(feat, self.melmat)
logmel_feat = (mel_feat + 1e-20).log()
@@ -151,6 +188,14 @@ class GlobalMVN(torch.nn.Module):
norm_vars: bool = True,
eps: float = 1.0e-20,
):
+ """Initialize GlobalMVN.
+
+ Args:
+ stats_file: TODO.
+ norm_means: TODO.
+ norm_vars: TODO.
+ eps: TODO.
+ """
super().__init__()
self.norm_means = norm_means
self.norm_vars = norm_vars
@@ -170,6 +215,7 @@ class GlobalMVN(torch.nn.Module):
self.register_buffer("scale", torch.from_numpy(1 / std.astype(np.float32)))
def extra_repr(self):
+ """Extra repr."""
return (
f"stats_file={self.stats_file}, "
f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"
@@ -179,6 +225,12 @@ class GlobalMVN(torch.nn.Module):
self, x: torch.Tensor, ilens: torch.LongTensor
) -> Tuple[torch.Tensor, torch.LongTensor]:
# feat: (B, T, D)
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
if self.norm_means:
x += self.bias.type_as(x)
x.masked_fill(make_pad_mask(ilens, x, 1), 0.0)
@@ -190,17 +242,31 @@ class GlobalMVN(torch.nn.Module):
class UtteranceMVN(torch.nn.Module):
def __init__(self, norm_means: bool = True, norm_vars: bool = False, eps: float = 1.0e-20):
+ """Initialize UtteranceMVN.
+
+ Args:
+ norm_means: TODO.
+ norm_vars: TODO.
+ eps: TODO.
+ """
super().__init__()
self.norm_means = norm_means
self.norm_vars = norm_vars
self.eps = eps
def extra_repr(self):
+ """Extra repr."""
return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"
def forward(
self, x: torch.Tensor, ilens: torch.LongTensor
) -> Tuple[torch.Tensor, torch.LongTensor]:
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
return utterance_mvn(
x, ilens, norm_means=self.norm_means, norm_vars=self.norm_vars, eps=self.eps
)
@@ -244,6 +310,12 @@ def utterance_mvn(
def feature_transform_for(args, n_fft):
+ """Feature transform for.
+
+ Args:
+ args: TODO.
+ n_fft: TODO.
+ """
return FeatureTransform(
# Mel options,
fs=args.fbank_fs,
diff --git a/funasr/frontends/utils/frontend.py b/funasr/frontends/utils/frontend.py
index a2a6cc120..924e8c2a7 100644
--- a/funasr/frontends/utils/frontend.py
+++ b/funasr/frontends/utils/frontend.py
@@ -37,6 +37,29 @@ class Frontend(nn.Module):
ref_channel: int = -1,
bdropout_rate=0.0,
):
+ """Initialize Frontend.
+
+ Args:
+ idim: TODO.
+ use_wpe: TODO.
+ wtype: TODO.
+ wlayers: TODO.
+ wunits: TODO.
+ wprojs: TODO.
+ wdropout_rate: TODO.
+ taps: TODO.
+ delay: TODO.
+ use_dnn_mask_for_wpe: TODO.
+ use_beamformer: TODO.
+ btype: TODO.
+ blayers: TODO.
+ bunits: TODO.
+ bprojs: TODO.
+ bnmask: TODO.
+ badim: TODO.
+ ref_channel: TODO.
+ bdropout_rate: TODO.
+ """
super().__init__()
self.use_beamformer = use_beamformer
@@ -88,6 +111,12 @@ class Frontend(nn.Module):
def forward(
self, x: ComplexTensor, ilens: Union[torch.LongTensor, numpy.ndarray, List[int]]
) -> Tuple[ComplexTensor, torch.LongTensor, Optional[ComplexTensor]]:
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
assert len(x) == len(ilens), (len(x), len(ilens))
# (B, T, F) or (B, T, C, F)
if x.dim() not in (3, 4):
@@ -126,6 +155,12 @@ class Frontend(nn.Module):
def frontend_for(args, idim):
+ """Frontend for.
+
+ Args:
+ args: TODO.
+ idim: TODO.
+ """
return Frontend(
idim=idim,
# WPE options
diff --git a/funasr/frontends/utils/log_mel.py b/funasr/frontends/utils/log_mel.py
index 611cf1eb3..dd66b7fcc 100644
--- a/funasr/frontends/utils/log_mel.py
+++ b/funasr/frontends/utils/log_mel.py
@@ -30,6 +30,17 @@ class LogMel(torch.nn.Module):
htk: bool = False,
log_base: float = None,
):
+ """Initialize LogMel.
+
+ Args:
+ fs: TODO.
+ n_fft: TODO.
+ n_mels: TODO.
+ fmin: TODO.
+ fmax: TODO.
+ htk: TODO.
+ log_base: TODO.
+ """
super().__init__()
fmin = 0 if fmin is None else fmin
@@ -51,6 +62,7 @@ class LogMel(torch.nn.Module):
self.register_buffer("melmat", torch.from_numpy(melmat.T).float())
def extra_repr(self):
+ """Extra repr."""
return ", ".join(f"{k}={v}" for k, v in self.mel_options.items())
def forward(
@@ -59,6 +71,12 @@ class LogMel(torch.nn.Module):
ilens: torch.Tensor = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
# feat: (B, T, D1) x melmat: (D1, D2) -> mel_feat: (B, T, D2)
+ """Forward pass for training.
+
+ Args:
+ feat: TODO.
+ ilens: TODO.
+ """
mel_feat = torch.matmul(feat, self.melmat)
mel_feat = torch.clamp(mel_feat, min=1e-10)
diff --git a/funasr/frontends/utils/mask_estimator.py b/funasr/frontends/utils/mask_estimator.py
index 29a21d97a..12b1294cb 100644
--- a/funasr/frontends/utils/mask_estimator.py
+++ b/funasr/frontends/utils/mask_estimator.py
@@ -12,6 +12,17 @@ from funasr.models.language_model.rnn.encoders import RNNP
class MaskEstimator(torch.nn.Module):
def __init__(self, type, idim, layers, units, projs, dropout, nmask=1):
+ """Initialize MaskEstimator.
+
+ Args:
+ type: TODO.
+ idim: TODO.
+ layers: TODO.
+ units: TODO.
+ projs: TODO.
+ dropout: TODO.
+ nmask: TODO.
+ """
super().__init__()
subsample = np.ones(layers + 1, dtype=np.int32)
diff --git a/funasr/frontends/utils/stft.py b/funasr/frontends/utils/stft.py
index 381f1e986..be2d4a3f1 100644
--- a/funasr/frontends/utils/stft.py
+++ b/funasr/frontends/utils/stft.py
@@ -32,6 +32,17 @@ class Stft(torch.nn.Module):
normalized: bool = False,
onesided: bool = True,
):
+ """Initialize Stft.
+
+ Args:
+ n_fft: TODO.
+ win_length: TODO.
+ hop_length: TODO.
+ window: TODO.
+ center: TODO.
+ normalized: TODO.
+ onesided: TODO.
+ """
super().__init__()
self.n_fft = n_fft
if win_length is None:
@@ -48,6 +59,7 @@ class Stft(torch.nn.Module):
self.window = window
def extra_repr(self):
+ """Extra repr."""
return (
f"n_fft={self.n_fft}, "
f"win_length={self.win_length}, "
diff --git a/funasr/frontends/wav_frontend.py b/funasr/frontends/wav_frontend.py
index c733d6663..072bebf82 100644
--- a/funasr/frontends/wav_frontend.py
+++ b/funasr/frontends/wav_frontend.py
@@ -13,6 +13,11 @@ from funasr.register import tables
def load_cmvn(cmvn_file):
+ """Load cmvn.
+
+ Args:
+ cmvn_file: TODO.
+ """
with open(cmvn_file, "r", encoding="utf-8") as f:
lines = f.readlines()
means_list = []
@@ -56,6 +61,13 @@ def apply_cmvn(inputs, cmvn): # noqa
def apply_lfr(inputs, lfr_m, lfr_n):
+ """Apply lfr.
+
+ Args:
+ inputs: TODO.
+ lfr_m: TODO.
+ lfr_n: TODO.
+ """
LFR_inputs = []
T = inputs.shape[0]
T_lfr = int(np.ceil(T / lfr_n))
@@ -96,6 +108,24 @@ class WavFrontend(nn.Module):
upsacle_samples: bool = True,
**kwargs,
):
+ """Initialize WavFrontend.
+
+ Args:
+ cmvn_file: TODO.
+ fs: TODO.
+ window: TODO.
+ n_mels: TODO.
+ frame_length: TODO.
+ frame_shift: TODO.
+ filter_length_min: TODO.
+ filter_length_max: TODO.
+ lfr_m: TODO.
+ lfr_n: TODO.
+ dither: TODO.
+ snip_edges: TODO.
+ upsacle_samples: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.fs = fs
self.window = window
@@ -113,6 +143,7 @@ class WavFrontend(nn.Module):
self.cmvn = None if self.cmvn_file is None else load_cmvn(self.cmvn_file)
def output_size(self) -> int:
+ """Output size."""
return self.n_mels * self.lfr_m
def forward(
@@ -121,6 +152,13 @@ class WavFrontend(nn.Module):
input_lengths,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ **kwargs: Additional keyword arguments.
+ """
batch_size = input.size(0)
feats = []
feats_lens = []
@@ -160,6 +198,12 @@ class WavFrontend(nn.Module):
def forward_fbank(
self, input: torch.Tensor, input_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward fbank.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
batch_size = input.size(0)
feats = []
feats_lens = []
@@ -190,6 +234,12 @@ class WavFrontend(nn.Module):
def forward_lfr_cmvn(
self, input: torch.Tensor, input_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward lfr cmvn.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
batch_size = input.size(0)
feats = []
feats_lens = []
@@ -229,6 +279,24 @@ class WavFrontendOnline(nn.Module):
upsacle_samples: bool = True,
**kwargs,
):
+ """Initialize WavFrontendOnline.
+
+ Args:
+ cmvn_file: TODO.
+ fs: TODO.
+ window: TODO.
+ n_mels: TODO.
+ frame_length: TODO.
+ frame_shift: TODO.
+ filter_length_min: TODO.
+ filter_length_max: TODO.
+ lfr_m: TODO.
+ lfr_n: TODO.
+ dither: TODO.
+ snip_edges: TODO.
+ upsacle_samples: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.fs = fs
self.window = window
@@ -254,6 +322,7 @@ class WavFrontendOnline(nn.Module):
# self.lfr_splice_cache = []
def output_size(self) -> int:
+ """Output size."""
return self.n_mels * self.lfr_m
@staticmethod
@@ -311,6 +380,13 @@ class WavFrontendOnline(nn.Module):
def compute_frame_num(
sample_length: int, frame_sample_length: int, frame_shift_sample_length: int
) -> int:
+ """Compute frame num.
+
+ Args:
+ sample_length: TODO.
+ frame_sample_length: TODO.
+ frame_shift_sample_length: TODO.
+ """
frame_num = int((sample_length - frame_sample_length) / frame_shift_sample_length + 1)
return frame_num if frame_num >= 1 and sample_length >= frame_sample_length else 0
@@ -321,6 +397,14 @@ class WavFrontendOnline(nn.Module):
cache: dict = None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """Forward fbank.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
batch_size = input.size(0)
@@ -383,6 +467,15 @@ class WavFrontendOnline(nn.Module):
cache: dict = None,
**kwargs,
):
+ """Forward lfr cmvn.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ is_final: Whether this is the final chunk in streaming.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
batch_size = input.size(0)
@@ -410,6 +503,13 @@ class WavFrontendOnline(nn.Module):
return feats_pad, feats_lens, lfr_splice_frame_idxs
def forward(self, input: torch.Tensor, input_lengths: torch.Tensor, **kwargs):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ **kwargs: Additional keyword arguments.
+ """
is_final = kwargs.get("is_final", False)
cache = kwargs.get("cache", {})
if len(cache) == 0:
@@ -492,6 +592,11 @@ class WavFrontendOnline(nn.Module):
return feats, feats_lengths
def init_cache(self, cache: dict = None):
+ """Init cache.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
cache["reserve_waveforms"] = torch.empty(0)
@@ -515,6 +620,16 @@ class WavFrontendMel23(nn.Module):
lfr_n: int = 1,
**kwargs,
):
+ """Initialize WavFrontendMel23.
+
+ Args:
+ fs: TODO.
+ frame_length: TODO.
+ frame_shift: TODO.
+ lfr_m: TODO.
+ lfr_n: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.fs = fs
self.frame_length = frame_length
@@ -524,11 +639,18 @@ class WavFrontendMel23(nn.Module):
self.n_mels = 23
def output_size(self) -> int:
+ """Output size."""
return self.n_mels * (2 * self.lfr_m + 1)
def forward(
self, input: torch.Tensor, input_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ """
batch_size = input.size(0)
feats = []
feats_lens = []
diff --git a/funasr/frontends/whisper_frontend.py b/funasr/frontends/whisper_frontend.py
index 1bd8aecdd..6fd10f696 100644
--- a/funasr/frontends/whisper_frontend.py
+++ b/funasr/frontends/whisper_frontend.py
@@ -23,6 +23,16 @@ class WhisperFrontend(nn.Module):
permute: bool = False,
**kwargs,
):
+ """Initialize WhisperFrontend.
+
+ Args:
+ fs: TODO.
+ whisper_model: Whisper Model instance.
+ do_pad_trim: TODO.
+ n_mels: TODO.
+ permute: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
assert fs == 16000
self.fs = fs
@@ -55,6 +65,7 @@ class WhisperFrontend(nn.Module):
# assert whisper_model in whisper.available_models()
def output_size(self) -> int:
+ """Output size."""
return self.n_mels
def log_mel_spectrogram(
@@ -62,6 +73,12 @@ class WhisperFrontend(nn.Module):
audio: torch.Tensor,
ilens: torch.Tensor = None,
) -> torch.Tensor:
+ """Log mel spectrogram.
+
+ Args:
+ audio: TODO.
+ ilens: TODO.
+ """
window = torch.hann_window(self.win_length).to(audio.device)
stft = torch.stft(audio, self.n_fft, self.hop_length, window=window, return_complex=True)
@@ -94,6 +111,13 @@ class WhisperFrontend(nn.Module):
input_lengths: torch.Tensor,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ input_lengths: Lengths of input.
+ **kwargs: Additional keyword arguments.
+ """
batch_size = input.size(0)
feats = []
feats_lens = []
diff --git a/funasr/losses/label_smoothing_loss.py b/funasr/losses/label_smoothing_loss.py
index ffc38dae7..42d5dd127 100644
--- a/funasr/losses/label_smoothing_loss.py
+++ b/funasr/losses/label_smoothing_loss.py
@@ -66,11 +66,24 @@ class LabelSmoothingLoss(nn.Module):
class SequenceBinaryCrossEntropy(nn.Module):
def __init__(self, normalize_length=False, criterion=nn.BCEWithLogitsLoss(reduction="none")):
+ """Initialize SequenceBinaryCrossEntropy.
+
+ Args:
+ normalize_length: TODO.
+ criterion: TODO.
+ """
super().__init__()
self.normalize_length = normalize_length
self.criterion = criterion
def forward(self, pred, label, lengths):
+ """Forward pass for training.
+
+ Args:
+ pred: TODO.
+ label: TODO.
+ lengths: TODO.
+ """
pad_mask = make_pad_mask(lengths, maxlen=pred.shape[1]).to(pred.device)
loss = self.criterion(pred, label)
denom = (~pad_mask).sum() if self.normalize_length else pred.shape[0]
diff --git a/funasr/metrics/compute_eer.py b/funasr/metrics/compute_eer.py
index 5180b1669..ea0e3a2f3 100644
--- a/funasr/metrics/compute_eer.py
+++ b/funasr/metrics/compute_eer.py
@@ -31,6 +31,12 @@ def _compute_eer(label, pred, positive_label=1):
def compute_eer(trials_path, scores_path):
+ """Compute eer.
+
+ Args:
+ trials_path: TODO.
+ scores_path: TODO.
+ """
labels = []
for one_line in open(trials_path, "r"):
labels.append(one_line.strip().rsplit(" ", 1)[-1] == "target")
@@ -46,6 +52,7 @@ def compute_eer(trials_path, scores_path):
def main():
+ """Main."""
parser = argparse.ArgumentParser()
parser.add_argument("trials", help="trial list")
parser.add_argument("scores", help="score file, normalized to [0, 1]")
diff --git a/funasr/metrics/compute_min_dcf.py b/funasr/metrics/compute_min_dcf.py
index 472f7b3e2..a1e3427db 100644
--- a/funasr/metrics/compute_min_dcf.py
+++ b/funasr/metrics/compute_min_dcf.py
@@ -16,6 +16,7 @@ import sys, argparse, os
def GetArgs():
+ """Getargs."""
parser = argparse.ArgumentParser(
description="Compute the minimum "
"detection cost function along with the threshold at which it occurs. "
@@ -61,6 +62,11 @@ def GetArgs():
def CheckArgs(args):
+ """Checkargs.
+
+ Args:
+ args: TODO.
+ """
if args.c_fa <= 0:
raise Exception("--c-fa must be greater than 0")
if args.c_miss <= 0:
@@ -77,6 +83,12 @@ def ComputeErrorRates(scores, labels):
# Sort the scores from smallest to largest, and also get the corresponding
# indexes of the sorted scores. We will treat the sorted scores as the
# thresholds at which the the error-rates are evaluated.
+ """Computeerrorrates.
+
+ Args:
+ scores: TODO.
+ labels: TODO.
+ """
sorted_indexes, thresholds = zip(
*sorted([(index, threshold) for index, threshold in enumerate(scores)], key=itemgetter(1))
)
@@ -113,6 +125,16 @@ def ComputeErrorRates(scores, labels):
# Computes the minimum of the detection cost function. The comments refer to
# equations in Section 3 of the NIST 2016 Speaker Recognition Evaluation Plan.
def ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa):
+ """Computemindcf.
+
+ Args:
+ fnrs: TODO.
+ fprs: TODO.
+ thresholds: TODO.
+ p_target: TODO.
+ c_miss: TODO.
+ c_fa: TODO.
+ """
min_c_det = float("inf")
min_c_det_threshold = thresholds[0]
for i in range(0, len(fnrs)):
@@ -129,6 +151,15 @@ def ComputeMinDcf(fnrs, fprs, thresholds, p_target, c_miss, c_fa):
def compute_min_dcf(scores_filename, trials_filename, c_miss=1, c_fa=1, p_target=0.01):
+ """Compute min dcf.
+
+ Args:
+ scores_filename: TODO.
+ trials_filename: TODO.
+ c_miss: TODO.
+ c_fa: TODO.
+ p_target: TODO.
+ """
scores_file = open(scores_filename, "r").readlines()
trials_file = open(trials_filename, "r").readlines()
c_miss = c_miss
@@ -162,6 +193,7 @@ def compute_min_dcf(scores_filename, trials_filename, c_miss=1, c_fa=1, p_target
def main():
+ """Main."""
args = GetArgs()
mindcf, threshold = compute_min_dcf(
args.scores_filename, args.trials_filename, args.c_miss, args.c_fa, args.p_target
diff --git a/funasr/metrics/wer.py b/funasr/metrics/wer.py
index 6bac30324..c32a262fa 100755
--- a/funasr/metrics/wer.py
+++ b/funasr/metrics/wer.py
@@ -11,6 +11,14 @@ def compute_wer(
cer_file,
cn_postprocess=False,
):
+ """Compute wer.
+
+ Args:
+ ref_file: TODO.
+ hyp_file: TODO.
+ cer_file: TODO.
+ cn_postprocess: TODO.
+ """
rst = {
"Wrd": 0,
"Corr": 0,
@@ -114,6 +122,12 @@ def compute_wer(
def compute_wer_by_line(hyp, ref):
+ """Compute wer by line.
+
+ Args:
+ hyp: TODO.
+ ref: TODO.
+ """
hyp = list(map(lambda x: x.lower(), hyp))
ref = list(map(lambda x: x.lower(), ref))
@@ -187,6 +201,11 @@ def compute_wer_by_line(hyp, ref):
def print_cer_detail(rst):
+ """Print cer detail.
+
+ Args:
+ rst: TODO.
+ """
return (
"("
+ "nwords="
@@ -208,6 +227,11 @@ def print_cer_detail(rst):
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
ref_file = cfg.get("ref_file", None)
hyp_file = cfg.get("hyp_file", None)
cer_file = cfg.get("cer_file", None)
diff --git a/funasr/models/bat/model.py b/funasr/models/bat/model.py
index ec2244337..5d64f752e 100644
--- a/funasr/models/bat/model.py
+++ b/funasr/models/bat/model.py
@@ -29,6 +29,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
diff --git a/funasr/models/bicif_paraformer/cif_predictor.py b/funasr/models/bicif_paraformer/cif_predictor.py
index 77d4e9659..95c100f79 100644
--- a/funasr/models/bicif_paraformer/cif_predictor.py
+++ b/funasr/models/bicif_paraformer/cif_predictor.py
@@ -1,544 +1,659 @@
-#!/usr/bin/env python3
-# -*- encoding: utf-8 -*-
-# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
-# MIT License (https://opensource.org/licenses/MIT)
-
-import torch
-
-from funasr.register import tables
-from funasr.models.transformer.utils.nets_utils import make_pad_mask
-
-
-class mae_loss(torch.nn.Module):
-
- def __init__(self, normalize_length=False):
- super(mae_loss, self).__init__()
- self.normalize_length = normalize_length
- self.criterion = torch.nn.L1Loss(reduction="sum")
-
- def forward(self, token_length, pre_token_length):
- loss_token_normalizer = token_length.size(0)
- if self.normalize_length:
- loss_token_normalizer = token_length.sum().type(torch.float32)
- loss = self.criterion(token_length, pre_token_length)
- loss = loss / loss_token_normalizer
- return loss
-
-
-def cif(hidden, alphas, threshold):
- batch_size, len_time, hidden_size = hidden.size()
-
- # loop varss
- integrate = torch.zeros([batch_size], device=hidden.device)
- frame = torch.zeros([batch_size, hidden_size], device=hidden.device)
- # intermediate vars along time
- list_fires = []
- list_frames = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
- distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(
- fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate
- )
- cur = torch.where(fire_place, distribution_completion, alpha)
- remainds = alpha - cur
-
- frame += cur[:, None] * hidden[:, t, :]
- list_frames.append(frame)
- frame = torch.where(
- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
- )
-
- fires = torch.stack(list_fires, 1)
- frames = torch.stack(list_frames, 1)
- list_ls = []
- len_labels = torch.round(alphas.sum(-1)).int()
- max_label_len = len_labels.max()
- for b in range(batch_size):
- fire = fires[b, :]
- l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze(-1))
- pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)
- list_ls.append(torch.cat([l, pad_l], 0))
- return torch.stack(list_ls, 0), fires
-
-
-def cif_wo_hidden(alphas, threshold):
- batch_size, len_time = alphas.size()
-
- # loop varss
- integrate = torch.zeros([batch_size], device=alphas.device)
- # intermediate vars along time
- list_fires = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(
- fire_place,
- integrate - torch.ones([batch_size], device=alphas.device) * threshold,
- integrate,
- )
-
- fires = torch.stack(list_fires, 1)
- return fires
-
-
[email protected]("predictor_classes", "CifPredictorV3")
-class CifPredictorV3(torch.nn.Module):
- def __init__(
- self,
- idim,
- l_order,
- r_order,
- threshold=1.0,
- dropout=0.1,
- smooth_factor=1.0,
- noise_threshold=0,
- tail_threshold=0.0,
- tf2torch_tensor_name_prefix_torch="predictor",
- tf2torch_tensor_name_prefix_tf="seq2seq/cif",
- smooth_factor2=1.0,
- noise_threshold2=0,
- upsample_times=5,
- upsample_type="cnn",
- use_cif1_cnn=True,
- tail_mask=True,
- ):
- super(CifPredictorV3, self).__init__()
-
- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)
- self.cif_output = torch.nn.Linear(idim, 1)
- self.dropout = torch.nn.Dropout(p=dropout)
- self.threshold = threshold
- self.smooth_factor = smooth_factor
- self.noise_threshold = noise_threshold
- self.tail_threshold = tail_threshold
- self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
- self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
-
- self.upsample_times = upsample_times
- self.upsample_type = upsample_type
- self.use_cif1_cnn = use_cif1_cnn
- if self.upsample_type == "cnn":
- self.upsample_cnn = torch.nn.ConvTranspose1d(
- idim, idim, self.upsample_times, self.upsample_times
- )
- self.cif_output2 = torch.nn.Linear(idim, 1)
- elif self.upsample_type == "cnn_blstm":
- self.upsample_cnn = torch.nn.ConvTranspose1d(
- idim, idim, self.upsample_times, self.upsample_times
- )
- self.blstm = torch.nn.LSTM(
- idim, idim, 1, bias=True, batch_first=True, dropout=0.0, bidirectional=True
- )
- self.cif_output2 = torch.nn.Linear(idim * 2, 1)
- elif self.upsample_type == "cnn_attn":
- self.upsample_cnn = torch.nn.ConvTranspose1d(
- idim, idim, self.upsample_times, self.upsample_times
- )
- from funasr.models.transformer.encoder import EncoderLayer as TransformerEncoderLayer
- from funasr.models.transformer.attention import MultiHeadedAttention
- from funasr.models.transformer.positionwise_feed_forward import PositionwiseFeedForward
-
- positionwise_layer_args = (
- idim,
- idim * 2,
- 0.1,
- )
- self.self_attn = TransformerEncoderLayer(
- idim,
- MultiHeadedAttention(4, idim, 0.1),
- PositionwiseFeedForward(*positionwise_layer_args),
- 0.1,
- True, # normalize_before,
- False, # concat_after,
- )
- self.cif_output2 = torch.nn.Linear(idim, 1)
- self.smooth_factor2 = smooth_factor2
- self.noise_threshold2 = noise_threshold2
-
- def forward(
- self,
- hidden,
- target_label=None,
- mask=None,
- ignore_id=-1,
- mask_chunk_predictor=None,
- target_label_length=None,
- ):
- h = hidden
- context = h.transpose(1, 2)
- queries = self.pad(context)
- output = torch.relu(self.cif_conv1d(queries))
-
- # alphas2 is an extra head for timestamp prediction
- if not self.use_cif1_cnn:
- _output = context
- else:
- _output = output
- if self.upsample_type == "cnn":
- output2 = self.upsample_cnn(_output)
- output2 = output2.transpose(1, 2)
- elif self.upsample_type == "cnn_blstm":
- output2 = self.upsample_cnn(_output)
- output2 = output2.transpose(1, 2)
- output2, (_, _) = self.blstm(output2)
- elif self.upsample_type == "cnn_attn":
- output2 = self.upsample_cnn(_output)
- output2 = output2.transpose(1, 2)
- output2, _ = self.self_attn(output2, mask)
-
- alphas2 = torch.sigmoid(self.cif_output2(output2))
- alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
- # repeat the mask in T demension to match the upsampled length
- if mask is not None:
- mask2 = (
- mask.repeat(1, self.upsample_times, 1)
- .transpose(-1, -2)
- .reshape(alphas2.shape[0], -1)
- )
- mask2 = mask2.unsqueeze(-1)
- alphas2 = alphas2 * mask2
- alphas2 = alphas2.squeeze(-1)
- token_num2 = alphas2.sum(-1)
-
- output = output.transpose(1, 2)
-
- output = self.cif_output(output)
- alphas = torch.sigmoid(output)
- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
- if mask is not None:
- mask = mask.transpose(-1, -2).float()
- alphas = alphas * mask
- if mask_chunk_predictor is not None:
- alphas = alphas * mask_chunk_predictor
- alphas = alphas.squeeze(-1)
- mask = mask.squeeze(-1)
- if target_label_length is not None:
- target_length = target_label_length
- elif target_label is not None:
- target_length = (target_label != ignore_id).float().sum(-1)
- else:
- target_length = None
- token_num = alphas.sum(-1)
-
- if target_length is not None:
- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
- elif self.tail_threshold > 0.0:
- hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=mask)
-
- acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)
- if target_length is None and self.tail_threshold > 0.0:
- token_num_int = torch.max(token_num).type(torch.int32).item()
- acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
- return acoustic_embeds, token_num, alphas, cif_peak, token_num2
-
- def get_upsample_timestamp(self, hidden, mask=None, token_num=None):
- h = hidden
- b = hidden.shape[0]
- context = h.transpose(1, 2)
- queries = self.pad(context)
- output = torch.relu(self.cif_conv1d(queries))
-
- # alphas2 is an extra head for timestamp prediction
- if not self.use_cif1_cnn:
- _output = context
- else:
- _output = output
- if self.upsample_type == "cnn":
- output2 = self.upsample_cnn(_output)
- output2 = output2.transpose(1, 2)
- elif self.upsample_type == "cnn_blstm":
- output2 = self.upsample_cnn(_output)
- output2 = output2.transpose(1, 2)
- output2, (_, _) = self.blstm(output2)
- elif self.upsample_type == "cnn_attn":
- output2 = self.upsample_cnn(_output)
- output2 = output2.transpose(1, 2)
- output2, _ = self.self_attn(output2, mask)
- alphas2 = torch.sigmoid(self.cif_output2(output2))
- alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
- # repeat the mask in T demension to match the upsampled length
- if mask is not None:
- mask2 = (
- mask.repeat(1, self.upsample_times, 1)
- .transpose(-1, -2)
- .reshape(alphas2.shape[0], -1)
- )
- mask2 = mask2.unsqueeze(-1)
- alphas2 = alphas2 * mask2
- alphas2 = alphas2.squeeze(-1)
- _token_num = alphas2.sum(-1)
- if token_num is not None:
- alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))
- # re-downsample
- ds_alphas = alphas2.reshape(b, -1, self.upsample_times).sum(-1)
- ds_cif_peak = cif_wo_hidden(ds_alphas, self.threshold - 1e-4)
- # upsampled alphas and cif_peak
- us_alphas = alphas2
- us_cif_peak = cif_wo_hidden(us_alphas, self.threshold - 1e-4)
- return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak
-
- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
- b, t, d = hidden.size()
- tail_threshold = self.tail_threshold
- if mask is not None:
- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
- ones_t = torch.ones_like(zeros_t)
- mask_1 = torch.cat([mask, zeros_t], dim=1)
- mask_2 = torch.cat([ones_t, mask], dim=1)
- mask = mask_2 - mask_1
- tail_threshold = mask * tail_threshold
- alphas = torch.cat([alphas, zeros_t], dim=1)
- alphas = torch.add(alphas, tail_threshold)
- else:
- tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)
- tail_threshold = torch.reshape(tail_threshold, (1, 1))
- alphas = torch.cat([alphas, tail_threshold], dim=1)
- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
- hidden = torch.cat([hidden, zeros], dim=1)
- token_num = alphas.sum(dim=-1)
- token_num_floor = torch.floor(token_num)
-
- return hidden, alphas, token_num_floor
-
- def gen_frame_alignments(
- self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None
- ):
- batch_size, maximum_length = alphas.size()
- int_type = torch.int32
-
- is_training = self.training
- if is_training:
- token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)
- else:
- token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)
-
- max_token_num = torch.max(token_num).item()
-
- alphas_cumsum = torch.cumsum(alphas, dim=1)
- alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)
- alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)
-
- index = torch.ones([batch_size, max_token_num], dtype=int_type)
- index = torch.cumsum(index, dim=1)
- index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)
-
- index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)
- index_div_bool_zeros = index_div.eq(0)
- index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1
- index_div_bool_zeros_count = torch.clamp(
- index_div_bool_zeros_count, 0, encoder_sequence_length.max()
- )
- token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)
- index_div_bool_zeros_count *= token_num_mask
-
- index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(
- 1, 1, maximum_length
- )
- ones = torch.ones_like(index_div_bool_zeros_count_tile)
- zeros = torch.zeros_like(index_div_bool_zeros_count_tile)
- ones = torch.cumsum(ones, dim=2)
- cond = index_div_bool_zeros_count_tile == ones
- index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)
-
- index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)
- index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)
- index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)
- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)
- predictor_mask = (
- (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))
- .type(int_type)
- .to(encoder_sequence_length.device)
- )
- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask
-
- predictor_alignments = index_div_bool_zeros_count_tile_out
- predictor_alignments_length = predictor_alignments.sum(-1).type(
- encoder_sequence_length.dtype
- )
- return predictor_alignments.detach(), predictor_alignments_length.detach()
-
-
[email protected]("predictor_classes", "CifPredictorV3Export")
-class CifPredictorV3Export(torch.nn.Module):
- def __init__(self, model, **kwargs):
- super().__init__()
-
- self.pad = model.pad
- self.cif_conv1d = model.cif_conv1d
- self.cif_output = model.cif_output
- self.threshold = model.threshold
- self.smooth_factor = model.smooth_factor
- self.noise_threshold = model.noise_threshold
- self.tail_threshold = model.tail_threshold
-
- self.upsample_times = model.upsample_times
- self.upsample_cnn = model.upsample_cnn
- self.blstm = model.blstm
- self.cif_output2 = model.cif_output2
- self.smooth_factor2 = model.smooth_factor2
- self.noise_threshold2 = model.noise_threshold2
-
- def forward(
- self,
- hidden: torch.Tensor,
- mask: torch.Tensor,
- ):
- h = hidden
- context = h.transpose(1, 2)
- queries = self.pad(context)
- output = torch.relu(self.cif_conv1d(queries))
- output = output.transpose(1, 2)
-
- output = self.cif_output(output)
- alphas = torch.sigmoid(output)
- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
- mask = mask.transpose(-1, -2).float()
- alphas = alphas * mask
- alphas = alphas.squeeze(-1)
- token_num = alphas.sum(-1)
-
- mask = mask.squeeze(-1)
- hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)
- acoustic_embeds, cif_peak = cif_export(hidden, alphas, self.threshold)
-
- return acoustic_embeds, token_num, alphas, cif_peak
-
- def get_upsample_timestmap(self, hidden, mask=None, token_num=None):
- h = hidden
- b = hidden.shape[0]
- context = h.transpose(1, 2)
-
- # generate alphas2
- _output = context
- output2 = self.upsample_cnn(_output)
- output2 = output2.transpose(1, 2)
- output2, (_, _) = self.blstm(output2)
- alphas2 = torch.sigmoid(self.cif_output2(output2))
- alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
-
- mask = (
- mask.repeat(1, self.upsample_times, 1).transpose(-1, -2).reshape(alphas2.shape[0], -1)
- )
- mask = mask.unsqueeze(-1)
- alphas2 = alphas2 * mask
- alphas2 = alphas2.squeeze(-1)
- _token_num = alphas2.sum(-1)
- alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))
- # upsampled alphas and cif_peak
- us_alphas = alphas2
- us_cif_peak = cif_wo_hidden_export(us_alphas, self.threshold - 1e-4)
- return us_alphas, us_cif_peak
-
- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
- b, t, d = hidden.size()
- tail_threshold = self.tail_threshold
-
- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
- ones_t = torch.ones_like(zeros_t)
-
- mask_1 = torch.cat([mask, zeros_t], dim=1)
- mask_2 = torch.cat([ones_t, mask], dim=1)
- mask = mask_2 - mask_1
- tail_threshold = mask * tail_threshold
- alphas = torch.cat([alphas, zeros_t], dim=1)
- alphas = torch.add(alphas, tail_threshold)
-
- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
- hidden = torch.cat([hidden, zeros], dim=1)
- token_num = alphas.sum(dim=-1)
- token_num_floor = torch.floor(token_num)
-
- return hidden, alphas, token_num_floor
-
-
-def cif_export(hidden, alphas, threshold: float):
- batch_size, len_time, hidden_size = hidden.size()
- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
-
- # loop varss
- integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)
- frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)
- # intermediate vars along time
- list_fires = []
- list_frames = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
- distribution_completion = (
- torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate
- )
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(
- fire_place,
- integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),
- integrate,
- )
- cur = torch.where(fire_place, distribution_completion, alpha)
- remainds = alpha - cur
-
- frame += cur[:, None] * hidden[:, t, :]
- list_frames.append(frame)
- frame = torch.where(
- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
- )
-
- fires = torch.stack(list_fires, 1)
- frames = torch.stack(list_frames, 1)
-
- fire_idxs = fires >= threshold
- frame_fires = torch.zeros_like(hidden)
- max_label_len = frames[0, fire_idxs[0]].size(0)
- for b in range(batch_size):
- frame_fire = frames[b, fire_idxs[b]]
- frame_len = frame_fire.size(0)
- frame_fires[b, :frame_len, :] = frame_fire
-
- if frame_len >= max_label_len:
- max_label_len = frame_len
- frame_fires = frame_fires[:, :max_label_len, :]
- return frame_fires, fires
-
-
-def cif_wo_hidden_export(alphas, threshold: float):
- batch_size, len_time = alphas.size()
-
- # loop varss
- integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=alphas.device)
- # intermediate vars along time
- list_fires = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(
- fire_place,
- integrate - torch.ones([batch_size], device=alphas.device) * threshold,
- integrate,
- )
-
- fires = torch.stack(list_fires, 1)
- return fires
-
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+# MIT License (https://opensource.org/licenses/MIT)
+
+import torch
+
+from funasr.register import tables
+from funasr.models.transformer.utils.nets_utils import make_pad_mask
+
+
+class mae_loss(torch.nn.Module):
+
+ def __init__(self, normalize_length=False):
+ """Initialize mae_loss.
+
+ Args:
+ normalize_length: TODO.
+ """
+ super(mae_loss, self).__init__()
+ self.normalize_length = normalize_length
+ self.criterion = torch.nn.L1Loss(reduction="sum")
+
+ def forward(self, token_length, pre_token_length):
+ """Forward pass for training.
+
+ Args:
+ token_length: TODO.
+ pre_token_length: TODO.
+ """
+ loss_token_normalizer = token_length.size(0)
+ if self.normalize_length:
+ loss_token_normalizer = token_length.sum().type(torch.float32)
+ loss = self.criterion(token_length, pre_token_length)
+ loss = loss / loss_token_normalizer
+ return loss
+
+
+def cif(hidden, alphas, threshold):
+ """Cif.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ threshold: TODO.
+ """
+ batch_size, len_time, hidden_size = hidden.size()
+
+ # loop varss
+ integrate = torch.zeros([batch_size], device=hidden.device)
+ frame = torch.zeros([batch_size, hidden_size], device=hidden.device)
+ # intermediate vars along time
+ list_fires = []
+ list_frames = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+ distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(
+ fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate
+ )
+ cur = torch.where(fire_place, distribution_completion, alpha)
+ remainds = alpha - cur
+
+ frame += cur[:, None] * hidden[:, t, :]
+ list_frames.append(frame)
+ frame = torch.where(
+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
+ )
+
+ fires = torch.stack(list_fires, 1)
+ frames = torch.stack(list_frames, 1)
+ list_ls = []
+ len_labels = torch.round(alphas.sum(-1)).int()
+ max_label_len = len_labels.max()
+ for b in range(batch_size):
+ fire = fires[b, :]
+ l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze(-1))
+ pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)
+ list_ls.append(torch.cat([l, pad_l], 0))
+ return torch.stack(list_ls, 0), fires
+
+
+def cif_wo_hidden(alphas, threshold):
+ """Cif wo hidden.
+
+ Args:
+ alphas: TODO.
+ threshold: TODO.
+ """
+ batch_size, len_time = alphas.size()
+
+ # loop varss
+ integrate = torch.zeros([batch_size], device=alphas.device)
+ # intermediate vars along time
+ list_fires = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(
+ fire_place,
+ integrate - torch.ones([batch_size], device=alphas.device) * threshold,
+ integrate,
+ )
+
+ fires = torch.stack(list_fires, 1)
+ return fires
+
+
[email protected]("predictor_classes", "CifPredictorV3")
+class CifPredictorV3(torch.nn.Module):
+ def __init__(
+ self,
+ idim,
+ l_order,
+ r_order,
+ threshold=1.0,
+ dropout=0.1,
+ smooth_factor=1.0,
+ noise_threshold=0,
+ tail_threshold=0.0,
+ tf2torch_tensor_name_prefix_torch="predictor",
+ tf2torch_tensor_name_prefix_tf="seq2seq/cif",
+ smooth_factor2=1.0,
+ noise_threshold2=0,
+ upsample_times=5,
+ upsample_type="cnn",
+ use_cif1_cnn=True,
+ tail_mask=True,
+ ):
+ """Initialize CifPredictorV3.
+
+ Args:
+ idim: TODO.
+ l_order: TODO.
+ r_order: TODO.
+ threshold: TODO.
+ dropout: TODO.
+ smooth_factor: TODO.
+ noise_threshold: TODO.
+ tail_threshold: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ smooth_factor2: TODO.
+ noise_threshold2: TODO.
+ upsample_times: TODO.
+ upsample_type: TODO.
+ use_cif1_cnn: TODO.
+ tail_mask: TODO.
+ """
+ super(CifPredictorV3, self).__init__()
+
+ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)
+ self.cif_output = torch.nn.Linear(idim, 1)
+ self.dropout = torch.nn.Dropout(p=dropout)
+ self.threshold = threshold
+ self.smooth_factor = smooth_factor
+ self.noise_threshold = noise_threshold
+ self.tail_threshold = tail_threshold
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
+
+ self.upsample_times = upsample_times
+ self.upsample_type = upsample_type
+ self.use_cif1_cnn = use_cif1_cnn
+ if self.upsample_type == "cnn":
+ self.upsample_cnn = torch.nn.ConvTranspose1d(
+ idim, idim, self.upsample_times, self.upsample_times
+ )
+ self.cif_output2 = torch.nn.Linear(idim, 1)
+ elif self.upsample_type == "cnn_blstm":
+ self.upsample_cnn = torch.nn.ConvTranspose1d(
+ idim, idim, self.upsample_times, self.upsample_times
+ )
+ self.blstm = torch.nn.LSTM(
+ idim, idim, 1, bias=True, batch_first=True, dropout=0.0, bidirectional=True
+ )
+ self.cif_output2 = torch.nn.Linear(idim * 2, 1)
+ elif self.upsample_type == "cnn_attn":
+ self.upsample_cnn = torch.nn.ConvTranspose1d(
+ idim, idim, self.upsample_times, self.upsample_times
+ )
+ from funasr.models.transformer.encoder import EncoderLayer as TransformerEncoderLayer
+ from funasr.models.transformer.attention import MultiHeadedAttention
+ from funasr.models.transformer.positionwise_feed_forward import PositionwiseFeedForward
+
+ positionwise_layer_args = (
+ idim,
+ idim * 2,
+ 0.1,
+ )
+ self.self_attn = TransformerEncoderLayer(
+ idim,
+ MultiHeadedAttention(4, idim, 0.1),
+ PositionwiseFeedForward(*positionwise_layer_args),
+ 0.1,
+ True, # normalize_before,
+ False, # concat_after,
+ )
+ self.cif_output2 = torch.nn.Linear(idim, 1)
+ self.smooth_factor2 = smooth_factor2
+ self.noise_threshold2 = noise_threshold2
+
+ def forward(
+ self,
+ hidden,
+ target_label=None,
+ mask=None,
+ ignore_id=-1,
+ mask_chunk_predictor=None,
+ target_label_length=None,
+ ):
+ """Forward pass for training.
+
+ Args:
+ hidden: TODO.
+ target_label: TODO.
+ mask: TODO.
+ ignore_id: TODO.
+ mask_chunk_predictor: TODO.
+ target_label_length: TODO.
+ """
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+
+ # alphas2 is an extra head for timestamp prediction
+ if not self.use_cif1_cnn:
+ _output = context
+ else:
+ _output = output
+ if self.upsample_type == "cnn":
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1, 2)
+ elif self.upsample_type == "cnn_blstm":
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1, 2)
+ output2, (_, _) = self.blstm(output2)
+ elif self.upsample_type == "cnn_attn":
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1, 2)
+ output2, _ = self.self_attn(output2, mask)
+
+ alphas2 = torch.sigmoid(self.cif_output2(output2))
+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
+ # repeat the mask in T demension to match the upsampled length
+ if mask is not None:
+ mask2 = (
+ mask.repeat(1, self.upsample_times, 1)
+ .transpose(-1, -2)
+ .reshape(alphas2.shape[0], -1)
+ )
+ mask2 = mask2.unsqueeze(-1)
+ alphas2 = alphas2 * mask2
+ alphas2 = alphas2.squeeze(-1)
+ token_num2 = alphas2.sum(-1)
+
+ output = output.transpose(1, 2)
+
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+ if mask is not None:
+ mask = mask.transpose(-1, -2).float()
+ alphas = alphas * mask
+ if mask_chunk_predictor is not None:
+ alphas = alphas * mask_chunk_predictor
+ alphas = alphas.squeeze(-1)
+ mask = mask.squeeze(-1)
+ if target_label_length is not None:
+ target_length = target_label_length
+ elif target_label is not None:
+ target_length = (target_label != ignore_id).float().sum(-1)
+ else:
+ target_length = None
+ token_num = alphas.sum(-1)
+
+ if target_length is not None:
+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
+ elif self.tail_threshold > 0.0:
+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, token_num, mask=mask)
+
+ acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)
+ if target_length is None and self.tail_threshold > 0.0:
+ token_num_int = torch.max(token_num).type(torch.int32).item()
+ acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
+ return acoustic_embeds, token_num, alphas, cif_peak, token_num2
+
+ def get_upsample_timestamp(self, hidden, mask=None, token_num=None):
+ """Get upsample timestamp.
+
+ Args:
+ hidden: TODO.
+ mask: TODO.
+ token_num: TODO.
+ """
+ h = hidden
+ b = hidden.shape[0]
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+
+ # alphas2 is an extra head for timestamp prediction
+ if not self.use_cif1_cnn:
+ _output = context
+ else:
+ _output = output
+ if self.upsample_type == "cnn":
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1, 2)
+ elif self.upsample_type == "cnn_blstm":
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1, 2)
+ output2, (_, _) = self.blstm(output2)
+ elif self.upsample_type == "cnn_attn":
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1, 2)
+ output2, _ = self.self_attn(output2, mask)
+ alphas2 = torch.sigmoid(self.cif_output2(output2))
+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
+ # repeat the mask in T demension to match the upsampled length
+ if mask is not None:
+ mask2 = (
+ mask.repeat(1, self.upsample_times, 1)
+ .transpose(-1, -2)
+ .reshape(alphas2.shape[0], -1)
+ )
+ mask2 = mask2.unsqueeze(-1)
+ alphas2 = alphas2 * mask2
+ alphas2 = alphas2.squeeze(-1)
+ _token_num = alphas2.sum(-1)
+ if token_num is not None:
+ alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))
+ # re-downsample
+ ds_alphas = alphas2.reshape(b, -1, self.upsample_times).sum(-1)
+ ds_cif_peak = cif_wo_hidden(ds_alphas, self.threshold - 1e-4)
+ # upsampled alphas and cif_peak
+ us_alphas = alphas2
+ us_cif_peak = cif_wo_hidden(us_alphas, self.threshold - 1e-4)
+ return ds_alphas, ds_cif_peak, us_alphas, us_cif_peak
+
+ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
+ """Tail process fn.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ token_num: TODO.
+ mask: TODO.
+ """
+ b, t, d = hidden.size()
+ tail_threshold = self.tail_threshold
+ if mask is not None:
+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
+ ones_t = torch.ones_like(zeros_t)
+ mask_1 = torch.cat([mask, zeros_t], dim=1)
+ mask_2 = torch.cat([ones_t, mask], dim=1)
+ mask = mask_2 - mask_1
+ tail_threshold = mask * tail_threshold
+ alphas = torch.cat([alphas, zeros_t], dim=1)
+ alphas = torch.add(alphas, tail_threshold)
+ else:
+ tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)
+ tail_threshold = torch.reshape(tail_threshold, (1, 1))
+ alphas = torch.cat([alphas, tail_threshold], dim=1)
+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
+ hidden = torch.cat([hidden, zeros], dim=1)
+ token_num = alphas.sum(dim=-1)
+ token_num_floor = torch.floor(token_num)
+
+ return hidden, alphas, token_num_floor
+
+ def gen_frame_alignments(
+ self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None
+ ):
+ """Gen frame alignments.
+
+ Args:
+ alphas: TODO.
+ encoder_sequence_length: TODO.
+ """
+ batch_size, maximum_length = alphas.size()
+ int_type = torch.int32
+
+ is_training = self.training
+ if is_training:
+ token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)
+ else:
+ token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)
+
+ max_token_num = torch.max(token_num).item()
+
+ alphas_cumsum = torch.cumsum(alphas, dim=1)
+ alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)
+ alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)
+
+ index = torch.ones([batch_size, max_token_num], dtype=int_type)
+ index = torch.cumsum(index, dim=1)
+ index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)
+
+ index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)
+ index_div_bool_zeros = index_div.eq(0)
+ index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1
+ index_div_bool_zeros_count = torch.clamp(
+ index_div_bool_zeros_count, 0, encoder_sequence_length.max()
+ )
+ token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)
+ index_div_bool_zeros_count *= token_num_mask
+
+ index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(
+ 1, 1, maximum_length
+ )
+ ones = torch.ones_like(index_div_bool_zeros_count_tile)
+ zeros = torch.zeros_like(index_div_bool_zeros_count_tile)
+ ones = torch.cumsum(ones, dim=2)
+ cond = index_div_bool_zeros_count_tile == ones
+ index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)
+
+ index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)
+ index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)
+ index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)
+ predictor_mask = (
+ (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))
+ .type(int_type)
+ .to(encoder_sequence_length.device)
+ )
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask
+
+ predictor_alignments = index_div_bool_zeros_count_tile_out
+ predictor_alignments_length = predictor_alignments.sum(-1).type(
+ encoder_sequence_length.dtype
+ )
+ return predictor_alignments.detach(), predictor_alignments_length.detach()
+
+
[email protected]("predictor_classes", "CifPredictorV3Export")
+class CifPredictorV3Export(torch.nn.Module):
+ def __init__(self, model, **kwargs):
+ """Initialize CifPredictorV3Export.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
+ super().__init__()
+
+ self.pad = model.pad
+ self.cif_conv1d = model.cif_conv1d
+ self.cif_output = model.cif_output
+ self.threshold = model.threshold
+ self.smooth_factor = model.smooth_factor
+ self.noise_threshold = model.noise_threshold
+ self.tail_threshold = model.tail_threshold
+
+ self.upsample_times = model.upsample_times
+ self.upsample_cnn = model.upsample_cnn
+ self.blstm = model.blstm
+ self.cif_output2 = model.cif_output2
+ self.smooth_factor2 = model.smooth_factor2
+ self.noise_threshold2 = model.noise_threshold2
+
+ def forward(
+ self,
+ hidden: torch.Tensor,
+ mask: torch.Tensor,
+ ):
+ """Forward pass for training.
+
+ Args:
+ hidden: TODO.
+ mask: TODO.
+ """
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+ output = output.transpose(1, 2)
+
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+ mask = mask.transpose(-1, -2).float()
+ alphas = alphas * mask
+ alphas = alphas.squeeze(-1)
+ token_num = alphas.sum(-1)
+
+ mask = mask.squeeze(-1)
+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)
+ acoustic_embeds, cif_peak = cif_export(hidden, alphas, self.threshold)
+
+ return acoustic_embeds, token_num, alphas, cif_peak
+
+ def get_upsample_timestmap(self, hidden, mask=None, token_num=None):
+ """Get upsample timestmap.
+
+ Args:
+ hidden: TODO.
+ mask: TODO.
+ token_num: TODO.
+ """
+ h = hidden
+ b = hidden.shape[0]
+ context = h.transpose(1, 2)
+
+ # generate alphas2
+ _output = context
+ output2 = self.upsample_cnn(_output)
+ output2 = output2.transpose(1, 2)
+ output2, (_, _) = self.blstm(output2)
+ alphas2 = torch.sigmoid(self.cif_output2(output2))
+ alphas2 = torch.nn.functional.relu(alphas2 * self.smooth_factor2 - self.noise_threshold2)
+
+ mask = (
+ mask.repeat(1, self.upsample_times, 1).transpose(-1, -2).reshape(alphas2.shape[0], -1)
+ )
+ mask = mask.unsqueeze(-1)
+ alphas2 = alphas2 * mask
+ alphas2 = alphas2.squeeze(-1)
+ _token_num = alphas2.sum(-1)
+ alphas2 *= (token_num / _token_num)[:, None].repeat(1, alphas2.size(1))
+ # upsampled alphas and cif_peak
+ us_alphas = alphas2
+ us_cif_peak = cif_wo_hidden_export(us_alphas, self.threshold - 1e-4)
+ return us_alphas, us_cif_peak
+
+ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
+ """Tail process fn.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ token_num: TODO.
+ mask: TODO.
+ """
+ b, t, d = hidden.size()
+ tail_threshold = self.tail_threshold
+
+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
+ ones_t = torch.ones_like(zeros_t)
+
+ mask_1 = torch.cat([mask, zeros_t], dim=1)
+ mask_2 = torch.cat([ones_t, mask], dim=1)
+ mask = mask_2 - mask_1
+ tail_threshold = mask * tail_threshold
+ alphas = torch.cat([alphas, zeros_t], dim=1)
+ alphas = torch.add(alphas, tail_threshold)
+
+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
+ hidden = torch.cat([hidden, zeros], dim=1)
+ token_num = alphas.sum(dim=-1)
+ token_num_floor = torch.floor(token_num)
+
+ return hidden, alphas, token_num_floor
+
+
+def cif_export(hidden, alphas, threshold: float):
+ """Cif export.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ threshold: TODO.
+ """
+ batch_size, len_time, hidden_size = hidden.size()
+ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
+
+ # loop varss
+ integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)
+ frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)
+ # intermediate vars along time
+ list_fires = []
+ list_frames = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+ distribution_completion = (
+ torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate
+ )
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(
+ fire_place,
+ integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),
+ integrate,
+ )
+ cur = torch.where(fire_place, distribution_completion, alpha)
+ remainds = alpha - cur
+
+ frame += cur[:, None] * hidden[:, t, :]
+ list_frames.append(frame)
+ frame = torch.where(
+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
+ )
+
+ fires = torch.stack(list_fires, 1)
+ frames = torch.stack(list_frames, 1)
+
+ fire_idxs = fires >= threshold
+ frame_fires = torch.zeros_like(hidden)
+ max_label_len = frames[0, fire_idxs[0]].size(0)
+ for b in range(batch_size):
+ frame_fire = frames[b, fire_idxs[b]]
+ frame_len = frame_fire.size(0)
+ frame_fires[b, :frame_len, :] = frame_fire
+
+ if frame_len >= max_label_len:
+ max_label_len = frame_len
+ frame_fires = frame_fires[:, :max_label_len, :]
+ return frame_fires, fires
+
+
+def cif_wo_hidden_export(alphas, threshold: float):
+ """Cif wo hidden export.
+
+ Args:
+ alphas: TODO.
+ threshold: TODO.
+ """
+ batch_size, len_time = alphas.size()
+
+ # loop varss
+ integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=alphas.device)
+ # intermediate vars along time
+ list_fires = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(
+ fire_place,
+ integrate - torch.ones([batch_size], device=alphas.device) * threshold,
+ integrate,
+ )
+
+ fires = torch.stack(list_fires, 1)
+ return fires
+
diff --git a/funasr/models/bicif_paraformer/export_meta.py b/funasr/models/bicif_paraformer/export_meta.py
index 75171f443..0bf291bde 100644
--- a/funasr/models/bicif_paraformer/export_meta.py
+++ b/funasr/models/bicif_paraformer/export_meta.py
@@ -10,6 +10,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
model.encoder = encoder_class(model.encoder, onnx=is_onnx)
@@ -41,6 +47,12 @@ def export_forward(
speech_lengths: torch.Tensor,
):
# a. To device
+ """Export forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
batch = {"speech": speech, "speech_lengths": speech_lengths}
enc, enc_len = self.encoder(**batch)
@@ -58,20 +70,24 @@ def export_forward(
def export_dummy_inputs(self):
+ """Export dummy inputs."""
speech = torch.randn(2, 30, 560)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
return (speech, speech_lengths)
def export_input_names(self):
+ """Export input names."""
return ["speech", "speech_lengths"]
def export_output_names(self):
+ """Export output names."""
return ["logits", "token_num", "us_alphas", "us_cif_peak"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"speech": {0: "batch_size", 1: "feats_length"},
"speech_lengths": {
@@ -84,4 +100,5 @@ def export_dynamic_axes(self):
def export_name(self):
+ """Export name."""
return "model.onnx"
diff --git a/funasr/models/bicif_paraformer/model.py b/funasr/models/bicif_paraformer/model.py
index 4db9c76cd..4944e5d36 100644
--- a/funasr/models/bicif_paraformer/model.py
+++ b/funasr/models/bicif_paraformer/model.py
@@ -31,6 +31,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -49,6 +54,12 @@ class BiCifParaformer(Paraformer):
*args,
**kwargs,
):
+ """Initialize BiCifParaformer.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def _calc_pre2_loss(
@@ -58,6 +69,14 @@ class BiCifParaformer(Paraformer):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc pre2 loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -80,6 +99,14 @@ class BiCifParaformer(Paraformer):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -124,6 +151,12 @@ class BiCifParaformer(Paraformer):
return loss_att, acc_att, cer_att, wer_att, loss_pre
def calc_predictor(self, encoder_out, encoder_out_lens):
+ """Calc predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -133,6 +166,13 @@ class BiCifParaformer(Paraformer):
return pre_acoustic_embeds, pre_token_length, alphas, pre_peak_index
def calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num):
+ """Calc predictor timestamp.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ token_num: TODO.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -230,6 +270,16 @@ class BiCifParaformer(Paraformer):
):
# init beamsearch
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None
is_use_lm = (
kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
@@ -369,6 +419,11 @@ class BiCifParaformer(Paraformer):
return results, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
if "max_seq_len" not in kwargs:
diff --git a/funasr/models/branchformer/cgmlp.py b/funasr/models/branchformer/cgmlp.py
index 17a10166e..3e7c69014 100644
--- a/funasr/models/branchformer/cgmlp.py
+++ b/funasr/models/branchformer/cgmlp.py
@@ -23,6 +23,15 @@ class ConvolutionalSpatialGatingUnit(torch.nn.Module):
use_linear_after_conv: bool,
gate_activation: str,
):
+ """Initialize ConvolutionalSpatialGatingUnit.
+
+ Args:
+ size: TODO.
+ kernel_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ use_linear_after_conv: TODO.
+ gate_activation: TODO.
+ """
super().__init__()
n_channels = size // 2 # split input channels
@@ -48,6 +57,7 @@ class ConvolutionalSpatialGatingUnit(torch.nn.Module):
self.dropout = torch.nn.Dropout(dropout_rate)
def espnet_initialization_fn(self):
+ """Espnet initialization fn."""
torch.nn.init.normal_(self.conv.weight, std=1e-6)
torch.nn.init.ones_(self.conv.bias)
if self.linear is not None:
@@ -93,6 +103,16 @@ class ConvolutionalGatingMLP(torch.nn.Module):
use_linear_after_conv: bool,
gate_activation: str,
):
+ """Initialize ConvolutionalGatingMLP.
+
+ Args:
+ size: TODO.
+ linear_units: TODO.
+ kernel_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ use_linear_after_conv: TODO.
+ gate_activation: TODO.
+ """
super().__init__()
self.channel_proj1 = torch.nn.Sequential(
@@ -108,6 +128,12 @@ class ConvolutionalGatingMLP(torch.nn.Module):
self.channel_proj2 = torch.nn.Linear(linear_units // 2, size)
def forward(self, x, mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ """
if isinstance(x, tuple):
xs_pad, pos_emb = x
else:
diff --git a/funasr/models/branchformer/encoder.py b/funasr/models/branchformer/encoder.py
index a15a3f282..95aff772d 100644
--- a/funasr/models/branchformer/encoder.py
+++ b/funasr/models/branchformer/encoder.py
@@ -73,6 +73,18 @@ class BranchformerEncoderLayer(torch.nn.Module):
attn_branch_drop_rate: float = 0.0,
stochastic_depth_rate: float = 0.0,
):
+ """Initialize BranchformerEncoderLayer.
+
+ Args:
+ size: TODO.
+ attn: TODO.
+ cgmlp: TODO.
+ dropout_rate: TODO.
+ merge_method: TODO.
+ cgmlp_weight: TODO.
+ attn_branch_drop_rate: TODO.
+ stochastic_depth_rate: TODO.
+ """
super().__init__()
assert (attn is not None) or (cgmlp is not None), "At least one branch should be valid"
@@ -305,6 +317,33 @@ class BranchformerEncoder(nn.Module):
padding_idx: int = -1,
stochastic_depth_rate: Union[float, List[float]] = 0.0,
):
+ """Initialize BranchformerEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ use_attn: TODO.
+ attention_heads: TODO.
+ attention_layer_type: TODO.
+ pos_enc_layer_type: TODO.
+ rel_pos_type: TODO.
+ use_cgmlp: TODO.
+ cgmlp_linear_units: TODO.
+ cgmlp_conv_kernel: TODO.
+ use_linear_after_conv: TODO.
+ gate_activation: TODO.
+ merge_method: TODO.
+ cgmlp_weight: TODO.
+ attn_branch_drop_rate: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ zero_triu: TODO.
+ padding_idx: TODO.
+ stochastic_depth_rate: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -472,6 +511,7 @@ class BranchformerEncoder(nn.Module):
self.after_norm = LayerNorm(output_size)
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
diff --git a/funasr/models/branchformer/fastformer.py b/funasr/models/branchformer/fastformer.py
index ede94dc30..435c486b0 100644
--- a/funasr/models/branchformer/fastformer.py
+++ b/funasr/models/branchformer/fastformer.py
@@ -20,6 +20,13 @@ class FastSelfAttention(torch.nn.Module):
attention_heads,
dropout_rate,
):
+ """Initialize FastSelfAttention.
+
+ Args:
+ size: TODO.
+ attention_heads: TODO.
+ dropout_rate: TODO.
+ """
super().__init__()
if size % attention_heads != 0:
raise ValueError(
@@ -37,9 +44,15 @@ class FastSelfAttention(torch.nn.Module):
self.dropout = torch.nn.Dropout(dropout_rate)
def espnet_initialization_fn(self):
+ """Espnet initialization fn."""
self.apply(self.init_weights)
def init_weights(self, module):
+ """Init weights.
+
+ Args:
+ module: TODO.
+ """
if isinstance(module, torch.nn.Linear):
module.weight.data.normal_(mean=0.0, std=0.02)
if isinstance(module, torch.nn.Linear) and module.bias is not None:
diff --git a/funasr/models/branchformer/model.py b/funasr/models/branchformer/model.py
index f7e4b3623..2dc88b23a 100644
--- a/funasr/models/branchformer/model.py
+++ b/funasr/models/branchformer/model.py
@@ -14,4 +14,10 @@ class Branchformer(Transformer):
**kwargs,
):
+ """Initialize Branchformer.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
diff --git a/funasr/models/campplus/cluster_backend.py b/funasr/models/campplus/cluster_backend.py
index 9e7042153..998ccd679 100644
--- a/funasr/models/campplus/cluster_backend.py
+++ b/funasr/models/campplus/cluster_backend.py
@@ -19,12 +19,25 @@ class SpectralCluster:
"""
def __init__(self, min_num_spks=1, max_num_spks=15, pval=0.022):
+ """Initialize SpectralCluster.
+
+ Args:
+ min_num_spks: TODO.
+ max_num_spks: TODO.
+ pval: TODO.
+ """
self.min_num_spks = min_num_spks
self.max_num_spks = max_num_spks
self.pval = pval
def __call__(self, X, oracle_num=None):
# Similarity matrix computation
+ """Internal: call .
+
+ Args:
+ X: TODO.
+ oracle_num: TODO.
+ """
sim_mat = self.get_sim_mat(X)
# Refining similarity matrix with pval
@@ -46,10 +59,20 @@ class SpectralCluster:
def get_sim_mat(self, X):
# Cosine similarities
+ """Get sim mat.
+
+ Args:
+ X: TODO.
+ """
M = sklearn.metrics.pairwise.cosine_similarity(X, X)
return M
def p_pruning(self, A):
+ """P pruning.
+
+ Args:
+ A: TODO.
+ """
if A.shape[0] * self.pval < 6:
pval = 6.0 / A.shape[0]
else:
@@ -67,6 +90,11 @@ class SpectralCluster:
return A
def get_laplacian(self, M):
+ """Get laplacian.
+
+ Args:
+ M: TODO.
+ """
M[np.diag_indices(M.shape[0])] = 0
D = np.sum(np.abs(M), axis=1)
D = np.diag(D)
@@ -74,6 +102,12 @@ class SpectralCluster:
return L
def get_spec_embs(self, L, k_oracle=None):
+ """Get spec embs.
+
+ Args:
+ L: TODO.
+ k_oracle: TODO.
+ """
lambdas, eig_vecs = scipy.linalg.eigh(L)
if k_oracle is not None:
@@ -88,10 +122,21 @@ class SpectralCluster:
return emb, num_of_spk
def cluster_embs(self, emb, k):
+ """Cluster embs.
+
+ Args:
+ emb: TODO.
+ k: TODO.
+ """
_, labels, _ = k_means(emb, k)
return labels
def getEigenGaps(self, eig_vals):
+ """Geteigengaps.
+
+ Args:
+ eig_vals: TODO.
+ """
eig_vals_gap_list = []
for i in range(len(eig_vals) - 1):
gap = float(eig_vals[i + 1]) - float(eig_vals[i])
@@ -109,6 +154,15 @@ class UmapHdbscan:
def __init__(
self, n_neighbors=20, n_components=60, min_samples=10, min_cluster_size=10, metric="cosine"
):
+ """Initialize UmapHdbscan.
+
+ Args:
+ n_neighbors: TODO.
+ n_components: TODO.
+ min_samples: TODO.
+ min_cluster_size: Size/dimension parameter.
+ metric: TODO.
+ """
self.n_neighbors = n_neighbors
self.n_components = n_components
self.min_samples = min_samples
@@ -116,6 +170,11 @@ class UmapHdbscan:
self.metric = metric
def __call__(self, X):
+ """Internal: call .
+
+ Args:
+ X: TODO.
+ """
import umap.umap_ as umap
umap_X = umap.UMAP(
@@ -140,6 +199,11 @@ class ClusterBackend(torch.nn.Module):
"""
def __init__(self, merge_thr=0.78):
+ """Initialize ClusterBackend.
+
+ Args:
+ merge_thr: TODO.
+ """
super().__init__()
self.model_config = {"merge_thr": merge_thr}
# self.other_config = kwargs
@@ -149,6 +213,12 @@ class ClusterBackend(torch.nn.Module):
def forward(self, X, **params):
# clustering and return the labels
+ """Forward pass for training.
+
+ Args:
+ X: TODO.
+ **params: Additional keyword arguments.
+ """
k = params["oracle_num"] if "oracle_num" in params else None
assert len(X.shape) == 2, "modelscope error: the shape of input should be [N, C]"
if X.shape[0] < 20:
@@ -166,6 +236,13 @@ class ClusterBackend(torch.nn.Module):
def merge_by_cos(self, labels, embs, cos_thr):
# merge the similar speakers by cosine similarity
+ """Merge by cos.
+
+ Args:
+ labels: TODO.
+ embs: TODO.
+ cos_thr: TODO.
+ """
assert cos_thr > 0 and cos_thr <= 1
while True:
spk_num = labels.max() + 1
diff --git a/funasr/models/campplus/components.py b/funasr/models/campplus/components.py
index 263b94334..c026cd4d5 100644
--- a/funasr/models/campplus/components.py
+++ b/funasr/models/campplus/components.py
@@ -13,6 +13,13 @@ class BasicResBlock(torch.nn.Module):
expansion = 1
def __init__(self, in_planes, planes, stride=1):
+ """Initialize BasicResBlock.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ """
super(BasicResBlock, self).__init__()
self.conv1 = torch.nn.Conv2d(
in_planes, planes, kernel_size=3, stride=(stride, 1), padding=1, bias=False
@@ -35,6 +42,11 @@ class BasicResBlock(torch.nn.Module):
)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
out = F.relu(self.bn1(self.conv1(x)))
out = self.bn2(self.conv2(out))
out += self.shortcut(x)
@@ -44,6 +56,14 @@ class BasicResBlock(torch.nn.Module):
class FCM(torch.nn.Module):
def __init__(self, block=BasicResBlock, num_blocks=[2, 2], m_channels=32, feat_dim=80):
+ """Initialize FCM.
+
+ Args:
+ block: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ """
super(FCM, self).__init__()
self.in_planes = m_channels
self.conv1 = torch.nn.Conv2d(1, m_channels, kernel_size=3, stride=1, padding=1, bias=False)
@@ -59,6 +79,14 @@ class FCM(torch.nn.Module):
self.out_channels = m_channels * (feat_dim // 8)
def _make_layer(self, block, planes, num_blocks, stride):
+ """Internal: make layer.
+
+ Args:
+ block: TODO.
+ planes: TODO.
+ num_blocks: TODO.
+ stride: TODO.
+ """
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
@@ -67,6 +95,11 @@ class FCM(torch.nn.Module):
return torch.nn.Sequential(*layers)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = x.unsqueeze(1)
out = F.relu(self.bn1(self.conv1(x)))
out = self.layer1(out)
@@ -79,6 +112,12 @@ class FCM(torch.nn.Module):
def get_nonlinear(config_str, channels):
+ """Get nonlinear.
+
+ Args:
+ config_str: TODO.
+ channels: TODO.
+ """
nonlinear = torch.nn.Sequential()
for name in config_str.split("-"):
if name == "relu":
@@ -95,6 +134,15 @@ def get_nonlinear(config_str, channels):
def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2):
+ """Statistics pooling.
+
+ Args:
+ x: TODO.
+ dim: TODO.
+ keepdim: TODO.
+ unbiased: TODO.
+ eps: TODO.
+ """
mean = x.mean(dim=dim)
std = x.std(dim=dim, unbiased=unbiased)
stats = torch.cat([mean, std], dim=-1)
@@ -105,6 +153,11 @@ def statistics_pooling(x, dim=-1, keepdim=False, unbiased=True, eps=1e-2):
class StatsPool(torch.nn.Module):
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return statistics_pooling(x)
@@ -120,6 +173,18 @@ class TDNNLayer(torch.nn.Module):
bias=False,
config_str="batchnorm-relu",
):
+ """Initialize TDNNLayer.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ padding: TODO.
+ dilation: TODO.
+ bias: TODO.
+ config_str: TODO.
+ """
super(TDNNLayer, self).__init__()
if padding < 0:
assert (
@@ -138,6 +203,11 @@ class TDNNLayer(torch.nn.Module):
self.nonlinear = get_nonlinear(config_str, out_channels)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = self.linear(x)
x = self.nonlinear(x)
return x
@@ -147,6 +217,18 @@ class CAMLayer(torch.nn.Module):
def __init__(
self, bn_channels, out_channels, kernel_size, stride, padding, dilation, bias, reduction=2
):
+ """Initialize CAMLayer.
+
+ Args:
+ bn_channels: TODO.
+ out_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ padding: TODO.
+ dilation: TODO.
+ bias: TODO.
+ reduction: TODO.
+ """
super(CAMLayer, self).__init__()
self.linear_local = torch.nn.Conv1d(
bn_channels,
@@ -163,6 +245,11 @@ class CAMLayer(torch.nn.Module):
self.sigmoid = torch.nn.Sigmoid()
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
y = self.linear_local(x)
context = x.mean(-1, keepdim=True) + self.seg_pooling(x)
context = self.relu(self.linear1(context))
@@ -170,6 +257,13 @@ class CAMLayer(torch.nn.Module):
return y * m
def seg_pooling(self, x, seg_len=100, stype="avg"):
+ """Seg pooling.
+
+ Args:
+ x: TODO.
+ seg_len: TODO.
+ stype: TODO.
+ """
if stype == "avg":
seg = F.avg_pool1d(x, kernel_size=seg_len, stride=seg_len, ceil_mode=True)
elif stype == "max":
@@ -195,6 +289,19 @@ class CAMDenseTDNNLayer(torch.nn.Module):
config_str="batchnorm-relu",
memory_efficient=False,
):
+ """Initialize CAMDenseTDNNLayer.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ bn_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ dilation: TODO.
+ bias: TODO.
+ config_str: TODO.
+ memory_efficient: TODO.
+ """
super(CAMDenseTDNNLayer, self).__init__()
assert kernel_size % 2 == 1, "Expect equal paddings, but got even kernel size ({})".format(
kernel_size
@@ -215,9 +322,19 @@ class CAMDenseTDNNLayer(torch.nn.Module):
)
def bn_function(self, x):
+ """Bn function.
+
+ Args:
+ x: TODO.
+ """
return self.linear1(self.nonlinear1(x))
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if self.training and self.memory_efficient:
x = cp.checkpoint(self.bn_function, x)
else:
@@ -240,6 +357,20 @@ class CAMDenseTDNNBlock(torch.nn.ModuleList):
config_str="batchnorm-relu",
memory_efficient=False,
):
+ """Initialize CAMDenseTDNNBlock.
+
+ Args:
+ num_layers: TODO.
+ in_channels: TODO.
+ out_channels: TODO.
+ bn_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ dilation: TODO.
+ bias: TODO.
+ config_str: TODO.
+ memory_efficient: TODO.
+ """
super(CAMDenseTDNNBlock, self).__init__()
for i in range(num_layers):
layer = CAMDenseTDNNLayer(
@@ -256,6 +387,11 @@ class CAMDenseTDNNBlock(torch.nn.ModuleList):
self.add_module("tdnnd%d" % (i + 1), layer)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
for layer in self:
x = torch.cat([x, layer(x)], dim=1)
return x
@@ -263,11 +399,24 @@ class CAMDenseTDNNBlock(torch.nn.ModuleList):
class TransitLayer(torch.nn.Module):
def __init__(self, in_channels, out_channels, bias=True, config_str="batchnorm-relu"):
+ """Initialize TransitLayer.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ bias: TODO.
+ config_str: TODO.
+ """
super(TransitLayer, self).__init__()
self.nonlinear = get_nonlinear(config_str, in_channels)
self.linear = torch.nn.Conv1d(in_channels, out_channels, 1, bias=bias)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = self.nonlinear(x)
x = self.linear(x)
return x
@@ -275,11 +424,24 @@ class TransitLayer(torch.nn.Module):
class DenseLayer(torch.nn.Module):
def __init__(self, in_channels, out_channels, bias=False, config_str="batchnorm-relu"):
+ """Initialize DenseLayer.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ bias: TODO.
+ config_str: TODO.
+ """
super(DenseLayer, self).__init__()
self.linear = torch.nn.Conv1d(in_channels, out_channels, 1, bias=bias)
self.nonlinear = get_nonlinear(config_str, out_channels)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if len(x.shape) == 2:
x = self.linear(x.unsqueeze(dim=-1)).squeeze(dim=-1)
else:
diff --git a/funasr/models/campplus/model.py b/funasr/models/campplus/model.py
index 6730ad589..12b739137 100644
--- a/funasr/models/campplus/model.py
+++ b/funasr/models/campplus/model.py
@@ -31,6 +31,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -56,6 +61,19 @@ class CAMPPlus(torch.nn.Module):
output_level="segment",
**kwargs,
):
+ """Initialize CAMPPlus.
+
+ Args:
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ growth_rate: TODO.
+ bn_size: Size/dimension parameter.
+ init_channels: TODO.
+ config_str: TODO.
+ memory_efficient: TODO.
+ output_level: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.head = FCM(feat_dim=feat_dim)
diff --git a/funasr/models/campplus/utils.py b/funasr/models/campplus/utils.py
index 4addd0a8a..e9f5eb4a1 100644
--- a/funasr/models/campplus/utils.py
+++ b/funasr/models/campplus/utils.py
@@ -22,6 +22,11 @@ from funasr.models.transformer.utils.nets_utils import pad_list
def check_audio_list(audio: list):
+ """Check audio list.
+
+ Args:
+ audio: TODO.
+ """
audio_dur = 0
for i in range(len(audio)):
seg = audio[i]
@@ -38,6 +43,11 @@ def check_audio_list(audio: list):
def sv_preprocess(inputs: Union[np.ndarray, list]):
+ """Sv preprocess.
+
+ Args:
+ inputs: TODO.
+ """
output = []
for i in range(len(inputs)):
if isinstance(inputs[i], str):
@@ -64,12 +74,23 @@ def sv_preprocess(inputs: Union[np.ndarray, list]):
def sv_chunk(vad_segments: list, fs=16000) -> list:
+ """Sv chunk.
+
+ Args:
+ vad_segments: TODO.
+ fs: TODO.
+ """
config = {
"seg_dur": 1.5,
"seg_shift": 0.75,
}
def seg_chunk(seg_data):
+ """Seg chunk.
+
+ Args:
+ seg_data: TODO.
+ """
seg_st = seg_data[0]
data = seg_data[2]
chunk_len = int(config["seg_dur"] * fs)
@@ -96,6 +117,11 @@ def sv_chunk(vad_segments: list, fs=16000) -> list:
def extract_feature(audio):
+ """Extract feature.
+
+ Args:
+ audio: TODO.
+ """
features = []
feature_times = []
feature_lengths = []
@@ -114,6 +140,14 @@ def extract_feature(audio):
def postprocess(
segments: list, vad_segments: list, labels: np.ndarray, embeddings: np.ndarray
) -> list:
+ """Postprocess.
+
+ Args:
+ segments: TODO.
+ vad_segments: TODO.
+ labels: TODO.
+ embeddings: TODO.
+ """
assert len(segments) == len(labels)
labels = correct_labels(labels)
distribute_res = []
@@ -130,6 +164,12 @@ def postprocess(
spk_embs = np.stack(spk_embs)
def is_overlapped(t1, t2):
+ """Is overlapped.
+
+ Args:
+ t1: TODO.
+ t2: TODO.
+ """
if t1 > t2 + 1e-4:
return True
return False
@@ -148,6 +188,11 @@ def postprocess(
def correct_labels(labels):
+ """Correct labels.
+
+ Args:
+ labels: TODO.
+ """
labels_id = 0
id2id = {}
new_labels = []
@@ -160,6 +205,11 @@ def correct_labels(labels):
def merge_seque(distribute_res):
+ """Merge seque.
+
+ Args:
+ distribute_res: TODO.
+ """
res = [distribute_res[0]]
for i in range(1, len(distribute_res)):
if distribute_res[i][2] != res[-1][2] or distribute_res[i][0] > res[-1][1]:
@@ -171,6 +221,12 @@ def merge_seque(distribute_res):
def smooth(res, mindur=0.7):
# if only one segment, return directly
+ """Smooth.
+
+ Args:
+ res: TODO.
+ mindur: TODO.
+ """
if len(res) < 2:
return res
# short segments are assigned to nearest speakers.
@@ -193,6 +249,12 @@ def smooth(res, mindur=0.7):
def distribute_spk(sentence_list, sd_time_list):
+ """Distribute spk.
+
+ Args:
+ sentence_list: TODO.
+ sd_time_list: TODO.
+ """
sd_time_list = [(spk_st * 1000, spk_ed * 1000, spk) for spk_st, spk_ed, spk in sd_time_list]
for d in sentence_list:
sentence_start = d['start']
@@ -220,18 +282,41 @@ class Storage(metaclass=ABCMeta):
@abstractmethod
def read(self, filepath: str):
+ """Read.
+
+ Args:
+ filepath: TODO.
+ """
pass
@abstractmethod
def read_text(self, filepath: str):
+ """Read text.
+
+ Args:
+ filepath: TODO.
+ """
pass
@abstractmethod
def write(self, obj: bytes, filepath: Union[str, Path]) -> None:
+ """Write.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ """
pass
@abstractmethod
def write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:
+ """Write text.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ encoding: TODO.
+ """
pass
@@ -315,11 +400,21 @@ class HTTPStorage(Storage):
def read(self, url):
# TODO @wenmeng.zwm add progress bar if file is too large
+ """Read.
+
+ Args:
+ url: TODO.
+ """
r = requests.get(url)
r.raise_for_status()
return r.content
def read_text(self, url):
+ """Read text.
+
+ Args:
+ url: TODO.
+ """
r = requests.get(url)
r.raise_for_status()
return r.text
@@ -351,9 +446,22 @@ class HTTPStorage(Storage):
os.remove(f.name)
def write(self, obj: bytes, url: Union[str, Path]) -> None:
+ """Write.
+
+ Args:
+ obj: TODO.
+ url: TODO.
+ """
raise NotImplementedError("write is not supported by HTTP Storage")
def write_text(self, obj: str, url: Union[str, Path], encoding: str = "utf-8") -> None:
+ """Write text.
+
+ Args:
+ obj: TODO.
+ url: TODO.
+ encoding: TODO.
+ """
raise NotImplementedError("write_text is not supported by HTTP Storage")
@@ -362,12 +470,28 @@ class OSSStorage(Storage):
def __init__(self, oss_config_file=None):
# read from config file or env var
+ """Initialize OSSStorage.
+
+ Args:
+ oss_config_file: TODO.
+ """
raise NotImplementedError("OSSStorage.__init__ to be implemented in the future")
def read(self, filepath):
+ """Read.
+
+ Args:
+ filepath: TODO.
+ """
raise NotImplementedError("OSSStorage.read to be implemented in the future")
def read_text(self, filepath, encoding="utf-8"):
+ """Read text.
+
+ Args:
+ filepath: TODO.
+ encoding: TODO.
+ """
raise NotImplementedError("OSSStorage.read_text to be implemented in the future")
@contextlib.contextmanager
@@ -397,9 +521,22 @@ class OSSStorage(Storage):
os.remove(f.name)
def write(self, obj: bytes, filepath: Union[str, Path]) -> None:
+ """Write.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ """
raise NotImplementedError("OSSStorage.write to be implemented in the future")
def write_text(self, obj: str, filepath: Union[str, Path], encoding: str = "utf-8") -> None:
+ """Write text.
+
+ Args:
+ obj: TODO.
+ filepath: TODO.
+ encoding: TODO.
+ """
raise NotImplementedError("OSSStorage.write_text to be implemented in the future")
@@ -416,6 +553,11 @@ class File(object):
@staticmethod
def _get_storage(uri):
+ """Internal: get storage.
+
+ Args:
+ uri: TODO.
+ """
assert isinstance(uri, str), f"uri should be str type, but got {type(uri)}"
if "://" not in uri:
diff --git a/funasr/models/conformer/encoder.py b/funasr/models/conformer/encoder.py
index 7c939b47c..7b1e61946 100644
--- a/funasr/models/conformer/encoder.py
+++ b/funasr/models/conformer/encoder.py
@@ -347,6 +347,35 @@ class ConformerEncoder(nn.Module):
interctc_use_conditioning: bool = False,
stochastic_depth_rate: Union[float, List[float]] = 0.0,
):
+ """Initialize ConformerEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ macaron_style: TODO.
+ rel_pos_type: TODO.
+ pos_enc_layer_type: TODO.
+ selfattention_layer_type: TODO.
+ activation_type: TODO.
+ use_cnn_module: TODO.
+ zero_triu: TODO.
+ cnn_module_kernel: TODO.
+ padding_idx: TODO.
+ interctc_layer_idx: TODO.
+ interctc_use_conditioning: TODO.
+ stochastic_depth_rate: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -524,6 +553,7 @@ class ConformerEncoder(nn.Module):
self.conditioning_layer = None
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
@@ -1004,6 +1034,7 @@ class ConformerChunkEncoder(torch.nn.Module):
self.time_reduction_factor = time_reduction_factor
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def get_encoder_input_raw_size(self, size: int, hop_length: int) -> int:
@@ -1181,6 +1212,15 @@ class ConformerChunkEncoder(torch.nn.Module):
left_context: int = 32,
right_context: int = 0,
) -> torch.Tensor:
+ """Simu chunk forward.
+
+ Args:
+ x: TODO.
+ x_len: TODO.
+ chunk_size: Size/dimension parameter.
+ left_context: TODO.
+ right_context: TODO.
+ """
short_status, limit_size = check_short_utt(self.embed.subsampling_factor, x.size(1))
if short_status:
diff --git a/funasr/models/conformer/model.py b/funasr/models/conformer/model.py
index d7e77b155..71710e2da 100644
--- a/funasr/models/conformer/model.py
+++ b/funasr/models/conformer/model.py
@@ -16,4 +16,10 @@ class Conformer(Transformer):
**kwargs,
):
+ """Initialize Conformer.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
diff --git a/funasr/models/conformer_rwkv/decoder.py b/funasr/models/conformer_rwkv/decoder.py
index 4c4104966..24805e502 100644
--- a/funasr/models/conformer_rwkv/decoder.py
+++ b/funasr/models/conformer_rwkv/decoder.py
@@ -31,6 +31,11 @@ from funasr.register import tables
class LayerNorm(nn.LayerNorm):
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return super().forward(x.float()).type(x.dtype)
@@ -216,6 +221,18 @@ class BaseTransformerDecoder(nn.Module, BatchScorerInterface):
pos_enc_class=PositionalEncoding,
normalize_before: bool = True,
):
+ """Initialize BaseTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ """
super().__init__()
attention_dim = encoder_output_size
@@ -395,6 +412,25 @@ class TransformerRWKVDecoder(BaseTransformerDecoder):
concat_after: bool = False,
**kwargs,
):
+ """Initialize TransformerRWKVDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
diff --git a/funasr/models/conformer_rwkv/model.py b/funasr/models/conformer_rwkv/model.py
index d7e77b155..71710e2da 100644
--- a/funasr/models/conformer_rwkv/model.py
+++ b/funasr/models/conformer_rwkv/model.py
@@ -16,4 +16,10 @@ class Conformer(Transformer):
**kwargs,
):
+ """Initialize Conformer.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
diff --git a/funasr/models/contextual_paraformer/decoder.py b/funasr/models/contextual_paraformer/decoder.py
index ba2ce9ade..74d378a7d 100644
--- a/funasr/models/contextual_paraformer/decoder.py
+++ b/funasr/models/contextual_paraformer/decoder.py
@@ -59,6 +59,15 @@ class ContextualDecoderLayer(torch.nn.Module):
cache=None,
):
# tgt = self.dropout(tgt)
+ """Forward pass for training.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
if isinstance(tgt, Tuple):
tgt, _ = tgt
residual = tgt
@@ -103,6 +112,15 @@ class ContextualBiasDecoder(torch.nn.Module):
self.normalize_before = normalize_before
def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Forward pass for training.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
x = tgt
if self.src_attn is not None:
if self.normalize_before:
@@ -139,6 +157,27 @@ class ContextualParaformerDecoder(ParaformerSANMDecoder):
kernel_size: int = 21,
sanm_shfit: int = 0,
):
+ """Initialize ContextualParaformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ att_layer_num: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -322,6 +361,15 @@ class ContextualParaformerDecoderExport(torch.nn.Module):
onnx: bool = True,
**kwargs,
):
+ """Initialize ContextualParaformerDecoderExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ model_name: TODO.
+ onnx: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
from funasr.utils.torch_function import sequence_mask
@@ -386,6 +434,11 @@ class ContextualParaformerDecoderExport(torch.nn.Module):
self.dropout = self.model.dropout
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -404,6 +457,15 @@ class ContextualParaformerDecoderExport(torch.nn.Module):
bias_embed: torch.Tensor,
):
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ bias_embed: TODO.
+ """
tgt = ys_in_pad
tgt_mask = self.make_pad_mask(ys_in_lens)
tgt_mask, _ = self.prepare_mask(tgt_mask)
diff --git a/funasr/models/contextual_paraformer/export_meta.py b/funasr/models/contextual_paraformer/export_meta.py
index 9d3a63b8a..325e768e8 100644
--- a/funasr/models/contextual_paraformer/export_meta.py
+++ b/funasr/models/contextual_paraformer/export_meta.py
@@ -12,12 +12,19 @@ from funasr.models.seaco_paraformer.export_meta import ContextualEmbedderExport
class ContextualEmbedderExport2(ContextualEmbedderExport):
def __init__(self, model, **kwargs):
+ """Initialize ContextualEmbedderExport2.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(model)
self.embedding = model.bias_embed
model.bias_encoder.batch_first = False
self.bias_encoder = model.bias_encoder
def export_dummy_inputs(self):
+ """Export dummy inputs."""
hotword = torch.tensor(
[
[10, 11, 12, 13, 14, 10, 11, 12, 13, 14],
@@ -34,6 +41,12 @@ class ContextualEmbedderExport2(ContextualEmbedderExport):
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
@@ -87,6 +100,13 @@ def export_backbone_forward(
speech_lengths: torch.Tensor,
bias_embed: torch.Tensor,
):
+ """Export backbone forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ bias_embed: TODO.
+ """
batch = {"speech": speech, "speech_lengths": speech_lengths}
enc, enc_len = self.encoder(**batch)
@@ -101,6 +121,7 @@ def export_backbone_forward(
def export_backbone_dummy_inputs(self):
+ """Export backbone dummy inputs."""
speech = torch.randn(2, 30, self.feats_dim)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
bias_embed = torch.randn(2, 1, 512)
@@ -108,14 +129,17 @@ def export_backbone_dummy_inputs(self):
def export_backbone_input_names(self):
+ """Export backbone input names."""
return ["speech", "speech_lengths", "bias_embed"]
def export_backbone_output_names(self):
+ """Export backbone output names."""
return ["logits", "token_num"]
def export_backbone_dynamic_axes(self):
+ """Export backbone dynamic axes."""
return {
"speech": {0: "batch_size", 1: "feats_length"},
"speech_lengths": {
@@ -127,4 +151,5 @@ def export_backbone_dynamic_axes(self):
def export_backbone_name(self):
+ """Export backbone name."""
return "model.onnx"
diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.py
index fd8822025..3f80be7f9 100644
--- a/funasr/models/contextual_paraformer/model.py
+++ b/funasr/models/contextual_paraformer/model.py
@@ -34,6 +34,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -50,6 +55,12 @@ class ContextualParaformer(Paraformer):
*args,
**kwargs,
):
+ """Initialize ContextualParaformer.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
self.target_buffer_length = kwargs.get("target_buffer_length", -1)
@@ -165,6 +176,16 @@ class ContextualParaformer(Paraformer):
hotword_pad: torch.Tensor,
hotword_lengths: torch.Tensor,
):
+ """Internal: calc att clas loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ hotword_pad: TODO.
+ hotword_lengths: Lengths of hotword.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -251,6 +272,16 @@ class ContextualParaformer(Paraformer):
pre_acoustic_embeds,
contextual_info,
):
+ """Sampler.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ pre_acoustic_embeds: TODO.
+ contextual_info: TODO.
+ """
tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(
ys_pad.device
)
@@ -304,6 +335,16 @@ class ContextualParaformer(Paraformer):
hw_list=None,
clas_scale=1.0,
):
+ """Cal decoder with predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ sematic_embeds: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ hw_list: TODO.
+ clas_scale: TODO.
+ """
if hw_list is None:
hw_list = [torch.Tensor([1]).long().to(encoder_out.device)] # empty hotword list
hw_list_pad = pad_list(hw_list, 0)
@@ -352,6 +393,16 @@ class ContextualParaformer(Paraformer):
):
# init beamsearch
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None
is_use_lm = (
kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
@@ -479,7 +530,19 @@ class ContextualParaformer(Paraformer):
return results, meta_data
def generate_hotwords_list(self, hotword_list_or_file, tokenizer=None, frontend=None):
+ """Generate hotwords list.
+
+ Args:
+ hotword_list_or_file: TODO.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ """
def load_seg_dict(seg_dict_file):
+ """Load seg dict.
+
+ Args:
+ seg_dict_file: TODO.
+ """
seg_dict = {}
assert isinstance(seg_dict_file, str)
with open(seg_dict_file, "r", encoding="utf8") as f:
@@ -492,6 +555,12 @@ class ContextualParaformer(Paraformer):
return seg_dict
def seg_tokenize(txt, seg_dict):
+ """Seg tokenize.
+
+ Args:
+ txt: TODO.
+ seg_dict: TODO.
+ """
pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$")
out_txt = ""
for word in txt:
@@ -589,6 +658,11 @@ class ContextualParaformer(Paraformer):
self,
**kwargs,
):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
if "max_seq_len" not in kwargs:
kwargs["max_seq_len"] = 512
from .export_meta import export_rebuild_model
diff --git a/funasr/models/ct_transformer/export_meta.py b/funasr/models/ct_transformer/export_meta.py
index c222008ce..cbba5c597 100644
--- a/funasr/models/ct_transformer/export_meta.py
+++ b/funasr/models/ct_transformer/export_meta.py
@@ -10,6 +10,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
model.encoder = encoder_class(model.encoder, onnx=is_onnx)
@@ -39,6 +45,7 @@ def export_forward(self, inputs: torch.Tensor, text_lengths: torch.Tensor):
def export_dummy_inputs(self):
+ """Export dummy inputs."""
length = 120
text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)).type(torch.int32)
text_lengths = torch.tensor([length - 20, length], dtype=torch.int32)
@@ -46,14 +53,17 @@ def export_dummy_inputs(self):
def export_input_names(self):
+ """Export input names."""
return ["inputs", "text_lengths"]
def export_output_names(self):
+ """Export output names."""
return ["logits"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"inputs": {0: "batch_size", 1: "feats_length"},
"text_lengths": {
@@ -64,4 +74,5 @@ def export_dynamic_axes(self):
def export_name(self):
+ """Export name."""
return "model.onnx"
diff --git a/funasr/models/ct_transformer/model.py b/funasr/models/ct_transformer/model.py
index ae2d318bc..ab70258a0 100644
--- a/funasr/models/ct_transformer/model.py
+++ b/funasr/models/ct_transformer/model.py
@@ -28,6 +28,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -65,6 +70,23 @@ class CTTransformer(torch.nn.Module):
sentence_end_id: int = 3,
**kwargs,
):
+ """Initialize CTTransformer.
+
+ Args:
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ vocab_size: Size/dimension parameter.
+ punc_list: TODO.
+ punc_weight: TODO.
+ embed_unit: TODO.
+ att_unit: TODO.
+ dropout_rate: TODO.
+ ignore_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ sentence_end_id: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
punc_size = len(punc_list)
@@ -103,6 +125,7 @@ class CTTransformer(torch.nn.Module):
return y, None
def with_vad(self):
+ """With vad."""
return False
def score(self, y: torch.Tensor, state: Any, x: torch.Tensor) -> Tuple[torch.Tensor, Any]:
@@ -245,6 +268,16 @@ class CTTransformer(torch.nn.Module):
vad_indexes: Optional[torch.Tensor] = None,
vad_indexes_lengths: Optional[torch.Tensor] = None,
):
+ """Forward pass for training.
+
+ Args:
+ text: Text tensor or string input.
+ punc: TODO.
+ text_lengths: Length of each text sample.
+ punc_lengths: Lengths of punc.
+ vad_indexes: TODO.
+ vad_indexes_lengths: Lengths of vad_indexes.
+ """
nll, y_lengths = self.nll(text, punc, text_lengths, punc_lengths, vad_indexes=vad_indexes)
ntokens = y_lengths.sum()
loss = nll.sum() / ntokens
@@ -263,6 +296,16 @@ class CTTransformer(torch.nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
assert len(data_in) == 1
if not data_in[0] or (isinstance(data_in[0], str) and not data_in[0].strip()):
meta_data = {"batch_data_time": -1}
@@ -427,6 +470,11 @@ class CTTransformer(torch.nn.Module):
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
models = export_rebuild_model(model=self, **kwargs)
diff --git a/funasr/models/ct_transformer/utils.py b/funasr/models/ct_transformer/utils.py
index b6e11e7af..465ca4225 100644
--- a/funasr/models/ct_transformer/utils.py
+++ b/funasr/models/ct_transformer/utils.py
@@ -7,6 +7,12 @@ import re
def split_to_mini_sentence(words: list, word_limit: int = 20):
+ """Split to mini sentence.
+
+ Args:
+ words: TODO.
+ word_limit: TODO.
+ """
assert word_limit > 1
if len(words) <= word_limit:
return [words]
@@ -21,6 +27,13 @@ def split_to_mini_sentence(words: list, word_limit: int = 20):
def split_words(text: str, jieba_usr_dict=None, **kwargs):
+ """Split words.
+
+ Args:
+ text: Text tensor or string input.
+ jieba_usr_dict: TODO.
+ **kwargs: Additional keyword arguments.
+ """
if jieba_usr_dict:
input_list = text.split()
token_list_all = []
@@ -80,6 +93,11 @@ def split_words(text: str, jieba_usr_dict=None, **kwargs):
def isEnglish(text: str):
+ """Isenglish.
+
+ Args:
+ text: Text tensor or string input.
+ """
if re.search("^[a-zA-Z']+$", text):
return True
else:
@@ -87,6 +105,11 @@ def isEnglish(text: str):
def join_chinese_and_english(input_list):
+ """Join chinese and english.
+
+ Args:
+ input_list: TODO.
+ """
line = ""
for token in input_list:
if isEnglish(token):
diff --git a/funasr/models/ct_transformer_streaming/attention.py b/funasr/models/ct_transformer_streaming/attention.py
index 97e770bbb..3b9de079e 100644
--- a/funasr/models/ct_transformer_streaming/attention.py
+++ b/funasr/models/ct_transformer_streaming/attention.py
@@ -9,9 +9,23 @@ from funasr.models.sanm.attention import MultiHeadedAttentionSANM
class MultiHeadedAttentionSANMwithMask(MultiHeadedAttentionSANM):
def __init__(self, *args, **kwargs):
+ """Initialize MultiHeadedAttentionSANMwithMask.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(self, x, mask, mask_shfit_chunk=None, mask_att_chunk_encoder=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ mask_shfit_chunk: TODO.
+ mask_att_chunk_encoder: TODO.
+ """
q_h, k_h, v_h, v = self.forward_qkv(x)
fsmn_memory = self.forward_fsmn(v, mask[0], mask_shfit_chunk)
q_h = q_h * self.d_k ** (-0.5)
diff --git a/funasr/models/ct_transformer_streaming/encoder.py b/funasr/models/ct_transformer_streaming/encoder.py
index a61319aa4..f0a4877fe 100644
--- a/funasr/models/ct_transformer_streaming/encoder.py
+++ b/funasr/models/ct_transformer_streaming/encoder.py
@@ -201,6 +201,30 @@ class SANMVadEncoder(torch.nn.Module):
sanm_shfit: int = 0,
selfattention_layer_type: str = "sanm",
):
+ """Initialize SANMVadEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ padding_idx: TODO.
+ interctc_layer_idx: TODO.
+ interctc_use_conditioning: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ selfattention_layer_type: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -325,6 +349,7 @@ class SANMVadEncoder(torch.nn.Module):
self.dropout = torch.nn.Dropout(dropout_rate)
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
@@ -420,6 +445,12 @@ class EncoderLayerSANMExport(torch.nn.Module):
def forward(self, x, mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ """
residual = x
x = self.norm1(x)
x = self.self_attn(x, mask)
@@ -443,6 +474,15 @@ class SANMVadEncoderExport(torch.nn.Module):
model_name="encoder",
onnx: bool = True,
):
+ """Initialize SANMVadEncoderExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ feats_dim: Size/dimension parameter.
+ model_name: TODO.
+ onnx: TODO.
+ """
super().__init__()
self.embed = model.embed
self.model = model
@@ -466,6 +506,12 @@ class SANMVadEncoderExport(torch.nn.Module):
self.model.encoders[i] = EncoderLayerSANMExport(d)
def prepare_mask(self, mask, sub_masks):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ sub_masks: TODO.
+ """
mask_3d_btd = mask[:, :, None]
mask_4d_bhlt = (1 - sub_masks) * -10000.0
@@ -478,6 +524,14 @@ class SANMVadEncoderExport(torch.nn.Module):
vad_masks: torch.Tensor,
sub_masks: torch.Tensor,
):
+ """Forward pass for training.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ vad_masks: TODO.
+ sub_masks: TODO.
+ """
speech = speech * self._output_size**0.5
mask = self.make_pad_mask(speech_lengths)
vad_masks = self.prepare_mask(mask, vad_masks)
@@ -503,4 +557,5 @@ class SANMVadEncoderExport(torch.nn.Module):
return xs_pad, speech_lengths
def get_output_size(self):
+ """Get output size."""
return self.model.encoders[0].size
diff --git a/funasr/models/ct_transformer_streaming/export_meta.py b/funasr/models/ct_transformer_streaming/export_meta.py
index ba0283e5d..f6ca99bb2 100644
--- a/funasr/models/ct_transformer_streaming/export_meta.py
+++ b/funasr/models/ct_transformer_streaming/export_meta.py
@@ -10,6 +10,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
model.encoder = encoder_class(model.encoder, onnx=is_onnx)
@@ -46,6 +52,7 @@ def export_forward(
def export_dummy_inputs(self):
+ """Export dummy inputs."""
length = 120
text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)).type(torch.int32)
text_lengths = torch.tensor([length], dtype=torch.int32)
@@ -56,14 +63,17 @@ def export_dummy_inputs(self):
def export_input_names(self):
+ """Export input names."""
return ["inputs", "text_lengths", "vad_masks", "sub_masks"]
def export_output_names(self):
+ """Export output names."""
return ["logits"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"inputs": {1: "feats_length"},
"vad_masks": {2: "feats_length1", 3: "feats_length2"},
@@ -73,4 +83,5 @@ def export_dynamic_axes(self):
def export_name(self):
+ """Export name."""
return "model.onnx"
diff --git a/funasr/models/ct_transformer_streaming/model.py b/funasr/models/ct_transformer_streaming/model.py
index 4394821d5..f3c63d5af 100644
--- a/funasr/models/ct_transformer_streaming/model.py
+++ b/funasr/models/ct_transformer_streaming/model.py
@@ -21,6 +21,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -37,6 +42,12 @@ class CTTransformerStreaming(CTTransformer):
*args,
**kwargs,
):
+ """Initialize CTTransformerStreaming.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def punc_forward(
@@ -56,6 +67,7 @@ class CTTransformerStreaming(CTTransformer):
return y, None
def with_vad(self):
+ """With vad."""
return True
def inference(
@@ -68,6 +80,17 @@ class CTTransformerStreaming(CTTransformer):
cache: dict = None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
assert len(data_in) == 1
@@ -189,6 +212,11 @@ class CTTransformerStreaming(CTTransformer):
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
models = export_rebuild_model(model=self, **kwargs)
diff --git a/funasr/models/ctc/ctc.py b/funasr/models/ctc/ctc.py
index 8eb64d159..f372d71f3 100644
--- a/funasr/models/ctc/ctc.py
+++ b/funasr/models/ctc/ctc.py
@@ -25,6 +25,17 @@ class CTC(torch.nn.Module):
ignore_nan_grad: bool = True,
extra_linear: bool = True,
):
+ """Initialize CTC.
+
+ Args:
+ odim: TODO.
+ encoder_output_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ ctc_type: TODO.
+ reduce: TODO.
+ ignore_nan_grad: TODO.
+ extra_linear: TODO.
+ """
super().__init__()
eprojs = encoder_output_size
self.dropout_rate = dropout_rate
@@ -51,6 +62,14 @@ class CTC(torch.nn.Module):
self.reduce = reduce
def loss_fn(self, th_pred, th_target, th_ilen, th_olen) -> torch.Tensor:
+ """Loss fn.
+
+ Args:
+ th_pred: TODO.
+ th_target: TODO.
+ th_ilen: TODO.
+ th_olen: TODO.
+ """
if self.ctc_type == "builtin":
th_pred = th_pred.log_softmax(2)
loss = self.ctc_loss(th_pred, th_target, th_ilen, th_olen)
diff --git a/funasr/models/ctc/model.py b/funasr/models/ctc/model.py
index e493c3b3a..439a23197 100644
--- a/funasr/models/ctc/model.py
+++ b/funasr/models/ctc/model.py
@@ -37,6 +37,25 @@ class Transformer(nn.Module):
**kwargs,
):
+ """Initialize Transformer.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ ctc_conf: Configuration dict for ctc.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ length_normalized_loss: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -149,6 +168,14 @@ class Transformer(nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -169,6 +196,16 @@ class Transformer(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
diff --git a/funasr/models/data2vec/data2vec.py b/funasr/models/data2vec/data2vec.py
index ed58d1c7c..c7386e35d 100644
--- a/funasr/models/data2vec/data2vec.py
+++ b/funasr/models/data2vec/data2vec.py
@@ -27,6 +27,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -42,6 +47,15 @@ class Data2VecPretrainModel(nn.Module):
preencoder=None,
):
+ """Initialize Data2VecPretrainModel.
+
+ Args:
+ frontend: Audio frontend for feature extraction.
+ specaug: TODO.
+ normalize: TODO.
+ encoder: TODO.
+ preencoder: TODO.
+ """
super().__init__()
self.frontend = frontend
@@ -91,6 +105,12 @@ class Data2VecPretrainModel(nn.Module):
def collect_feats(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
) -> Dict[str, torch.Tensor]:
+ """Collect feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
feats, feats_lengths = self._extract_feats(speech, speech_lengths)
return {"feats": feats, "feats_lengths": feats_lengths}
@@ -130,6 +150,12 @@ class Data2VecPretrainModel(nn.Module):
def _extract_feats(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Internal: extract feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
assert speech_lengths.dim() == 1, speech_lengths.shape
# for data-parallel
@@ -147,7 +173,13 @@ class Data2VecPretrainModel(nn.Module):
return feats, feats_lengths
def set_num_updates(self, num_updates):
+ """Set num updates.
+
+ Args:
+ num_updates: TODO.
+ """
self.num_updates = num_updates
def get_num_updates(self):
+ """Get num updates."""
return self.num_updates
diff --git a/funasr/models/data2vec/data2vec_encoder.py b/funasr/models/data2vec/data2vec_encoder.py
index f591dd6a3..09d161b62 100644
--- a/funasr/models/data2vec/data2vec_encoder.py
+++ b/funasr/models/data2vec/data2vec_encoder.py
@@ -22,6 +22,14 @@ from funasr.models.transformer.utils.nets_utils import make_pad_mask
def get_annealed_rate(start, end, curr_step, total_steps):
+ """Get annealed rate.
+
+ Args:
+ start: TODO.
+ end: TODO.
+ curr_step: TODO.
+ total_steps: TODO.
+ """
r = end - start
pct_remaining = 1 - curr_step / total_steps
return end - r * pct_remaining
@@ -95,6 +103,63 @@ class Data2VecEncoder(nn.Module):
# FP16 optimization
required_seq_len_multiple: int = 2,
):
+ """Initialize Data2VecEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ extractor_mode: TODO.
+ conv_feature_layers: TODO.
+ layer_type: TODO.
+ layer_norm_first: TODO.
+ encoder_layers: TODO.
+ encoder_embed_dim: Size/dimension parameter.
+ encoder_ffn_embed_dim: Size/dimension parameter.
+ encoder_attention_heads: TODO.
+ activation_fn: TODO.
+ dropout: TODO.
+ attention_dropout: TODO.
+ activation_dropout: TODO.
+ encoder_layerdrop: TODO.
+ dropout_input: TODO.
+ dropout_features: TODO.
+ feature_grad_mult: TODO.
+ mask_prob: TODO.
+ mask_length: TODO.
+ mask_selection: TODO.
+ mask_other: TODO.
+ no_mask_overlap: TODO.
+ mask_min_space: TODO.
+ require_same_masks: TODO.
+ mask_dropout: TODO.
+ mask_channel_length: TODO.
+ mask_channel_prob: TODO.
+ mask_channel_before: TODO.
+ mask_channel_selection: TODO.
+ mask_channel_other: TODO.
+ no_mask_channel_overlap: TODO.
+ mask_channel_min_space: TODO.
+ conv_pos: TODO.
+ conv_pos_groups: TODO.
+ pos_conv_depth: TODO.
+ max_positions: TODO.
+ average_top_k_layers: TODO.
+ layer_norm_target_layer: TODO.
+ instance_norm_target_layer: TODO.
+ instance_norm_targets: TODO.
+ layer_norm_targets: TODO.
+ batch_norm_target_layer: TODO.
+ group_norm_target_layer: TODO.
+ ema_decay: TODO.
+ ema_end_decay: TODO.
+ ema_anneal_end_step: TODO.
+ ema_transformer_only: TODO.
+ ema_layers_only: TODO.
+ min_target_var: TODO.
+ min_pred_var: TODO.
+ loss_beta: TODO.
+ loss_scale: TODO.
+ required_seq_len_multiple: TODO.
+ """
super().__init__()
# ConvFeatureExtractionModel
@@ -206,6 +271,7 @@ class Data2VecEncoder(nn.Module):
logging.info("Data2VecEncoder settings: {}".format(self.__dict__))
def make_ema_teacher(self):
+ """Make ema teacher."""
skip_keys = set()
if self.ema_layers_only:
self.ema_transformer_only = True
@@ -220,6 +286,11 @@ class Data2VecEncoder(nn.Module):
)
def set_num_updates(self, num_updates):
+ """Set num updates.
+
+ Args:
+ num_updates: TODO.
+ """
if self.ema is None and self.final_proj is not None:
logging.info("Making EMA Teacher")
self.make_ema_teacher()
@@ -247,6 +318,14 @@ class Data2VecEncoder(nn.Module):
mask_indices=None,
mask_channel_indices=None,
):
+ """Apply mask.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ mask_indices: TODO.
+ mask_channel_indices: TODO.
+ """
B, T, C = x.shape
if self.mask_channel_prob > 0 and self.mask_channel_before:
@@ -313,6 +392,13 @@ class Data2VecEncoder(nn.Module):
"""
def _conv_out_length(input_length, kernel_size, stride):
+ """Internal: conv out length.
+
+ Args:
+ input_length: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ """
return torch.floor((input_length - kernel_size).to(torch.float32) / stride + 1)
conv_cfg_list = eval(self.conv_feature_layers)
@@ -336,6 +422,18 @@ class Data2VecEncoder(nn.Module):
padding_count=None,
):
# create padding_mask by ilens
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ mask: TODO.
+ features_only: TODO.
+ layer: TODO.
+ mask_indices: TODO.
+ mask_channel_indices: TODO.
+ padding_count: TODO.
+ """
if ilens is not None:
padding_mask = make_pad_mask(lengths=ilens).to(xs_pad.device)
else:
@@ -526,6 +624,11 @@ class Data2VecEncoder(nn.Module):
@staticmethod
def compute_var(y):
+ """Compute var.
+
+ Args:
+ y: TODO.
+ """
y = y.view(-1, y.size(-1))
if dist.is_initialized():
zc = torch.tensor(y.size(0)).cuda()
@@ -542,6 +645,14 @@ class Data2VecEncoder(nn.Module):
return torch.sqrt(y.var(dim=0) + 1e-6).mean()
def extract_features(self, xs_pad, ilens, mask=False, layer=None):
+ """Extract features.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ mask: TODO.
+ layer: TODO.
+ """
res = self.forward(
xs_pad,
ilens,
@@ -552,6 +663,11 @@ class Data2VecEncoder(nn.Module):
return res
def remove_pretraining_modules(self, last_layer=None):
+ """Remove pretraining modules.
+
+ Args:
+ last_layer: TODO.
+ """
self.final_proj = None
self.ema = None
if last_layer is not None:
@@ -560,4 +676,5 @@ class Data2VecEncoder(nn.Module):
)
def output_size(self) -> int:
+ """Output size."""
return self.encoder_embed_dim
diff --git a/funasr/models/data2vec/data_utils.py b/funasr/models/data2vec/data_utils.py
index f486edf5a..8d87a70a8 100644
--- a/funasr/models/data2vec/data_utils.py
+++ b/funasr/models/data2vec/data_utils.py
@@ -90,6 +90,14 @@ def compute_mask_indices(
mask_idc = []
def arrange(s, e, length, keep_length):
+ """Arrange.
+
+ Args:
+ s: TODO.
+ e: TODO.
+ length: TODO.
+ keep_length: TODO.
+ """
span_start = np.random.randint(s, e - length)
mask_idc.extend(span_start + i for i in range(length))
diff --git a/funasr/models/data2vec/ema_module.py b/funasr/models/data2vec/ema_module.py
index 3d7c21e98..bf60dbe4b 100644
--- a/funasr/models/data2vec/ema_module.py
+++ b/funasr/models/data2vec/ema_module.py
@@ -59,6 +59,11 @@ class EMAModule:
state_dict = self.model.state_dict()
def _to_float(t):
+ """Internal: to float.
+
+ Args:
+ t: TODO.
+ """
return t.float() if torch.is_floating_point(t) else t
for param_key in state_dict:
@@ -74,9 +79,15 @@ class EMAModule:
self.build_fp32_params(state_dict)
def set_decay(self, decay):
+ """Set decay.
+
+ Args:
+ decay: TODO.
+ """
self.decay = decay
def get_decay(self):
+ """Get decay."""
return self.decay
def _step_internal(self, new_model):
@@ -115,6 +126,11 @@ class EMAModule:
self.restore(ema_state_dict, build_fp32_params=False)
def step(self, new_model):
+ """Step.
+
+ Args:
+ new_model: New Model instance.
+ """
self._step_internal(new_model)
def reverse(self, model):
diff --git a/funasr/models/data2vec/grad_multiply.py b/funasr/models/data2vec/grad_multiply.py
index 08d15f55d..6041e0305 100644
--- a/funasr/models/data2vec/grad_multiply.py
+++ b/funasr/models/data2vec/grad_multiply.py
@@ -9,10 +9,23 @@ import torch
class GradMultiply(torch.autograd.Function):
@staticmethod
def forward(ctx, x, scale):
+ """Forward pass for training.
+
+ Args:
+ ctx: TODO.
+ x: TODO.
+ scale: TODO.
+ """
ctx.scale = scale
res = x.new(x)
return res
@staticmethod
def backward(ctx, grad):
+ """Backward.
+
+ Args:
+ ctx: TODO.
+ grad: TODO.
+ """
return grad * ctx.scale, None
diff --git a/funasr/models/data2vec/multihead_attention.py b/funasr/models/data2vec/multihead_attention.py
index 85bd0387d..7b977b067 100644
--- a/funasr/models/data2vec/multihead_attention.py
+++ b/funasr/models/data2vec/multihead_attention.py
@@ -17,12 +17,24 @@ from funasr.models.data2vec.quant_noise import quant_noise
class FairseqDropout(nn.Module):
def __init__(self, p, module_name=None):
+ """Initialize FairseqDropout.
+
+ Args:
+ p: TODO.
+ module_name: TODO.
+ """
super().__init__()
self.p = p
self.module_name = module_name
self.apply_during_inference = False
def forward(self, x, inplace: bool = False):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ inplace: TODO.
+ """
if self.p > 0 and (self.training or self.apply_during_inference):
return F.dropout(x, p=self.p, training=True, inplace=inplace)
else:
@@ -35,6 +47,14 @@ class FairseqDropout(nn.Module):
retain_dropout_modules: Optional[List[str]] = None,
**kwargs,
):
+ """Make generation fast .
+
+ Args:
+ name: TODO.
+ retain_dropout: TODO.
+ retain_dropout_modules: TODO.
+ **kwargs: Additional keyword arguments.
+ """
if retain_dropout:
if retain_dropout_modules is not None and self.module_name is None:
logging.warning(
@@ -72,6 +92,22 @@ class MultiheadAttention(nn.Module):
q_noise=0.0,
qn_block_size=8,
):
+ """Initialize MultiheadAttention.
+
+ Args:
+ embed_dim: Size/dimension parameter.
+ num_heads: TODO.
+ kdim: TODO.
+ vdim: TODO.
+ dropout: TODO.
+ bias: TODO.
+ add_bias_kv: TODO.
+ add_zero_attn: TODO.
+ self_attention: TODO.
+ encoder_decoder_attention: TODO.
+ q_noise: TODO.
+ qn_block_size: Size/dimension parameter.
+ """
super().__init__()
self.embed_dim = embed_dim
self.kdim = kdim if kdim is not None else embed_dim
@@ -122,9 +158,11 @@ class MultiheadAttention(nn.Module):
self.skip_embed_dim_check = False
def prepare_for_onnx_export_(self):
+ """Prepare for onnx export ."""
self.onnx_trace = True
def reset_parameters(self):
+ """Reset parameters."""
if self.qkv_same_dim:
# Empirically observed the convergence to be much better with
# the scaled initialization
@@ -145,6 +183,11 @@ class MultiheadAttention(nn.Module):
nn.init.xavier_normal_(self.bias_v)
def _get_reserve_head_index(self, num_heads_to_keep: int):
+ """Internal: get reserve head index.
+
+ Args:
+ num_heads_to_keep: TODO.
+ """
k_proj_heads_norm = []
q_proj_heads_norm = []
v_proj_heads_norm = []
@@ -178,6 +221,11 @@ class MultiheadAttention(nn.Module):
return reserve_head_index
def _adaptive_prune_heads(self, reserve_head_index: List[Tuple[int, int]]):
+ """Internal: adaptive prune heads.
+
+ Args:
+ reserve_head_index: TODO.
+ """
new_q_weight = []
new_q_bias = []
new_k_weight = []
@@ -236,6 +284,7 @@ class MultiheadAttention(nn.Module):
self.v_proj.out_features = self.embed_dim
def _set_skip_embed_dim_check(self):
+ """Internal: set skip embed dim check."""
self.skip_embed_dim_check = True
def forward(
@@ -506,6 +555,15 @@ class MultiheadAttention(nn.Module):
static_kv: bool,
) -> Optional[Tensor]:
# saved key padding masks have shape (bsz, seq_len)
+ """Internal: append prev key padding mask.
+
+ Args:
+ key_padding_mask: TODO.
+ prev_key_padding_mask: TODO.
+ batch_size: Number of samples per batch.
+ src_len: TODO.
+ static_kv: TODO.
+ """
if prev_key_padding_mask is not None and static_kv:
new_key_padding_mask = prev_key_padding_mask
elif prev_key_padding_mask is not None and key_padding_mask is not None:
@@ -562,6 +620,11 @@ class MultiheadAttention(nn.Module):
def _get_input_buffer(
self, incremental_state: Optional[Dict[str, Dict[str, Optional[Tensor]]]]
) -> Dict[str, Optional[Tensor]]:
+ """Internal: get input buffer.
+
+ Args:
+ incremental_state: TODO.
+ """
result = self.get_incremental_state(incremental_state, "attn_state")
if result is not None:
return result
@@ -574,12 +637,32 @@ class MultiheadAttention(nn.Module):
incremental_state: Dict[str, Dict[str, Optional[Tensor]]],
buffer: Dict[str, Optional[Tensor]],
):
+ """Internal: set input buffer.
+
+ Args:
+ incremental_state: TODO.
+ buffer: TODO.
+ """
return self.set_incremental_state(incremental_state, "attn_state", buffer)
def apply_sparse_mask(self, attn_weights, tgt_len: int, src_len: int, bsz: int):
+ """Apply sparse mask.
+
+ Args:
+ attn_weights: TODO.
+ tgt_len: TODO.
+ src_len: TODO.
+ bsz: TODO.
+ """
return attn_weights
def upgrade_state_dict_named(self, state_dict, name):
+ """Upgrade state dict named.
+
+ Args:
+ state_dict: TODO.
+ name: TODO.
+ """
prefix = name + "." if name != "" else ""
items_to_add = {}
keys_to_remove = []
diff --git a/funasr/models/data2vec/quant_noise.py b/funasr/models/data2vec/quant_noise.py
index 50b2c1528..42b9674f6 100644
--- a/funasr/models/data2vec/quant_noise.py
+++ b/funasr/models/data2vec/quant_noise.py
@@ -57,6 +57,12 @@ def quant_noise(module, p, block_size):
def _forward_pre_hook(mod, input):
# no noise for evaluation
+ """Internal: forward pre hook.
+
+ Args:
+ mod: TODO.
+ input: Input audio/text data.
+ """
if mod.training:
if not is_conv:
# gather weight and sizes
diff --git a/funasr/models/data2vec/utils.py b/funasr/models/data2vec/utils.py
index b4c160358..f8b55b73f 100644
--- a/funasr/models/data2vec/utils.py
+++ b/funasr/models/data2vec/utils.py
@@ -14,9 +14,20 @@ from funasr.models.data2vec.multihead_attention import MultiheadAttention
class Fp32LayerNorm(nn.LayerNorm):
def __init__(self, *args, **kwargs):
+ """Initialize Fp32LayerNorm.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = F.layer_norm(
input.float(),
self.normalized_shape,
@@ -29,9 +40,20 @@ class Fp32LayerNorm(nn.LayerNorm):
class Fp32GroupNorm(nn.GroupNorm):
def __init__(self, *args, **kwargs):
+ """Initialize Fp32GroupNorm.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = F.group_norm(
input.float(),
self.num_groups,
@@ -44,10 +66,20 @@ class Fp32GroupNorm(nn.GroupNorm):
class TransposeLast(nn.Module):
def __init__(self, deconstruct_idx=None):
+ """Initialize TransposeLast.
+
+ Args:
+ deconstruct_idx: TODO.
+ """
super().__init__()
self.deconstruct_idx = deconstruct_idx
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if self.deconstruct_idx is not None:
x = x[self.deconstruct_idx]
return x.transpose(-2, -1)
@@ -55,6 +87,12 @@ class TransposeLast(nn.Module):
class SamePad(nn.Module):
def __init__(self, kernel_size, causal=False):
+ """Initialize SamePad.
+
+ Args:
+ kernel_size: Size/dimension parameter.
+ causal: TODO.
+ """
super().__init__()
if causal:
self.remove = kernel_size - 1
@@ -62,6 +100,11 @@ class SamePad(nn.Module):
self.remove = 1 if kernel_size % 2 == 0 else 0
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if self.remove > 0:
x = x[:, :, : -self.remove]
return x
@@ -69,6 +112,14 @@ class SamePad(nn.Module):
def pad_to_multiple(x, multiple, dim=-1, value=0):
# Inspired from https://github.com/lucidrains/local-attention/blob/master/local_attention/local_attention.py#L41
+ """Pad to multiple.
+
+ Args:
+ x: TODO.
+ multiple: TODO.
+ dim: TODO.
+ value: TODO.
+ """
if x is None:
return None, 0
tsz = x.size(dim)
@@ -82,16 +133,27 @@ def pad_to_multiple(x, multiple, dim=-1, value=0):
def gelu_accurate(x):
+ """Gelu accurate.
+
+ Args:
+ x: TODO.
+ """
if not hasattr(gelu_accurate, "_a"):
gelu_accurate._a = math.sqrt(2 / math.pi)
return 0.5 * x * (1 + torch.tanh(gelu_accurate._a * (x + 0.044715 * torch.pow(x, 3))))
def gelu(x: torch.Tensor) -> torch.Tensor:
+ """Gelu.
+
+ Args:
+ x: TODO.
+ """
return torch.nn.functional.gelu(x.float()).type_as(x)
def get_available_activation_fns():
+ """Get available activation fns."""
return [
"relu",
"gelu",
@@ -138,6 +200,11 @@ def init_bert_params(module):
def normal_(data):
# with FSDP, module params will be on CUDA, so we cast them back to CPU
# so that the RNG is consistent with and without FSDP
+ """Normal .
+
+ Args:
+ data: TODO.
+ """
data.copy_(data.cpu().normal_(mean=0.0, std=0.02).to(data.device))
if isinstance(module, nn.Linear):
diff --git a/funasr/models/data2vec/wav2vec2.py b/funasr/models/data2vec/wav2vec2.py
index a3e837ad3..ad1d86566 100644
--- a/funasr/models/data2vec/wav2vec2.py
+++ b/funasr/models/data2vec/wav2vec2.py
@@ -25,6 +25,15 @@ class ConvFeatureExtractionModel(nn.Module):
conv_bias: bool = False,
in_d: int = 1,
):
+ """Initialize ConvFeatureExtractionModel.
+
+ Args:
+ conv_layers: TODO.
+ dropout: TODO.
+ mode: TODO.
+ conv_bias: TODO.
+ in_d: TODO.
+ """
super().__init__()
assert mode in {"default", "layer_norm"}
@@ -38,7 +47,19 @@ class ConvFeatureExtractionModel(nn.Module):
is_group_norm=False,
conv_bias=False,
):
+ """Block.
+
+ Args:
+ n_in: TODO.
+ n_out: TODO.
+ k: TODO.
+ stride: TODO.
+ is_layer_norm: Boolean flag for layer norm.
+ is_group_norm: Boolean flag for group norm.
+ conv_bias: TODO.
+ """
def make_conv():
+ """Make conv."""
conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias)
nn.init.kaiming_normal_(conv.weight)
return conv
@@ -87,6 +108,11 @@ class ConvFeatureExtractionModel(nn.Module):
in_d = dim
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if len(x.shape) == 2:
x = x.unsqueeze(1)
else:
@@ -98,6 +124,13 @@ class ConvFeatureExtractionModel(nn.Module):
def make_conv_pos(e, k, g):
+ """Make conv pos.
+
+ Args:
+ e: TODO.
+ k: TODO.
+ g: TODO.
+ """
pos_conv = nn.Conv1d(
e,
e,
@@ -118,6 +151,7 @@ def make_conv_pos(e, k, g):
class TransformerEncoder(nn.Module):
def build_encoder_layer(self):
+ """Build encoder layer."""
if self.layer_type == "transformer":
layer = TransformerSentenceEncoderLayer(
embedding_dim=self.embedding_dim,
@@ -154,6 +188,26 @@ class TransformerEncoder(nn.Module):
encoder_layerdrop,
max_positions,
):
+ """Initialize TransformerEncoder.
+
+ Args:
+ dropout: TODO.
+ encoder_embed_dim: Size/dimension parameter.
+ required_seq_len_multiple: TODO.
+ pos_conv_depth: TODO.
+ conv_pos: TODO.
+ conv_pos_groups: TODO.
+ layer_type: TODO.
+ encoder_layers: TODO.
+ encoder_ffn_embed_dim: Size/dimension parameter.
+ encoder_attention_heads: TODO.
+ attention_dropout: TODO.
+ activation_dropout: TODO.
+ activation_fn: TODO.
+ layer_norm_first: TODO.
+ encoder_layerdrop: TODO.
+ max_positions: TODO.
+ """
super().__init__()
# position
@@ -165,6 +219,14 @@ class TransformerEncoder(nn.Module):
k = max(3, conv_pos // num_layers)
def make_conv_block(e, k, g, l):
+ """Make conv block.
+
+ Args:
+ e: TODO.
+ k: TODO.
+ g: TODO.
+ l: TODO.
+ """
return nn.Sequential(
*[
nn.Sequential(
@@ -210,6 +272,13 @@ class TransformerEncoder(nn.Module):
self.apply(utils.init_bert_params)
def forward(self, x, padding_mask=None, layer=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ layer: TODO.
+ """
x, layer_results = self.extract_features(x, padding_mask, layer)
if self.layer_norm_first and layer is None:
@@ -225,6 +294,14 @@ class TransformerEncoder(nn.Module):
min_layer=0,
):
+ """Extract features.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ tgt_layer: TODO.
+ min_layer: TODO.
+ """
if padding_mask is not None:
x[padding_mask] = 0
@@ -272,6 +349,13 @@ class TransformerEncoder(nn.Module):
x = x[:, :-pad_length]
def undo_pad(a, b, c):
+ """Undo pad.
+
+ Args:
+ a: TODO.
+ b: TODO.
+ c: TODO.
+ """
return (
a[:-pad_length],
b[:-pad_length] if b is not None else b,
@@ -309,6 +393,18 @@ class TransformerSentenceEncoderLayer(nn.Module):
layer_norm_first: bool = False,
) -> None:
+ """Initialize TransformerSentenceEncoderLayer.
+
+ Args:
+ embedding_dim: Size/dimension parameter.
+ ffn_embedding_dim: Size/dimension parameter.
+ num_attention_heads: TODO.
+ dropout: TODO.
+ attention_dropout: TODO.
+ activation_dropout: TODO.
+ activation_fn: TODO.
+ layer_norm_first: TODO.
+ """
super().__init__()
# Initialize parameters
self.embedding_dim = embedding_dim
diff --git a/funasr/models/e_branchformer/encoder.py b/funasr/models/e_branchformer/encoder.py
index adc6642f4..1e32fb84c 100644
--- a/funasr/models/e_branchformer/encoder.py
+++ b/funasr/models/e_branchformer/encoder.py
@@ -68,6 +68,17 @@ class EBranchformerEncoderLayer(torch.nn.Module):
dropout_rate: float,
merge_conv_kernel: int = 3,
):
+ """Initialize EBranchformerEncoderLayer.
+
+ Args:
+ size: TODO.
+ attn: TODO.
+ cgmlp: TODO.
+ feed_forward: TODO.
+ feed_forward_macaron: TODO.
+ dropout_rate: TODO.
+ merge_conv_kernel: TODO.
+ """
super().__init__()
self.size = size
@@ -210,6 +221,37 @@ class EBranchformerEncoder(nn.Module):
interctc_layer_idx=None,
interctc_use_conditioning: bool = False,
):
+ """Initialize EBranchformerEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ attention_layer_type: TODO.
+ pos_enc_layer_type: TODO.
+ rel_pos_type: TODO.
+ cgmlp_linear_units: TODO.
+ cgmlp_conv_kernel: TODO.
+ use_linear_after_conv: TODO.
+ gate_activation: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ zero_triu: TODO.
+ padding_idx: TODO.
+ layer_drop_rate: TODO.
+ max_pos_emb_len: TODO.
+ use_ffn: TODO.
+ macaron_ffn: TODO.
+ ffn_activation_type: TODO.
+ linear_units: TODO.
+ positionwise_layer_type: TODO.
+ merge_conv_kernel: TODO.
+ interctc_layer_idx: TODO.
+ interctc_use_conditioning: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -375,6 +417,7 @@ class EBranchformerEncoder(nn.Module):
self.conditioning_layer = None
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
diff --git a/funasr/models/e_branchformer/model.py b/funasr/models/e_branchformer/model.py
index 9853f7352..e8f791afa 100644
--- a/funasr/models/e_branchformer/model.py
+++ b/funasr/models/e_branchformer/model.py
@@ -14,4 +14,10 @@ class EBranchformer(Transformer):
**kwargs,
):
+ """Initialize EBranchformer.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
diff --git a/funasr/models/e_paraformer/decoder.py b/funasr/models/e_paraformer/decoder.py
index 7edd91a2a..f784e0914 100644
--- a/funasr/models/e_paraformer/decoder.py
+++ b/funasr/models/e_paraformer/decoder.py
@@ -121,6 +121,15 @@ class DecoderLayerSANM(torch.nn.Module):
return x, tgt_mask, memory, memory_mask, cache
def get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Get attn mat.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt = self.feed_forward(tgt)
@@ -257,6 +266,35 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
tf2torch_tensor_name_prefix_torch: str = "decoder",
tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",
):
+ """Initialize ParaformerSANMDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ wo_input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ att_layer_num: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ lora_list: TODO.
+ lora_rank: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ chunk_multiply_factor: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -426,6 +464,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
ys_in_lens: torch.Tensor,
):
+ """Forward asf2.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -444,6 +490,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
ys_in_lens: torch.Tensor,
):
+ """Forward asf6.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -592,6 +646,11 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
class DecoderLayerSANMExport(torch.nn.Module):
def __init__(self, model):
+ """Initialize DecoderLayerSANMExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.self_attn = model.self_attn
self.src_attn = model.src_attn
@@ -603,6 +662,15 @@ class DecoderLayerSANMExport(torch.nn.Module):
def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Forward pass for training.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt = self.feed_forward(tgt)
@@ -621,6 +689,15 @@ class DecoderLayerSANMExport(torch.nn.Module):
return x, tgt_mask, memory, memory_mask, cache
def get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Get attn mat.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt = self.feed_forward(tgt)
@@ -640,6 +717,15 @@ class DecoderLayerSANMExport(torch.nn.Module):
@tables.register("decoder_classes", "ParaformerSANMDecoderExport")
class ParaformerSANMDecoderExport(torch.nn.Module):
def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):
+ """Initialize ParaformerSANMDecoderExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ model_name: TODO.
+ onnx: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# self.embed = model.embed #Embedding(model.embed, max_seq_len)
@@ -673,6 +759,11 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
self.model_name = model_name
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -692,6 +783,16 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
return_both: bool = False,
):
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ return_hidden: TODO.
+ return_both: TODO.
+ """
tgt = ys_in_pad
tgt_mask = self.make_pad_mask(ys_in_lens)
tgt_mask, _ = self.prepare_mask(tgt_mask)
@@ -728,6 +829,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
ys_in_lens: torch.Tensor,
):
+ """Forward asf2.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -749,6 +858,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
ys_in_lens: torch.Tensor,
):
+ """Forward asf6.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -829,6 +946,15 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
@tables.register("decoder_classes", "ParaformerSANMDecoderOnlineExport")
class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):
+ """Initialize ParaformerSANMDecoderOnlineExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ model_name: TODO.
+ onnx: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# self.embed = model.embed #Embedding(model.embed, max_seq_len)
self.model = model
@@ -863,6 +989,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
self.model_name = model_name
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -881,6 +1012,15 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
*args,
):
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ *args: Variable positional arguments.
+ """
tgt = ys_in_pad
tgt_mask = self.make_pad_mask(ys_in_lens)
tgt_mask, _ = self.prepare_mask(tgt_mask)
@@ -913,6 +1053,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
return x, out_caches
def get_dummy_inputs(self, enc_size):
+ """Get dummy inputs.
+
+ Args:
+ enc_size: Size/dimension parameter.
+ """
enc = torch.randn(2, 100, enc_size).type(torch.float32)
enc_len = torch.tensor([30, 100], dtype=torch.int32)
acoustic_embeds = torch.randn(2, 10, enc_size).type(torch.float32)
@@ -930,6 +1075,7 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
return (enc, enc_len, acoustic_embeds, acoustic_embeds_len, *cache)
def get_input_names(self):
+ """Get input names."""
cache_num = len(self.model.decoders)
if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:
cache_num += len(self.model.decoders2)
@@ -938,12 +1084,14 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
]
def get_output_names(self):
+ """Get output names."""
cache_num = len(self.model.decoders)
if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:
cache_num += len(self.model.decoders2)
return ["logits", "sample_ids"] + ["out_cache_%d" % i for i in range(cache_num)]
def get_dynamic_axes(self):
+ """Get dynamic axes."""
ret = {
"enc": {0: "batch_size", 1: "enc_length"},
"acoustic_embeds": {0: "batch_size", 1: "token_length"},
@@ -1004,6 +1152,25 @@ class ParaformerSANDecoder(BaseTransformerDecoder):
concat_after: bool = False,
embeds_id: int = -1,
):
+ """Initialize ParaformerSANDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ embeds_id: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -1093,6 +1260,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):
model_name="decoder",
onnx: bool = True,
):
+ """Initialize ParaformerDecoderSANExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ model_name: TODO.
+ onnx: TODO.
+ """
super().__init__()
# self.embed = model.embed #Embedding(model.embed, max_seq_len)
self.model = model
@@ -1116,6 +1291,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):
self.model_name = model_name
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -1133,6 +1313,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):
ys_in_lens: torch.Tensor,
):
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = self.make_pad_mask(ys_in_lens)
tgt_mask, _ = self.prepare_mask(tgt_mask)
@@ -1151,6 +1339,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):
return x, ys_in_lens
def get_dummy_inputs(self, enc_size):
+ """Get dummy inputs.
+
+ Args:
+ enc_size: Size/dimension parameter.
+ """
tgt = torch.LongTensor([0]).unsqueeze(0)
memory = torch.randn(1, 100, enc_size)
pre_acoustic_embeds = torch.randn(1, 1, enc_size)
@@ -1164,17 +1357,21 @@ class ParaformerDecoderSANExport(torch.nn.Module):
return (tgt, memory, pre_acoustic_embeds, cache)
def is_optimizable(self):
+ """Is optimizable."""
return True
def get_input_names(self):
+ """Get input names."""
cache_num = len(self.model.decoders) + len(self.model.decoders2)
return ["tgt", "memory", "pre_acoustic_embeds"] + ["cache_%d" % i for i in range(cache_num)]
def get_output_names(self):
+ """Get output names."""
cache_num = len(self.model.decoders) + len(self.model.decoders2)
return ["y"] + ["out_cache_%d" % i for i in range(cache_num)]
def get_dynamic_axes(self):
+ """Get dynamic axes."""
ret = {
"tgt": {0: "tgt_batch", 1: "tgt_length"},
"memory": {0: "memory_batch", 1: "memory_length"},
diff --git a/funasr/models/e_paraformer/export_meta.py b/funasr/models/e_paraformer/export_meta.py
index db9385519..eb0678847 100644
--- a/funasr/models/e_paraformer/export_meta.py
+++ b/funasr/models/e_paraformer/export_meta.py
@@ -9,6 +9,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
model.device = kwargs.get("device")
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
@@ -41,6 +47,12 @@ def export_forward(
speech_lengths: torch.Tensor,
):
# a. To device
+ """Export forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
batch = {"speech": speech, "speech_lengths": speech_lengths}
# batch = to_device(batch, device=self.device)
@@ -57,20 +69,24 @@ def export_forward(
def export_dummy_inputs(self):
+ """Export dummy inputs."""
speech = torch.randn(2, 30, 560)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
return (speech, speech_lengths)
def export_input_names(self):
+ """Export input names."""
return ["speech", "speech_lengths"]
def export_output_names(self):
+ """Export output names."""
return ["logits", "token_num"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"speech": {0: "batch_size", 1: "feats_length"},
"speech_lengths": {
@@ -83,4 +99,5 @@ def export_dynamic_axes(self):
def export_name(
self,
):
+ """Export name."""
return "model.onnx"
diff --git a/funasr/models/e_paraformer/model.py b/funasr/models/e_paraformer/model.py
index 47b01901d..34ab3e375 100644
--- a/funasr/models/e_paraformer/model.py
+++ b/funasr/models/e_paraformer/model.py
@@ -77,6 +77,37 @@ class EParaformer(torch.nn.Module):
**kwargs,
):
+ """Initialize EParaformer.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ predictor: TODO.
+ predictor_conf: Configuration dict for predictor.
+ ctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ predictor_weight: TODO.
+ predictor_bias: TODO.
+ sampling_ratio: TODO.
+ share_embedding: TODO.
+ use_1st_decoder_loss: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -282,6 +313,12 @@ class EParaformer(torch.nn.Module):
def calc_predictor(self, encoder_out, encoder_out_lens):
+ """Calc predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -294,6 +331,14 @@ class EParaformer(torch.nn.Module):
self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
):
+ """Cal decoder with predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ sematic_embeds: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
decoder_outs = self.decoder(encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens)
decoder_out = decoder_outs[0]
decoder_out = torch.log_softmax(decoder_out, dim=-1)
@@ -306,6 +351,14 @@ class EParaformer(torch.nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -364,6 +417,15 @@ class EParaformer(torch.nn.Module):
def sampler(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):
+ """Sampler.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ pre_acoustic_embeds: TODO.
+ """
tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(
ys_pad.device
)
@@ -404,6 +466,15 @@ class EParaformer(torch.nn.Module):
def sampler_with_grad(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):
+ """Sampler with grad.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ pre_acoustic_embeds: TODO.
+ """
tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(
ys_pad.device
)
@@ -450,6 +521,14 @@ class EParaformer(torch.nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -463,6 +542,11 @@ class EParaformer(torch.nn.Module):
self,
**kwargs,
):
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.paraformer.search import BeamSearchPara
from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
from funasr.models.transformer.scorers.length_bonus import LengthBonus
@@ -516,6 +600,16 @@ class EParaformer(torch.nn.Module):
**kwargs,
):
# init beamsearch
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None
is_use_lm = (
kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
@@ -662,6 +756,11 @@ class EParaformer(torch.nn.Module):
return results, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
if "max_seq_len" not in kwargs:
diff --git a/funasr/models/e_paraformer/pif_predictor.py b/funasr/models/e_paraformer/pif_predictor.py
index 8d350f6ec..6cd666486 100644
--- a/funasr/models/e_paraformer/pif_predictor.py
+++ b/funasr/models/e_paraformer/pif_predictor.py
@@ -1,107 +1,131 @@
-#!/usr/bin/env python3
-# -*- encoding: utf-8 -*-
-# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
-# Copyright 2024 Kun Zou ([email protected]). All Rights Reserved.
-# MIT License (https://opensource.org/licenses/MIT)
-
-import torch
-import logging
-import numpy as np
-
-from funasr.register import tables
-from funasr.train_utils.device_funcs import to_device
-from funasr.models.transformer.utils.nets_utils import make_pad_mask
-from torch.cuda.amp import autocast
-
-
[email protected]("predictor_classes", "PifPredictor")
-class PifPredictor(torch.nn.Module):
- """
- Author: Kun Zou, [email protected]
- E-Paraformer: A Faster and Better Parallel Transformer for Non-autoregressive End-to-End Mandarin Speech Recognition
- https://www.isca-archive.org/interspeech_2024/zou24_interspeech.pdf
- """
- def __init__(
- self,
- idim,
- l_order,
- r_order,
- threshold=1.0,
- dropout=0.1,
- smooth_factor=1.0,
- noise_threshold=0,
- sigma=0.5,
- bias=0.0,
- sigma_heads=4,
- ):
- super().__init__()
-
- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)
- self.cif_output = torch.nn.Linear(idim, 1)
- self.dropout = torch.nn.Dropout(p=dropout)
- self.threshold = threshold
- self.smooth_factor = smooth_factor
- self.noise_threshold = noise_threshold
- self.sigma = torch.nn.Parameter(torch.tensor([sigma]*sigma_heads))
- self.bias = torch.nn.Parameter(torch.tensor([bias]*sigma_heads))
- self.sigma_heads = sigma_heads
-
- def forward(
- self,
- hidden,
- target_label=None,
- mask=None,
- ignore_id=-1,
- mask_chunk_predictor=None,
- target_label_length=None,
- ):
-
- with autocast(False):
- h = hidden
- context = h.transpose(1, 2)
- queries = self.pad(context)
- memory = self.cif_conv1d(queries)
- output = memory + context
- output = self.dropout(output)
- output = output.transpose(1, 2)
- output = torch.relu(output)
- output = self.cif_output(output)
- alphas = torch.sigmoid(output)
- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
- if mask is not None:
- mask = mask.transpose(-1, -2).float()
- alphas = alphas * mask
- if mask_chunk_predictor is not None:
- alphas = alphas * mask_chunk_predictor
- alphas = alphas.squeeze(-1)
- mask = mask.squeeze(-1)
- if target_label_length is not None:
- target_length = target_label_length
- elif target_label is not None:
- target_mask = (target_label != ignore_id).float()
- target_length = target_mask.sum(-1)
- else:
- target_mask = None
- target_length = None
- token_num = alphas.sum(-1)
- if target_length is not None:
- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
- max_token_num = torch.max(target_length)
- else:
- token_num_int = token_num.round()
- alphas *=(token_num_int / token_num)[:, None]
- max_token_num = torch.max(token_num_int)
- alignment = torch.cumsum(alphas, dim=-1)
- fire_positions = (torch.arange(max_token_num) + 0.5).type_as(alphas).unsqueeze(0)
- scores = - ((fire_positions[:, None, :, None] - alignment[:, None, None, :]) * self.sigma[None, :, None, None]) **2 + self.bias[None, :, None, None]
- scores = scores.masked_fill(~(mask[:, None, None, :].to(torch.bool)), float("-inf"))
- weights = torch.softmax(scores, dim=-1)
- n_hidden = hidden.view(hidden.size(0), -1, self.sigma_heads, hidden.size(-1) // self.sigma_heads).transpose(1, 2)
- acoustic_embeds = torch.matmul(weights, n_hidden).transpose(1,2).contiguous().view(hidden.size(0), -1, hidden.size(-1))
-
- if target_mask is not None:
- acoustic_embeds *= target_mask[:, :, None]
- cif_peak = None
- return acoustic_embeds, token_num, alphas, cif_peak
-
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+# Copyright 2024 Kun Zou ([email protected]). All Rights Reserved.
+# MIT License (https://opensource.org/licenses/MIT)
+
+import torch
+import logging
+import numpy as np
+
+from funasr.register import tables
+from funasr.train_utils.device_funcs import to_device
+from funasr.models.transformer.utils.nets_utils import make_pad_mask
+from torch.cuda.amp import autocast
+
+
[email protected]("predictor_classes", "PifPredictor")
+class PifPredictor(torch.nn.Module):
+ """
+ Author: Kun Zou, [email protected]
+ E-Paraformer: A Faster and Better Parallel Transformer for Non-autoregressive End-to-End Mandarin Speech Recognition
+ https://www.isca-archive.org/interspeech_2024/zou24_interspeech.pdf
+ """
+ def __init__(
+ self,
+ idim,
+ l_order,
+ r_order,
+ threshold=1.0,
+ dropout=0.1,
+ smooth_factor=1.0,
+ noise_threshold=0,
+ sigma=0.5,
+ bias=0.0,
+ sigma_heads=4,
+ ):
+ """Initialize PifPredictor.
+
+ Args:
+ idim: TODO.
+ l_order: TODO.
+ r_order: TODO.
+ threshold: TODO.
+ dropout: TODO.
+ smooth_factor: TODO.
+ noise_threshold: TODO.
+ sigma: TODO.
+ bias: TODO.
+ sigma_heads: TODO.
+ """
+ super().__init__()
+
+ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)
+ self.cif_output = torch.nn.Linear(idim, 1)
+ self.dropout = torch.nn.Dropout(p=dropout)
+ self.threshold = threshold
+ self.smooth_factor = smooth_factor
+ self.noise_threshold = noise_threshold
+ self.sigma = torch.nn.Parameter(torch.tensor([sigma]*sigma_heads))
+ self.bias = torch.nn.Parameter(torch.tensor([bias]*sigma_heads))
+ self.sigma_heads = sigma_heads
+
+ def forward(
+ self,
+ hidden,
+ target_label=None,
+ mask=None,
+ ignore_id=-1,
+ mask_chunk_predictor=None,
+ target_label_length=None,
+ ):
+
+ """Forward pass for training.
+
+ Args:
+ hidden: TODO.
+ target_label: TODO.
+ mask: TODO.
+ ignore_id: TODO.
+ mask_chunk_predictor: TODO.
+ target_label_length: TODO.
+ """
+ with autocast(False):
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ memory = self.cif_conv1d(queries)
+ output = memory + context
+ output = self.dropout(output)
+ output = output.transpose(1, 2)
+ output = torch.relu(output)
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+ if mask is not None:
+ mask = mask.transpose(-1, -2).float()
+ alphas = alphas * mask
+ if mask_chunk_predictor is not None:
+ alphas = alphas * mask_chunk_predictor
+ alphas = alphas.squeeze(-1)
+ mask = mask.squeeze(-1)
+ if target_label_length is not None:
+ target_length = target_label_length
+ elif target_label is not None:
+ target_mask = (target_label != ignore_id).float()
+ target_length = target_mask.sum(-1)
+ else:
+ target_mask = None
+ target_length = None
+ token_num = alphas.sum(-1)
+ if target_length is not None:
+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
+ max_token_num = torch.max(target_length)
+ else:
+ token_num_int = token_num.round()
+ alphas *=(token_num_int / token_num)[:, None]
+ max_token_num = torch.max(token_num_int)
+ alignment = torch.cumsum(alphas, dim=-1)
+ fire_positions = (torch.arange(max_token_num) + 0.5).type_as(alphas).unsqueeze(0)
+ scores = - ((fire_positions[:, None, :, None] - alignment[:, None, None, :]) * self.sigma[None, :, None, None]) **2 + self.bias[None, :, None, None]
+ scores = scores.masked_fill(~(mask[:, None, None, :].to(torch.bool)), float("-inf"))
+ weights = torch.softmax(scores, dim=-1)
+ n_hidden = hidden.view(hidden.size(0), -1, self.sigma_heads, hidden.size(-1) // self.sigma_heads).transpose(1, 2)
+ acoustic_embeds = torch.matmul(weights, n_hidden).transpose(1,2).contiguous().view(hidden.size(0), -1, hidden.size(-1))
+
+ if target_mask is not None:
+ acoustic_embeds *= target_mask[:, :, None]
+ cif_peak = None
+ return acoustic_embeds, token_num, alphas, cif_peak
+
diff --git a/funasr/models/eend/e2e_diar_eend_ola.py b/funasr/models/eend/e2e_diar_eend_ola.py
index cae5d1f23..a3dac8e99 100644
--- a/funasr/models/eend/e2e_diar_eend_ola.py
+++ b/funasr/models/eend/e2e_diar_eend_ola.py
@@ -25,10 +25,21 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
def pad_attractor(att, max_n_speakers):
+ """Pad attractor.
+
+ Args:
+ att: TODO.
+ max_n_speakers: TODO.
+ """
C, D = att.shape
if C < max_n_speakers:
att = torch.cat(
@@ -38,6 +49,12 @@ def pad_attractor(att, max_n_speakers):
def pad_labels(ts, out_size):
+ """Pad labels.
+
+ Args:
+ ts: TODO.
+ out_size: Size/dimension parameter.
+ """
for i, t in enumerate(ts):
if t.shape[1] < out_size:
ts[i] = F.pad(t, (0, out_size - t.shape[1], 0, 0), mode="constant", value=0.0)
@@ -45,6 +62,12 @@ def pad_labels(ts, out_size):
def pad_results(ys, out_size):
+ """Pad results.
+
+ Args:
+ ys: TODO.
+ out_size: Size/dimension parameter.
+ """
ys_padded = []
for i, y in enumerate(ys):
if y.shape[1] < out_size:
@@ -78,6 +101,18 @@ class DiarEENDOLAModel(nn.Module):
mapping_dict=None,
**kwargs,
):
+ """Initialize DiarEENDOLAModel.
+
+ Args:
+ frontend: Audio frontend for feature extraction.
+ encoder: TODO.
+ encoder_decoder_attractor: TODO.
+ n_units: TODO.
+ max_n_speaker: TODO.
+ attractor_loss_weight: TODO.
+ mapping_dict: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.frontend = frontend
self.enc = encoder
@@ -92,6 +127,12 @@ class DiarEENDOLAModel(nn.Module):
self.output_layer = nn.Linear(n_units, mapping_dict["oov"] + 1)
def forward_encoder(self, xs, ilens):
+ """Forward encoder.
+
+ Args:
+ xs: TODO.
+ ilens: TODO.
+ """
xs = nn.utils.rnn.pad_sequence(xs, batch_first=True, padding_value=-1)
pad_shape = xs.shape
xs_mask = [torch.ones(ilen).to(xs.device) for ilen in ilens]
@@ -104,6 +145,12 @@ class DiarEENDOLAModel(nn.Module):
return emb
def forward_post_net(self, logits, ilens):
+ """Forward post net.
+
+ Args:
+ logits: TODO.
+ ilens: TODO.
+ """
maxlen = torch.max(ilens).to(torch.int).item()
logits = nn.utils.rnn.pad_sequence(logits, batch_first=True, padding_value=-1)
logits = nn.utils.rnn.pack_padded_sequence(
@@ -125,6 +172,13 @@ class DiarEENDOLAModel(nn.Module):
) -> Tuple[torch.Tensor, Dict[str, torch.Tensor], torch.Tensor]:
# Check that batch_size is unified
+ """Forward pass for training.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speaker_labels: TODO.
+ orders: TODO.
+ """
assert len(speech) == len(speaker_labels), (len(speech), len(speaker_labels))
speech_lengths = torch.tensor([len(sph) for sph in speech]).to(torch.int64)
speaker_labels_lengths = torch.tensor([spk.shape[-1] for spk in speaker_labels]).to(
@@ -186,6 +240,15 @@ class DiarEENDOLAModel(nn.Module):
threshold: float = 0.5,
**kwargs,
):
+ """Estimate sequential.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ n_speakers: TODO.
+ shuffle: TODO.
+ threshold: TODO.
+ **kwargs: Additional keyword arguments.
+ """
speech_lengths = torch.tensor([len(sph) for sph in speech]).to(torch.int64)
emb = self.forward_encoder(speech, speech_lengths)
if shuffle:
@@ -231,6 +294,12 @@ class DiarEENDOLAModel(nn.Module):
return ys, emb, attractors, raw_n_speakers
def recover_y_from_powerlabel(self, logit, n_speaker):
+ """Recover y from powerlabel.
+
+ Args:
+ logit: TODO.
+ n_speaker: TODO.
+ """
pred = torch.argmax(torch.softmax(logit, dim=-1), dim=-1)
oov_index = torch.where(pred == self.mapping_dict["oov"])[0]
for i in oov_index:
@@ -252,6 +321,11 @@ class DiarEENDOLAModel(nn.Module):
def inv_mapping_func(self, label):
+ """Inv mapping func.
+
+ Args:
+ label: TODO.
+ """
if not isinstance(label, int):
label = int(label)
if label in self.mapping_dict["label2dec"].keys():
@@ -261,4 +335,9 @@ class DiarEENDOLAModel(nn.Module):
return num
def collect_feats(self, **batch: torch.Tensor) -> Dict[str, torch.Tensor]:
+ """Collect feats.
+
+ Args:
+ **batch: Additional keyword arguments.
+ """
pass
diff --git a/funasr/models/eend/eend_ola_dataloader.py b/funasr/models/eend/eend_ola_dataloader.py
index 43f00fb65..d8099634b 100644
--- a/funasr/models/eend/eend_ola_dataloader.py
+++ b/funasr/models/eend/eend_ola_dataloader.py
@@ -8,6 +8,11 @@ from torch.utils.data import Dataset
def custom_collate(batch):
+ """Custom collate.
+
+ Args:
+ batch: TODO.
+ """
keys, speech, speaker_labels, orders = zip(*batch)
speech = [torch.from_numpy(np.copy(sph)).to(torch.float32) for sph in speech]
speaker_labels = [torch.from_numpy(np.copy(spk)).to(torch.float32) for spk in speaker_labels]
@@ -22,6 +27,11 @@ class EENDOLADataset(Dataset):
self,
data_file,
):
+ """Initialize EENDOLADataset.
+
+ Args:
+ data_file: TODO.
+ """
self.data_file = data_file
with open(data_file) as f:
lines = f.readlines()
@@ -29,9 +39,15 @@ class EENDOLADataset(Dataset):
logging.info("total samples: {}".format(len(self.samples)))
def __len__(self):
+ """Internal: len ."""
return len(self.samples)
def __getitem__(self, idx):
+ """Internal: getitem .
+
+ Args:
+ idx: TODO.
+ """
key, speech_path, speaker_label_path = self.samples[idx]
speech = kaldiio.load_mat(speech_path)
speaker_label = kaldiio.load_mat(speaker_label_path).reshape(speech.shape[0], -1)
@@ -44,6 +60,14 @@ class EENDOLADataset(Dataset):
class EENDOLADataLoader:
def __init__(self, data_file, batch_size, shuffle=True, num_workers=8):
+ """Initialize EENDOLADataLoader.
+
+ Args:
+ data_file: TODO.
+ batch_size: Number of samples per batch.
+ shuffle: TODO.
+ num_workers: TODO.
+ """
dataset = EENDOLADataset(data_file)
self.data_loader = DataLoader(
dataset,
@@ -54,4 +78,9 @@ class EENDOLADataLoader:
)
def build_iter(self, epoch):
+ """Build iter.
+
+ Args:
+ epoch: TODO.
+ """
return self.data_loader
diff --git a/funasr/models/eend/encoder.py b/funasr/models/eend/encoder.py
index 0dbd98fdf..b7e51632e 100644
--- a/funasr/models/eend/encoder.py
+++ b/funasr/models/eend/encoder.py
@@ -7,6 +7,13 @@ from torch import nn
class MultiHeadSelfAttention(nn.Module):
def __init__(self, n_units, h=8, dropout_rate=0.1):
+ """Initialize MultiHeadSelfAttention.
+
+ Args:
+ n_units: TODO.
+ h: TODO.
+ dropout_rate: TODO.
+ """
super().__init__()
self.linearQ = nn.Linear(n_units, n_units)
self.linearK = nn.Linear(n_units, n_units)
@@ -17,6 +24,13 @@ class MultiHeadSelfAttention(nn.Module):
self.dropout = nn.Dropout(dropout_rate)
def __call__(self, x, batch_size, x_mask):
+ """Internal: call .
+
+ Args:
+ x: TODO.
+ batch_size: Number of samples per batch.
+ x_mask: TODO.
+ """
q = self.linearQ(x).view(batch_size, -1, self.h, self.d_k)
k = self.linearK(x).view(batch_size, -1, self.h, self.d_k)
v = self.linearV(x).view(batch_size, -1, self.h, self.d_k)
@@ -33,17 +47,37 @@ class MultiHeadSelfAttention(nn.Module):
class PositionwiseFeedForward(nn.Module):
def __init__(self, n_units, d_units, dropout_rate):
+ """Initialize PositionwiseFeedForward.
+
+ Args:
+ n_units: TODO.
+ d_units: TODO.
+ dropout_rate: TODO.
+ """
super(PositionwiseFeedForward, self).__init__()
self.linear1 = nn.Linear(n_units, d_units)
self.linear2 = nn.Linear(d_units, n_units)
self.dropout = nn.Dropout(dropout_rate)
def __call__(self, x):
+ """Internal: call .
+
+ Args:
+ x: TODO.
+ """
return self.linear2(self.dropout(F.relu(self.linear1(x))))
class PositionalEncoding(torch.nn.Module):
def __init__(self, d_model, dropout_rate, max_len=5000, reverse=False):
+ """Initialize PositionalEncoding.
+
+ Args:
+ d_model: D Model instance.
+ dropout_rate: TODO.
+ max_len: TODO.
+ reverse: TODO.
+ """
super(PositionalEncoding, self).__init__()
self.d_model = d_model
self.reverse = reverse
@@ -53,6 +87,11 @@ class PositionalEncoding(torch.nn.Module):
self.extend_pe(torch.tensor(0.0).expand(1, max_len))
def extend_pe(self, x):
+ """Extend pe.
+
+ Args:
+ x: TODO.
+ """
if self.pe is not None:
if self.pe.size(1) >= x.size(1):
if self.pe.dtype != x.dtype or self.pe.device != x.device:
@@ -73,6 +112,11 @@ class PositionalEncoding(torch.nn.Module):
self.pe = pe.to(device=x.device, dtype=x.dtype)
def forward(self, x: torch.Tensor):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
self.extend_pe(x)
x = x * self.xscale + self.pe[:, : x.size(1)]
return self.dropout(x)
@@ -89,6 +133,17 @@ class EENDOLATransformerEncoder(nn.Module):
dropout_rate: float = 0.1,
use_pos_emb: bool = False,
):
+ """Initialize EENDOLATransformerEncoder.
+
+ Args:
+ idim: TODO.
+ n_layers: TODO.
+ n_units: TODO.
+ e_units: TODO.
+ h: TODO.
+ dropout_rate: TODO.
+ use_pos_emb: TODO.
+ """
super(EENDOLATransformerEncoder, self).__init__()
self.linear_in = nn.Linear(idim, n_units)
self.lnorm_in = nn.LayerNorm(n_units)
@@ -106,6 +161,12 @@ class EENDOLATransformerEncoder(nn.Module):
self.lnorm_out = nn.LayerNorm(n_units)
def __call__(self, x, x_mask=None):
+ """Internal: call .
+
+ Args:
+ x: TODO.
+ x_mask: TODO.
+ """
BT_size = x.shape[0] * x.shape[1]
e = self.linear_in(x.reshape(BT_size, -1))
for i in range(self.n_layers):
diff --git a/funasr/models/eend/encoder_decoder_attractor.py b/funasr/models/eend/encoder_decoder_attractor.py
index 65007912d..7961a71a4 100644
--- a/funasr/models/eend/encoder_decoder_attractor.py
+++ b/funasr/models/eend/encoder_decoder_attractor.py
@@ -7,6 +7,13 @@ from torch import nn
class EncoderDecoderAttractor(nn.Module):
def __init__(self, n_units, encoder_dropout=0.1, decoder_dropout=0.1):
+ """Initialize EncoderDecoderAttractor.
+
+ Args:
+ n_units: TODO.
+ encoder_dropout: TODO.
+ decoder_dropout: TODO.
+ """
super(EncoderDecoderAttractor, self).__init__()
self.enc0_dropout = nn.Dropout(encoder_dropout)
self.encoder = nn.LSTM(n_units, n_units, 1, batch_first=True, dropout=encoder_dropout)
@@ -16,6 +23,12 @@ class EncoderDecoderAttractor(nn.Module):
self.n_units = n_units
def forward_core(self, xs, zeros):
+ """Forward core.
+
+ Args:
+ xs: TODO.
+ zeros: TODO.
+ """
ilens = torch.from_numpy(np.array([x.shape[0] for x in xs])).to(torch.int64)
xs = [self.enc0_dropout(x) for x in xs]
xs = nn.utils.rnn.pad_sequence(xs, batch_first=True, padding_value=-1)
@@ -36,6 +49,12 @@ class EncoderDecoderAttractor(nn.Module):
return attractors
def forward(self, xs, n_speakers):
+ """Forward pass for training.
+
+ Args:
+ xs: TODO.
+ n_speakers: TODO.
+ """
zeros = [
torch.zeros(n_spk + 1, self.n_units).to(torch.float32).to(xs[0].device)
for n_spk in n_speakers
@@ -56,6 +75,12 @@ class EncoderDecoderAttractor(nn.Module):
return loss, attractors
def estimate(self, xs, max_n_speakers=15):
+ """Estimate.
+
+ Args:
+ xs: TODO.
+ max_n_speakers: TODO.
+ """
zeros = [
torch.zeros(max_n_speakers, self.n_units).to(torch.float32).to(xs[0].device) for _ in xs
]
diff --git a/funasr/models/eend/utils/feature.py b/funasr/models/eend/utils/feature.py
index 1dd55dc05..d62bb4104 100644
--- a/funasr/models/eend/utils/feature.py
+++ b/funasr/models/eend/utils/feature.py
@@ -12,6 +12,13 @@ def get_input_dim(
context_size,
transform_type,
):
+ """Get input dim.
+
+ Args:
+ frame_size: Size/dimension parameter.
+ context_size: Size/dimension parameter.
+ transform_type: TODO.
+ """
if transform_type.startswith("logmel23"):
frame_size = 23
elif transform_type.startswith("logmel"):
@@ -155,6 +162,13 @@ def stft(data, frame_size=1024, frame_shift=256):
def _count_frames(data_len, size, shift):
# HACK: Assuming librosa.stft(..., center=True)
+ """Internal: count frames.
+
+ Args:
+ data_len: TODO.
+ size: TODO.
+ shift: TODO.
+ """
n_frames = 1 + int(data_len / shift)
if data_len % shift == 0:
n_frames = n_frames - 1
diff --git a/funasr/models/eend/utils/kaldi_data.py b/funasr/models/eend/utils/kaldi_data.py
index 59e7a16f8..25f658b80 100644
--- a/funasr/models/eend/utils/kaldi_data.py
+++ b/funasr/models/eend/utils/kaldi_data.py
@@ -26,6 +26,11 @@ def load_segments(segments_file):
def load_segments_hash(segments_file):
+ """Load segments hash.
+
+ Args:
+ segments_file: TODO.
+ """
ret = {}
if not os.path.exists(segments_file):
return None
@@ -36,6 +41,11 @@ def load_segments_hash(segments_file):
def load_segments_rechash(segments_file):
+ """Load segments rechash.
+
+ Args:
+ segments_file: TODO.
+ """
ret = {}
if not os.path.exists(segments_file):
return None
@@ -140,6 +150,11 @@ def extract_segments(wavs, segments=None):
class KaldiData:
def __init__(self, data_dir):
+ """Initialize KaldiData.
+
+ Args:
+ data_dir: TODO.
+ """
self.data_dir = data_dir
self.segments = load_segments_rechash(os.path.join(self.data_dir, "segments"))
self.utt2spk = load_utt2spk(os.path.join(self.data_dir, "utt2spk"))
@@ -148,5 +163,12 @@ class KaldiData:
self.spk2utt = load_spk2utt(os.path.join(self.data_dir, "spk2utt"))
def load_wav(self, recid, start=0, end=None):
+ """Load wav.
+
+ Args:
+ recid: TODO.
+ start: TODO.
+ end: TODO.
+ """
data, rate = load_wav(self.wavs[recid], start, end)
return data, rate
diff --git a/funasr/models/eend/utils/losses.py b/funasr/models/eend/utils/losses.py
index 82fb57358..09086161e 100644
--- a/funasr/models/eend/utils/losses.py
+++ b/funasr/models/eend/utils/losses.py
@@ -5,6 +5,12 @@ from scipy.optimize import linear_sum_assignment
def standard_loss(ys, ts):
+ """Standard loss.
+
+ Args:
+ ys: TODO.
+ ts: TODO.
+ """
losses = [F.binary_cross_entropy(torch.sigmoid(y), t) * len(y) for y, t in zip(ys, ts)]
loss = torch.sum(torch.stack(losses))
n_frames = (
@@ -17,6 +23,12 @@ def standard_loss(ys, ts):
def fast_batch_pit_n_speaker_loss(ys, ts):
+ """Fast batch pit n speaker loss.
+
+ Args:
+ ys: TODO.
+ ts: TODO.
+ """
with torch.no_grad():
bs = len(ys)
indices = []
@@ -35,6 +47,12 @@ def fast_batch_pit_n_speaker_loss(ys, ts):
def cal_power_loss(logits, power_ts):
+ """Cal power loss.
+
+ Args:
+ logits: TODO.
+ power_ts: TODO.
+ """
losses = [
F.cross_entropy(input=logit, target=power_t.to(torch.long)) * len(logit)
for logit, power_t in zip(logits, power_ts)
diff --git a/funasr/models/eend/utils/power.py b/funasr/models/eend/utils/power.py
index b6b32c556..9db0a92d1 100644
--- a/funasr/models/eend/utils/power.py
+++ b/funasr/models/eend/utils/power.py
@@ -7,6 +7,12 @@ from itertools import permutations
def generate_mapping_dict(max_speaker_num=6, max_olp_speaker_num=3):
+ """Generate mapping dict.
+
+ Args:
+ max_speaker_num: TODO.
+ max_olp_speaker_num: TODO.
+ """
all_kinds = []
all_kinds.append(0)
for i in range(max_olp_speaker_num):
@@ -32,6 +38,12 @@ def generate_mapping_dict(max_speaker_num=6, max_olp_speaker_num=3):
def raw_dec_trans(x, max_speaker_num):
+ """Raw dec trans.
+
+ Args:
+ x: TODO.
+ max_speaker_num: TODO.
+ """
num_list = []
for i in range(max_speaker_num):
num_list.append(x[:, i])
@@ -45,6 +57,12 @@ def raw_dec_trans(x, max_speaker_num):
def mapping_func(num, mapping_dict):
+ """Mapping func.
+
+ Args:
+ num: TODO.
+ mapping_dict: TODO.
+ """
if num in mapping_dict["dec2label"].keys():
label = mapping_dict["dec2label"][num]
else:
@@ -53,6 +71,13 @@ def mapping_func(num, mapping_dict):
def dec_trans(x, max_speaker_num, mapping_dict):
+ """Dec trans.
+
+ Args:
+ x: TODO.
+ max_speaker_num: TODO.
+ mapping_dict: TODO.
+ """
num_list = []
for i in range(max_speaker_num):
num_list.append(x[:, i])
@@ -67,6 +92,14 @@ def dec_trans(x, max_speaker_num, mapping_dict):
def create_powerlabel(label, mapping_dict, max_speaker_num=6, max_olp_speaker_num=3):
+ """Create powerlabel.
+
+ Args:
+ label: TODO.
+ mapping_dict: TODO.
+ max_speaker_num: TODO.
+ max_olp_speaker_num: TODO.
+ """
T, C = label.shape
padding_label = np.zeros((T, max_speaker_num))
padding_label[:, :C] = label
@@ -76,6 +109,15 @@ def create_powerlabel(label, mapping_dict, max_speaker_num=6, max_olp_speaker_nu
def generate_perm_pse(label, n_speaker, mapping_dict, max_speaker_num, max_olp_speaker_num=3):
+ """Generate perm pse.
+
+ Args:
+ label: TODO.
+ n_speaker: TODO.
+ mapping_dict: TODO.
+ max_speaker_num: TODO.
+ max_olp_speaker_num: TODO.
+ """
perms = np.array(list(permutations(range(n_speaker)))).astype(np.float32)
perms = torch.from_numpy(perms).to(label.device).to(torch.int64)
perm_labels = [label[:, perm] for perm in perms]
@@ -91,6 +133,16 @@ def generate_perm_pse(label, n_speaker, mapping_dict, max_speaker_num, max_olp_s
def generate_min_pse(
label, n_speaker, mapping_dict, max_speaker_num, pse_logit, max_olp_speaker_num=3
):
+ """Generate min pse.
+
+ Args:
+ label: TODO.
+ n_speaker: TODO.
+ mapping_dict: TODO.
+ max_speaker_num: TODO.
+ pse_logit: TODO.
+ max_olp_speaker_num: TODO.
+ """
perm_labels, perm_pse_labels = generate_perm_pse(
label, n_speaker, mapping_dict, max_speaker_num, max_olp_speaker_num=max_olp_speaker_num
)
diff --git a/funasr/models/eend/utils/report.py b/funasr/models/eend/utils/report.py
index eb1bdacd1..d8a6abf60 100644
--- a/funasr/models/eend/utils/report.py
+++ b/funasr/models/eend/utils/report.py
@@ -17,6 +17,12 @@ metrics = [
def recover_prediction(y, n_speaker):
+ """Recover prediction.
+
+ Args:
+ y: TODO.
+ n_speaker: TODO.
+ """
if n_speaker <= 1:
return y
elif n_speaker == 2:
@@ -72,6 +78,13 @@ def recover_prediction(y, n_speaker):
class PowerReporter:
def __init__(self, valid_data_loader, mapping_dict, max_n_speaker):
+ """Initialize PowerReporter.
+
+ Args:
+ valid_data_loader: TODO.
+ mapping_dict: TODO.
+ max_n_speaker: TODO.
+ """
valid_data_loader_cp = copy.deepcopy(valid_data_loader)
self.valid_data_loader = valid_data_loader_cp
del valid_data_loader
@@ -79,9 +92,23 @@ class PowerReporter:
self.max_n_speaker = max_n_speaker
def report(self, model, eidx, device):
+ """Report.
+
+ Args:
+ model: Model instance or model name.
+ eidx: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
self.report_val(model, eidx, device)
def report_val(self, model, eidx, device):
+ """Report val.
+
+ Args:
+ model: Model instance or model name.
+ eidx: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
model.eval()
ud_valid_start = time.time()
valid_res, valid_loss, stats_keys, vad_valid_accuracy = self.report_core(
@@ -105,6 +132,12 @@ class PowerReporter:
print("Valid cost time ... ", ud_valid)
def inv_mapping_func(self, label, mapping_dict):
+ """Inv mapping func.
+
+ Args:
+ label: TODO.
+ mapping_dict: TODO.
+ """
if not isinstance(label, int):
label = int(label)
if label in mapping_dict["label2dec"].keys():
@@ -114,6 +147,13 @@ class PowerReporter:
return num
def report_core(self, model, data_loader, device):
+ """Report core.
+
+ Args:
+ model: Model instance or model name.
+ data_loader: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
res = {}
for item in metrics:
res[item[0]] = 0.0
@@ -162,6 +202,13 @@ class PowerReporter:
return res, loss_s, stats.keys(), vad_acc
def calc_diarization_error(self, decisions, label, label_delay=0):
+ """Calc diarization error.
+
+ Args:
+ decisions: TODO.
+ label: TODO.
+ label_delay: TODO.
+ """
label = label[: len(label) - label_delay, ...]
n_ref = torch.sum(label, dim=-1)
n_sys = torch.sum(decisions, dim=-1)
diff --git a/funasr/models/emotion2vec/audio.py b/funasr/models/emotion2vec/audio.py
index 260957343..0e83277e5 100644
--- a/funasr/models/emotion2vec/audio.py
+++ b/funasr/models/emotion2vec/audio.py
@@ -32,6 +32,16 @@ class AudioEncoder(ModalitySpecificEncoder):
alibi_biases: Dict,
):
+ """Initialize AudioEncoder.
+
+ Args:
+ modality_cfg: TODO.
+ embed_dim: Size/dimension parameter.
+ make_block: TODO.
+ norm_layer: TODO.
+ layer_norm_first: TODO.
+ alibi_biases: TODO.
+ """
self.feature_enc_layers = eval(modality_cfg.feature_encoder_spec)
feature_embed_dim = self.feature_enc_layers[-1][0]
@@ -108,12 +118,25 @@ class AudioEncoder(ModalitySpecificEncoder):
)
def convert_padding_mask(self, x, padding_mask):
+ """Convert padding mask.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ """
def get_feat_extract_output_lengths(input_lengths: torch.LongTensor):
"""
Computes the output length of the convolutional layers
"""
def _conv_out_length(input_length, kernel_size, stride):
+ """Internal: conv out length.
+
+ Args:
+ input_length: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ """
return torch.floor((input_length - kernel_size) / stride + 1)
for i in range(len(self.feature_enc_layers)):
@@ -148,6 +171,7 @@ class AudioEncoder(ModalitySpecificEncoder):
return padding_mask
def reset_parameters(self):
+ """Reset parameters."""
super().reset_parameters()
for mod in self.project_features.children():
if isinstance(mod, nn.Linear):
diff --git a/funasr/models/emotion2vec/base.py b/funasr/models/emotion2vec/base.py
index 1b4301ea3..7a9659355 100644
--- a/funasr/models/emotion2vec/base.py
+++ b/funasr/models/emotion2vec/base.py
@@ -39,6 +39,19 @@ class ModalitySpecificEncoder(nn.Module):
decoder: nn.Module,
get_alibi_bias: Optional[Callable[[int, int, str, str], torch.Tensor]],
):
+ """Initialize ModalitySpecificEncoder.
+
+ Args:
+ modality_cfg: TODO.
+ embed_dim: Size/dimension parameter.
+ local_encoder: TODO.
+ project_features: TODO.
+ fixed_positional_encoder: TODO.
+ relative_positional_encoder: TODO.
+ context_encoder: TODO.
+ decoder: TODO.
+ get_alibi_bias: TODO.
+ """
super().__init__()
self.modality_cfg = modality_cfg
@@ -102,6 +115,12 @@ class ModalitySpecificEncoder(nn.Module):
self.get_alibi_bias = partial(_learned_alibi_bias, alibi_bias=self.alibi_bias)
def upgrade_state_dict_named(self, state_dict, name):
+ """Upgrade state dict named.
+
+ Args:
+ state_dict: TODO.
+ name: TODO.
+ """
k = f"{name}.alibi_scale"
if k in state_dict and state_dict[k].dim() == 4:
state_dict[k] = state_dict[k].unsqueeze(0)
@@ -109,9 +128,21 @@ class ModalitySpecificEncoder(nn.Module):
return state_dict
def convert_padding_mask(self, x, padding_mask):
+ """Convert padding mask.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ """
return padding_mask
def decoder_input(self, x, mask_info: MaskInfo):
+ """Decoder input.
+
+ Args:
+ x: TODO.
+ mask_info: TODO.
+ """
inp_drop = self.modality_cfg.decoder.input_dropout
if inp_drop > 0:
x = F.dropout(x, inp_drop, training=self.training, inplace=True)
@@ -144,6 +175,11 @@ class ModalitySpecificEncoder(nn.Module):
return x, mask_info
def local_features(self, features):
+ """Local features.
+
+ Args:
+ features: TODO.
+ """
if self.local_grad_mult > 0:
if self.local_grad_mult == 1.0:
x = self.local_encoder(features)
@@ -167,6 +203,17 @@ class ModalitySpecificEncoder(nn.Module):
precomputed_mask=None,
):
+ """Contextualized features.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ mask: TODO.
+ remove_masked: TODO.
+ clone_batch: TODO.
+ mask_seeds: TODO.
+ precomputed_mask: TODO.
+ """
if padding_mask is not None:
padding_mask = self.convert_padding_mask(x, padding_mask)
@@ -290,6 +337,17 @@ class ModalitySpecificEncoder(nn.Module):
mask_seeds: Optional[torch.Tensor] = None,
precomputed_mask=None,
):
+ """Forward pass for training.
+
+ Args:
+ features: TODO.
+ padding_mask: TODO.
+ mask: TODO.
+ remove_masked: TODO.
+ clone_batch: TODO.
+ mask_seeds: TODO.
+ precomputed_mask: TODO.
+ """
x = self.local_features(features)
return self.contextualized_features(
x,
@@ -302,6 +360,7 @@ class ModalitySpecificEncoder(nn.Module):
)
def reset_parameters(self):
+ """Reset parameters."""
pass
def compute_mask(
@@ -312,6 +371,15 @@ class ModalitySpecificEncoder(nn.Module):
apply,
precomputed_mask,
):
+ """Compute mask.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ mask_seed: TODO.
+ apply: TODO.
+ precomputed_mask: TODO.
+ """
if precomputed_mask is not None:
mask = precomputed_mask
mask_info = self.make_maskinfo(x, mask)
@@ -362,6 +430,13 @@ class ModalitySpecificEncoder(nn.Module):
return x, mask_info
def make_maskinfo(self, x, mask, shape=None):
+ """Make maskinfo.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ shape: TODO.
+ """
if shape is None:
B, T, D = x.shape
else:
@@ -392,6 +467,12 @@ class ModalitySpecificEncoder(nn.Module):
return mask_info
def apply_mask(self, x, mask_info):
+ """Apply mask.
+
+ Args:
+ x: TODO.
+ mask_info: TODO.
+ """
cfg = self.modality_cfg
B, T, C = x.shape
@@ -417,11 +498,24 @@ class ModalitySpecificEncoder(nn.Module):
return x
def remove_pretraining_modules(self, keep_decoder=False):
+ """Remove pretraining modules.
+
+ Args:
+ keep_decoder: TODO.
+ """
if not keep_decoder:
self.decoder = None
def get_annealed_rate(start, end, curr_step, total_steps):
+ """Get annealed rate.
+
+ Args:
+ start: TODO.
+ end: TODO.
+ curr_step: TODO.
+ total_steps: TODO.
+ """
if curr_step >= total_steps:
return end
r = end - start
@@ -431,6 +525,13 @@ def get_annealed_rate(start, end, curr_step, total_steps):
# adapted from MAE
def random_masking(x, mask_ratio, mask_seed: Optional[MaskSeed]):
+ """Random masking.
+
+ Args:
+ x: TODO.
+ mask_ratio: TODO.
+ mask_seed: TODO.
+ """
N, L, D = x.shape # batch, length, dim
len_keep = int(L * (1 - mask_ratio))
@@ -463,6 +564,12 @@ def random_masking(x, mask_ratio, mask_seed: Optional[MaskSeed]):
def gather_unmasked(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor:
+ """Gather unmasked.
+
+ Args:
+ x: TODO.
+ mask_info: TODO.
+ """
return torch.gather(
x,
dim=1,
@@ -471,6 +578,12 @@ def gather_unmasked(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor:
def gather_unmasked_mask(x: torch.Tensor, mask_info: MaskInfo) -> torch.Tensor:
+ """Gather unmasked mask.
+
+ Args:
+ x: TODO.
+ mask_info: TODO.
+ """
return torch.gather(
x,
dim=1,
@@ -484,8 +597,26 @@ def get_alibi(
dims: int = 1,
distance: str = "manhattan",
):
+ """Get alibi.
+
+ Args:
+ max_positions: TODO.
+ attention_heads: TODO.
+ dims: TODO.
+ distance: TODO.
+ """
def get_slopes(n):
+ """Get slopes.
+
+ Args:
+ n: TODO.
+ """
def get_slopes_power_of_2(n):
+ """Get slopes power of 2.
+
+ Args:
+ n: TODO.
+ """
start = 2 ** (-(2 ** -(math.log2(n) - 3)))
ratio = start
return [start * ratio**i for i in range(n)]
@@ -552,6 +683,18 @@ def get_alibi_bias(
dims=1,
distance="manhattan",
):
+ """Get alibi bias.
+
+ Args:
+ alibi_biases: TODO.
+ batch_size: Number of samples per batch.
+ time_steps: TODO.
+ heads: TODO.
+ dtype: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ dims: TODO.
+ distance: TODO.
+ """
cache_key = f"{dims}_{heads}_{distance}"
buffered = alibi_biases.get(cache_key, None)
@@ -589,6 +732,17 @@ def _learned_alibi_bias(
dtype,
device,
):
+ """Internal: learned alibi bias.
+
+ Args:
+ alibi_bias: TODO.
+ batch_size: Number of samples per batch.
+ time_steps: TODO.
+ heads: TODO.
+ scale: TODO.
+ dtype: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
assert alibi_bias.size(1) == heads, alibi_bias.shape
assert alibi_bias.dtype == dtype, alibi_bias.dtype
assert alibi_bias.device == device, alibi_bias.device
@@ -602,6 +756,12 @@ def _learned_alibi_bias(
def masked_alibi(alibi_bias, mask_info):
+ """Masked alibi.
+
+ Args:
+ alibi_bias: TODO.
+ mask_info: TODO.
+ """
H = alibi_bias.size(1)
orig_bias = alibi_bias
diff --git a/funasr/models/emotion2vec/export_meta.py b/funasr/models/emotion2vec/export_meta.py
index f0c5f434b..3bd53d64c 100644
--- a/funasr/models/emotion2vec/export_meta.py
+++ b/funasr/models/emotion2vec/export_meta.py
@@ -9,6 +9,12 @@ import torch.nn.functional as F
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
model.device = kwargs.get("device")
# store original forward since self.extract_features is calling it
@@ -25,6 +31,11 @@ def export_rebuild_model(model, **kwargs):
def export_forward(self, x: torch.Tensor):
+ """Export forward.
+
+ Args:
+ x: TODO.
+ """
with torch.no_grad():
if self.cfg.normalize:
mean = torch.mean(x, dim=1, keepdim=True)
@@ -44,18 +55,22 @@ def export_forward(self, x: torch.Tensor):
def export_dummy_inputs(self):
+ """Export dummy inputs."""
return (torch.randn(1, 16000),)
def export_input_names(self):
+ """Export input names."""
return ["input"]
def export_output_names(self):
+ """Export output names."""
return ["output"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"input": {
0: "batch_size",
@@ -66,4 +81,5 @@ def export_dynamic_axes(self):
def export_name(self):
+ """Export name."""
return "emotion2vec"
diff --git a/funasr/models/emotion2vec/fairseq_modules.py b/funasr/models/emotion2vec/fairseq_modules.py
index 209dedaa3..3f7887225 100644
--- a/funasr/models/emotion2vec/fairseq_modules.py
+++ b/funasr/models/emotion2vec/fairseq_modules.py
@@ -6,11 +6,25 @@ import numpy as np
def LayerNorm(normalized_shape, eps=1e-5, elementwise_affine=True, export=False):
+ """Layernorm.
+
+ Args:
+ normalized_shape: TODO.
+ eps: TODO.
+ elementwise_affine: TODO.
+ export: TODO.
+ """
return torch.nn.LayerNorm(normalized_shape, eps, elementwise_affine)
class SamePad(nn.Module):
def __init__(self, kernel_size, causal=False):
+ """Initialize SamePad.
+
+ Args:
+ kernel_size: Size/dimension parameter.
+ causal: TODO.
+ """
super().__init__()
if causal:
self.remove = kernel_size - 1
@@ -18,6 +32,11 @@ class SamePad(nn.Module):
self.remove = 1 if kernel_size % 2 == 0 else 0
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if self.remove > 0:
x = x[:, :, : -self.remove]
return x
@@ -25,10 +44,20 @@ class SamePad(nn.Module):
class TransposeLast(nn.Module):
def __init__(self, deconstruct_idx=None):
+ """Initialize TransposeLast.
+
+ Args:
+ deconstruct_idx: TODO.
+ """
super().__init__()
self.deconstruct_idx = deconstruct_idx
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if self.deconstruct_idx is not None:
x = x[self.deconstruct_idx]
return x.transpose(-2, -1)
@@ -36,9 +65,20 @@ class TransposeLast(nn.Module):
class Fp32LayerNorm(nn.LayerNorm):
def __init__(self, *args, **kwargs):
+ """Initialize Fp32LayerNorm.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = F.layer_norm(
input.float(),
self.normalized_shape,
@@ -51,9 +91,20 @@ class Fp32LayerNorm(nn.LayerNorm):
class Fp32GroupNorm(nn.GroupNorm):
def __init__(self, *args, **kwargs):
+ """Initialize Fp32GroupNorm.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = F.group_norm(
input.float(),
self.num_groups,
@@ -72,6 +123,14 @@ class ConvFeatureExtractionModel(nn.Module):
mode: str = "default",
conv_bias: bool = False,
):
+ """Initialize ConvFeatureExtractionModel.
+
+ Args:
+ conv_layers: TODO.
+ dropout: TODO.
+ mode: TODO.
+ conv_bias: TODO.
+ """
super().__init__()
assert mode in {"default", "layer_norm"}
@@ -85,7 +144,19 @@ class ConvFeatureExtractionModel(nn.Module):
is_group_norm=False,
conv_bias=False,
):
+ """Block.
+
+ Args:
+ n_in: TODO.
+ n_out: TODO.
+ k: TODO.
+ stride: TODO.
+ is_layer_norm: Boolean flag for layer norm.
+ is_group_norm: Boolean flag for group norm.
+ conv_bias: TODO.
+ """
def make_conv():
+ """Make conv."""
conv = nn.Conv1d(n_in, n_out, k, stride=stride, bias=conv_bias)
nn.init.kaiming_normal_(conv.weight)
return conv
@@ -137,6 +208,11 @@ class ConvFeatureExtractionModel(nn.Module):
def forward(self, x):
# BxT -> BxCxT
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = x.unsqueeze(1)
for conv in self.conv_layers:
@@ -225,6 +301,14 @@ def compute_mask_indices(
mask_idc = []
def arrange(s, e, length, keep_length):
+ """Arrange.
+
+ Args:
+ s: TODO.
+ e: TODO.
+ length: TODO.
+ keep_length: TODO.
+ """
span_start = np.random.randint(s, e - length)
mask_idc.extend(span_start + i for i in range(length))
@@ -279,20 +363,45 @@ def compute_mask_indices(
class GradMultiply(torch.autograd.Function):
@staticmethod
def forward(ctx, x, scale):
+ """Forward pass for training.
+
+ Args:
+ ctx: TODO.
+ x: TODO.
+ scale: TODO.
+ """
ctx.scale = scale
res = x.new(x)
return res
@staticmethod
def backward(ctx, grad):
+ """Backward.
+
+ Args:
+ ctx: TODO.
+ grad: TODO.
+ """
return grad * ctx.scale, None
def is_xla_tensor(tensor):
+ """Is xla tensor.
+
+ Args:
+ tensor: TODO.
+ """
return torch.is_tensor(tensor) and tensor.device.type == "xla"
def index_put(tensor, indices, value):
+ """Index put.
+
+ Args:
+ tensor: TODO.
+ indices: TODO.
+ value: TODO.
+ """
if is_xla_tensor(tensor):
for _ in range(indices.dim(), tensor.dim()):
indices = indices.unsqueeze(-1)
diff --git a/funasr/models/emotion2vec/model.py b/funasr/models/emotion2vec/model.py
index 75ede4c0c..3ecace8eb 100644
--- a/funasr/models/emotion2vec/model.py
+++ b/funasr/models/emotion2vec/model.py
@@ -28,6 +28,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -40,6 +45,11 @@ class Emotion2vec(torch.nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize Emotion2vec.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# import pdb; pdb.set_trace()
cfg = OmegaConf.create(kwargs["model_conf"])
@@ -50,6 +60,13 @@ class Emotion2vec(torch.nn.Module):
)
def make_block(drop_path, dim=None, heads=None):
+ """Make block.
+
+ Args:
+ drop_path: TODO.
+ dim: TODO.
+ heads: TODO.
+ """
return AltBlock(
cfg.get("embed_dim") if dim is None else dim,
cfg.get("num_heads") if heads is None else heads,
@@ -116,6 +133,21 @@ class Emotion2vec(torch.nn.Module):
**kwargs,
):
+ """Forward pass for training.
+
+ Args:
+ source: TODO.
+ target: TODO.
+ id: TODO.
+ mode: TODO.
+ padding_mask: TODO.
+ mask: TODO.
+ features_only: TODO.
+ force_remove_masked: TODO.
+ remove_extra_tokens: TODO.
+ precomputed_mask: TODO.
+ **kwargs: Additional keyword arguments.
+ """
feature_extractor = self.modality_encoders["AUDIO"]
mask_seeds = None
@@ -180,6 +212,15 @@ class Emotion2vec(torch.nn.Module):
def extract_features(
self, source, mode=None, padding_mask=None, mask=False, remove_extra_tokens=True
):
+ """Extract features.
+
+ Args:
+ source: TODO.
+ mode: TODO.
+ padding_mask: TODO.
+ mask: TODO.
+ remove_extra_tokens: TODO.
+ """
res = self.forward(
source,
mode=mode,
@@ -205,6 +246,16 @@ class Emotion2vec(torch.nn.Module):
# channel = sf.info(source_file).channels
# assert sr == 16e3, "Sample rate should be 16kHz, but got {}in file {}".format(sr, source_file)
# assert channel == 1, "Channel should be 1, but got {} in file {}".format(channel, source_file)
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
granularity = kwargs.get("granularity", "utterance")
extract_embedding = kwargs.get("extract_embedding", True)
if self.proj is None:
@@ -267,6 +318,11 @@ class Emotion2vec(torch.nn.Module):
return results, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
models = export_rebuild_model(model=self, **kwargs)
diff --git a/funasr/models/emotion2vec/modules.py b/funasr/models/emotion2vec/modules.py
index 6e3640948..2cb8db6e5 100644
--- a/funasr/models/emotion2vec/modules.py
+++ b/funasr/models/emotion2vec/modules.py
@@ -38,10 +38,21 @@ class D2vDecoderConfig:
class FixedPositionalEncoder(nn.Module):
def __init__(self, pos_embed):
+ """Initialize FixedPositionalEncoder.
+
+ Args:
+ pos_embed: TODO.
+ """
super().__init__()
self.positions = pos_embed
def forward(self, x, padding_mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ """
return self.positions
@@ -52,17 +63,37 @@ class TextFeatPositionalEncoder(nn.Module):
"""
def __init__(self, pos_encoder):
+ """Initialize TextFeatPositionalEncoder.
+
+ Args:
+ pos_encoder: TODO.
+ """
super().__init__()
self.pos_encoder = pos_encoder
def forward(self, x, padding_mask):
# assume padded token embeddings are 0s
# TODO: consider using padding_mask as input
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ """
return self.pos_encoder(x[..., 0])
class BlockEncoder(nn.Module):
def __init__(self, blocks, norm_layer, layer_norm_first, layerdrop, dropout):
+ """Initialize BlockEncoder.
+
+ Args:
+ blocks: TODO.
+ norm_layer: TODO.
+ layer_norm_first: TODO.
+ layerdrop: TODO.
+ dropout: TODO.
+ """
super().__init__()
self.blocks = blocks
self.norm = norm_layer
@@ -71,6 +102,14 @@ class BlockEncoder(nn.Module):
self.dropout = nn.Dropout(dropout, inplace=True)
def forward(self, x, padding_mask, alibi_bias, alibi_scale):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ alibi_bias: TODO.
+ alibi_scale: TODO.
+ """
if self.norm is not None and not self.layer_norm_first:
x = self.norm(x)
@@ -94,16 +133,30 @@ class DecoderBase(nn.Module):
decoder_cfg: D2vDecoderConfig
def __init__(self, cfg: D2vDecoderConfig):
+ """Initialize DecoderBase.
+
+ Args:
+ cfg: Configuration overrides.
+ """
super().__init__()
self.decoder_cfg = cfg
def reset_parameters(self):
+ """Reset parameters."""
for mod in self.proj.modules():
if isinstance(mod, nn.Linear):
mod.reset_parameters()
def add_residual(self, x, residual, i, mask_info):
+ """Add residual.
+
+ Args:
+ x: TODO.
+ residual: TODO.
+ i: TODO.
+ mask_info: TODO.
+ """
if (
residual is None
or not self.decoder_cfg.decoder_residual
@@ -118,9 +171,20 @@ class DecoderBase(nn.Module):
class Decoder1d(DecoderBase):
def __init__(self, cfg: D2vDecoderConfig, input_dim):
+ """Initialize Decoder1d.
+
+ Args:
+ cfg: Configuration overrides.
+ input_dim: Size/dimension parameter.
+ """
super().__init__(cfg)
def make_block(in_dim):
+ """Make block.
+
+ Args:
+ in_dim: Size/dimension parameter.
+ """
block = [
nn.Conv1d(
in_dim,
@@ -160,6 +224,12 @@ class Decoder1d(DecoderBase):
def forward(self, x, mask_info):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask_info: TODO.
+ """
x = x.transpose(1, 2)
residual = x
@@ -193,6 +263,25 @@ class AltBlock(nn.Module):
ffn_targets=False,
cosine_attention=False,
):
+ """Initialize AltBlock.
+
+ Args:
+ dim: TODO.
+ num_heads: TODO.
+ mlp_ratio: TODO.
+ qkv_bias: TODO.
+ qk_scale: TODO.
+ drop: TODO.
+ attn_drop: TODO.
+ mlp_drop: TODO.
+ post_mlp_drop: TODO.
+ drop_path: TODO.
+ act_layer: TODO.
+ norm_layer: TODO.
+ layer_norm_first: TODO.
+ ffn_targets: TODO.
+ cosine_attention: TODO.
+ """
super().__init__()
self.layer_norm_first = layer_norm_first
@@ -223,6 +312,13 @@ class AltBlock(nn.Module):
self.post_mlp_dropout = nn.Dropout(post_mlp_drop, inplace=False)
def forward(self, x, padding_mask=None, alibi_bias=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ alibi_bias: TODO.
+ """
if self.layer_norm_first:
x = x + self.drop_path(self.attn(self.norm1(x), padding_mask, alibi_bias))
r = x = self.mlp(self.norm2(x))
@@ -253,6 +349,17 @@ class AltAttention(nn.Module):
proj_drop=0.0,
cosine_attention=False,
):
+ """Initialize AltAttention.
+
+ Args:
+ dim: TODO.
+ num_heads: TODO.
+ qkv_bias: TODO.
+ qk_scale: TODO.
+ attn_drop: TODO.
+ proj_drop: TODO.
+ cosine_attention: TODO.
+ """
super().__init__()
self.num_heads = num_heads
head_dim = dim // num_heads
@@ -271,6 +378,13 @@ class AltAttention(nn.Module):
)
def forward(self, x, padding_mask=None, alibi_bias=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ padding_mask: TODO.
+ alibi_bias: TODO.
+ """
B, N, C = x.shape
qkv = (
self.qkv(x)
diff --git a/funasr/models/emotion2vec/timm_modules.py b/funasr/models/emotion2vec/timm_modules.py
index 416d2cbaf..04d9cffc8 100644
--- a/funasr/models/emotion2vec/timm_modules.py
+++ b/funasr/models/emotion2vec/timm_modules.py
@@ -28,20 +28,42 @@ class DropPath(nn.Module):
"""Drop paths (Stochastic Depth) per sample (when applied in main path of residual blocks)."""
def __init__(self, drop_prob: float = 0.0, scale_by_keep: bool = True):
+ """Initialize DropPath.
+
+ Args:
+ drop_prob: TODO.
+ scale_by_keep: TODO.
+ """
super(DropPath, self).__init__()
self.drop_prob = drop_prob
self.scale_by_keep = scale_by_keep
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return drop_path(x, self.drop_prob, self.training, self.scale_by_keep)
def extra_repr(self):
+ """Extra repr."""
return f"drop_prob={round(self.drop_prob,3):0.3f}"
# From PyTorch internals
def _ntuple(n):
+ """Internal: ntuple.
+
+ Args:
+ n: TODO.
+ """
def parse(x):
+ """Parse.
+
+ Args:
+ x: TODO.
+ """
if isinstance(x, collections.abc.Iterable) and not isinstance(x, str):
return tuple(x)
return tuple(repeat(x, n))
@@ -70,6 +92,18 @@ class Mlp(nn.Module):
drop=0.0,
use_conv=False,
):
+ """Initialize Mlp.
+
+ Args:
+ in_features: TODO.
+ hidden_features: TODO.
+ out_features: TODO.
+ act_layer: TODO.
+ norm_layer: TODO.
+ bias: TODO.
+ drop: TODO.
+ use_conv: TODO.
+ """
super().__init__()
out_features = out_features or in_features
hidden_features = hidden_features or in_features
@@ -85,6 +119,11 @@ class Mlp(nn.Module):
self.drop2 = nn.Dropout(drop_probs[1])
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = self.fc1(x)
x = self.act(x)
x = self.drop1(x)
diff --git a/funasr/models/eres2net/eres2net.py b/funasr/models/eres2net/eres2net.py
index de6c2bbdd..fe6a65941 100644
--- a/funasr/models/eres2net/eres2net.py
+++ b/funasr/models/eres2net/eres2net.py
@@ -23,9 +23,15 @@ from funasr.models.eres2net.fusion import AFF
class ReLU(nn.Hardtanh):
def __init__(self, inplace=False):
+ """Initialize ReLU.
+
+ Args:
+ inplace: TODO.
+ """
super(ReLU, self).__init__(0, 20, inplace)
def __repr__(self):
+ """Internal: repr ."""
inplace_str = "inplace" if self.inplace else ""
return self.__class__.__name__ + " (" + inplace_str + ")"
@@ -44,6 +50,15 @@ class BasicBlockERes2Net(nn.Module):
expansion = 2
def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):
+ """Initialize BasicBlockERes2Net.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockERes2Net, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -74,6 +89,11 @@ class BasicBlockERes2Net(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -106,6 +126,15 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):
expansion = 2
def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):
+ """Initialize BasicBlockERes2Net_diff_AFF.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockERes2Net_diff_AFF, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -141,6 +170,11 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -182,6 +216,18 @@ class ERes2Net(nn.Module):
pooling_func="TSTP",
two_emb_layer=False,
):
+ """Initialize ERes2Net.
+
+ Args:
+ block: TODO.
+ block_fuse: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ """
super(ERes2Net, self).__init__()
self.in_planes = m_channels
self.feat_dim = feat_dim
@@ -223,6 +269,14 @@ class ERes2Net(nn.Module):
self.seg_2 = nn.Identity()
def _make_layer(self, block, planes, num_blocks, stride):
+ """Internal: make layer.
+
+ Args:
+ block: TODO.
+ planes: TODO.
+ num_blocks: TODO.
+ stride: TODO.
+ """
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
@@ -231,6 +285,11 @@ class ERes2Net(nn.Module):
return nn.Sequential(*layers)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)
x = x.unsqueeze_(1)
out = F.relu(self.bn1(self.conv1(x)))
@@ -260,6 +319,15 @@ class BasicBlockRes2Net(nn.Module):
expansion = 2
def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):
+ """Initialize BasicBlockRes2Net.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockRes2Net, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -289,6 +357,11 @@ class BasicBlockRes2Net(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -330,6 +403,17 @@ class Res2Net(nn.Module):
pooling_func="TSTP",
two_emb_layer=False,
):
+ """Initialize Res2Net.
+
+ Args:
+ block: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ """
super(Res2Net, self).__init__()
self.in_planes = m_channels
self.feat_dim = feat_dim
@@ -355,6 +439,14 @@ class Res2Net(nn.Module):
self.seg_2 = nn.Identity()
def _make_layer(self, block, planes, num_blocks, stride):
+ """Internal: make layer.
+
+ Args:
+ block: TODO.
+ planes: TODO.
+ num_blocks: TODO.
+ stride: TODO.
+ """
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
@@ -363,6 +455,11 @@ class Res2Net(nn.Module):
return nn.Sequential(*layers)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)
x = x.unsqueeze_(1)
diff --git a/funasr/models/eres2net/eres2net_aug.py b/funasr/models/eres2net/eres2net_aug.py
index f700b9c98..36217892f 100644
--- a/funasr/models/eres2net/eres2net_aug.py
+++ b/funasr/models/eres2net/eres2net_aug.py
@@ -23,9 +23,15 @@ from funasr.models.eres2net.fusion import AFF
class ReLU(nn.Hardtanh):
def __init__(self, inplace=False):
+ """Initialize ReLU.
+
+ Args:
+ inplace: TODO.
+ """
super(ReLU, self).__init__(0, 20, inplace)
def __repr__(self):
+ """Internal: repr ."""
inplace_str = "inplace" if self.inplace else ""
return self.__class__.__name__ + " (" + inplace_str + ")"
@@ -44,6 +50,15 @@ class BasicBlockERes2Net(nn.Module):
expansion = 4
def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3):
+ """Initialize BasicBlockERes2Net.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockERes2Net, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -74,6 +89,11 @@ class BasicBlockERes2Net(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -106,6 +126,15 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):
expansion = 4
def __init__(self, in_planes, planes, stride=1, baseWidth=24, scale=3):
+ """Initialize BasicBlockERes2Net_diff_AFF.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockERes2Net_diff_AFF, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -142,6 +171,11 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -183,6 +217,18 @@ class ERes2NetAug(nn.Module):
pooling_func="TSTP",
two_emb_layer=False,
):
+ """Initialize ERes2NetAug.
+
+ Args:
+ block: TODO.
+ block_fuse: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ """
super(ERes2NetAug, self).__init__()
self.in_planes = m_channels
self.feat_dim = feat_dim
@@ -221,6 +267,14 @@ class ERes2NetAug(nn.Module):
self.seg_2 = nn.Identity()
def _make_layer(self, block, planes, num_blocks, stride):
+ """Internal: make layer.
+
+ Args:
+ block: TODO.
+ planes: TODO.
+ num_blocks: TODO.
+ stride: TODO.
+ """
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
@@ -229,6 +283,11 @@ class ERes2NetAug(nn.Module):
return nn.Sequential(*layers)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)
x = x.unsqueeze_(1)
diff --git a/funasr/models/eres2net/eres2netv2.py b/funasr/models/eres2net/eres2netv2.py
index 4720ab8c9..44b3b0dc8 100644
--- a/funasr/models/eres2net/eres2netv2.py
+++ b/funasr/models/eres2net/eres2netv2.py
@@ -14,9 +14,15 @@ from funasr.models.eres2net.fusion import AFF
class ReLU(nn.Hardtanh):
def __init__(self, inplace=False):
+ """Initialize ReLU.
+
+ Args:
+ inplace: TODO.
+ """
super(ReLU, self).__init__(0, 20, inplace)
def __repr__(self):
+ """Internal: repr ."""
inplace_str = "inplace" if self.inplace else ""
return self.__class__.__name__ + " (" + inplace_str + ")"
@@ -24,6 +30,16 @@ class ReLU(nn.Hardtanh):
class BasicBlockERes2NetV2(nn.Module):
def __init__(self, in_planes, planes, stride=1, baseWidth=26, scale=2, expansion=2):
+ """Initialize BasicBlockERes2NetV2.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ expansion: TODO.
+ """
super(BasicBlockERes2NetV2, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = nn.Conv2d(in_planes, width * scale, kernel_size=1, stride=stride, bias=False)
@@ -53,6 +69,11 @@ class BasicBlockERes2NetV2(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -84,6 +105,16 @@ class BasicBlockERes2NetV2(nn.Module):
class BasicBlockERes2NetV2AFF(nn.Module):
def __init__(self, in_planes, planes, stride=1, baseWidth=26, scale=2, expansion=2):
+ """Initialize BasicBlockERes2NetV2AFF.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ expansion: TODO.
+ """
super(BasicBlockERes2NetV2AFF, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = nn.Conv2d(in_planes, width * scale, kernel_size=1, stride=stride, bias=False)
@@ -118,6 +149,11 @@ class BasicBlockERes2NetV2AFF(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -161,6 +197,21 @@ class ERes2NetV2(nn.Module):
pooling_func="TSTP",
two_emb_layer=False,
):
+ """Initialize ERes2NetV2.
+
+ Args:
+ block: TODO.
+ block_fuse: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ baseWidth: TODO.
+ scale: TODO.
+ expansion: TODO.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ """
super(ERes2NetV2, self).__init__()
self.in_planes = m_channels
self.feat_dim = feat_dim
@@ -195,6 +246,14 @@ class ERes2NetV2(nn.Module):
self.seg_2 = nn.Identity()
def _make_layer(self, block, planes, num_blocks, stride):
+ """Internal: make layer.
+
+ Args:
+ block: TODO.
+ planes: TODO.
+ num_blocks: TODO.
+ stride: TODO.
+ """
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
@@ -205,6 +264,11 @@ class ERes2NetV2(nn.Module):
return nn.Sequential(*layers)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)
x = x.unsqueeze_(1)
out = F.relu(self.bn1(self.conv1(x)))
diff --git a/funasr/models/eres2net/fusion.py b/funasr/models/eres2net/fusion.py
index fbe699efb..74886e293 100644
--- a/funasr/models/eres2net/fusion.py
+++ b/funasr/models/eres2net/fusion.py
@@ -8,6 +8,12 @@ import torch.nn as nn
class AFF(nn.Module):
def __init__(self, channels=64, r=4):
+ """Initialize AFF.
+
+ Args:
+ channels: TODO.
+ r: TODO.
+ """
super(AFF, self).__init__()
inter_channels = int(channels // r)
@@ -20,6 +26,12 @@ class AFF(nn.Module):
)
def forward(self, x, ds_y):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ds_y: TODO.
+ """
xa = torch.cat((x, ds_y), dim=1)
x_att = self.local_att(xa)
x_att = 1.0 + torch.tanh(x_att)
diff --git a/funasr/models/eres2net/model.py b/funasr/models/eres2net/model.py
index e2549a2b7..ebf624ecc 100644
--- a/funasr/models/eres2net/model.py
+++ b/funasr/models/eres2net/model.py
@@ -32,6 +32,20 @@ class ERes2NetV2SV(torch.nn.Module):
two_emb_layer=False,
**kwargs,
):
+ """Initialize ERes2NetV2SV.
+
+ Args:
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ m_channels: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ expansion: TODO.
+ num_blocks: TODO.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.model = ERes2NetV2(
feat_dim=feat_dim,
@@ -56,6 +70,11 @@ class ERes2NetV2SV(torch.nn.Module):
self._load_pretrained(init_param)
def _load_pretrained(self, path):
+ """Internal: load pretrained.
+
+ Args:
+ path: TODO.
+ """
state_dict = torch.load(path, map_location="cpu")
if "state_dict" in state_dict:
state_dict = state_dict["state_dict"]
@@ -65,6 +84,11 @@ class ERes2NetV2SV(torch.nn.Module):
logging.info(f"ERes2NetV2 loaded pretrained weights from {path}")
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return self.model(x)
def inference(
@@ -76,6 +100,16 @@ class ERes2NetV2SV(torch.nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
time1 = time.perf_counter()
audio_sample_list = load_audio_text_image_video(
diff --git a/funasr/models/fsmn_kws/encoder.py b/funasr/models/fsmn_kws/encoder.py
index 2c316871f..ff4e2892d 100755
--- a/funasr/models/fsmn_kws/encoder.py
+++ b/funasr/models/fsmn_kws/encoder.py
@@ -11,6 +11,11 @@ from funasr.register import tables
def toKaldiMatrix(np_mat):
+ """Tokaldimatrix.
+
+ Args:
+ np_mat: TODO.
+ """
np.set_printoptions(threshold=np.inf, linewidth=np.nan)
out_str = str(np_mat)
out_str = out_str.replace('[', '')
@@ -21,17 +26,29 @@ def toKaldiMatrix(np_mat):
class LinearTransform(nn.Module):
def __init__(self, input_dim, output_dim):
+ """Initialize LinearTransform.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ """
super(LinearTransform, self).__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.linear = nn.Linear(input_dim, output_dim, bias=False)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = self.linear(input)
return output
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
re_str += '<LinearTransform> %d %d\n' % (self.output_dim,
self.input_dim)
@@ -44,6 +61,11 @@ class LinearTransform(nn.Module):
return re_str
def to_pytorch_net(self, fread):
+ """To pytorch net.
+
+ Args:
+ fread: TODO.
+ """
linear_line = fread.readline()
linear_split = linear_line.strip().split()
assert len(linear_split) == 3
@@ -74,17 +96,29 @@ class LinearTransform(nn.Module):
class AffineTransform(nn.Module):
def __init__(self, input_dim, output_dim):
+ """Initialize AffineTransform.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ """
super(AffineTransform, self).__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.linear = nn.Linear(input_dim, output_dim)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = self.linear(input)
return output
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
re_str += '<AffineTransform> %d %d\n' % (self.output_dim,
self.input_dim)
@@ -101,6 +135,11 @@ class AffineTransform(nn.Module):
return re_str
def to_pytorch_net(self, fread):
+ """To pytorch net.
+
+ Args:
+ fread: TODO.
+ """
affine_line = fread.readline()
affine_split = affine_line.strip().split()
assert len(affine_split) == 3
@@ -143,21 +182,38 @@ class AffineTransform(nn.Module):
class RectifiedLinear(nn.Module):
def __init__(self, input_dim, output_dim):
+ """Initialize RectifiedLinear.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ """
super(RectifiedLinear, self).__init__()
self.dim = input_dim
self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
out = self.relu(input)
return out
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
re_str += '<RectifiedLinear> %d %d\n' % (self.dim, self.dim)
return re_str
def to_pytorch_net(self, fread):
+ """To pytorch net.
+
+ Args:
+ fread: TODO.
+ """
line = fread.readline()
splits = line.strip().split()
assert len(splits) == 3
@@ -178,6 +234,16 @@ class FSMNBlock(nn.Module):
lstride=1,
rstride=1,
):
+ """Initialize FSMNBlock.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ """
super(FSMNBlock, self).__init__()
self.dim = input_dim
@@ -202,6 +268,12 @@ class FSMNBlock(nn.Module):
self.conv_right = None
def forward(self, input: torch.Tensor, cache: torch.Tensor = None):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ cache: State cache dict for streaming inference.
+ """
x = torch.unsqueeze(input, 1)
x_per = x.permute(0, 3, 2, 1) # B D T C
@@ -228,6 +300,7 @@ class FSMNBlock(nn.Module):
return output, cache
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
re_str += '<Fsmn> %d %d\n' % (self.dim, self.dim)
re_str += '<LearnRateCoef> %d <LOrder> %d <ROrder> %d <LStride> %d <RStride> %d <MaxNorm> 0\n' % (
@@ -246,6 +319,11 @@ class FSMNBlock(nn.Module):
return re_str
def to_pytorch_net(self, fread):
+ """To pytorch net.
+
+ Args:
+ fread: TODO.
+ """
fsmn_line = fread.readline()
fsmn_split = fsmn_line.strip().split()
assert len(fsmn_split) == 3
@@ -320,6 +398,17 @@ class BasicBlock(nn.Module):
rstride: int,
stack_layer: int,
):
+ """Initialize BasicBlock.
+
+ Args:
+ linear_dim: Size/dimension parameter.
+ proj_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ stack_layer: TODO.
+ """
super(BasicBlock, self).__init__()
self.lorder = lorder
self.rorder = rorder
@@ -332,6 +421,12 @@ class BasicBlock(nn.Module):
self.relu = RectifiedLinear(linear_dim, linear_dim)
def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor] = None):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ cache: State cache dict for streaming inference.
+ """
x1 = self.linear(input) # B T D
if cache is not None:
@@ -348,6 +443,7 @@ class BasicBlock(nn.Module):
return x4
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
re_str += self.linear.to_kaldi_net()
re_str += self.fsmn_block.to_kaldi_net()
@@ -357,6 +453,11 @@ class BasicBlock(nn.Module):
return re_str
def to_pytorch_net(self, fread):
+ """To pytorch net.
+
+ Args:
+ fread: TODO.
+ """
self.linear.to_pytorch_net(fread)
self.fsmn_block.to_pytorch_net(fread)
self.affine.to_pytorch_net(fread)
@@ -368,6 +469,11 @@ class BasicBlock_export(nn.Module):
self,
model,
):
+ """Initialize BasicBlock_export.
+
+ Args:
+ model: Model instance or model name.
+ """
super(BasicBlock_export, self).__init__()
self.linear = model.linear
self.fsmn_block = model.fsmn_block
@@ -375,6 +481,12 @@ class BasicBlock_export(nn.Module):
self.relu = model.relu
def forward(self, input: torch.Tensor, in_cache: torch.Tensor):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ in_cache: TODO.
+ """
x = self.linear(input) # B T D
# cache_layer_name = 'cache_layer_{}'.format(self.stack_layer)
# if cache_layer_name not in in_cache:
@@ -387,15 +499,27 @@ class BasicBlock_export(nn.Module):
class FsmnStack(nn.Sequential):
def __init__(self, *args):
+ """Initialize FsmnStack.
+
+ Args:
+ *args: Variable positional arguments.
+ """
super(FsmnStack, self).__init__(*args)
def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor]):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ cache: State cache dict for streaming inference.
+ """
x = input
for module in self._modules.values():
x = module(x, cache)
return x
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
for module in self._modules.values():
re_str += module.to_kaldi_net()
@@ -403,6 +527,11 @@ class FsmnStack(nn.Sequential):
return re_str
def to_pytorch_net(self, fread):
+ """To pytorch net.
+
+ Args:
+ fread: TODO.
+ """
for module in self._modules.values():
module.to_pytorch_net(fread)
@@ -436,6 +565,22 @@ class FSMNConvert(nn.Module):
output_dim: int,
use_softmax: bool = True,
):
+ """Initialize FSMNConvert.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ input_affine_dim: Size/dimension parameter.
+ fsmn_layers: TODO.
+ linear_dim: Size/dimension parameter.
+ proj_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ output_affine_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ use_softmax: TODO.
+ """
super().__init__()
self.input_dim = input_dim
@@ -463,6 +608,7 @@ class FSMNConvert(nn.Module):
self.softmax = nn.Softmax(dim=-1)
def output_size(self) -> int:
+ """Output size."""
return self.output_dim
def forward(
@@ -491,6 +637,7 @@ class FSMNConvert(nn.Module):
return x6
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
re_str += '<Nnet>\n'
re_str += self.in_linear1.to_kaldi_net()
@@ -508,6 +655,11 @@ class FSMNConvert(nn.Module):
return re_str
def to_pytorch_net(self, kaldi_file):
+ """To pytorch net.
+
+ Args:
+ kaldi_file: TODO.
+ """
with open(kaldi_file, 'r', encoding='utf8') as fread:
fread = open(kaldi_file, 'r')
nnet_start_line = fread.readline()
diff --git a/funasr/models/fsmn_kws/model.py b/funasr/models/fsmn_kws/model.py
index 066730f1c..40d6bd98e 100644
--- a/funasr/models/fsmn_kws/model.py
+++ b/funasr/models/fsmn_kws/model.py
@@ -48,6 +48,24 @@ class FsmnKWS(torch.nn.Module):
blank_id: int = 0,
**kwargs,
):
+ """Initialize FsmnKWS.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -158,6 +176,14 @@ class FsmnKWS(torch.nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -177,6 +203,16 @@ class FsmnKWS(torch.nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
keywords = kwargs.get("keywords")
from funasr.utils.kws_utils import KwsCtcPrefixDecoder
self.kws_decoder = KwsCtcPrefixDecoder(
@@ -257,6 +293,19 @@ class FsmnKWSConvert(torch.nn.Module):
blank_id: int = 0,
**kwargs,
):
+ """Initialize FsmnKWSConvert.
+
+ Args:
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ blank_id: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
encoder_class = tables.encoder_classes.get(encoder)
@@ -278,8 +327,14 @@ class FsmnKWSConvert(torch.nn.Module):
self.error_calculator = None
def to_kaldi_net(self):
+ """To kaldi net."""
return self.encoder.to_kaldi_net()
def to_pytorch_net(self, kaldi_file):
+ """To pytorch net.
+
+ Args:
+ kaldi_file: TODO.
+ """
return self.encoder.to_pytorch_net(kaldi_file)
diff --git a/funasr/models/fsmn_kws_mt/encoder.py b/funasr/models/fsmn_kws_mt/encoder.py
index 4b36d839a..b46dfd9eb 100755
--- a/funasr/models/fsmn_kws_mt/encoder.py
+++ b/funasr/models/fsmn_kws_mt/encoder.py
@@ -42,6 +42,23 @@ class FSMNMT(nn.Module):
output_dim2: int,
use_softmax: bool = True,
):
+ """Initialize FSMNMT.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ input_affine_dim: Size/dimension parameter.
+ fsmn_layers: TODO.
+ linear_dim: Size/dimension parameter.
+ proj_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ output_affine_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ output_dim2: TODO.
+ use_softmax: TODO.
+ """
super().__init__()
self.input_dim = input_dim
@@ -68,9 +85,11 @@ class FSMNMT(nn.Module):
self.softmax = nn.Softmax(dim=-1)
def output_size(self) -> int:
+ """Output size."""
return self.output_dim
def output_size2(self) -> int:
+ """Output size2."""
return self.output_dim2
def forward(
@@ -121,6 +140,23 @@ class FSMNMTConvert(nn.Module):
output_dim2: int,
use_softmax: bool = True,
):
+ """Initialize FSMNMTConvert.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ input_affine_dim: Size/dimension parameter.
+ fsmn_layers: TODO.
+ linear_dim: Size/dimension parameter.
+ proj_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ output_affine_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ output_dim2: TODO.
+ use_softmax: TODO.
+ """
super().__init__()
self.input_dim = input_dim
@@ -147,12 +183,15 @@ class FSMNMTConvert(nn.Module):
self.softmax = nn.Softmax(dim=-1)
def output_size(self) -> int:
+ """Output size."""
return self.output_dim
def output_size2(self) -> int:
+ """Output size2."""
return self.output_dim2
def to_kaldi_net(self):
+ """To kaldi net."""
re_str = ''
re_str += '<Nnet>\n'
re_str += self.in_linear1.to_kaldi_net()
@@ -170,6 +209,7 @@ class FSMNMTConvert(nn.Module):
return re_str
def to_kaldi_net2(self):
+ """To kaldi net2."""
re_str = ''
re_str += '<Nnet>\n'
re_str += self.in_linear1.to_kaldi_net()
@@ -187,6 +227,11 @@ class FSMNMTConvert(nn.Module):
return re_str
def to_pytorch_net(self, kaldi_file):
+ """To pytorch net.
+
+ Args:
+ kaldi_file: TODO.
+ """
with open(kaldi_file, 'r', encoding='utf8') as fread:
fread = open(kaldi_file, 'r')
nnet_start_line = fread.readline()
diff --git a/funasr/models/fsmn_kws_mt/model.py b/funasr/models/fsmn_kws_mt/model.py
index 3fa728a21..c8ca992c0 100644
--- a/funasr/models/fsmn_kws_mt/model.py
+++ b/funasr/models/fsmn_kws_mt/model.py
@@ -46,6 +46,22 @@ class FsmnKWSMT(torch.nn.Module):
blank_id: int = 0,
**kwargs,
):
+ """Initialize FsmnKWSMT.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ ctc_conf: Configuration dict for ctc.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -173,6 +189,14 @@ class FsmnKWSMT(torch.nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -190,6 +214,14 @@ class FsmnKWSMT(torch.nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc2 loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc2(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -209,6 +241,16 @@ class FsmnKWSMT(torch.nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
keywords = kwargs.get("keywords")
from funasr.utils.kws_utils import KwsCtcPrefixDecoder
self.kws_decoder = KwsCtcPrefixDecoder(
@@ -317,6 +359,17 @@ class FsmnKWSMTConvert(torch.nn.Module):
blank_id: int = 0,
**kwargs,
):
+ """Initialize FsmnKWSMTConvert.
+
+ Args:
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ blank_id: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
encoder_class = tables.encoder_classes.get(encoder)
@@ -328,10 +381,17 @@ class FsmnKWSMTConvert(torch.nn.Module):
self.error_calculator = None
def to_kaldi_net(self):
+ """To kaldi net."""
return self.encoder.to_kaldi_net()
def to_kaldi_net2(self):
+ """To kaldi net2."""
return self.encoder.to_kaldi_net2()
def to_pytorch_net(self, kaldi_file):
+ """To pytorch net.
+
+ Args:
+ kaldi_file: TODO.
+ """
return self.encoder.to_pytorch_net(kaldi_file)
diff --git a/funasr/models/fsmn_vad_streaming/encoder.py b/funasr/models/fsmn_vad_streaming/encoder.py
index 14c2f5f10..9e44eea10 100755
--- a/funasr/models/fsmn_vad_streaming/encoder.py
+++ b/funasr/models/fsmn_vad_streaming/encoder.py
@@ -13,12 +13,23 @@ from funasr.register import tables
class LinearTransform(nn.Module):
def __init__(self, input_dim, output_dim):
+ """Initialize LinearTransform.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ """
super(LinearTransform, self).__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.linear = nn.Linear(input_dim, output_dim, bias=False)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = self.linear(input)
return output
@@ -27,12 +38,23 @@ class LinearTransform(nn.Module):
class AffineTransform(nn.Module):
def __init__(self, input_dim, output_dim):
+ """Initialize AffineTransform.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ """
super(AffineTransform, self).__init__()
self.input_dim = input_dim
self.output_dim = output_dim
self.linear = nn.Linear(input_dim, output_dim)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = self.linear(input)
return output
@@ -41,12 +63,23 @@ class AffineTransform(nn.Module):
class RectifiedLinear(nn.Module):
def __init__(self, input_dim, output_dim):
+ """Initialize RectifiedLinear.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ """
super(RectifiedLinear, self).__init__()
self.dim = input_dim
self.relu = nn.ReLU()
self.dropout = nn.Dropout(0.1)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
out = self.relu(input)
return out
@@ -62,6 +95,16 @@ class FSMNBlock(nn.Module):
lstride=1,
rstride=1,
):
+ """Initialize FSMNBlock.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ """
super(FSMNBlock, self).__init__()
self.dim = input_dim
@@ -86,6 +129,12 @@ class FSMNBlock(nn.Module):
self.conv_right = None
def forward(self, input: torch.Tensor, cache: torch.Tensor = None):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ cache: State cache dict for streaming inference.
+ """
x = torch.unsqueeze(input, 1)
x_per = x.permute(0, 3, 2, 1) # B D T C
@@ -123,6 +172,17 @@ class BasicBlock(nn.Module):
rstride: int,
stack_layer: int,
):
+ """Initialize BasicBlock.
+
+ Args:
+ linear_dim: Size/dimension parameter.
+ proj_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ stack_layer: TODO.
+ """
super(BasicBlock, self).__init__()
self.lorder = lorder
self.rorder = rorder
@@ -135,6 +195,12 @@ class BasicBlock(nn.Module):
self.relu = RectifiedLinear(linear_dim, linear_dim)
def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor] = None):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ cache: State cache dict for streaming inference.
+ """
x1 = self.linear(input) # B T D
if cache is not None:
@@ -156,6 +222,11 @@ class BasicBlock_export(nn.Module):
self,
model,
):
+ """Initialize BasicBlock_export.
+
+ Args:
+ model: Model instance or model name.
+ """
super(BasicBlock_export, self).__init__()
self.linear = model.linear
self.fsmn_block = model.fsmn_block
@@ -163,6 +234,12 @@ class BasicBlock_export(nn.Module):
self.relu = model.relu
def forward(self, input: torch.Tensor, in_cache: torch.Tensor):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ in_cache: TODO.
+ """
x = self.linear(input) # B T D
# cache_layer_name = 'cache_layer_{}'.format(self.stack_layer)
# if cache_layer_name not in in_cache:
@@ -175,9 +252,20 @@ class BasicBlock_export(nn.Module):
class FsmnStack(nn.Sequential):
def __init__(self, *args):
+ """Initialize FsmnStack.
+
+ Args:
+ *args: Variable positional arguments.
+ """
super(FsmnStack, self).__init__(*args)
def forward(self, input: torch.Tensor, cache: Dict[str, torch.Tensor]):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ cache: State cache dict for streaming inference.
+ """
x = input
for module in self._modules.values():
x = module(x, cache)
@@ -213,6 +301,22 @@ class FSMN(nn.Module):
output_dim: int,
use_softmax: bool = True,
):
+ """Initialize FSMN.
+
+ Args:
+ input_dim: Size/dimension parameter.
+ input_affine_dim: Size/dimension parameter.
+ fsmn_layers: TODO.
+ linear_dim: Size/dimension parameter.
+ proj_dim: Size/dimension parameter.
+ lorder: TODO.
+ rorder: TODO.
+ lstride: TODO.
+ rstride: TODO.
+ output_affine_dim: Size/dimension parameter.
+ output_dim: Size/dimension parameter.
+ use_softmax: TODO.
+ """
super().__init__()
self.input_dim = input_dim
@@ -240,9 +344,11 @@ class FSMN(nn.Module):
self.softmax = nn.Softmax(dim=-1)
def fuse_modules(self):
+ """Fuse modules."""
pass
def output_size(self) -> int:
+ """Output size."""
return self.output_dim
def forward(
@@ -278,6 +384,12 @@ class FSMNExport(nn.Module):
model,
**kwargs,
):
+ """Initialize FSMNExport.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# self.input_dim = input_dim
@@ -310,6 +422,7 @@ class FSMNExport(nn.Module):
self.fsmn[i] = BasicBlock_export(d)
def fuse_modules(self):
+ """Fuse modules."""
pass
def forward(
diff --git a/funasr/models/fsmn_vad_streaming/export_meta.py b/funasr/models/fsmn_vad_streaming/export_meta.py
index f76016931..7865cc5a9 100644
--- a/funasr/models/fsmn_vad_streaming/export_meta.py
+++ b/funasr/models/fsmn_vad_streaming/export_meta.py
@@ -9,6 +9,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
model.encoder = encoder_class(model.encoder, onnx=is_onnx)
@@ -25,12 +31,25 @@ def export_rebuild_model(model, **kwargs):
def export_forward(self, feats: torch.Tensor, *args, **kwargs):
+ """Export forward.
+
+ Args:
+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
scores, out_caches = self.encoder(feats, *args)
return scores, out_caches
def export_dummy_inputs(self, data_in=None, frame=30):
+ """Export dummy inputs.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ frame: TODO.
+ """
if data_in is None:
speech = torch.randn(1, frame, self.encoder_conf.get("input_dim"))
else:
@@ -46,14 +65,17 @@ def export_dummy_inputs(self, data_in=None, frame=30):
def export_input_names(self):
+ """Export input names."""
return ["speech", "in_cache0", "in_cache1", "in_cache2", "in_cache3"]
def export_output_names(self):
+ """Export output names."""
return ["logits", "out_cache0", "out_cache1", "out_cache2", "out_cache3"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"speech": {1: "feats_length"},
}
@@ -62,4 +84,5 @@ def export_dynamic_axes(self):
def export_name(
self,
):
+ """Export name."""
return "model.onnx"
diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py
index d4c91c8d5..3acd481e1 100644
--- a/funasr/models/fsmn_vad_streaming/model.py
+++ b/funasr/models/fsmn_vad_streaming/model.py
@@ -86,6 +86,40 @@ class VADXOptions:
frame_length_ms: int = 25,
**kwargs,
):
+ """Initialize VADXOptions.
+
+ Args:
+ sample_rate: TODO.
+ detect_mode: TODO.
+ snr_mode: TODO.
+ max_end_silence_time: TODO.
+ max_start_silence_time: TODO.
+ do_start_point_detection: TODO.
+ do_end_point_detection: TODO.
+ window_size_ms: TODO.
+ sil_to_speech_time_thres: TODO.
+ speech_to_sil_time_thres: TODO.
+ speech_2_noise_ratio: TODO.
+ do_extend: TODO.
+ lookback_time_start_point: TODO.
+ lookahead_time_end_point: TODO.
+ max_single_segment_time: TODO.
+ nn_eval_block_size: Size/dimension parameter.
+ dcd_block_size: Size/dimension parameter.
+ snr_thres: TODO.
+ noise_frame_num_used_for_snr: TODO.
+ decibel_thres: TODO.
+ speech_noise_thres: TODO.
+ fe_prior_thres: TODO.
+ silence_pdf_num: TODO.
+ sil_pdf_ids: TODO.
+ speech_noise_thresh_low: TODO.
+ speech_noise_thresh_high: TODO.
+ output_frame_probs: TODO.
+ frame_in_ms: TODO.
+ frame_length_ms: TODO.
+ **kwargs: Additional keyword arguments.
+ """
self.sample_rate = sample_rate
self.detect_mode = detect_mode
self.snr_mode = snr_mode
@@ -125,6 +159,7 @@ class E2EVadSpeechBufWithDoa(object):
"""
def __init__(self):
+ """Initialize E2EVadSpeechBufWithDoa."""
self.start_ms = 0
self.end_ms = 0
self.buffer = []
@@ -133,6 +168,7 @@ class E2EVadSpeechBufWithDoa(object):
self.doa = 0
def Reset(self):
+ """Reset."""
self.start_ms = 0
self.end_ms = 0
self.buffer = []
@@ -149,6 +185,7 @@ class E2EVadFrameProb(object):
"""
def __init__(self):
+ """Initialize E2EVadFrameProb."""
self.noise_prob = 0.0
self.speech_prob = 0.0
self.score = 0.0
@@ -170,6 +207,14 @@ class WindowDetector(object):
speech_to_sil_time: int,
frame_size_ms: int,
):
+ """Initialize WindowDetector.
+
+ Args:
+ window_size_ms: TODO.
+ sil_to_speech_time: TODO.
+ speech_to_sil_time: TODO.
+ frame_size_ms: TODO.
+ """
self.window_size_ms = window_size_ms
self.sil_to_speech_time = sil_to_speech_time
self.speech_to_sil_time = speech_to_sil_time
@@ -190,6 +235,7 @@ class WindowDetector(object):
self.hydre_frame_count = 0
def Reset(self) -> None:
+ """Reset."""
self.cur_win_pos = 0
self.win_sum = 0
self.win_state = [0] * self.win_size_frame
@@ -200,11 +246,19 @@ class WindowDetector(object):
self.hydre_frame_count = 0
def GetWinSize(self) -> int:
+ """Getwinsize."""
return int(self.win_size_frame)
def DetectOneFrame(
self, frameState: FrameState, frame_count: int, cache: dict = None
) -> AudioChangeState:
+ """Detectoneframe.
+
+ Args:
+ frameState: TODO.
+ frame_count: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
cur_frame_state = FrameState.kFrameStateSil
@@ -240,6 +294,7 @@ class WindowDetector(object):
return AudioChangeState.kChangeStateInvalid
def FrameSizeMs(self) -> int:
+ """Framesizems."""
return int(self.frame_size_ms)
@@ -250,6 +305,13 @@ class Stats(object):
max_end_sil_frame_cnt_thresh,
speech_noise_thres,
):
+ """Initialize Stats.
+
+ Args:
+ sil_pdf_ids: TODO.
+ max_end_sil_frame_cnt_thresh: TODO.
+ speech_noise_thres: TODO.
+ """
self.data_buf_start_frame = 0
self.frm_cnt = 0
self.latest_confirmed_speech_frame = 0
@@ -301,6 +363,14 @@ class FsmnVADStreaming(nn.Module):
vad_post_args: Dict[str, Any] = None,
**kwargs,
):
+ """Initialize FsmnVADStreaming.
+
+ Args:
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ vad_post_args: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.vad_opts = VADXOptions(**kwargs)
@@ -310,6 +380,11 @@ class FsmnVADStreaming(nn.Module):
self.encoder_conf = encoder_conf
def ResetDetection(self, cache: dict = None):
+ """Resetdetection.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
cache["stats"].continous_silence_frame_count = 0
@@ -335,6 +410,11 @@ class FsmnVADStreaming(nn.Module):
cache["stats"].scores = cache["stats"].scores[:, real_drop_frames:, :]
def ComputeDecibel(self, cache: dict = None) -> None:
+ """Computedecibel.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
frame_sample_length = int(self.vad_opts.frame_length_ms * self.vad_opts.sample_rate / 1000)
@@ -361,6 +441,12 @@ class FsmnVADStreaming(nn.Module):
def ComputeScores(self, feats: torch.Tensor, cache: dict = None) -> None:
+ """Computescores.
+
+ Args:
+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
scores = self.encoder(feats, cache=cache["encoder"]) # return B * T * D
@@ -375,6 +461,12 @@ class FsmnVADStreaming(nn.Module):
cache["stats"].scores = torch.cat((cache["stats"].scores, scores), dim=1)
def PopDataBufTillFrame(self, frame_idx: int, cache: dict = None) -> None: # need check again
+ """Popdatabuftillframe.
+
+ Args:
+ frame_idx: TODO.
+ cache: State cache dict for streaming inference.
+ """
while cache["stats"].data_buf_start_frame < frame_idx:
if len(cache["stats"].data_buf) >= int(
self.vad_opts.frame_in_ms * self.vad_opts.sample_rate / 1000
@@ -396,6 +488,16 @@ class FsmnVADStreaming(nn.Module):
end_point_is_sent_end: bool,
cache: dict = None,
) -> None:
+ """Popdatatooutputbuf.
+
+ Args:
+ start_frm: TODO.
+ frm_cnt: TODO.
+ first_frm_is_start_point: TODO.
+ last_frm_is_end_point: TODO.
+ end_point_is_sent_end: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
self.PopDataBufTillFrame(start_frm, cache=cache)
@@ -448,6 +550,12 @@ class FsmnVADStreaming(nn.Module):
cur_seg.contain_seg_end_point = True
def OnSilenceDetected(self, valid_frame: int, cache: dict = None):
+ """Onsilencedetected.
+
+ Args:
+ valid_frame: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
cache["stats"].lastest_confirmed_silence_frame = valid_frame
@@ -458,12 +566,25 @@ class FsmnVADStreaming(nn.Module):
# pass
def OnVoiceDetected(self, valid_frame: int, cache: dict = None) -> None:
+ """Onvoicedetected.
+
+ Args:
+ valid_frame: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
cache["stats"].latest_confirmed_speech_frame = valid_frame
self.PopDataToOutputBuf(valid_frame, 1, False, False, False, cache=cache)
def OnVoiceStart(self, start_frame: int, fake_result: bool = False, cache: dict = None) -> None:
+ """Onvoicestart.
+
+ Args:
+ start_frame: TODO.
+ fake_result: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
if self.vad_opts.do_start_point_detection:
@@ -484,6 +605,14 @@ class FsmnVADStreaming(nn.Module):
def OnVoiceEnd(
self, end_frame: int, fake_result: bool, is_last_frame: bool, cache: dict = None
) -> None:
+ """Onvoiceend.
+
+ Args:
+ end_frame: TODO.
+ fake_result: TODO.
+ is_last_frame: Boolean flag for last frame.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
for t in range(cache["stats"].latest_confirmed_speech_frame + 1, end_frame):
@@ -504,6 +633,13 @@ class FsmnVADStreaming(nn.Module):
def MaybeOnVoiceEndIfLastFrame(
self, is_final_frame: bool, cur_frm_idx: int, cache: dict = None
) -> None:
+ """Maybeonvoiceendiflastframe.
+
+ Args:
+ is_final_frame: Boolean flag for final frame.
+ cur_frm_idx: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
if is_final_frame:
@@ -511,11 +647,21 @@ class FsmnVADStreaming(nn.Module):
cache["stats"].vad_state_machine = VadStateMachine.kVadInStateEndPointDetected
def GetLatency(self, cache: dict = None) -> int:
+ """Getlatency.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
return int(self.LatencyFrmNumAtStartPoint(cache=cache) * self.vad_opts.frame_in_ms)
def LatencyFrmNumAtStartPoint(self, cache: dict = None) -> int:
+ """Latencyfrmnumatstartpoint.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
vad_latency = cache["windows_detector"].GetWinSize()
@@ -524,6 +670,12 @@ class FsmnVADStreaming(nn.Module):
return vad_latency
def GetFrameState(self, t: int, cache: dict = None):
+ """Getframestate.
+
+ Args:
+ t: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
frame_state = FrameState.kFrameStateInvalid
@@ -590,6 +742,15 @@ class FsmnVADStreaming(nn.Module):
is_final: bool = False,
**kwargs,
):
+ """Forward pass for training.
+
+ Args:
+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).
+ waveform: TODO.
+ cache: State cache dict for streaming inference.
+ is_final: Whether this is the final chunk in streaming.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
# if len(cache) == 0:
@@ -657,6 +818,12 @@ class FsmnVADStreaming(nn.Module):
return segments
def init_cache(self, cache: dict = None, **kwargs):
+ """Init cache.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
@@ -696,6 +863,17 @@ class FsmnVADStreaming(nn.Module):
cache: dict = None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
if len(cache) == 0:
@@ -791,12 +969,22 @@ class FsmnVADStreaming(nn.Module):
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
models = export_rebuild_model(model=self, **kwargs)
return models
def DetectCommonFrames(self, cache: dict = None) -> int:
+ """Detectcommonframes.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected:
@@ -811,6 +999,11 @@ class FsmnVADStreaming(nn.Module):
return 0
def DetectLastFrames(self, cache: dict = None) -> int:
+ """Detectlastframes.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
if cache["stats"].vad_state_machine == VadStateMachine.kVadInStateEndPointDetected:
@@ -830,6 +1023,14 @@ class FsmnVADStreaming(nn.Module):
def DetectOneFrame(
self, cur_frm_state: FrameState, cur_frm_idx: int, is_final_frame: bool, cache: dict = None
) -> None:
+ """Detectoneframe.
+
+ Args:
+ cur_frm_state: TODO.
+ cur_frm_idx: TODO.
+ is_final_frame: Boolean flag for final frame.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
tmp_cur_frm_state = FrameState.kFrameStateInvalid
diff --git a/funasr/models/fun_asr_nano/ctc.py b/funasr/models/fun_asr_nano/ctc.py
index f2d592759..6b07a02a4 100644
--- a/funasr/models/fun_asr_nano/ctc.py
+++ b/funasr/models/fun_asr_nano/ctc.py
@@ -21,6 +21,16 @@ class CTC(torch.nn.Module):
blank_id: int = 0,
**kwargs,
):
+ """Initialize CTC.
+
+ Args:
+ odim: TODO.
+ encoder_output_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ reduce: TODO.
+ blank_id: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
eprojs = encoder_output_size
self.dropout_rate = dropout_rate
diff --git a/funasr/models/fun_asr_nano/model.py b/funasr/models/fun_asr_nano/model.py
index f78d0e755..b7e6ad5de 100644
--- a/funasr/models/fun_asr_nano/model.py
+++ b/funasr/models/fun_asr_nano/model.py
@@ -57,6 +57,19 @@ class FunASRNano(nn.Module):
length_normalized_loss: bool = False,
**kwargs,
):
+ """Initialize FunASRNano.
+
+ Args:
+ audio_encoder: TODO.
+ audio_encoder_conf: Configuration dict for audio_encoder.
+ audio_adaptor: TODO.
+ audio_adaptor_conf: Configuration dict for audio_adaptor.
+ llm: TODO.
+ llm_conf: Configuration dict for llm.
+ input_size: Size/dimension parameter.
+ length_normalized_loss: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# audio encoder
@@ -189,6 +202,18 @@ class FunASRNano(nn.Module):
fbank_mask: torch.Tensor = None,
**kwargs,
):
+ """Forward pass for training.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ input_ids: TODO.
+ attention_mask: TODO.
+ labels_ids: TODO.
+ fbank_beg: TODO.
+ fbank_mask: TODO.
+ **kwargs: Additional keyword arguments.
+ """
batch_size, token_num = input_ids.shape
stats = {}
input_ids[input_ids < 0] = 0
@@ -290,17 +315,35 @@ class FunASRNano(nn.Module):
return loss, stats, weight
def forward_export(self, speech, speech_lengths, **kwargs):
+ """Forward export.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ **kwargs: Additional keyword arguments.
+ """
x, olens = self.audio_encoder(speech, speech_lengths)
encoder_out, encoder_out_lens = self.audio_adaptor(x, olens)
return encoder_out, encoder_out_lens
def encode(self, speech, speech_lengths):
# audio encoder
+ """Encode.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)
return encoder_out, encoder_out_lens
def data_template(self, data):
+ """Data template.
+
+ Args:
+ data: TODO.
+ """
system, user, assistant = [], [], []
for i, item in enumerate(data):
role = item["role"]
@@ -326,6 +369,15 @@ class FunASRNano(nn.Module):
return contents
def data_load_speech(self, contents: dict, tokenizer, frontend, meta_data={}, **kwargs):
+ """Data load speech.
+
+ Args:
+ contents: TODO.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ meta_data: TODO.
+ **kwargs: Additional keyword arguments.
+ """
system = contents["system"]
user = contents["user"]
assistant = contents["assistant"]
@@ -487,6 +539,16 @@ class FunASRNano(nn.Module):
frontend=None,
**kwargs,
):
+ """Inference prepare.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
if len(data_in) > 1:
@@ -568,6 +630,13 @@ class FunASRNano(nn.Module):
return inputs_embeds, contents, batch, source_ids, meta_data
def get_prompt(self, hotwords: list[str], language: str = None, itn: bool = True):
+ """Get prompt.
+
+ Args:
+ hotwords: TODO.
+ language: Language identifier.
+ itn: TODO.
+ """
if len(hotwords) > 0:
hotwords = ", ".join(hotwords)
prompt = f"请结合上下文信息,更加准确地完成语音转写任务。如果没有相关信息,我们会留空。\n\n\n**上下文信息:**\n\n\n"
@@ -583,6 +652,12 @@ class FunASRNano(nn.Module):
return prompt + ":"
def generate_chatml(self, prompt: str, data: Union[str, torch.Tensor]):
+ """Generate chatml.
+
+ Args:
+ prompt: TODO.
+ data: TODO.
+ """
if isinstance(data, str):
return [
{"role": "system", "content": "You are a helpful assistant."},
@@ -609,6 +684,16 @@ class FunASRNano(nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
prompt = self.get_prompt(
kwargs.get("hotwords", []), kwargs.get("language", None), kwargs.get("itn", True)
)
@@ -638,6 +723,16 @@ class FunASRNano(nn.Module):
frontend=None,
**kwargs,
):
+ """Inference llm.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
inputs_embeds, contents, batch, source_ids, meta_data = self.inference_prepare(
data_in, data_lengths, key, tokenizer, frontend, **kwargs
)
@@ -759,6 +854,12 @@ class FunASRNano(nn.Module):
@staticmethod
def from_pretrained(model: str = None, **kwargs):
+ """From pretrained.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
from funasr import AutoModel
model, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)
diff --git a/funasr/models/fun_asr_nano/tools/cn_tn.py b/funasr/models/fun_asr_nano/tools/cn_tn.py
index 4aaccd040..d76bc43f8 100644
--- a/funasr/models/fun_asr_nano/tools/cn_tn.py
+++ b/funasr/models/fun_asr_nano/tools/cn_tn.py
@@ -408,14 +408,22 @@ class ChineseChar(object):
"""
def __init__(self, simplified, traditional):
+ """Initialize ChineseChar.
+
+ Args:
+ simplified: TODO.
+ traditional: TODO.
+ """
self.simplified = simplified
self.traditional = traditional
# self.__repr__ = self.__str__
def __str__(self):
+ """Internal: str ."""
return self.simplified or self.traditional or None
def __repr__(self):
+ """Internal: repr ."""
return self.__str__()
@@ -427,16 +435,34 @@ class ChineseNumberUnit(ChineseChar):
"""
def __init__(self, power, simplified, traditional, big_s, big_t):
+ """Initialize ChineseNumberUnit.
+
+ Args:
+ power: TODO.
+ simplified: TODO.
+ traditional: TODO.
+ big_s: TODO.
+ big_t: TODO.
+ """
super(ChineseNumberUnit, self).__init__(simplified, traditional)
self.power = power
self.big_s = big_s
self.big_t = big_t
def __str__(self):
+ """Internal: str ."""
return "10^{}".format(self.power)
@classmethod
def create(cls, index, value, numbering_type=NUMBERING_TYPES[1], small_unit=False):
+ """Create.
+
+ Args:
+ index: TODO.
+ value: TODO.
+ numbering_type: TODO.
+ small_unit: TODO.
+ """
if small_unit:
return ChineseNumberUnit(
power=index + 1,
@@ -479,6 +505,17 @@ class ChineseNumberDigit(ChineseChar):
"""
def __init__(self, value, simplified, traditional, big_s, big_t, alt_s=None, alt_t=None):
+ """Initialize ChineseNumberDigit.
+
+ Args:
+ value: TODO.
+ simplified: TODO.
+ traditional: TODO.
+ big_s: TODO.
+ big_t: TODO.
+ alt_s: TODO.
+ alt_t: TODO.
+ """
super(ChineseNumberDigit, self).__init__(simplified, traditional)
self.value = value
self.big_s = big_s
@@ -487,10 +524,17 @@ class ChineseNumberDigit(ChineseChar):
self.alt_t = alt_t
def __str__(self):
+ """Internal: str ."""
return str(self.value)
@classmethod
def create(cls, i, v):
+ """Create.
+
+ Args:
+ i: TODO.
+ v: TODO.
+ """
return ChineseNumberDigit(i, v[0], v[1], v[2], v[3])
@@ -500,6 +544,14 @@ class ChineseMath(ChineseChar):
"""
def __init__(self, simplified, traditional, symbol, expression=None):
+ """Initialize ChineseMath.
+
+ Args:
+ simplified: TODO.
+ traditional: TODO.
+ symbol: TODO.
+ expression: TODO.
+ """
super(ChineseMath, self).__init__(simplified, traditional)
self.symbol = symbol
self.expression = expression
@@ -527,11 +579,19 @@ class MathSymbol(object):
"""
def __init__(self, positive, negative, point):
+ """Initialize MathSymbol.
+
+ Args:
+ positive: TODO.
+ negative: TODO.
+ point: TODO.
+ """
self.positive = positive
self.negative = negative
self.point = point
def __iter__(self):
+ """Internal: iter ."""
for v in self.__dict__.values():
yield v
@@ -600,7 +660,19 @@ def create_system(numbering_type=NUMBERING_TYPES[1]):
def chn2num(chinese_string, numbering_type=NUMBERING_TYPES[1]):
+ """Chn2num.
+
+ Args:
+ chinese_string: TODO.
+ numbering_type: TODO.
+ """
def get_symbol(char, system):
+ """Get symbol.
+
+ Args:
+ char: TODO.
+ system: TODO.
+ """
for u in system.units:
if char in [u.traditional, u.simplified, u.big_s, u.big_t]:
return u
@@ -619,6 +691,12 @@ def chn2num(chinese_string, numbering_type=NUMBERING_TYPES[1]):
return m
def string2symbols(chinese_string, system):
+ """String2symbols.
+
+ Args:
+ chinese_string: TODO.
+ system: TODO.
+ """
int_string, dec_string = chinese_string, ""
for p in [system.math.point.simplified, system.math.point.traditional]:
if p in chinese_string:
@@ -705,7 +783,26 @@ def num2chn(
use_zeros=True,
use_units=True,
):
+ """Num2chn.
+
+ Args:
+ number_string: TODO.
+ numbering_type: TODO.
+ big: TODO.
+ traditional: TODO.
+ alt_zero: TODO.
+ alt_one: TODO.
+ alt_two: TODO.
+ use_zeros: TODO.
+ use_units: TODO.
+ """
def get_value(value_string, use_zeros=True):
+ """Get value.
+
+ Args:
+ value_string: TODO.
+ use_zeros: TODO.
+ """
striped_string = value_string.lstrip("0")
# record nothing if all zeros
@@ -818,13 +915,21 @@ class Cardinal:
"""
def __init__(self, cardinal=None, chntext=None):
+ """Initialize Cardinal.
+
+ Args:
+ cardinal: TODO.
+ chntext: TODO.
+ """
self.cardinal = cardinal
self.chntext = chntext
def chntext2cardinal(self):
+ """Chntext2cardinal."""
return chn2num(self.chntext)
def cardinal2chntext(self):
+ """Cardinal2chntext."""
return num2chn(self.cardinal)
@@ -834,6 +939,12 @@ class Digit:
"""
def __init__(self, digit=None, chntext=None):
+ """Initialize Digit.
+
+ Args:
+ digit: TODO.
+ chntext: TODO.
+ """
self.digit = digit
self.chntext = chntext
@@ -841,6 +952,7 @@ class Digit:
# return chn2num(self.chntext)
def digit2chntext(self):
+ """Digit2chntext."""
return num2chn(self.digit, alt_two=False, use_units=False)
@@ -850,6 +962,13 @@ class TelePhone:
"""
def __init__(self, telephone=None, raw_chntext=None, chntext=None):
+ """Initialize TelePhone.
+
+ Args:
+ telephone: TODO.
+ raw_chntext: TODO.
+ chntext: TODO.
+ """
self.telephone = telephone
self.raw_chntext = raw_chntext
self.chntext = chntext
@@ -862,6 +981,11 @@ class TelePhone:
# return self.telephone
def telephone2chntext(self, fixed=False):
+ """Telephone2chntext.
+
+ Args:
+ fixed: TODO.
+ """
if fixed:
sil_parts = self.telephone.split("-")
self.raw_chntext = "<SIL>".join([num2chn(part, alt_two=False, use_units=False) for part in sil_parts])
@@ -879,14 +1003,22 @@ class Fraction:
"""
def __init__(self, fraction=None, chntext=None):
+ """Initialize Fraction.
+
+ Args:
+ fraction: TODO.
+ chntext: TODO.
+ """
self.fraction = fraction
self.chntext = chntext
def chntext2fraction(self):
+ """Chntext2fraction."""
denominator, numerator = self.chntext.split("分之")
return chn2num(numerator) + "/" + chn2num(denominator)
def fraction2chntext(self):
+ """Fraction2chntext."""
numerator, denominator = self.fraction.split("/")
return num2chn(denominator) + "分之" + num2chn(numerator)
@@ -897,6 +1029,12 @@ class Date:
"""
def __init__(self, date=None, chntext=None):
+ """Initialize Date.
+
+ Args:
+ date: TODO.
+ chntext: TODO.
+ """
self.date = date
self.chntext = chntext
@@ -925,6 +1063,7 @@ class Date:
# return self.date
def date2chntext(self):
+ """Date2chntext."""
date = self.date
try:
year, other = date.strip().split("年", 1)
@@ -955,6 +1094,12 @@ class Money:
"""
def __init__(self, money=None, chntext=None):
+ """Initialize Money.
+
+ Args:
+ money: TODO.
+ chntext: TODO.
+ """
self.money = money
self.chntext = chntext
@@ -962,6 +1107,7 @@ class Money:
# return self.money
def money2chntext(self):
+ """Money2chntext."""
money = self.money
pattern = re.compile(r"(\d+(\.\d+)?)")
matchers = pattern.findall(money)
@@ -978,17 +1124,30 @@ class Percentage:
"""
def __init__(self, percentage=None, chntext=None):
+ """Initialize Percentage.
+
+ Args:
+ percentage: TODO.
+ chntext: TODO.
+ """
self.percentage = percentage
self.chntext = chntext
def chntext2percentage(self):
+ """Chntext2percentage."""
return chn2num(self.chntext.strip().strip("百分之")) + "%"
def percentage2chntext(self):
+ """Percentage2chntext."""
return "百分之" + num2chn(self.percentage.strip().strip("%"))
def normalize_nsw(raw_text):
+ """Normalize nsw.
+
+ Args:
+ raw_text: TODO.
+ """
text = "^" + raw_text + "$"
# 规范化日期
@@ -1111,6 +1270,11 @@ def remove_erhua(text):
def remove_space(text):
+ """Remove space.
+
+ Args:
+ text: Text tensor or string input.
+ """
tokens = text.split()
new = []
for k, t in enumerate(tokens):
@@ -1132,6 +1296,17 @@ class TextNorm:
check_chars: bool = False,
remove_space: bool = False,
):
+ """Initialize TextNorm.
+
+ Args:
+ to_banjiao: TODO.
+ to_upper: TODO.
+ to_lower: TODO.
+ remove_fillers: TODO.
+ remove_erhua: TODO.
+ check_chars: TODO.
+ remove_space: TODO.
+ """
self.to_banjiao = to_banjiao
self.to_upper = to_upper
self.to_lower = to_lower
@@ -1141,6 +1316,11 @@ class TextNorm:
self.remove_space = remove_space
def __call__(self, text):
+ """Internal: call .
+
+ Args:
+ text: Text tensor or string input.
+ """
if self.to_banjiao:
text = text.translate(QJ2BJ_TRANSFORM)
diff --git a/funasr/models/fun_asr_nano/tools/format5res.py b/funasr/models/fun_asr_nano/tools/format5res.py
index 7f1f92873..3b11bb83a 100644
--- a/funasr/models/fun_asr_nano/tools/format5res.py
+++ b/funasr/models/fun_asr_nano/tools/format5res.py
@@ -7,6 +7,13 @@ import sys
def scoreformat(name, line, flag=1):
+ """Scoreformat.
+
+ Args:
+ name: TODO.
+ line: TODO.
+ flag: TODO.
+ """
newline = ""
for i in range(0, len(line)):
curr = line[i]
@@ -47,6 +54,11 @@ def scoreformat(name, line, flag=1):
def recoformat(line):
+ """Recoformat.
+
+ Args:
+ line: TODO.
+ """
newline = ""
en_flag = 0 # 0: no-english 1 : english 2: former
for i in range(0, len(line)):
@@ -86,6 +98,11 @@ def recoformat(line):
def numbersingle(line):
+ """Numbersingle.
+
+ Args:
+ line: TODO.
+ """
chnu = ["零", "一", "二", "两", "三", "四", "五", "六", "七", "八", "九", "点"]
newline = ""
for id in range(len(line)):
@@ -118,6 +135,11 @@ def numbersingle(line):
def ch_number2digit(line):
+ """Ch number2digit.
+
+ Args:
+ line: TODO.
+ """
number_flag = 0
zero_flag = 0
bits = {
@@ -224,6 +246,11 @@ def ch_number2digit(line):
def special(line):
+ """Special.
+
+ Args:
+ line: TODO.
+ """
newline = ""
for e in range(len(line)):
if ord(line[e]) == 247:
@@ -255,6 +282,11 @@ def special(line):
def all_convert(content):
+ """All convert.
+
+ Args:
+ content: TODO.
+ """
content = recoformat(content)
content = numbersingle(content)
content = ch_number2digit(content)
diff --git a/funasr/models/fun_asr_nano/tools/scp2jsonl.py b/funasr/models/fun_asr_nano/tools/scp2jsonl.py
index 5d269cb18..9b04d0a44 100644
--- a/funasr/models/fun_asr_nano/tools/scp2jsonl.py
+++ b/funasr/models/fun_asr_nano/tools/scp2jsonl.py
@@ -15,10 +15,20 @@ from omegaconf import DictConfig, OmegaConf, ListConfig
class LineProcessor:
def __init__(self, tokenizer):
+ """Initialize LineProcessor.
+
+ Args:
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ """
self.tokenizer = tokenizer
self.lock = threading.Lock()
def process_line(self, line_pair: Tuple[str, str]) -> Optional[Dict]:
+ """Process line.
+
+ Args:
+ line_pair: TODO.
+ """
line1, line2 = line_pair
line1, line2 = line1.strip(), line2.strip()
@@ -67,7 +77,17 @@ class LineProcessor:
@hydra.main(config_name=None, version_base=None)
def main_hydra(cfg: DictConfig):
+ """Main hydra.
+
+ Args:
+ cfg: Configuration overrides.
+ """
def to_plain_list(cfg_item):
+ """To plain list.
+
+ Args:
+ cfg_item: TODO.
+ """
if isinstance(cfg_item, ListConfig):
return OmegaConf.to_container(cfg_item, resolve=True)
elif isinstance(cfg_item, DictConfig):
diff --git a/funasr/models/fun_asr_nano/tools/utils.py b/funasr/models/fun_asr_nano/tools/utils.py
index 7125bca5c..bc388cbb3 100644
--- a/funasr/models/fun_asr_nano/tools/utils.py
+++ b/funasr/models/fun_asr_nano/tools/utils.py
@@ -7,6 +7,14 @@ import torchaudio.functional as F
def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float = None):
+ """Load audio.
+
+ Args:
+ wav_path: TODO.
+ rate: TODO.
+ offset: TODO.
+ duration: TODO.
+ """
with sf.SoundFile(wav_path) as f:
start_frame = int(offset * f.samplerate)
if duration is None:
@@ -29,6 +37,13 @@ def load_audio(wav_path, rate: int = None, offset: float = 0, duration: float =
def forced_align(log_probs: torch.Tensor, targets: torch.Tensor, blank: int = 0):
+ """Forced align.
+
+ Args:
+ log_probs: TODO.
+ targets: TODO.
+ blank: TODO.
+ """
items = []
try:
# The current version only supports batch_size==1.
diff --git a/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py b/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py
index 632b52906..3bb63d6e9 100644
--- a/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py
+++ b/funasr/models/fun_asr_nano/tools/whisper_mix_normalize.py
@@ -18,6 +18,11 @@ english_normalizer = EnglishTextNormalizer()
def is_only_chinese_and_english(s):
# 定义正则表达式模式,匹配中文字符范围和英文字母(包括大小写)
+ """Is only chinese and english.
+
+ Args:
+ s: TODO.
+ """
pattern = r"^[\u4e00-\u9fa5A-Za-z0-9,\.!\?:;,。!?:;、%\'\s\-\~]+$"
# 使用正则表达式进行匹配
return re.match(pattern, s) is not None
@@ -25,6 +30,11 @@ def is_only_chinese_and_english(s):
def is_only_english(s):
# 定义正则表达式模式,匹配中文字符范围和英文字母(包括大小写)
+ """Is only english.
+
+ Args:
+ s: TODO.
+ """
pattern = r"^[A-Za-z0-9,\.!\?:;,。!?:;、%\'\s\-\~]+$"
# 使用正则表达式进行匹配
return re.match(pattern, s) is not None
@@ -32,12 +42,24 @@ def is_only_english(s):
def is_number(s):
# 定义正则表达式模式,匹配中文字符范围和英文字母(包括大小写)
+ """Is number.
+
+ Args:
+ s: TODO.
+ """
pattern = r"^[0-9,\.!\?:;,。!?:;、%\'\s]+$"
# 使用正则表达式进行匹配
return re.match(pattern, s) is not None
def safe_ja_g2p(text, kana=True, max_length=100):
+ """Safe ja g2p.
+
+ Args:
+ text: Text tensor or string input.
+ kana: TODO.
+ max_length: TODO.
+ """
if len(text) > max_length:
# 如果文本过长,分段处理
parts = []
@@ -57,6 +79,13 @@ def safe_ja_g2p(text, kana=True, max_length=100):
def normalize_text(srcfn, dstfn, kana=False):
+ """Normalize text.
+
+ Args:
+ srcfn: TODO.
+ dstfn: TODO.
+ kana: TODO.
+ """
with open(srcfn, "r") as f_read, open(dstfn, "w") as f_write:
all_lines = f_read.readlines()
for line in all_lines:
diff --git a/funasr/models/glm_asr/model.py b/funasr/models/glm_asr/model.py
index bc5673232..54bef5143 100644
--- a/funasr/models/glm_asr/model.py
+++ b/funasr/models/glm_asr/model.py
@@ -14,6 +14,11 @@ from funasr.register import tables
class GLMASR(nn.Module):
def __init__(self, **kwargs):
+ """Initialize GLMASR.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
model_path = kwargs.get("model_path", kwargs.get("model", "zai-org/GLM-ASR-Nano-2512"))
device = kwargs.get("device", "cuda:0")
@@ -43,6 +48,13 @@ class GLMASR(nn.Module):
logging.info(f"GLM-ASR model loaded from {model_path}")
def _resolve_model_path(self, model_path, hub, kwargs):
+ """Internal: resolve model path.
+
+ Args:
+ model_path: TODO.
+ hub: TODO.
+ kwargs: Additional keyword arguments.
+ """
if os.path.exists(model_path):
return model_path
@@ -59,6 +71,11 @@ class GLMASR(nn.Module):
return model_path
def forward(self, **kwargs):
+ """Forward pass for training.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
raise NotImplementedError("GLMASR only supports inference mode")
def inference(
@@ -70,6 +87,16 @@ class GLMASR(nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
time1 = time.perf_counter()
diff --git a/funasr/models/language_model/rnn/attentions.py b/funasr/models/language_model/rnn/attentions.py
index f5c450c10..d9f146ce8 100644
--- a/funasr/models/language_model/rnn/attentions.py
+++ b/funasr/models/language_model/rnn/attentions.py
@@ -45,6 +45,7 @@ class NoAtt(torch.nn.Module):
"""No attention"""
def __init__(self):
+ """Initialize NoAtt."""
super(NoAtt, self).__init__()
self.h_length = None
self.enc_h = None
@@ -98,6 +99,14 @@ class AttDot(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, han_mode=False):
+ """Initialize AttDot.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ han_mode: TODO.
+ """
super(AttDot, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim)
@@ -174,6 +183,14 @@ class AttAdd(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, han_mode=False):
+ """Initialize AttAdd.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ han_mode: TODO.
+ """
super(AttAdd, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -258,6 +275,16 @@ class AttLoc(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False):
+ """Initialize AttLoc.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ han_mode: TODO.
+ """
super(AttLoc, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -381,6 +408,14 @@ class AttCov(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, han_mode=False):
+ """Initialize AttCov.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ han_mode: TODO.
+ """
super(AttCov, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -481,6 +516,17 @@ class AttLoc2D(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, att_win, aconv_chans, aconv_filts, han_mode=False):
+ """Initialize AttLoc2D.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ att_win: TODO.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ han_mode: TODO.
+ """
super(AttLoc2D, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -597,6 +643,16 @@ class AttLocRec(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False):
+ """Initialize AttLocRec.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ han_mode: TODO.
+ """
super(AttLocRec, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -716,6 +772,16 @@ class AttCovLoc(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts, han_mode=False):
+ """Initialize AttCovLoc.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ han_mode: TODO.
+ """
super(AttCovLoc, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -827,6 +893,16 @@ class AttMultiHeadDot(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False):
+ """Initialize AttMultiHeadDot.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ aheads: TODO.
+ att_dim_k: TODO.
+ att_dim_v: TODO.
+ han_mode: TODO.
+ """
super(AttMultiHeadDot, self).__init__()
self.mlp_q = torch.nn.ModuleList()
self.mlp_k = torch.nn.ModuleList()
@@ -935,6 +1011,16 @@ class AttMultiHeadAdd(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, aheads, att_dim_k, att_dim_v, han_mode=False):
+ """Initialize AttMultiHeadAdd.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ aheads: TODO.
+ att_dim_k: TODO.
+ att_dim_v: TODO.
+ han_mode: TODO.
+ """
super(AttMultiHeadAdd, self).__init__()
self.mlp_q = torch.nn.ModuleList()
self.mlp_k = torch.nn.ModuleList()
@@ -1055,6 +1141,18 @@ class AttMultiHeadLoc(torch.nn.Module):
aconv_filts,
han_mode=False,
):
+ """Initialize AttMultiHeadLoc.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ aheads: TODO.
+ att_dim_k: TODO.
+ att_dim_v: TODO.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ han_mode: TODO.
+ """
super(AttMultiHeadLoc, self).__init__()
self.mlp_q = torch.nn.ModuleList()
self.mlp_k = torch.nn.ModuleList()
@@ -1205,6 +1303,18 @@ class AttMultiHeadMultiResLoc(torch.nn.Module):
aconv_filts,
han_mode=False,
):
+ """Initialize AttMultiHeadMultiResLoc.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ aheads: TODO.
+ att_dim_k: TODO.
+ att_dim_v: TODO.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ han_mode: TODO.
+ """
super(AttMultiHeadMultiResLoc, self).__init__()
self.mlp_q = torch.nn.ModuleList()
self.mlp_k = torch.nn.ModuleList()
@@ -1333,6 +1443,15 @@ class AttForward(torch.nn.Module):
"""
def __init__(self, eprojs, dunits, att_dim, aconv_chans, aconv_filts):
+ """Initialize AttForward.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ """
super(AttForward, self).__init__()
self.mlp_enc = torch.nn.Linear(eprojs, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -1460,6 +1579,16 @@ class AttForwardTA(torch.nn.Module):
"""
def __init__(self, eunits, dunits, att_dim, aconv_chans, aconv_filts, odim):
+ """Initialize AttForwardTA.
+
+ Args:
+ eunits: TODO.
+ dunits: TODO.
+ att_dim: Size/dimension parameter.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ odim: TODO.
+ """
super(AttForwardTA, self).__init__()
self.mlp_enc = torch.nn.Linear(eunits, att_dim)
self.mlp_dec = torch.nn.Linear(dunits, att_dim, bias=False)
@@ -1483,6 +1612,7 @@ class AttForwardTA(torch.nn.Module):
self.trans_agent_prob = 0.5
def reset(self):
+ """Reset."""
self.h_length = None
self.enc_h = None
self.pre_compute_enc_h = None
diff --git a/funasr/models/language_model/rnn/decoders.py b/funasr/models/language_model/rnn/decoders.py
index cdae9e376..ec9f92b26 100644
--- a/funasr/models/language_model/rnn/decoders.py
+++ b/funasr/models/language_model/rnn/decoders.py
@@ -66,6 +66,27 @@ class Decoder(torch.nn.Module, ScorerInterface):
num_encs=1,
):
+ """Initialize Decoder.
+
+ Args:
+ eprojs: TODO.
+ odim: TODO.
+ dtype: TODO.
+ dlayers: TODO.
+ dunits: TODO.
+ sos: TODO.
+ eos: TODO.
+ att: TODO.
+ verbose: TODO.
+ char_list: TODO.
+ labeldist: TODO.
+ lsm_weight: TODO.
+ sampling_probability: TODO.
+ dropout: TODO.
+ context_residual: TODO.
+ replace_sos: TODO.
+ num_encs: TODO.
+ """
torch.nn.Module.__init__(self)
self.dtype = dtype
self.dunits = dunits
@@ -124,9 +145,23 @@ class Decoder(torch.nn.Module, ScorerInterface):
self.logzero = -10000000000.0
def zero_state(self, hs_pad):
+ """Zero state.
+
+ Args:
+ hs_pad: TODO.
+ """
return hs_pad.new_zeros(hs_pad.size(0), self.dunits)
def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev):
+ """Rnn forward.
+
+ Args:
+ ey: TODO.
+ z_list: TODO.
+ c_list: TODO.
+ z_prev: TODO.
+ c_prev: TODO.
+ """
if self.dtype == "lstm":
z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0]))
for i in six.moves.range(1, self.dlayers):
@@ -609,6 +644,19 @@ class Decoder(torch.nn.Module, ScorerInterface):
):
# to support mutiple encoder asr mode, in single encoder mode,
# convert torch.Tensor to List of torch.Tensor
+ """Recognize beam batch.
+
+ Args:
+ h: TODO.
+ hlens: TODO.
+ lpz: TODO.
+ recog_args: TODO.
+ char_list: TODO.
+ rnnlm: TODO.
+ normalize_score: TODO.
+ strm_idx: TODO.
+ lang_ids: TODO.
+ """
if self.num_encs == 1:
h = [h]
hlens = [hlens]
@@ -1009,6 +1057,11 @@ class Decoder(torch.nn.Module, ScorerInterface):
@staticmethod
def _get_last_yseq(exp_yseq):
+ """Internal: get last yseq.
+
+ Args:
+ exp_yseq: TODO.
+ """
last = []
for y_seq in exp_yseq:
last.append(y_seq[-1])
@@ -1016,6 +1069,12 @@ class Decoder(torch.nn.Module, ScorerInterface):
@staticmethod
def _append_ids(yseq, ids):
+ """Internal: append ids.
+
+ Args:
+ yseq: TODO.
+ ids: TODO.
+ """
if isinstance(ids, list):
for i, j in enumerate(ids):
yseq[i].append(j)
@@ -1026,6 +1085,12 @@ class Decoder(torch.nn.Module, ScorerInterface):
@staticmethod
def _index_select_list(yseq, lst):
+ """Internal: index select list.
+
+ Args:
+ yseq: TODO.
+ lst: TODO.
+ """
new_yseq = []
for i in lst:
new_yseq.append(yseq[i][:])
@@ -1033,6 +1098,13 @@ class Decoder(torch.nn.Module, ScorerInterface):
@staticmethod
def _index_select_lm_state(rnnlm_state, dim, vidx):
+ """Internal: index select lm state.
+
+ Args:
+ rnnlm_state: TODO.
+ dim: TODO.
+ vidx: TODO.
+ """
if isinstance(rnnlm_state, dict):
new_state = {}
for k, v in rnnlm_state.items():
@@ -1047,6 +1119,11 @@ class Decoder(torch.nn.Module, ScorerInterface):
def init_state(self, x):
# to support mutiple encoder asr mode, in single encoder mode,
# convert torch.Tensor to List of torch.Tensor
+ """Init state.
+
+ Args:
+ x: TODO.
+ """
if self.num_encs == 1:
x = [x]
@@ -1075,6 +1152,13 @@ class Decoder(torch.nn.Module, ScorerInterface):
def score(self, yseq, state, x):
# to support mutiple encoder asr mode, in single encoder mode,
# convert torch.Tensor to List of torch.Tensor
+ """Score.
+
+ Args:
+ yseq: TODO.
+ state: TODO.
+ x: TODO.
+ """
if self.num_encs == 1:
x = [x]
@@ -1124,6 +1208,16 @@ class Decoder(torch.nn.Module, ScorerInterface):
def decoder_for(args, odim, sos, eos, att, labeldist):
+ """Decoder for.
+
+ Args:
+ args: TODO.
+ odim: TODO.
+ sos: TODO.
+ eos: TODO.
+ att: TODO.
+ labeldist: TODO.
+ """
return Decoder(
args.eprojs,
odim,
diff --git a/funasr/models/language_model/rnn/encoders.py b/funasr/models/language_model/rnn/encoders.py
index fff376650..906b3c9fb 100644
--- a/funasr/models/language_model/rnn/encoders.py
+++ b/funasr/models/language_model/rnn/encoders.py
@@ -25,6 +25,17 @@ class RNNP(torch.nn.Module):
"""
def __init__(self, idim, elayers, cdim, hdim, subsample, dropout, typ="blstm"):
+ """Initialize RNNP.
+
+ Args:
+ idim: TODO.
+ elayers: TODO.
+ cdim: TODO.
+ hdim: TODO.
+ subsample: TODO.
+ dropout: TODO.
+ typ: TODO.
+ """
super(RNNP, self).__init__()
bidir = typ[0] == "b"
for i in six.moves.range(elayers):
@@ -100,6 +111,16 @@ class RNN(torch.nn.Module):
"""
def __init__(self, idim, elayers, cdim, hdim, dropout, typ="blstm"):
+ """Initialize RNN.
+
+ Args:
+ idim: TODO.
+ elayers: TODO.
+ cdim: TODO.
+ hdim: TODO.
+ dropout: TODO.
+ typ: TODO.
+ """
super(RNN, self).__init__()
bidir = typ[0] == "b"
self.nbrnn = (
@@ -176,6 +197,11 @@ class VGG2L(torch.nn.Module):
"""
def __init__(self, in_channel=1):
+ """Initialize VGG2L.
+
+ Args:
+ in_channel: TODO.
+ """
super(VGG2L, self).__init__()
# CNN layer (VGG motivated)
self.conv1_1 = torch.nn.Conv2d(in_channel, 64, 3, stride=1, padding=1)
@@ -243,6 +269,18 @@ class Encoder(torch.nn.Module):
"""
def __init__(self, etype, idim, elayers, eunits, eprojs, subsample, dropout, in_channel=1):
+ """Initialize Encoder.
+
+ Args:
+ etype: TODO.
+ idim: TODO.
+ elayers: TODO.
+ eunits: TODO.
+ eprojs: TODO.
+ subsample: TODO.
+ dropout: TODO.
+ in_channel: TODO.
+ """
super(Encoder, self).__init__()
typ = etype.lstrip("vgg").rstrip("p")
if typ not in ["lstm", "gru", "blstm", "bgru"]:
diff --git a/funasr/models/language_model/seq_rnn_lm.py b/funasr/models/language_model/seq_rnn_lm.py
index baf63bc30..86462205d 100644
--- a/funasr/models/language_model/seq_rnn_lm.py
+++ b/funasr/models/language_model/seq_rnn_lm.py
@@ -27,6 +27,18 @@ class SequentialRNNLM(AbsLM):
rnn_type: str = "lstm",
ignore_id: int = 0,
):
+ """Initialize SequentialRNNLM.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ unit: TODO.
+ nhid: TODO.
+ nlayers: TODO.
+ dropout_rate: TODO.
+ tie_weights: TODO.
+ rnn_type: TODO.
+ ignore_id: TODO.
+ """
super().__init__()
ninp = unit
@@ -87,6 +99,12 @@ class SequentialRNNLM(AbsLM):
def forward(
self, input: torch.Tensor, hidden: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ hidden: TODO.
+ """
emb = self.drop(self.encoder(input))
output, hidden = self.rnn(emb, hidden)
output = self.drop(output)
diff --git a/funasr/models/language_model/transformer_lm.py b/funasr/models/language_model/transformer_lm.py
index 3b90e3dc8..919309e51 100644
--- a/funasr/models/language_model/transformer_lm.py
+++ b/funasr/models/language_model/transformer_lm.py
@@ -23,12 +23,30 @@ class TransformerLM(AbsLM):
layer: int = 4,
dropout_rate: float = 0.5,
):
+ """Initialize TransformerLM.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ pos_enc: TODO.
+ embed_unit: TODO.
+ att_unit: TODO.
+ head: TODO.
+ unit: TODO.
+ layer: TODO.
+ dropout_rate: TODO.
+ """
super().__init__()
if pos_enc == "sinusoidal":
pos_enc_class = PositionalEncoding
elif pos_enc is None:
def pos_enc_class(*args, **kwargs):
+ """Pos enc class.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
return nn.Sequential() # indentity
else:
@@ -48,6 +66,11 @@ class TransformerLM(AbsLM):
self.decoder = nn.Linear(att_unit, vocab_size)
def _target_mask(self, ys_in_pad):
+ """Internal: target mask.
+
+ Args:
+ ys_in_pad: TODO.
+ """
ys_mask = ys_in_pad != 0
m = subsequent_mask(ys_mask.size(-1), device=ys_mask.device).unsqueeze(0)
return ys_mask.unsqueeze(-2) & m
diff --git a/funasr/models/lcbnet/encoder.py b/funasr/models/lcbnet/encoder.py
index f5f249776..01138db91 100644
--- a/funasr/models/lcbnet/encoder.py
+++ b/funasr/models/lcbnet/encoder.py
@@ -167,6 +167,21 @@ class TransformerTextEncoder(nn.Module):
normalize_before: bool = True,
concat_after: bool = False,
):
+ """Initialize TransformerTextEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -198,6 +213,7 @@ class TransformerTextEncoder(nn.Module):
self.after_norm = LayerNorm(output_size)
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
@@ -365,6 +381,16 @@ class ConvPredictor(nn.Module):
attention_dropout_rate=0.1,
linear_units=2048,
):
+ """Initialize ConvPredictor.
+
+ Args:
+ size: TODO.
+ l_order: TODO.
+ r_order: TODO.
+ attention_heads: TODO.
+ attention_dropout_rate: TODO.
+ linear_units: TODO.
+ """
super().__init__()
self.atten = MultiHeadedAttention(attention_heads, size, attention_dropout_rate)
self.norm1 = LayerNorm(size)
@@ -376,6 +402,12 @@ class ConvPredictor(nn.Module):
def forward(self, text_enc, asr_enc):
# stage1 cross-attention
+ """Forward pass for training.
+
+ Args:
+ text_enc: TODO.
+ asr_enc: TODO.
+ """
residual = text_enc
text_enc = residual + self.atten(text_enc, asr_enc, asr_enc, None)
diff --git a/funasr/models/lcbnet/model.py b/funasr/models/lcbnet/model.py
index 7b2038e6f..c964c58aa 100644
--- a/funasr/models/lcbnet/model.py
+++ b/funasr/models/lcbnet/model.py
@@ -74,6 +74,45 @@ class LCBNet(nn.Module):
**kwargs,
):
+ """Initialize LCBNet.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ text_encoder: TODO.
+ text_encoder_conf: Configuration dict for text_encoder.
+ bias_predictor: TODO.
+ bias_predictor_conf: Configuration dict for bias_predictor.
+ fusion_encoder: TODO.
+ fusion_encoder_conf: Configuration dict for fusion_encoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ interctc_weight: TODO.
+ select_num: TODO.
+ select_length: TODO.
+ insert_blank: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -300,6 +339,14 @@ class LCBNet(nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -331,6 +378,14 @@ class LCBNet(nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -344,6 +399,11 @@ class LCBNet(nn.Module):
self,
**kwargs,
):
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.transformer.search import BeamSearch
from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
from funasr.models.transformer.scorers.length_bonus import LengthBonus
@@ -395,6 +455,16 @@ class LCBNet(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
diff --git a/funasr/models/llm_asr/adaptor.py b/funasr/models/llm_asr/adaptor.py
index 43482134e..b849422e4 100644
--- a/funasr/models/llm_asr/adaptor.py
+++ b/funasr/models/llm_asr/adaptor.py
@@ -9,6 +9,15 @@ from funasr.register import tables
@tables.register("adaptor_classes", "Linear")
class Linear(nn.Module):
def __init__(self, downsample_rate, encoder_dim, llm_dim, ffn_dim: int = 2048, **kwargs):
+ """Initialize Linear.
+
+ Args:
+ downsample_rate: TODO.
+ encoder_dim: Size/dimension parameter.
+ llm_dim: Size/dimension parameter.
+ ffn_dim: Size/dimension parameter.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.k = downsample_rate
self.encoder_dim = encoder_dim
@@ -18,6 +27,11 @@ class Linear(nn.Module):
self.linear2 = nn.Linear(ffn_dim, self.llm_dim)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
batch_size, seq_len, dim = x.size()
num_frames_to_discard = seq_len % self.k
if num_frames_to_discard > 0:
@@ -35,6 +49,15 @@ class Linear(nn.Module):
@tables.register("adaptor_classes", "QFormer")
class EncoderProjectorQFormer(nn.Module):
def __init__(self, downsample_rate, encoder_dim, llm_dim, ffn_dim: int = 2048, **kwargs):
+ """Initialize EncoderProjectorQFormer.
+
+ Args:
+ downsample_rate: TODO.
+ encoder_dim: Size/dimension parameter.
+ llm_dim: Size/dimension parameter.
+ ffn_dim: Size/dimension parameter.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.encoder_dim = encoder_dim
self.llm_dim = llm_dim
@@ -56,6 +79,11 @@ class EncoderProjectorQFormer(nn.Module):
self.second_stride = 0.333333
def split_frames(self, speech_embeds):
+ """Split frames.
+
+ Args:
+ speech_embeds: TODO.
+ """
B, T, C = speech_embeds.shape
kernel = round(T * self.second_per_frame / 30.0)
stride = round(T * self.second_stride / 30.0)
@@ -71,6 +99,11 @@ class EncoderProjectorQFormer(nn.Module):
return speech_embeds, speech_atts
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
B, T, C = x.size()
encoder_out_feat, attention_mask = self.split_frames(x)
query = self.query.expand(encoder_out_feat.shape[0], -1, -1)
@@ -94,6 +127,15 @@ class Transformer(nn.Module):
def __init__(
self, downsample_rate=2, encoder_dim=1280, llm_dim=4096, ffn_dim: int = 2048, **kwargs
):
+ """Initialize Transformer.
+
+ Args:
+ downsample_rate: TODO.
+ encoder_dim: Size/dimension parameter.
+ llm_dim: Size/dimension parameter.
+ ffn_dim: Size/dimension parameter.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.k = downsample_rate
self.encoder_dim = encoder_dim
@@ -129,6 +171,12 @@ class Transformer(nn.Module):
def forward(self, x, ilens=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
batch_size, seq_len, dim = x.size()
# num_frames_to_discard = seq_len % self.k
chunk_num = (seq_len - 1) // self.k + 1
diff --git a/funasr/models/llm_asr/model.py b/funasr/models/llm_asr/model.py
index fa1d2c3c2..3bc77299b 100644
--- a/funasr/models/llm_asr/model.py
+++ b/funasr/models/llm_asr/model.py
@@ -64,6 +64,39 @@ class LLMASR(nn.Module):
**kwargs,
):
+ """Initialize LLMASR.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ audio_encoder: TODO.
+ audio_encoder_conf: Configuration dict for audio_encoder.
+ audio_adaptor: TODO.
+ audio_adaptor_conf: Configuration dict for audio_adaptor.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ llm: TODO.
+ llm_conf: Configuration dict for llm.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -224,6 +257,13 @@ class LLMASR(nn.Module):
speech_lengths: torch.Tensor,
**kwargs,
):
+ """Encode.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ **kwargs: Additional keyword arguments.
+ """
speech = speech.permute(0, 2, 1)
res = self.audio_encoder(speech)
if isinstance(res, (list, tuple)):
@@ -242,6 +282,16 @@ class LLMASR(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
prompt = kwargs.get("prompt", "Transcribe speech to text.")
if kwargs.get("batch_size", 1) > 1:
@@ -385,6 +435,39 @@ class LLMASR2(nn.Module):
**kwargs,
):
+ """Initialize LLMASR2.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ audio_encoder: TODO.
+ audio_encoder_conf: Configuration dict for audio_encoder.
+ audio_adaptor: TODO.
+ audio_adaptor_conf: Configuration dict for audio_adaptor.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ llm: TODO.
+ llm_conf: Configuration dict for llm.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# audio encoder
@@ -566,11 +649,22 @@ class LLMASR2(nn.Module):
def encode(self, speech, speech_lengths):
# audio encoder
+ """Encode.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
encoder_out, encoder_out_lens = self.audio_encoder(speech.permute(0, 2, 1), speech_lengths)
return encoder_out, encoder_out_lens
def data_template(self, data):
+ """Data template.
+
+ Args:
+ data: TODO.
+ """
system, user, assistant = [], [], []
for i, item in enumerate(data):
role = item["role"]
@@ -594,6 +688,15 @@ class LLMASR2(nn.Module):
def data_load_speech(self, contents: dict, tokenizer, frontend, meta_data={}, **kwargs):
+ """Data load speech.
+
+ Args:
+ contents: TODO.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ meta_data: TODO.
+ **kwargs: Additional keyword arguments.
+ """
system = contents["system"]
user = contents["user"]
assistant = contents["assistant"]
@@ -720,6 +823,16 @@ class LLMASR2(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
prompt = kwargs.get("prompt", None)
@@ -836,10 +949,22 @@ class LLMASR3(LLMASR2):
**kwargs,
):
+ """Initialize LLMASR3.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def encode(self, speech, speech_lengths):
# audio encoder
+ """Encode.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
encoder_out, encoder_out_lens = self.audio_encoder(speech, speech_lengths)
return encoder_out, encoder_out_lens
@@ -884,6 +1009,39 @@ class LLMASR4(nn.Module):
**kwargs,
):
+ """Initialize LLMASR4.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ audio_encoder: TODO.
+ audio_encoder_conf: Configuration dict for audio_encoder.
+ audio_adaptor: TODO.
+ audio_adaptor_conf: Configuration dict for audio_adaptor.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ llm: TODO.
+ llm_conf: Configuration dict for llm.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# audio encoder
@@ -1083,11 +1241,22 @@ class LLMASR4(nn.Module):
def encode(self, speech, speech_lengths):
# audio encoder
+ """Encode.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
encoder_out, encoder_out_lens = self.audio_encoder(speech.permute(0, 2, 1), speech_lengths)
return encoder_out, encoder_out_lens
def data_template(self, data):
+ """Data template.
+
+ Args:
+ data: TODO.
+ """
system, user, assistant = [], [], []
for i, item in enumerate(data):
role = item["role"]
@@ -1111,6 +1280,15 @@ class LLMASR4(nn.Module):
def data_load_speech(self, contents: dict, tokenizer, frontend, meta_data={}, **kwargs):
+ """Data load speech.
+
+ Args:
+ contents: TODO.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ meta_data: TODO.
+ **kwargs: Additional keyword arguments.
+ """
system = contents["system"]
user = contents["user"]
assistant = contents["assistant"]
@@ -1258,6 +1436,16 @@ class LLMASR4(nn.Module):
**kwargs,
):
+ """Inference prepare.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
prompt = kwargs.get("prompt", None)
@@ -1340,6 +1528,16 @@ class LLMASR4(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
inputs_embeds, contents, batch, source_ids, meta_data = self.inference_prepare(
data_in, data_lengths, key, tokenizer, frontend, **kwargs
)
diff --git a/funasr/models/llm_asr_nar/adaptor.py b/funasr/models/llm_asr_nar/adaptor.py
index 0d73a1a0e..54ed8bbf1 100644
--- a/funasr/models/llm_asr_nar/adaptor.py
+++ b/funasr/models/llm_asr_nar/adaptor.py
@@ -7,6 +7,15 @@ from funasr.register import tables
@tables.register("adaptor_classes", "Linear")
class Linear(nn.Module):
def __init__(self, downsample_rate, encoder_dim, llm_dim, ffn_dim: int = 2048, **kwargs):
+ """Initialize Linear.
+
+ Args:
+ downsample_rate: TODO.
+ encoder_dim: Size/dimension parameter.
+ llm_dim: Size/dimension parameter.
+ ffn_dim: Size/dimension parameter.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.k = downsample_rate
self.encoder_dim = encoder_dim
@@ -16,6 +25,11 @@ class Linear(nn.Module):
self.linear2 = nn.Linear(ffn_dim, self.llm_dim)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
batch_size, seq_len, dim = x.size()
num_frames_to_discard = seq_len % self.k
if num_frames_to_discard > 0:
diff --git a/funasr/models/llm_asr_nar/model.py b/funasr/models/llm_asr_nar/model.py
index 192c1990b..b0d4f10d3 100644
--- a/funasr/models/llm_asr_nar/model.py
+++ b/funasr/models/llm_asr_nar/model.py
@@ -62,6 +62,39 @@ class LLMASRNAR(nn.Module):
**kwargs,
):
+ """Initialize LLMASRNAR.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ llm: TODO.
+ llm_conf: Configuration dict for llm.
+ adaptor: TODO.
+ adaptor_conf: Configuration dict for adaptor.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -224,6 +257,13 @@ class LLMASRNAR(nn.Module):
**kwargs,
):
+ """Encode.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ **kwargs: Additional keyword arguments.
+ """
audio_mask = kwargs.get("audio_mask", None)
audio_token_lengths = audio_mask.sum(-1) if audio_mask is not None else None
text_token_int = kwargs.get("text_token_int", None)
@@ -252,6 +292,16 @@ class LLMASRNAR(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
prompt = kwargs.get("prompt", "Transcribe speech to text.")
if kwargs.get("batch_size", 1) > 1:
@@ -408,6 +458,40 @@ class LLMASRNARPrompt(nn.Module):
**kwargs,
):
+ """Initialize LLMASRNARPrompt.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ llm: TODO.
+ llm_conf: Configuration dict for llm.
+ adaptor: TODO.
+ adaptor_conf: Configuration dict for adaptor.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ predictor_weight: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -611,6 +695,13 @@ class LLMASRNARPrompt(nn.Module):
**kwargs,
):
+ """Encode.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ **kwargs: Additional keyword arguments.
+ """
audio_mask = kwargs.get("audio_mask", None)
audio_token_lengths = audio_mask.sum(-1) if audio_mask is not None else None
text_token_int = kwargs.get("text_token_int", None)
@@ -642,6 +733,14 @@ class LLMASRNARPrompt(nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -661,6 +760,16 @@ class LLMASRNARPrompt(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
prompt = kwargs.get("prompt", "Transcribe speech to text.")
if kwargs.get("batch_size", 1) > 1:
diff --git a/funasr/models/lora/layers.py b/funasr/models/lora/layers.py
index b91253d56..9880b0f58 100644
--- a/funasr/models/lora/layers.py
+++ b/funasr/models/lora/layers.py
@@ -18,6 +18,14 @@ class LoRALayer:
lora_dropout: float,
merge_weights: bool,
):
+ """Initialize LoRALayer.
+
+ Args:
+ r: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ merge_weights: TODO.
+ """
self.r = r
self.lora_alpha = lora_alpha
# Optional dropout
@@ -41,6 +49,16 @@ class Embedding(nn.Embedding, LoRALayer):
merge_weights: bool = True,
**kwargs
):
+ """Initialize Embedding.
+
+ Args:
+ num_embeddings: TODO.
+ embedding_dim: Size/dimension parameter.
+ r: TODO.
+ lora_alpha: TODO.
+ merge_weights: TODO.
+ **kwargs: Additional keyword arguments.
+ """
nn.Embedding.__init__(self, num_embeddings, embedding_dim, **kwargs)
LoRALayer.__init__(
self, r=r, lora_alpha=lora_alpha, lora_dropout=0, merge_weights=merge_weights
@@ -55,6 +73,7 @@ class Embedding(nn.Embedding, LoRALayer):
self.reset_parameters()
def reset_parameters(self):
+ """Reset parameters."""
nn.Embedding.reset_parameters(self)
if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
@@ -62,6 +81,11 @@ class Embedding(nn.Embedding, LoRALayer):
nn.init.normal_(self.lora_B)
def train(self, mode: bool = True):
+ """Train.
+
+ Args:
+ mode: TODO.
+ """
nn.Embedding.train(self, mode)
if self.merge_weights and self.merged:
# Make sure that the weights are not merged
@@ -70,6 +94,7 @@ class Embedding(nn.Embedding, LoRALayer):
self.merged = False
def eval(self):
+ """Eval."""
nn.Linear.eval(self)
if self.merge_weights and not self.merged:
# Merge the weights and mark it
@@ -78,6 +103,11 @@ class Embedding(nn.Embedding, LoRALayer):
self.merged = True
def forward(self, x: torch.Tensor):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if self.r > 0 and not self.merged:
result = nn.Embedding.forward(self, x)
if self.r > 0:
@@ -109,6 +139,18 @@ class Linear(nn.Linear, LoRALayer):
merge_weights: bool = True,
**kwargs
):
+ """Initialize Linear.
+
+ Args:
+ in_features: TODO.
+ out_features: TODO.
+ r: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ fan_in_fan_out: TODO.
+ merge_weights: TODO.
+ **kwargs: Additional keyword arguments.
+ """
nn.Linear.__init__(self, in_features, out_features, **kwargs)
LoRALayer.__init__(
self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights
@@ -127,6 +169,7 @@ class Linear(nn.Linear, LoRALayer):
self.weight.data = self.weight.data.T
def reset_parameters(self):
+ """Reset parameters."""
nn.Linear.reset_parameters(self)
if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
@@ -134,7 +177,17 @@ class Linear(nn.Linear, LoRALayer):
nn.init.zeros_(self.lora_B)
def train(self, mode: bool = True):
+ """Train.
+
+ Args:
+ mode: TODO.
+ """
def T(w):
+ """T.
+
+ Args:
+ w: TODO.
+ """
return w.T if self.fan_in_fan_out else w
nn.Linear.train(self, mode)
@@ -145,7 +198,13 @@ class Linear(nn.Linear, LoRALayer):
self.merged = False
def eval(self):
+ """Eval."""
def T(w):
+ """T.
+
+ Args:
+ w: TODO.
+ """
return w.T if self.fan_in_fan_out else w
nn.Linear.eval(self)
@@ -156,7 +215,17 @@ class Linear(nn.Linear, LoRALayer):
self.merged = True
def forward(self, x: torch.Tensor):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
def T(w):
+ """T.
+
+ Args:
+ w: TODO.
+ """
return w.T if self.fan_in_fan_out else w
if self.r > 0 and not self.merged:
@@ -182,6 +251,19 @@ class MergedLinear(nn.Linear, LoRALayer):
merge_weights: bool = True,
**kwargs
):
+ """Initialize MergedLinear.
+
+ Args:
+ in_features: TODO.
+ out_features: TODO.
+ r: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ enable_lora: TODO.
+ fan_in_fan_out: TODO.
+ merge_weights: TODO.
+ **kwargs: Additional keyword arguments.
+ """
nn.Linear.__init__(self, in_features, out_features, **kwargs)
LoRALayer.__init__(
self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights
@@ -211,6 +293,7 @@ class MergedLinear(nn.Linear, LoRALayer):
self.weight.data = self.weight.data.T
def reset_parameters(self):
+ """Reset parameters."""
nn.Linear.reset_parameters(self)
if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
@@ -218,6 +301,11 @@ class MergedLinear(nn.Linear, LoRALayer):
nn.init.zeros_(self.lora_B)
def zero_pad(self, x):
+ """Zero pad.
+
+ Args:
+ x: TODO.
+ """
result = x.new_zeros((*x.shape[:-1], self.out_features))
result = result.view(-1, self.out_features)
result[:, self.lora_ind] = x.reshape(
@@ -226,7 +314,17 @@ class MergedLinear(nn.Linear, LoRALayer):
return result.view((*x.shape[:-1], self.out_features))
def train(self, mode: bool = True):
+ """Train.
+
+ Args:
+ mode: TODO.
+ """
def T(w):
+ """T.
+
+ Args:
+ w: TODO.
+ """
return w.T if self.fan_in_fan_out else w
nn.Linear.train(self, mode)
@@ -242,7 +340,13 @@ class MergedLinear(nn.Linear, LoRALayer):
self.merged = False
def eval(self):
+ """Eval."""
def T(w):
+ """T.
+
+ Args:
+ w: TODO.
+ """
return w.T if self.fan_in_fan_out else w
nn.Linear.eval(self)
@@ -258,7 +362,17 @@ class MergedLinear(nn.Linear, LoRALayer):
self.merged = True
def forward(self, x: torch.Tensor):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
def T(w):
+ """T.
+
+ Args:
+ w: TODO.
+ """
return w.T if self.fan_in_fan_out else w
if self.merged:
@@ -289,6 +403,18 @@ class Conv2d(nn.Conv2d, LoRALayer):
merge_weights: bool = True,
**kwargs
):
+ """Initialize Conv2d.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ r: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ merge_weights: TODO.
+ **kwargs: Additional keyword arguments.
+ """
nn.Conv2d.__init__(self, in_channels, out_channels, kernel_size, **kwargs)
LoRALayer.__init__(
self, r=r, lora_alpha=lora_alpha, lora_dropout=lora_dropout, merge_weights=merge_weights
@@ -308,6 +434,7 @@ class Conv2d(nn.Conv2d, LoRALayer):
self.reset_parameters()
def reset_parameters(self):
+ """Reset parameters."""
nn.Conv2d.reset_parameters(self)
if hasattr(self, "lora_A"):
# initialize A the same way as the default for nn.Linear and B to zero
@@ -315,6 +442,11 @@ class Conv2d(nn.Conv2d, LoRALayer):
nn.init.zeros_(self.lora_B)
def train(self, mode: bool = True):
+ """Train.
+
+ Args:
+ mode: TODO.
+ """
nn.Conv2d.train(self, mode)
if self.merge_weights and self.merged:
# Make sure that the weights are not merged
@@ -322,6 +454,7 @@ class Conv2d(nn.Conv2d, LoRALayer):
self.merged = False
def eval(self):
+ """Eval."""
nn.Conv2d.eval(self)
if self.merge_weights and not self.merged:
# Merge the weights and mark it
@@ -329,6 +462,11 @@ class Conv2d(nn.Conv2d, LoRALayer):
self.merged = True
def forward(self, x: torch.Tensor):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if self.r > 0 and not self.merged:
return F.conv2d(
x,
diff --git a/funasr/models/lora/utils.py b/funasr/models/lora/utils.py
index 670c1dcc0..586341367 100644
--- a/funasr/models/lora/utils.py
+++ b/funasr/models/lora/utils.py
@@ -11,6 +11,12 @@ from .layers import LoRALayer
def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None:
+ """Mark only lora as trainable.
+
+ Args:
+ model: Model instance or model name.
+ bias: TODO.
+ """
for n, p in model.named_parameters():
if "lora_" not in n and "cif" not in n:
p.requires_grad = False
@@ -29,6 +35,12 @@ def mark_only_lora_as_trainable(model: nn.Module, bias: str = "none") -> None:
def lora_state_dict(model: nn.Module, bias: str = "none") -> Dict[str, torch.Tensor]:
+ """Lora state dict.
+
+ Args:
+ model: Model instance or model name.
+ bias: TODO.
+ """
my_state_dict = model.state_dict()
if bias == "none":
return {k: my_state_dict[k] for k in my_state_dict if "lora_" in k}
diff --git a/funasr/models/mfcca/e2e_asr_mfcca.py b/funasr/models/mfcca/e2e_asr_mfcca.py
index 681d1f61f..f6cbb089f 100644
--- a/funasr/models/mfcca/e2e_asr_mfcca.py
+++ b/funasr/models/mfcca/e2e_asr_mfcca.py
@@ -30,6 +30,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -67,6 +72,29 @@ class MFCCA(FunASRModel):
sym_blank: str = "<blank>",
preencoder: Optional[AbsPreEncoder] = None,
):
+ """Initialize MFCCA.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ token_list: TODO.
+ frontend: Audio frontend for feature extraction.
+ specaug: TODO.
+ normalize: TODO.
+ encoder: TODO.
+ decoder: TODO.
+ ctc: TODO.
+ rnnt_decoder: TODO.
+ ctc_weight: TODO.
+ ignore_id: TODO.
+ lsm_weight: TODO.
+ mask_ratio: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ preencoder: TODO.
+ """
assert 0.0 <= ctc_weight <= 1.0, ctc_weight
assert rnnt_decoder is None, "Not implemented"
@@ -198,6 +226,14 @@ class MFCCA(FunASRModel):
text: torch.Tensor,
text_lengths: torch.Tensor,
) -> Dict[str, torch.Tensor]:
+ """Collect feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ text: Text tensor or string input.
+ text_lengths: Length of each text sample.
+ """
feats, feats_lengths, channel_size = self._extract_feats(speech, speech_lengths)
return {"feats": feats, "feats_lengths": feats_lengths}
@@ -246,6 +282,12 @@ class MFCCA(FunASRModel):
def _extract_feats(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Internal: extract feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
assert speech_lengths.dim() == 1, speech_lengths.shape
# for data-parallel
speech = speech[:, : speech_lengths.max()]
@@ -268,6 +310,14 @@ class MFCCA(FunASRModel):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -299,6 +349,14 @@ class MFCCA(FunASRModel):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
if encoder_out.dim() == 4:
encoder_out = encoder_out.mean(1)
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
@@ -317,4 +375,12 @@ class MFCCA(FunASRModel):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc rnnt loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
raise NotImplementedError
diff --git a/funasr/models/mfcca/mfcca_encoder.py b/funasr/models/mfcca/mfcca_encoder.py
index a0bb58ee0..aa16d08f6 100644
--- a/funasr/models/mfcca/mfcca_encoder.py
+++ b/funasr/models/mfcca/mfcca_encoder.py
@@ -159,6 +159,32 @@ class MFCCAEncoder(AbsEncoder):
cnn_module_kernel: int = 31,
padding_idx: int = -1,
):
+ """Initialize MFCCAEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ macaron_style: TODO.
+ rel_pos_type: TODO.
+ pos_enc_layer_type: TODO.
+ selfattention_layer_type: TODO.
+ activation_type: TODO.
+ use_cnn_module: TODO.
+ zero_triu: TODO.
+ cnn_module_kernel: TODO.
+ padding_idx: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -319,6 +345,7 @@ class MFCCAEncoder(AbsEncoder):
self.conv4 = torch.nn.Conv2d(16, 1, [5, 7], stride=[1, 1], padding=(2, 3))
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
diff --git a/funasr/models/monotonic_aligner/model.py b/funasr/models/monotonic_aligner/model.py
index 4754d1fb3..be1284b75 100644
--- a/funasr/models/monotonic_aligner/model.py
+++ b/funasr/models/monotonic_aligner/model.py
@@ -44,6 +44,22 @@ class MonotonicAligner(torch.nn.Module):
length_normalized_loss: bool = False,
**kwargs,
):
+ """Initialize MonotonicAligner.
+
+ Args:
+ input_size: Size/dimension parameter.
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ predictor: TODO.
+ predictor_conf: Configuration dict for predictor.
+ predictor_bias: TODO.
+ length_normalized_loss: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -116,6 +132,13 @@ class MonotonicAligner(torch.nn.Module):
return loss, stats, weight
def calc_predictor_timestamp(self, encoder_out, encoder_out_lens, token_num):
+ """Calc predictor timestamp.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ token_num: TODO.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -162,6 +185,16 @@ class MonotonicAligner(torch.nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
# extract fbank feats
time1 = time.perf_counter()
diff --git a/funasr/models/mossformer/e2e_ss.py b/funasr/models/mossformer/e2e_ss.py
index 40d30ca89..9f765d274 100644
--- a/funasr/models/mossformer/e2e_ss.py
+++ b/funasr/models/mossformer/e2e_ss.py
@@ -45,6 +45,19 @@ class MossFormer(FunASRModel):
use_global_pos_enc=True,
max_length=20000,
):
+ """Initialize MossFormer.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ num_blocks: TODO.
+ kernel_size: Size/dimension parameter.
+ norm: TODO.
+ num_spks: TODO.
+ skip_around_intra: TODO.
+ use_global_pos_enc: TODO.
+ max_length: TODO.
+ """
super(MossFormer, self).__init__()
self.num_spks = num_spks
# Encoding
@@ -72,6 +85,11 @@ class MossFormer(FunASRModel):
)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
x = self.enc(input)
mask = self.mask_net(x)
x = torch.stack([x] * self.num_spks)
diff --git a/funasr/models/mossformer/mossformer.py b/funasr/models/mossformer/mossformer.py
index 30aab481a..9237c5857 100644
--- a/funasr/models/mossformer/mossformer.py
+++ b/funasr/models/mossformer/mossformer.py
@@ -5,24 +5,54 @@ from einops import rearrange
def identity(t, *args, **kwargs):
+ """Identity.
+
+ Args:
+ t: TODO.
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
return t
def append_dims(x, num_dims):
+ """Append dims.
+
+ Args:
+ x: TODO.
+ num_dims: TODO.
+ """
if num_dims <= 0:
return x
return x.view(*x.shape, *((1,) * num_dims))
def exists(val):
+ """Exists.
+
+ Args:
+ val: TODO.
+ """
return val is not None
def default(val, d):
+ """Default.
+
+ Args:
+ val: TODO.
+ d: TODO.
+ """
return val if exists(val) else d
def padding_to_multiple_of(n, mult):
+ """Padding to multiple of.
+
+ Args:
+ n: TODO.
+ mult: TODO.
+ """
remainder = n % mult
if remainder == 0:
return 0
@@ -33,10 +63,20 @@ class Transpose(nn.Module):
"""Wrapper class of torch.transpose() for Sequential module."""
def __init__(self, shape: tuple):
+ """Initialize Transpose.
+
+ Args:
+ shape: TODO.
+ """
super(Transpose, self).__init__()
self.shape = shape
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return x.transpose(*self.shape)
@@ -66,6 +106,16 @@ class DepthwiseConv1d(nn.Module):
padding: int = 0,
bias: bool = False,
) -> None:
+ """Initialize DepthwiseConv1d.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ padding: TODO.
+ bias: TODO.
+ """
super(DepthwiseConv1d, self).__init__()
assert (
out_channels % in_channels == 0
@@ -81,6 +131,11 @@ class DepthwiseConv1d(nn.Module):
)
def forward(self, inputs):
+ """Forward pass for training.
+
+ Args:
+ inputs: TODO.
+ """
return self.conv(inputs)
@@ -106,6 +161,14 @@ class ConvModule(nn.Module):
expansion_factor: int = 2,
dropout_p: float = 0.1,
) -> None:
+ """Initialize ConvModule.
+
+ Args:
+ in_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ expansion_factor: TODO.
+ dropout_p: TODO.
+ """
super(ConvModule, self).__init__()
assert (kernel_size - 1) % 2 == 0, "kernel_size should be a odd number for 'SAME' padding"
assert expansion_factor == 2, "Currently, Only Supports expansion_factor 2"
@@ -118,23 +181,47 @@ class ConvModule(nn.Module):
)
def forward(self, inputs):
+ """Forward pass for training.
+
+ Args:
+ inputs: TODO.
+ """
return inputs + self.sequential(inputs).transpose(1, 2)
class OffsetScale(nn.Module):
def __init__(self, dim, heads=1):
+ """Initialize OffsetScale.
+
+ Args:
+ dim: TODO.
+ heads: TODO.
+ """
super().__init__()
self.gamma = nn.Parameter(torch.ones(heads, dim))
self.beta = nn.Parameter(torch.zeros(heads, dim))
nn.init.normal_(self.gamma, std=0.02)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
out = einsum("... d, h d -> ... h d", x, self.gamma) + self.beta
return out.unbind(dim=-2)
class FFConvM(nn.Module):
def __init__(self, dim_in, dim_out, norm_klass=nn.LayerNorm, dropout=0.1):
+ """Initialize FFConvM.
+
+ Args:
+ dim_in: TODO.
+ dim_out: TODO.
+ norm_klass: TODO.
+ dropout: TODO.
+ """
super().__init__()
self.mdl = nn.Sequential(
norm_klass(dim_in),
@@ -148,6 +235,11 @@ class FFConvM(nn.Module):
self,
x,
):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
output = self.mdl(x)
return output
@@ -166,6 +258,7 @@ class FLASH_ShareA_FFConvM(nn.Module):
norm_klass=nn.LayerNorm,
shift_tokens=True
):
+ """Initialize FLASH_ShareA_FFConvM."""
super().__init__()
hidden_dim = int(dim * expansion_factor)
self.group_size = group_size
@@ -236,6 +329,18 @@ class FLASH_ShareA_FFConvM(nn.Module):
return x
def cal_attention(self, x, quad_q, lin_q, quad_k, lin_k, v, u, mask=None):
+ """Cal attention.
+
+ Args:
+ x: TODO.
+ quad_q: TODO.
+ lin_q: TODO.
+ quad_k: TODO.
+ lin_k: TODO.
+ v: TODO.
+ u: TODO.
+ mask: TODO.
+ """
b, n, device, g = x.shape[0], x.shape[-2], x.device, self.group_size
if exists(mask):
diff --git a/funasr/models/mossformer/mossformer_decoder.py b/funasr/models/mossformer/mossformer_decoder.py
index 56ad2ecb3..6b447c782 100644
--- a/funasr/models/mossformer/mossformer_decoder.py
+++ b/funasr/models/mossformer/mossformer_decoder.py
@@ -25,6 +25,12 @@ class MossFormerDecoder(nn.ConvTranspose1d):
"""
def __init__(self, *args, **kwargs):
+ """Initialize MossFormerDecoder.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super(MossFormerDecoder, self).__init__(*args, **kwargs)
def forward(self, x):
diff --git a/funasr/models/mossformer/mossformer_encoder.py b/funasr/models/mossformer/mossformer_encoder.py
index a691d8c62..2642ace67 100644
--- a/funasr/models/mossformer/mossformer_encoder.py
+++ b/funasr/models/mossformer/mossformer_encoder.py
@@ -41,6 +41,7 @@ class MossformerBlock(nn.Module):
norm_type="scalenorm",
shift_tokens=True
):
+ """Initialize MossformerBlock."""
super().__init__()
assert norm_type in (
"scalenorm",
@@ -74,6 +75,11 @@ class MossformerBlock(nn.Module):
)
def forward(self, x, *, mask=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
ii = 0
for flash in self.layers:
x = flash(x, mask=mask)
@@ -124,6 +130,18 @@ class MossFormer_MaskNet(nn.Module):
use_global_pos_enc=True,
max_length=20000,
):
+ """Initialize MossFormer_MaskNet.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ num_blocks: TODO.
+ norm: TODO.
+ num_spks: TODO.
+ skip_around_intra: TODO.
+ use_global_pos_enc: TODO.
+ max_length: TODO.
+ """
super(MossFormer_MaskNet, self).__init__()
self.num_spks = num_spks
self.num_blocks = num_blocks
@@ -236,6 +254,13 @@ class MossFormerEncoder(nn.Module):
"""
def __init__(self, kernel_size=2, out_channels=64, in_channels=1):
+ """Initialize MossFormerEncoder.
+
+ Args:
+ kernel_size: Size/dimension parameter.
+ out_channels: TODO.
+ in_channels: TODO.
+ """
super(MossFormerEncoder, self).__init__()
self.conv1d = nn.Conv1d(
in_channels=in_channels,
@@ -314,6 +339,17 @@ class MossFormerM(nn.Module):
expansion_factor=4.0,
attn_dropout=0.1,
):
+ """Initialize MossFormerM.
+
+ Args:
+ num_blocks: TODO.
+ d_model: D Model instance.
+ causal: TODO.
+ group_size: Size/dimension parameter.
+ query_key_dim: Size/dimension parameter.
+ expansion_factor: TODO.
+ attn_dropout: TODO.
+ """
super().__init__()
self.mossformerM = MossformerBlock(
@@ -379,6 +415,14 @@ class Computation_Block(nn.Module):
norm="ln",
skip_around_intra=True,
):
+ """Initialize Computation_Block.
+
+ Args:
+ num_blocks: TODO.
+ out_channels: TODO.
+ norm: TODO.
+ skip_around_intra: TODO.
+ """
super(Computation_Block, self).__init__()
##MossFormer2M: MossFormer with recurrence
diff --git a/funasr/models/normalize/global_mvn.py b/funasr/models/normalize/global_mvn.py
index 55a728706..c12da5110 100644
--- a/funasr/models/normalize/global_mvn.py
+++ b/funasr/models/normalize/global_mvn.py
@@ -27,6 +27,14 @@ class GlobalMVN(torch.nn.Module):
norm_vars: bool = True,
eps: float = 1.0e-20,
):
+ """Initialize GlobalMVN.
+
+ Args:
+ stats_file: TODO.
+ norm_means: TODO.
+ norm_vars: TODO.
+ eps: TODO.
+ """
super().__init__()
self.norm_means = norm_means
self.norm_vars = norm_vars
@@ -53,6 +61,7 @@ class GlobalMVN(torch.nn.Module):
self.register_buffer("std", torch.from_numpy(std))
def extra_repr(self):
+ """Extra repr."""
return (
f"stats_file={self.stats_file}, "
f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"
@@ -93,6 +102,12 @@ class GlobalMVN(torch.nn.Module):
def inverse(
self, x: torch.Tensor, ilens: torch.Tensor = None
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Inverse.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
if ilens is None:
ilens = x.new_full([x.size(0)], x.size(1))
norm_means = self.norm_means
diff --git a/funasr/models/normalize/utterance_mvn.py b/funasr/models/normalize/utterance_mvn.py
index 8216328c8..9128c6d04 100644
--- a/funasr/models/normalize/utterance_mvn.py
+++ b/funasr/models/normalize/utterance_mvn.py
@@ -14,12 +14,20 @@ class UtteranceMVN(torch.nn.Module):
norm_vars: bool = False,
eps: float = 1.0e-20,
):
+ """Initialize UtteranceMVN.
+
+ Args:
+ norm_means: TODO.
+ norm_vars: TODO.
+ eps: TODO.
+ """
super().__init__()
self.norm_means = norm_means
self.norm_vars = norm_vars
self.eps = eps
def extra_repr(self):
+ """Extra repr."""
return f"norm_means={self.norm_means}, norm_vars={self.norm_vars}"
def forward(
diff --git a/funasr/models/paraformer/cif_predictor.py b/funasr/models/paraformer/cif_predictor.py
index d59705032..d72bd0f7b 100644
--- a/funasr/models/paraformer/cif_predictor.py
+++ b/funasr/models/paraformer/cif_predictor.py
@@ -1,760 +1,920 @@
-#!/usr/bin/env python3
-# -*- encoding: utf-8 -*-
-# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
-# MIT License (https://opensource.org/licenses/MIT)
-
-import torch
-import logging
-import numpy as np
-
-from funasr.register import tables
-from funasr.train_utils.device_funcs import to_device
-from funasr.models.transformer.utils.nets_utils import make_pad_mask
-from torch.cuda.amp import autocast
-
-
[email protected]("predictor_classes", "CifPredictor")
-class CifPredictor(torch.nn.Module):
- def __init__(
- self,
- idim,
- l_order,
- r_order,
- threshold=1.0,
- dropout=0.1,
- smooth_factor=1.0,
- noise_threshold=0,
- tail_threshold=0.45,
- ):
- super().__init__()
-
- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)
- self.cif_output = torch.nn.Linear(idim, 1)
- self.dropout = torch.nn.Dropout(p=dropout)
- self.threshold = threshold
- self.smooth_factor = smooth_factor
- self.noise_threshold = noise_threshold
- self.tail_threshold = tail_threshold
-
- def forward(
- self,
- hidden,
- target_label=None,
- mask=None,
- ignore_id=-1,
- mask_chunk_predictor=None,
- target_label_length=None,
- ):
-
- with autocast(False):
- h = hidden
- context = h.transpose(1, 2)
- queries = self.pad(context)
- memory = self.cif_conv1d(queries)
- output = memory + context
- output = self.dropout(output)
- output = output.transpose(1, 2)
- output = torch.relu(output)
- output = self.cif_output(output)
- alphas = torch.sigmoid(output)
- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
- if mask is not None:
- mask = mask.transpose(-1, -2).float()
- alphas = alphas * mask
- if mask_chunk_predictor is not None:
- alphas = alphas * mask_chunk_predictor
- alphas = alphas.squeeze(-1)
- mask = mask.squeeze(-1)
- if target_label_length is not None:
- target_length = target_label_length
- elif target_label is not None:
- target_length = (target_label != ignore_id).float().sum(-1)
- else:
- target_length = None
- token_num = alphas.sum(-1)
- if target_length is not None:
- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
- elif self.tail_threshold > 0.0:
- hidden, alphas, token_num = self.tail_process_fn(
- hidden, alphas, token_num, mask=mask
- )
-
- acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)
-
- if target_length is None and self.tail_threshold > 0.0:
- token_num_int = torch.max(token_num).type(torch.int32).item()
- acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
-
- return acoustic_embeds, token_num, alphas, cif_peak
-
- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
- b, t, d = hidden.size()
- tail_threshold = self.tail_threshold
- if mask is not None:
- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
- ones_t = torch.ones_like(zeros_t)
- mask_1 = torch.cat([mask, zeros_t], dim=1)
- mask_2 = torch.cat([ones_t, mask], dim=1)
- mask = mask_2 - mask_1
- tail_threshold = mask * tail_threshold
- alphas = torch.cat([alphas, zeros_t], dim=1)
- alphas = torch.add(alphas, tail_threshold)
- else:
- tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)
- tail_threshold = torch.reshape(tail_threshold, (1, 1))
- alphas = torch.cat([alphas, tail_threshold], dim=1)
- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
- hidden = torch.cat([hidden, zeros], dim=1)
- token_num = alphas.sum(dim=-1)
- token_num_floor = torch.floor(token_num)
-
- return hidden, alphas, token_num_floor
-
- def gen_frame_alignments(
- self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None
- ):
- batch_size, maximum_length = alphas.size()
- int_type = torch.int32
-
- is_training = self.training
- if is_training:
- token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)
- else:
- token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)
-
- max_token_num = torch.max(token_num).item()
-
- alphas_cumsum = torch.cumsum(alphas, dim=1)
- alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)
- alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)
-
- index = torch.ones([batch_size, max_token_num], dtype=int_type)
- index = torch.cumsum(index, dim=1)
- index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)
-
- index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)
- index_div_bool_zeros = index_div.eq(0)
- index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1
- index_div_bool_zeros_count = torch.clamp(
- index_div_bool_zeros_count, 0, encoder_sequence_length.max()
- )
- token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)
- index_div_bool_zeros_count *= token_num_mask
-
- index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(
- 1, 1, maximum_length
- )
- ones = torch.ones_like(index_div_bool_zeros_count_tile)
- zeros = torch.zeros_like(index_div_bool_zeros_count_tile)
- ones = torch.cumsum(ones, dim=2)
- cond = index_div_bool_zeros_count_tile == ones
- index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)
-
- index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)
- index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)
- index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)
- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)
- predictor_mask = (
- (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))
- .type(int_type)
- .to(encoder_sequence_length.device)
- )
- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask
-
- predictor_alignments = index_div_bool_zeros_count_tile_out
- predictor_alignments_length = predictor_alignments.sum(-1).type(
- encoder_sequence_length.dtype
- )
- return predictor_alignments.detach(), predictor_alignments_length.detach()
-
-
[email protected]("predictor_classes", "CifPredictorV2")
-class CifPredictorV2(torch.nn.Module):
- def __init__(
- self,
- idim,
- l_order,
- r_order,
- threshold=1.0,
- dropout=0.1,
- smooth_factor=1.0,
- noise_threshold=0,
- tail_threshold=0.0,
- tf2torch_tensor_name_prefix_torch="predictor",
- tf2torch_tensor_name_prefix_tf="seq2seq/cif",
- tail_mask=True,
- ):
- super().__init__()
-
- self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
- self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)
- self.cif_output = torch.nn.Linear(idim, 1)
- self.dropout = torch.nn.Dropout(p=dropout)
- self.threshold = threshold
- self.smooth_factor = smooth_factor
- self.noise_threshold = noise_threshold
- self.tail_threshold = tail_threshold
- self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
- self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
- self.tail_mask = tail_mask
-
- def forward(
- self,
- hidden,
- target_label=None,
- mask=None,
- ignore_id=-1,
- mask_chunk_predictor=None,
- target_label_length=None,
- ):
-
- with autocast(False):
- h = hidden
- context = h.transpose(1, 2)
- queries = self.pad(context)
- output = torch.relu(self.cif_conv1d(queries))
- output = output.transpose(1, 2)
-
- output = self.cif_output(output)
- alphas = torch.sigmoid(output)
- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
- if mask is not None:
- mask = mask.transpose(-1, -2).float()
- alphas = alphas * mask
- if mask_chunk_predictor is not None:
- alphas = alphas * mask_chunk_predictor
- alphas = alphas.squeeze(-1)
- mask = mask.squeeze(-1)
- if target_label_length is not None:
- target_length = target_label_length.squeeze(-1)
- elif target_label is not None:
- target_length = (target_label != ignore_id).float().sum(-1)
- else:
- target_length = None
- token_num = alphas.sum(-1)
- if target_length is not None:
- alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
- elif self.tail_threshold > 0.0:
- if self.tail_mask:
- hidden, alphas, token_num = self.tail_process_fn(
- hidden, alphas, token_num, mask=mask
- )
- else:
- hidden, alphas, token_num = self.tail_process_fn(
- hidden, alphas, token_num, mask=None
- )
-
- acoustic_embeds, cif_peak = cif_v1(hidden, alphas, self.threshold)
- if target_length is None and self.tail_threshold > 0.0:
- token_num_int = torch.max(token_num).type(torch.int32).item()
- acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
-
- return acoustic_embeds, token_num, alphas, cif_peak
-
- def forward_chunk(self, hidden, cache=None, **kwargs):
- is_final = kwargs.get("is_final", False)
- batch_size, len_time, hidden_size = hidden.shape
- h = hidden
- context = h.transpose(1, 2)
- queries = self.pad(context)
- output = torch.relu(self.cif_conv1d(queries))
- output = output.transpose(1, 2)
- output = self.cif_output(output)
- alphas = torch.sigmoid(output)
- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
-
- alphas = alphas.squeeze(-1)
-
- token_length = []
- list_fires = []
- list_frames = []
- cache_alphas = []
- cache_hiddens = []
-
- if cache is not None and "chunk_size" in cache:
- alphas[:, : cache["chunk_size"][0]] = 0.0
- if not is_final:
- alphas[:, sum(cache["chunk_size"][:2]) :] = 0.0
- if cache is not None and "cif_alphas" in cache and "cif_hidden" in cache:
- cache["cif_hidden"] = to_device(cache["cif_hidden"], device=hidden.device)
- cache["cif_alphas"] = to_device(cache["cif_alphas"], device=alphas.device)
- hidden = torch.cat((cache["cif_hidden"], hidden), dim=1)
- alphas = torch.cat((cache["cif_alphas"], alphas), dim=1)
- if cache is not None and is_final:
- tail_hidden = torch.zeros((batch_size, 1, hidden_size), device=hidden.device)
- tail_alphas = torch.tensor([[self.tail_threshold]], device=alphas.device)
- tail_alphas = torch.tile(tail_alphas, (batch_size, 1))
- hidden = torch.cat((hidden, tail_hidden), dim=1)
- alphas = torch.cat((alphas, tail_alphas), dim=1)
-
- len_time = alphas.shape[1]
- for b in range(batch_size):
- integrate = 0.0
- frames = torch.zeros((hidden_size), device=hidden.device)
- list_frame = []
- list_fire = []
- for t in range(len_time):
- alpha = alphas[b][t]
- if alpha + integrate < self.threshold:
- integrate += alpha
- list_fire.append(integrate)
- frames += alpha * hidden[b][t]
- else:
- frames += (self.threshold - integrate) * hidden[b][t]
- list_frame.append(frames)
- integrate += alpha
- list_fire.append(integrate)
- integrate -= self.threshold
- frames = integrate * hidden[b][t]
-
- cache_alphas.append(integrate)
- if integrate > 0.0:
- cache_hiddens.append(frames / integrate)
- else:
- cache_hiddens.append(frames)
-
- token_length.append(torch.tensor(len(list_frame), device=alphas.device))
- list_fires.append(list_fire)
- list_frames.append(list_frame)
-
- cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)
- cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)
- cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)
- cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)
-
- max_token_len = max(token_length)
- if max_token_len == 0:
- return hidden, torch.stack(token_length, 0), None, None
- list_ls = []
- for b in range(batch_size):
- pad_frames = torch.zeros(
- (max_token_len - token_length[b], hidden_size), device=alphas.device
- )
- if token_length[b] == 0:
- list_ls.append(pad_frames)
- else:
- list_frames[b] = torch.stack(list_frames[b])
- list_ls.append(torch.cat((list_frames[b], pad_frames), dim=0))
-
- cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)
- cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)
- cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)
- cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)
- return torch.stack(list_ls, 0), torch.stack(token_length, 0), None, None
-
- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
- b, t, d = hidden.size()
- tail_threshold = self.tail_threshold
- if mask is not None:
- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
- ones_t = torch.ones_like(zeros_t)
- mask_1 = torch.cat([mask, zeros_t], dim=1)
- mask_2 = torch.cat([ones_t, mask], dim=1)
- mask = mask_2 - mask_1
- tail_threshold = mask * tail_threshold
- alphas = torch.cat([alphas, zeros_t], dim=1)
- alphas = torch.add(alphas, tail_threshold)
- else:
- tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)
- tail_threshold = torch.reshape(tail_threshold, (1, 1))
- if b > 1:
- alphas = torch.cat([alphas, tail_threshold.repeat(b, 1)], dim=1)
- else:
- alphas = torch.cat([alphas, tail_threshold], dim=1)
- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
- hidden = torch.cat([hidden, zeros], dim=1)
- token_num = alphas.sum(dim=-1)
- token_num_floor = torch.floor(token_num)
-
- return hidden, alphas, token_num_floor
-
- def gen_frame_alignments(
- self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None
- ):
- batch_size, maximum_length = alphas.size()
- int_type = torch.int32
-
- is_training = self.training
- if is_training:
- token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)
- else:
- token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)
-
- max_token_num = torch.max(token_num).item()
-
- alphas_cumsum = torch.cumsum(alphas, dim=1)
- alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)
- alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)
-
- index = torch.ones([batch_size, max_token_num], dtype=int_type)
- index = torch.cumsum(index, dim=1)
- index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)
-
- index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)
- index_div_bool_zeros = index_div.eq(0)
- index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1
- index_div_bool_zeros_count = torch.clamp(
- index_div_bool_zeros_count, 0, encoder_sequence_length.max()
- )
- token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)
- index_div_bool_zeros_count *= token_num_mask
-
- index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(
- 1, 1, maximum_length
- )
- ones = torch.ones_like(index_div_bool_zeros_count_tile)
- zeros = torch.zeros_like(index_div_bool_zeros_count_tile)
- ones = torch.cumsum(ones, dim=2)
- cond = index_div_bool_zeros_count_tile == ones
- index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)
-
- index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)
- index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)
- index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)
- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)
- predictor_mask = (
- (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))
- .type(int_type)
- .to(encoder_sequence_length.device)
- )
- index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask
-
- predictor_alignments = index_div_bool_zeros_count_tile_out
- predictor_alignments_length = predictor_alignments.sum(-1).type(
- encoder_sequence_length.dtype
- )
- return predictor_alignments.detach(), predictor_alignments_length.detach()
-
-
[email protected]("predictor_classes", "CifPredictorV2Export")
-class CifPredictorV2Export(torch.nn.Module):
- def __init__(self, model, **kwargs):
- super().__init__()
-
- self.pad = model.pad
- self.cif_conv1d = model.cif_conv1d
- self.cif_output = model.cif_output
- self.threshold = model.threshold
- self.smooth_factor = model.smooth_factor
- self.noise_threshold = model.noise_threshold
- self.tail_threshold = model.tail_threshold
-
- def forward(
- self,
- hidden: torch.Tensor,
- mask: torch.Tensor,
- ):
- alphas, token_num = self.forward_cnn(hidden, mask)
- mask = mask.transpose(-1, -2).float()
- mask = mask.squeeze(-1)
- hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)
- acoustic_embeds, cif_peak = cif_v1_export(hidden, alphas, self.threshold)
-
- return acoustic_embeds, token_num, alphas, cif_peak
-
- def forward_cnn(
- self,
- hidden: torch.Tensor,
- mask: torch.Tensor,
- ):
- h = hidden
- context = h.transpose(1, 2)
- queries = self.pad(context)
- output = torch.relu(self.cif_conv1d(queries))
- output = output.transpose(1, 2)
-
- output = self.cif_output(output)
- alphas = torch.sigmoid(output)
- alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
- mask = mask.transpose(-1, -2).float()
- alphas = alphas * mask
- alphas = alphas.squeeze(-1)
- token_num = alphas.sum(-1)
-
- return alphas, token_num
-
- def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
- b, t, d = hidden.size()
- tail_threshold = self.tail_threshold
-
- zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
- ones_t = torch.ones_like(zeros_t)
-
- mask_1 = torch.cat([mask, zeros_t], dim=1)
- mask_2 = torch.cat([ones_t, mask], dim=1)
- mask = mask_2 - mask_1
- tail_threshold = mask * tail_threshold
- alphas = torch.cat([alphas, zeros_t], dim=1)
- alphas = torch.add(alphas, tail_threshold)
-
- zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
- hidden = torch.cat([hidden, zeros], dim=1)
- token_num = alphas.sum(dim=-1)
- token_num_floor = torch.floor(token_num)
-
- return hidden, alphas, token_num_floor
-
-
-def cif_v1_export(hidden, alphas, threshold: float):
- device = hidden.device
- dtype = hidden.dtype
- batch_size, len_time, hidden_size = hidden.size()
- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
-
- frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)
- fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)
-
- # prefix_sum = torch.cumsum(alphas, dim=1)
- prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(
- torch.float32
- ) # cumsum precision degradation cause wrong result in extreme
- prefix_sum_floor = torch.floor(prefix_sum)
- dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)
- dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)
-
- dislocation_prefix_sum_floor[:, 0] = 0
- dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor
-
- fire_idxs = dislocation_diff > 0
- fires[fire_idxs] = 1
- fires = fires + prefix_sum - prefix_sum_floor
-
- # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)
- prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)
- frames = prefix_sum_hidden[fire_idxs]
- shift_frames = torch.roll(frames, 1, dims=0)
-
- batch_len = fire_idxs.sum(1)
- batch_idxs = torch.cumsum(batch_len, dim=0)
- shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)
- shift_batch_idxs[0] = 0
- shift_frames[shift_batch_idxs] = 0
-
- remains = fires - torch.floor(fires)
- # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]
- remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]
-
- shift_remain_frames = torch.roll(remain_frames, 1, dims=0)
- shift_remain_frames[shift_batch_idxs] = 0
-
- frames = frames - shift_frames + shift_remain_frames - remain_frames
-
- # max_label_len = batch_len.max()
- max_label_len = alphas.sum(dim=-1)
- max_label_len = torch.floor(max_label_len).max().to(dtype=torch.int64)
-
- # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
- frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
- indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)
- frame_fires_idxs = indices < batch_len.unsqueeze(1)
- frame_fires[frame_fires_idxs] = frames
- return frame_fires, fires
-
-
-def cif_export(hidden, alphas, threshold: float):
- batch_size, len_time, hidden_size = hidden.size()
- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
-
- # loop varss
- integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)
- frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)
- # intermediate vars along time
- list_fires = []
- list_frames = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
- distribution_completion = (
- torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate
- )
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(
- fire_place,
- integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),
- integrate,
- )
- cur = torch.where(fire_place, distribution_completion, alpha)
- remainds = alpha - cur
-
- frame += cur[:, None] * hidden[:, t, :]
- list_frames.append(frame)
- frame = torch.where(
- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
- )
-
- fires = torch.stack(list_fires, 1)
- frames = torch.stack(list_frames, 1)
-
- fire_idxs = fires >= threshold
- frame_fires = torch.zeros_like(hidden)
- max_label_len = frames[0, fire_idxs[0]].size(0)
- for b in range(batch_size):
- frame_fire = frames[b, fire_idxs[b]]
- frame_len = frame_fire.size(0)
- frame_fires[b, :frame_len, :] = frame_fire
-
- if frame_len >= max_label_len:
- max_label_len = frame_len
- frame_fires = frame_fires[:, :max_label_len, :]
- return frame_fires, fires
-
-
-class mae_loss(torch.nn.Module):
-
- def __init__(self, normalize_length=False):
- super(mae_loss, self).__init__()
- self.normalize_length = normalize_length
- self.criterion = torch.nn.L1Loss(reduction="sum")
-
- def forward(self, token_length, pre_token_length):
- loss_token_normalizer = token_length.size(0)
- if self.normalize_length:
- loss_token_normalizer = token_length.sum().type(torch.float32)
- loss = self.criterion(token_length, pre_token_length)
- loss = loss / loss_token_normalizer
- return loss
-
-
-def cif(hidden, alphas, threshold):
- batch_size, len_time, hidden_size = hidden.size()
-
- # loop varss
- integrate = torch.zeros([batch_size], device=hidden.device)
- frame = torch.zeros([batch_size, hidden_size], device=hidden.device)
- # intermediate vars along time
- list_fires = []
- list_frames = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
- distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(
- fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate
- )
- cur = torch.where(fire_place, distribution_completion, alpha)
- remainds = alpha - cur
-
- frame += cur[:, None] * hidden[:, t, :]
- list_frames.append(frame)
- frame = torch.where(
- fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
- )
-
- fires = torch.stack(list_fires, 1)
- frames = torch.stack(list_frames, 1)
- list_ls = []
- len_labels = torch.round(alphas.sum(-1)).int()
- max_label_len = len_labels.max()
- for b in range(batch_size):
- fire = fires[b, :]
- l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze())
- pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)
- list_ls.append(torch.cat([l, pad_l], 0))
- return torch.stack(list_ls, 0), fires
-
-
-def cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=False):
- batch_size, len_time = alphas.size()
- device = alphas.device
- dtype = alphas.dtype
-
- threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
-
- fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)
-
- # prefix_sum = torch.cumsum(alphas, dim=1)
- prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(
- torch.float32
- ) # cumsum precision degradation cause wrong result in extreme
- prefix_sum_floor = torch.floor(prefix_sum)
- dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)
- dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)
-
- dislocation_prefix_sum_floor[:, 0] = 0
- dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor
-
- fire_idxs = dislocation_diff > 0
- fires[fire_idxs] = 1
- fires = fires + prefix_sum - prefix_sum_floor
- if return_fire_idxs:
- return fires, fire_idxs
- return fires
-
-
-def cif_v1(hidden, alphas, threshold):
- fires, fire_idxs = cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=True)
-
- device = hidden.device
- dtype = hidden.dtype
- batch_size, len_time, hidden_size = hidden.size()
- # frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)
- # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)
- frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)
- prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)
-
- frames = prefix_sum_hidden[fire_idxs]
- shift_frames = torch.roll(frames, 1, dims=0)
-
- batch_len = fire_idxs.sum(1)
- batch_idxs = torch.cumsum(batch_len, dim=0)
- shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)
- shift_batch_idxs[0] = 0
- shift_frames[shift_batch_idxs] = 0
-
- remains = fires - torch.floor(fires)
- # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]
- remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]
-
- shift_remain_frames = torch.roll(remain_frames, 1, dims=0)
- shift_remain_frames[shift_batch_idxs] = 0
-
- frames = frames - shift_frames + shift_remain_frames - remain_frames
-
- # max_label_len = batch_len.max()
- max_label_len = (
- torch.round(alphas.sum(-1)).int().max()
- ) # torch.round to calculate the max length
-
- # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
- frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
- indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)
- frame_fires_idxs = indices < batch_len.unsqueeze(1)
- frame_fires[frame_fires_idxs] = frames
- return frame_fires, fires
-
-
-def cif_wo_hidden(alphas, threshold):
- batch_size, len_time = alphas.size()
-
- # loop varss
- integrate = torch.zeros([batch_size], device=alphas.device)
- # intermediate vars along time
- list_fires = []
-
- for t in range(len_time):
- alpha = alphas[:, t]
-
- integrate += alpha
- list_fires.append(integrate)
-
- fire_place = integrate >= threshold
- integrate = torch.where(
- fire_place,
- integrate - torch.ones([batch_size], device=alphas.device) * threshold,
- integrate,
- )
-
- fires = torch.stack(list_fires, 1)
- return fires
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+# MIT License (https://opensource.org/licenses/MIT)
+
+import torch
+import logging
+import numpy as np
+
+from funasr.register import tables
+from funasr.train_utils.device_funcs import to_device
+from funasr.models.transformer.utils.nets_utils import make_pad_mask
+from torch.cuda.amp import autocast
+
+
[email protected]("predictor_classes", "CifPredictor")
+class CifPredictor(torch.nn.Module):
+ def __init__(
+ self,
+ idim,
+ l_order,
+ r_order,
+ threshold=1.0,
+ dropout=0.1,
+ smooth_factor=1.0,
+ noise_threshold=0,
+ tail_threshold=0.45,
+ ):
+ """Initialize CifPredictor.
+
+ Args:
+ idim: TODO.
+ l_order: TODO.
+ r_order: TODO.
+ threshold: TODO.
+ dropout: TODO.
+ smooth_factor: TODO.
+ noise_threshold: TODO.
+ tail_threshold: TODO.
+ """
+ super().__init__()
+
+ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1, groups=idim)
+ self.cif_output = torch.nn.Linear(idim, 1)
+ self.dropout = torch.nn.Dropout(p=dropout)
+ self.threshold = threshold
+ self.smooth_factor = smooth_factor
+ self.noise_threshold = noise_threshold
+ self.tail_threshold = tail_threshold
+
+ def forward(
+ self,
+ hidden,
+ target_label=None,
+ mask=None,
+ ignore_id=-1,
+ mask_chunk_predictor=None,
+ target_label_length=None,
+ ):
+
+ """Forward pass for training.
+
+ Args:
+ hidden: TODO.
+ target_label: TODO.
+ mask: TODO.
+ ignore_id: TODO.
+ mask_chunk_predictor: TODO.
+ target_label_length: TODO.
+ """
+ with autocast(False):
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ memory = self.cif_conv1d(queries)
+ output = memory + context
+ output = self.dropout(output)
+ output = output.transpose(1, 2)
+ output = torch.relu(output)
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+ if mask is not None:
+ mask = mask.transpose(-1, -2).float()
+ alphas = alphas * mask
+ if mask_chunk_predictor is not None:
+ alphas = alphas * mask_chunk_predictor
+ alphas = alphas.squeeze(-1)
+ mask = mask.squeeze(-1)
+ if target_label_length is not None:
+ target_length = target_label_length
+ elif target_label is not None:
+ target_length = (target_label != ignore_id).float().sum(-1)
+ else:
+ target_length = None
+ token_num = alphas.sum(-1)
+ if target_length is not None:
+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
+ elif self.tail_threshold > 0.0:
+ hidden, alphas, token_num = self.tail_process_fn(
+ hidden, alphas, token_num, mask=mask
+ )
+
+ acoustic_embeds, cif_peak = cif(hidden, alphas, self.threshold)
+
+ if target_length is None and self.tail_threshold > 0.0:
+ token_num_int = torch.max(token_num).type(torch.int32).item()
+ acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
+
+ return acoustic_embeds, token_num, alphas, cif_peak
+
+ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
+ """Tail process fn.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ token_num: TODO.
+ mask: TODO.
+ """
+ b, t, d = hidden.size()
+ tail_threshold = self.tail_threshold
+ if mask is not None:
+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
+ ones_t = torch.ones_like(zeros_t)
+ mask_1 = torch.cat([mask, zeros_t], dim=1)
+ mask_2 = torch.cat([ones_t, mask], dim=1)
+ mask = mask_2 - mask_1
+ tail_threshold = mask * tail_threshold
+ alphas = torch.cat([alphas, zeros_t], dim=1)
+ alphas = torch.add(alphas, tail_threshold)
+ else:
+ tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)
+ tail_threshold = torch.reshape(tail_threshold, (1, 1))
+ alphas = torch.cat([alphas, tail_threshold], dim=1)
+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
+ hidden = torch.cat([hidden, zeros], dim=1)
+ token_num = alphas.sum(dim=-1)
+ token_num_floor = torch.floor(token_num)
+
+ return hidden, alphas, token_num_floor
+
+ def gen_frame_alignments(
+ self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None
+ ):
+ """Gen frame alignments.
+
+ Args:
+ alphas: TODO.
+ encoder_sequence_length: TODO.
+ """
+ batch_size, maximum_length = alphas.size()
+ int_type = torch.int32
+
+ is_training = self.training
+ if is_training:
+ token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)
+ else:
+ token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)
+
+ max_token_num = torch.max(token_num).item()
+
+ alphas_cumsum = torch.cumsum(alphas, dim=1)
+ alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)
+ alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)
+
+ index = torch.ones([batch_size, max_token_num], dtype=int_type)
+ index = torch.cumsum(index, dim=1)
+ index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)
+
+ index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)
+ index_div_bool_zeros = index_div.eq(0)
+ index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1
+ index_div_bool_zeros_count = torch.clamp(
+ index_div_bool_zeros_count, 0, encoder_sequence_length.max()
+ )
+ token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)
+ index_div_bool_zeros_count *= token_num_mask
+
+ index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(
+ 1, 1, maximum_length
+ )
+ ones = torch.ones_like(index_div_bool_zeros_count_tile)
+ zeros = torch.zeros_like(index_div_bool_zeros_count_tile)
+ ones = torch.cumsum(ones, dim=2)
+ cond = index_div_bool_zeros_count_tile == ones
+ index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)
+
+ index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)
+ index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)
+ index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)
+ predictor_mask = (
+ (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))
+ .type(int_type)
+ .to(encoder_sequence_length.device)
+ )
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask
+
+ predictor_alignments = index_div_bool_zeros_count_tile_out
+ predictor_alignments_length = predictor_alignments.sum(-1).type(
+ encoder_sequence_length.dtype
+ )
+ return predictor_alignments.detach(), predictor_alignments_length.detach()
+
+
[email protected]("predictor_classes", "CifPredictorV2")
+class CifPredictorV2(torch.nn.Module):
+ def __init__(
+ self,
+ idim,
+ l_order,
+ r_order,
+ threshold=1.0,
+ dropout=0.1,
+ smooth_factor=1.0,
+ noise_threshold=0,
+ tail_threshold=0.0,
+ tf2torch_tensor_name_prefix_torch="predictor",
+ tf2torch_tensor_name_prefix_tf="seq2seq/cif",
+ tail_mask=True,
+ ):
+ """Initialize CifPredictorV2.
+
+ Args:
+ idim: TODO.
+ l_order: TODO.
+ r_order: TODO.
+ threshold: TODO.
+ dropout: TODO.
+ smooth_factor: TODO.
+ noise_threshold: TODO.
+ tail_threshold: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ tail_mask: TODO.
+ """
+ super().__init__()
+
+ self.pad = torch.nn.ConstantPad1d((l_order, r_order), 0)
+ self.cif_conv1d = torch.nn.Conv1d(idim, idim, l_order + r_order + 1)
+ self.cif_output = torch.nn.Linear(idim, 1)
+ self.dropout = torch.nn.Dropout(p=dropout)
+ self.threshold = threshold
+ self.smooth_factor = smooth_factor
+ self.noise_threshold = noise_threshold
+ self.tail_threshold = tail_threshold
+ self.tf2torch_tensor_name_prefix_torch = tf2torch_tensor_name_prefix_torch
+ self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
+ self.tail_mask = tail_mask
+
+ def forward(
+ self,
+ hidden,
+ target_label=None,
+ mask=None,
+ ignore_id=-1,
+ mask_chunk_predictor=None,
+ target_label_length=None,
+ ):
+
+ """Forward pass for training.
+
+ Args:
+ hidden: TODO.
+ target_label: TODO.
+ mask: TODO.
+ ignore_id: TODO.
+ mask_chunk_predictor: TODO.
+ target_label_length: TODO.
+ """
+ with autocast(False):
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+ output = output.transpose(1, 2)
+
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+ if mask is not None:
+ mask = mask.transpose(-1, -2).float()
+ alphas = alphas * mask
+ if mask_chunk_predictor is not None:
+ alphas = alphas * mask_chunk_predictor
+ alphas = alphas.squeeze(-1)
+ mask = mask.squeeze(-1)
+ if target_label_length is not None:
+ target_length = target_label_length.squeeze(-1)
+ elif target_label is not None:
+ target_length = (target_label != ignore_id).float().sum(-1)
+ else:
+ target_length = None
+ token_num = alphas.sum(-1)
+ if target_length is not None:
+ alphas *= (target_length / token_num)[:, None].repeat(1, alphas.size(1))
+ elif self.tail_threshold > 0.0:
+ if self.tail_mask:
+ hidden, alphas, token_num = self.tail_process_fn(
+ hidden, alphas, token_num, mask=mask
+ )
+ else:
+ hidden, alphas, token_num = self.tail_process_fn(
+ hidden, alphas, token_num, mask=None
+ )
+
+ acoustic_embeds, cif_peak = cif_v1(hidden, alphas, self.threshold)
+ if target_length is None and self.tail_threshold > 0.0:
+ token_num_int = torch.max(token_num).type(torch.int32).item()
+ acoustic_embeds = acoustic_embeds[:, :token_num_int, :]
+
+ return acoustic_embeds, token_num, alphas, cif_peak
+
+ def forward_chunk(self, hidden, cache=None, **kwargs):
+ """Forward chunk.
+
+ Args:
+ hidden: TODO.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
+ is_final = kwargs.get("is_final", False)
+ batch_size, len_time, hidden_size = hidden.shape
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+ output = output.transpose(1, 2)
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+
+ alphas = alphas.squeeze(-1)
+
+ token_length = []
+ list_fires = []
+ list_frames = []
+ cache_alphas = []
+ cache_hiddens = []
+
+ if cache is not None and "chunk_size" in cache:
+ alphas[:, : cache["chunk_size"][0]] = 0.0
+ if not is_final:
+ alphas[:, sum(cache["chunk_size"][:2]) :] = 0.0
+ if cache is not None and "cif_alphas" in cache and "cif_hidden" in cache:
+ cache["cif_hidden"] = to_device(cache["cif_hidden"], device=hidden.device)
+ cache["cif_alphas"] = to_device(cache["cif_alphas"], device=alphas.device)
+ hidden = torch.cat((cache["cif_hidden"], hidden), dim=1)
+ alphas = torch.cat((cache["cif_alphas"], alphas), dim=1)
+ if cache is not None and is_final:
+ tail_hidden = torch.zeros((batch_size, 1, hidden_size), device=hidden.device)
+ tail_alphas = torch.tensor([[self.tail_threshold]], device=alphas.device)
+ tail_alphas = torch.tile(tail_alphas, (batch_size, 1))
+ hidden = torch.cat((hidden, tail_hidden), dim=1)
+ alphas = torch.cat((alphas, tail_alphas), dim=1)
+
+ len_time = alphas.shape[1]
+ for b in range(batch_size):
+ integrate = 0.0
+ frames = torch.zeros((hidden_size), device=hidden.device)
+ list_frame = []
+ list_fire = []
+ for t in range(len_time):
+ alpha = alphas[b][t]
+ if alpha + integrate < self.threshold:
+ integrate += alpha
+ list_fire.append(integrate)
+ frames += alpha * hidden[b][t]
+ else:
+ frames += (self.threshold - integrate) * hidden[b][t]
+ list_frame.append(frames)
+ integrate += alpha
+ list_fire.append(integrate)
+ integrate -= self.threshold
+ frames = integrate * hidden[b][t]
+
+ cache_alphas.append(integrate)
+ if integrate > 0.0:
+ cache_hiddens.append(frames / integrate)
+ else:
+ cache_hiddens.append(frames)
+
+ token_length.append(torch.tensor(len(list_frame), device=alphas.device))
+ list_fires.append(list_fire)
+ list_frames.append(list_frame)
+
+ cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)
+ cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)
+ cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)
+ cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)
+
+ max_token_len = max(token_length)
+ if max_token_len == 0:
+ return hidden, torch.stack(token_length, 0), None, None
+ list_ls = []
+ for b in range(batch_size):
+ pad_frames = torch.zeros(
+ (max_token_len - token_length[b], hidden_size), device=alphas.device
+ )
+ if token_length[b] == 0:
+ list_ls.append(pad_frames)
+ else:
+ list_frames[b] = torch.stack(list_frames[b])
+ list_ls.append(torch.cat((list_frames[b], pad_frames), dim=0))
+
+ cache["cif_alphas"] = torch.stack(cache_alphas, axis=0)
+ cache["cif_alphas"] = torch.unsqueeze(cache["cif_alphas"], axis=0)
+ cache["cif_hidden"] = torch.stack(cache_hiddens, axis=0)
+ cache["cif_hidden"] = torch.unsqueeze(cache["cif_hidden"], axis=0)
+ return torch.stack(list_ls, 0), torch.stack(token_length, 0), None, None
+
+ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
+ """Tail process fn.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ token_num: TODO.
+ mask: TODO.
+ """
+ b, t, d = hidden.size()
+ tail_threshold = self.tail_threshold
+ if mask is not None:
+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
+ ones_t = torch.ones_like(zeros_t)
+ mask_1 = torch.cat([mask, zeros_t], dim=1)
+ mask_2 = torch.cat([ones_t, mask], dim=1)
+ mask = mask_2 - mask_1
+ tail_threshold = mask * tail_threshold
+ alphas = torch.cat([alphas, zeros_t], dim=1)
+ alphas = torch.add(alphas, tail_threshold)
+ else:
+ tail_threshold = torch.tensor([tail_threshold], dtype=alphas.dtype).to(alphas.device)
+ tail_threshold = torch.reshape(tail_threshold, (1, 1))
+ if b > 1:
+ alphas = torch.cat([alphas, tail_threshold.repeat(b, 1)], dim=1)
+ else:
+ alphas = torch.cat([alphas, tail_threshold], dim=1)
+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
+ hidden = torch.cat([hidden, zeros], dim=1)
+ token_num = alphas.sum(dim=-1)
+ token_num_floor = torch.floor(token_num)
+
+ return hidden, alphas, token_num_floor
+
+ def gen_frame_alignments(
+ self, alphas: torch.Tensor = None, encoder_sequence_length: torch.Tensor = None
+ ):
+ """Gen frame alignments.
+
+ Args:
+ alphas: TODO.
+ encoder_sequence_length: TODO.
+ """
+ batch_size, maximum_length = alphas.size()
+ int_type = torch.int32
+
+ is_training = self.training
+ if is_training:
+ token_num = torch.round(torch.sum(alphas, dim=1)).type(int_type)
+ else:
+ token_num = torch.floor(torch.sum(alphas, dim=1)).type(int_type)
+
+ max_token_num = torch.max(token_num).item()
+
+ alphas_cumsum = torch.cumsum(alphas, dim=1)
+ alphas_cumsum = torch.floor(alphas_cumsum).type(int_type)
+ alphas_cumsum = alphas_cumsum[:, None, :].repeat(1, max_token_num, 1)
+
+ index = torch.ones([batch_size, max_token_num], dtype=int_type)
+ index = torch.cumsum(index, dim=1)
+ index = index[:, :, None].repeat(1, 1, maximum_length).to(alphas_cumsum.device)
+
+ index_div = torch.floor(torch.true_divide(alphas_cumsum, index)).type(int_type)
+ index_div_bool_zeros = index_div.eq(0)
+ index_div_bool_zeros_count = torch.sum(index_div_bool_zeros, dim=-1) + 1
+ index_div_bool_zeros_count = torch.clamp(
+ index_div_bool_zeros_count, 0, encoder_sequence_length.max()
+ )
+ token_num_mask = (~make_pad_mask(token_num, maxlen=max_token_num)).to(token_num.device)
+ index_div_bool_zeros_count *= token_num_mask
+
+ index_div_bool_zeros_count_tile = index_div_bool_zeros_count[:, :, None].repeat(
+ 1, 1, maximum_length
+ )
+ ones = torch.ones_like(index_div_bool_zeros_count_tile)
+ zeros = torch.zeros_like(index_div_bool_zeros_count_tile)
+ ones = torch.cumsum(ones, dim=2)
+ cond = index_div_bool_zeros_count_tile == ones
+ index_div_bool_zeros_count_tile = torch.where(cond, zeros, ones)
+
+ index_div_bool_zeros_count_tile_bool = index_div_bool_zeros_count_tile.type(torch.bool)
+ index_div_bool_zeros_count_tile = 1 - index_div_bool_zeros_count_tile_bool.type(int_type)
+ index_div_bool_zeros_count_tile_out = torch.sum(index_div_bool_zeros_count_tile, dim=1)
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out.type(int_type)
+ predictor_mask = (
+ (~make_pad_mask(encoder_sequence_length, maxlen=encoder_sequence_length.max()))
+ .type(int_type)
+ .to(encoder_sequence_length.device)
+ )
+ index_div_bool_zeros_count_tile_out = index_div_bool_zeros_count_tile_out * predictor_mask
+
+ predictor_alignments = index_div_bool_zeros_count_tile_out
+ predictor_alignments_length = predictor_alignments.sum(-1).type(
+ encoder_sequence_length.dtype
+ )
+ return predictor_alignments.detach(), predictor_alignments_length.detach()
+
+
[email protected]("predictor_classes", "CifPredictorV2Export")
+class CifPredictorV2Export(torch.nn.Module):
+ def __init__(self, model, **kwargs):
+ """Initialize CifPredictorV2Export.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
+ super().__init__()
+
+ self.pad = model.pad
+ self.cif_conv1d = model.cif_conv1d
+ self.cif_output = model.cif_output
+ self.threshold = model.threshold
+ self.smooth_factor = model.smooth_factor
+ self.noise_threshold = model.noise_threshold
+ self.tail_threshold = model.tail_threshold
+
+ def forward(
+ self,
+ hidden: torch.Tensor,
+ mask: torch.Tensor,
+ ):
+ """Forward pass for training.
+
+ Args:
+ hidden: TODO.
+ mask: TODO.
+ """
+ alphas, token_num = self.forward_cnn(hidden, mask)
+ mask = mask.transpose(-1, -2).float()
+ mask = mask.squeeze(-1)
+ hidden, alphas, token_num = self.tail_process_fn(hidden, alphas, mask=mask)
+ acoustic_embeds, cif_peak = cif_v1_export(hidden, alphas, self.threshold)
+
+ return acoustic_embeds, token_num, alphas, cif_peak
+
+ def forward_cnn(
+ self,
+ hidden: torch.Tensor,
+ mask: torch.Tensor,
+ ):
+ """Forward cnn.
+
+ Args:
+ hidden: TODO.
+ mask: TODO.
+ """
+ h = hidden
+ context = h.transpose(1, 2)
+ queries = self.pad(context)
+ output = torch.relu(self.cif_conv1d(queries))
+ output = output.transpose(1, 2)
+
+ output = self.cif_output(output)
+ alphas = torch.sigmoid(output)
+ alphas = torch.nn.functional.relu(alphas * self.smooth_factor - self.noise_threshold)
+ mask = mask.transpose(-1, -2).float()
+ alphas = alphas * mask
+ alphas = alphas.squeeze(-1)
+ token_num = alphas.sum(-1)
+
+ return alphas, token_num
+
+ def tail_process_fn(self, hidden, alphas, token_num=None, mask=None):
+ """Tail process fn.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ token_num: TODO.
+ mask: TODO.
+ """
+ b, t, d = hidden.size()
+ tail_threshold = self.tail_threshold
+
+ zeros_t = torch.zeros((b, 1), dtype=torch.float32, device=alphas.device)
+ ones_t = torch.ones_like(zeros_t)
+
+ mask_1 = torch.cat([mask, zeros_t], dim=1)
+ mask_2 = torch.cat([ones_t, mask], dim=1)
+ mask = mask_2 - mask_1
+ tail_threshold = mask * tail_threshold
+ alphas = torch.cat([alphas, zeros_t], dim=1)
+ alphas = torch.add(alphas, tail_threshold)
+
+ zeros = torch.zeros((b, 1, d), dtype=hidden.dtype).to(hidden.device)
+ hidden = torch.cat([hidden, zeros], dim=1)
+ token_num = alphas.sum(dim=-1)
+ token_num_floor = torch.floor(token_num)
+
+ return hidden, alphas, token_num_floor
+
+
+def cif_v1_export(hidden, alphas, threshold: float):
+ """Cif v1 export.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ threshold: TODO.
+ """
+ device = hidden.device
+ dtype = hidden.dtype
+ batch_size, len_time, hidden_size = hidden.size()
+ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
+
+ frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)
+ fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)
+
+ # prefix_sum = torch.cumsum(alphas, dim=1)
+ prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(
+ torch.float32
+ ) # cumsum precision degradation cause wrong result in extreme
+ prefix_sum_floor = torch.floor(prefix_sum)
+ dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)
+ dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)
+
+ dislocation_prefix_sum_floor[:, 0] = 0
+ dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor
+
+ fire_idxs = dislocation_diff > 0
+ fires[fire_idxs] = 1
+ fires = fires + prefix_sum - prefix_sum_floor
+
+ # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)
+ prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)
+ frames = prefix_sum_hidden[fire_idxs]
+ shift_frames = torch.roll(frames, 1, dims=0)
+
+ batch_len = fire_idxs.sum(1)
+ batch_idxs = torch.cumsum(batch_len, dim=0)
+ shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)
+ shift_batch_idxs[0] = 0
+ shift_frames[shift_batch_idxs] = 0
+
+ remains = fires - torch.floor(fires)
+ # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]
+ remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]
+
+ shift_remain_frames = torch.roll(remain_frames, 1, dims=0)
+ shift_remain_frames[shift_batch_idxs] = 0
+
+ frames = frames - shift_frames + shift_remain_frames - remain_frames
+
+ # max_label_len = batch_len.max()
+ max_label_len = alphas.sum(dim=-1)
+ max_label_len = torch.floor(max_label_len).max().to(dtype=torch.int64)
+
+ # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
+ frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
+ indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)
+ frame_fires_idxs = indices < batch_len.unsqueeze(1)
+ frame_fires[frame_fires_idxs] = frames
+ return frame_fires, fires
+
+
+def cif_export(hidden, alphas, threshold: float):
+ """Cif export.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ threshold: TODO.
+ """
+ batch_size, len_time, hidden_size = hidden.size()
+ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
+
+ # loop varss
+ integrate = torch.zeros([batch_size], dtype=alphas.dtype, device=hidden.device)
+ frame = torch.zeros([batch_size, hidden_size], dtype=hidden.dtype, device=hidden.device)
+ # intermediate vars along time
+ list_fires = []
+ list_frames = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+ distribution_completion = (
+ torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device) - integrate
+ )
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(
+ fire_place,
+ integrate - torch.ones([batch_size], dtype=alphas.dtype, device=hidden.device),
+ integrate,
+ )
+ cur = torch.where(fire_place, distribution_completion, alpha)
+ remainds = alpha - cur
+
+ frame += cur[:, None] * hidden[:, t, :]
+ list_frames.append(frame)
+ frame = torch.where(
+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
+ )
+
+ fires = torch.stack(list_fires, 1)
+ frames = torch.stack(list_frames, 1)
+
+ fire_idxs = fires >= threshold
+ frame_fires = torch.zeros_like(hidden)
+ max_label_len = frames[0, fire_idxs[0]].size(0)
+ for b in range(batch_size):
+ frame_fire = frames[b, fire_idxs[b]]
+ frame_len = frame_fire.size(0)
+ frame_fires[b, :frame_len, :] = frame_fire
+
+ if frame_len >= max_label_len:
+ max_label_len = frame_len
+ frame_fires = frame_fires[:, :max_label_len, :]
+ return frame_fires, fires
+
+
+class mae_loss(torch.nn.Module):
+
+ def __init__(self, normalize_length=False):
+ """Initialize mae_loss.
+
+ Args:
+ normalize_length: TODO.
+ """
+ super(mae_loss, self).__init__()
+ self.normalize_length = normalize_length
+ self.criterion = torch.nn.L1Loss(reduction="sum")
+
+ def forward(self, token_length, pre_token_length):
+ """Forward pass for training.
+
+ Args:
+ token_length: TODO.
+ pre_token_length: TODO.
+ """
+ loss_token_normalizer = token_length.size(0)
+ if self.normalize_length:
+ loss_token_normalizer = token_length.sum().type(torch.float32)
+ loss = self.criterion(token_length, pre_token_length)
+ loss = loss / loss_token_normalizer
+ return loss
+
+
+def cif(hidden, alphas, threshold):
+ """Cif.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ threshold: TODO.
+ """
+ batch_size, len_time, hidden_size = hidden.size()
+
+ # loop varss
+ integrate = torch.zeros([batch_size], device=hidden.device)
+ frame = torch.zeros([batch_size, hidden_size], device=hidden.device)
+ # intermediate vars along time
+ list_fires = []
+ list_frames = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+ distribution_completion = torch.ones([batch_size], device=hidden.device) - integrate
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(
+ fire_place, integrate - torch.ones([batch_size], device=hidden.device), integrate
+ )
+ cur = torch.where(fire_place, distribution_completion, alpha)
+ remainds = alpha - cur
+
+ frame += cur[:, None] * hidden[:, t, :]
+ list_frames.append(frame)
+ frame = torch.where(
+ fire_place[:, None].repeat(1, hidden_size), remainds[:, None] * hidden[:, t, :], frame
+ )
+
+ fires = torch.stack(list_fires, 1)
+ frames = torch.stack(list_frames, 1)
+ list_ls = []
+ len_labels = torch.round(alphas.sum(-1)).int()
+ max_label_len = len_labels.max()
+ for b in range(batch_size):
+ fire = fires[b, :]
+ l = torch.index_select(frames[b, :, :], 0, torch.nonzero(fire >= threshold).squeeze())
+ pad_l = torch.zeros([max_label_len - l.size(0), hidden_size], device=hidden.device)
+ list_ls.append(torch.cat([l, pad_l], 0))
+ return torch.stack(list_ls, 0), fires
+
+
+def cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=False):
+ """Cif wo hidden v1.
+
+ Args:
+ alphas: TODO.
+ threshold: TODO.
+ return_fire_idxs: TODO.
+ """
+ batch_size, len_time = alphas.size()
+ device = alphas.device
+ dtype = alphas.dtype
+
+ threshold = torch.tensor([threshold], dtype=alphas.dtype).to(alphas.device)
+
+ fires = torch.zeros(batch_size, len_time, dtype=dtype, device=device)
+
+ # prefix_sum = torch.cumsum(alphas, dim=1)
+ prefix_sum = torch.cumsum(alphas, dim=1, dtype=torch.float64).to(
+ torch.float32
+ ) # cumsum precision degradation cause wrong result in extreme
+ prefix_sum_floor = torch.floor(prefix_sum)
+ dislocation_prefix_sum = torch.roll(prefix_sum, 1, dims=1)
+ dislocation_prefix_sum_floor = torch.floor(dislocation_prefix_sum)
+
+ dislocation_prefix_sum_floor[:, 0] = 0
+ dislocation_diff = prefix_sum_floor - dislocation_prefix_sum_floor
+
+ fire_idxs = dislocation_diff > 0
+ fires[fire_idxs] = 1
+ fires = fires + prefix_sum - prefix_sum_floor
+ if return_fire_idxs:
+ return fires, fire_idxs
+ return fires
+
+
+def cif_v1(hidden, alphas, threshold):
+ """Cif v1.
+
+ Args:
+ hidden: TODO.
+ alphas: TODO.
+ threshold: TODO.
+ """
+ fires, fire_idxs = cif_wo_hidden_v1(alphas, threshold, return_fire_idxs=True)
+
+ device = hidden.device
+ dtype = hidden.dtype
+ batch_size, len_time, hidden_size = hidden.size()
+ # frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)
+ # prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).tile((1, 1, hidden_size)) * hidden, dim=1)
+ frames = torch.zeros(batch_size, len_time, hidden_size, dtype=dtype, device=device)
+ prefix_sum_hidden = torch.cumsum(alphas.unsqueeze(-1).repeat((1, 1, hidden_size)) * hidden, dim=1)
+
+ frames = prefix_sum_hidden[fire_idxs]
+ shift_frames = torch.roll(frames, 1, dims=0)
+
+ batch_len = fire_idxs.sum(1)
+ batch_idxs = torch.cumsum(batch_len, dim=0)
+ shift_batch_idxs = torch.roll(batch_idxs, 1, dims=0)
+ shift_batch_idxs[0] = 0
+ shift_frames[shift_batch_idxs] = 0
+
+ remains = fires - torch.floor(fires)
+ # remain_frames = remains[fire_idxs].unsqueeze(-1).tile((1, hidden_size)) * hidden[fire_idxs]
+ remain_frames = remains[fire_idxs].unsqueeze(-1).repeat((1, hidden_size)) * hidden[fire_idxs]
+
+ shift_remain_frames = torch.roll(remain_frames, 1, dims=0)
+ shift_remain_frames[shift_batch_idxs] = 0
+
+ frames = frames - shift_frames + shift_remain_frames - remain_frames
+
+ # max_label_len = batch_len.max()
+ max_label_len = (
+ torch.round(alphas.sum(-1)).int().max()
+ ) # torch.round to calculate the max length
+
+ # frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
+ frame_fires = torch.zeros(batch_size, max_label_len, hidden_size, dtype=dtype, device=device)
+ indices = torch.arange(max_label_len, device=device).expand(batch_size, -1)
+ frame_fires_idxs = indices < batch_len.unsqueeze(1)
+ frame_fires[frame_fires_idxs] = frames
+ return frame_fires, fires
+
+
+def cif_wo_hidden(alphas, threshold):
+ """Cif wo hidden.
+
+ Args:
+ alphas: TODO.
+ threshold: TODO.
+ """
+ batch_size, len_time = alphas.size()
+
+ # loop varss
+ integrate = torch.zeros([batch_size], device=alphas.device)
+ # intermediate vars along time
+ list_fires = []
+
+ for t in range(len_time):
+ alpha = alphas[:, t]
+
+ integrate += alpha
+ list_fires.append(integrate)
+
+ fire_place = integrate >= threshold
+ integrate = torch.where(
+ fire_place,
+ integrate - torch.ones([batch_size], device=alphas.device) * threshold,
+ integrate,
+ )
+
+ fires = torch.stack(list_fires, 1)
+ return fires
diff --git a/funasr/models/paraformer/decoder.py b/funasr/models/paraformer/decoder.py
index 7edd91a2a..f784e0914 100644
--- a/funasr/models/paraformer/decoder.py
+++ b/funasr/models/paraformer/decoder.py
@@ -121,6 +121,15 @@ class DecoderLayerSANM(torch.nn.Module):
return x, tgt_mask, memory, memory_mask, cache
def get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Get attn mat.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt = self.feed_forward(tgt)
@@ -257,6 +266,35 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
tf2torch_tensor_name_prefix_torch: str = "decoder",
tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",
):
+ """Initialize ParaformerSANMDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ wo_input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ att_layer_num: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ lora_list: TODO.
+ lora_rank: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ chunk_multiply_factor: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -426,6 +464,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
ys_in_lens: torch.Tensor,
):
+ """Forward asf2.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -444,6 +490,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
ys_in_lens: torch.Tensor,
):
+ """Forward asf6.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -592,6 +646,11 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
class DecoderLayerSANMExport(torch.nn.Module):
def __init__(self, model):
+ """Initialize DecoderLayerSANMExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.self_attn = model.self_attn
self.src_attn = model.src_attn
@@ -603,6 +662,15 @@ class DecoderLayerSANMExport(torch.nn.Module):
def forward(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Forward pass for training.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt = self.feed_forward(tgt)
@@ -621,6 +689,15 @@ class DecoderLayerSANMExport(torch.nn.Module):
return x, tgt_mask, memory, memory_mask, cache
def get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Get attn mat.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt = self.feed_forward(tgt)
@@ -640,6 +717,15 @@ class DecoderLayerSANMExport(torch.nn.Module):
@tables.register("decoder_classes", "ParaformerSANMDecoderExport")
class ParaformerSANMDecoderExport(torch.nn.Module):
def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):
+ """Initialize ParaformerSANMDecoderExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ model_name: TODO.
+ onnx: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# self.embed = model.embed #Embedding(model.embed, max_seq_len)
@@ -673,6 +759,11 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
self.model_name = model_name
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -692,6 +783,16 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
return_both: bool = False,
):
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ return_hidden: TODO.
+ return_both: TODO.
+ """
tgt = ys_in_pad
tgt_mask = self.make_pad_mask(ys_in_lens)
tgt_mask, _ = self.prepare_mask(tgt_mask)
@@ -728,6 +829,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
ys_in_lens: torch.Tensor,
):
+ """Forward asf2.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -749,6 +858,14 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
ys_in_lens: torch.Tensor,
):
+ """Forward asf6.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -829,6 +946,15 @@ class ParaformerSANMDecoderExport(torch.nn.Module):
@tables.register("decoder_classes", "ParaformerSANMDecoderOnlineExport")
class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
def __init__(self, model, max_seq_len=512, model_name="decoder", onnx: bool = True, **kwargs):
+ """Initialize ParaformerSANMDecoderOnlineExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ model_name: TODO.
+ onnx: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
# self.embed = model.embed #Embedding(model.embed, max_seq_len)
self.model = model
@@ -863,6 +989,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
self.model_name = model_name
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -881,6 +1012,15 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
*args,
):
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ *args: Variable positional arguments.
+ """
tgt = ys_in_pad
tgt_mask = self.make_pad_mask(ys_in_lens)
tgt_mask, _ = self.prepare_mask(tgt_mask)
@@ -913,6 +1053,11 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
return x, out_caches
def get_dummy_inputs(self, enc_size):
+ """Get dummy inputs.
+
+ Args:
+ enc_size: Size/dimension parameter.
+ """
enc = torch.randn(2, 100, enc_size).type(torch.float32)
enc_len = torch.tensor([30, 100], dtype=torch.int32)
acoustic_embeds = torch.randn(2, 10, enc_size).type(torch.float32)
@@ -930,6 +1075,7 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
return (enc, enc_len, acoustic_embeds, acoustic_embeds_len, *cache)
def get_input_names(self):
+ """Get input names."""
cache_num = len(self.model.decoders)
if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:
cache_num += len(self.model.decoders2)
@@ -938,12 +1084,14 @@ class ParaformerSANMDecoderOnlineExport(torch.nn.Module):
]
def get_output_names(self):
+ """Get output names."""
cache_num = len(self.model.decoders)
if hasattr(self.model, "decoders2") and self.model.decoders2 is not None:
cache_num += len(self.model.decoders2)
return ["logits", "sample_ids"] + ["out_cache_%d" % i for i in range(cache_num)]
def get_dynamic_axes(self):
+ """Get dynamic axes."""
ret = {
"enc": {0: "batch_size", 1: "enc_length"},
"acoustic_embeds": {0: "batch_size", 1: "token_length"},
@@ -1004,6 +1152,25 @@ class ParaformerSANDecoder(BaseTransformerDecoder):
concat_after: bool = False,
embeds_id: int = -1,
):
+ """Initialize ParaformerSANDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ embeds_id: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -1093,6 +1260,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):
model_name="decoder",
onnx: bool = True,
):
+ """Initialize ParaformerDecoderSANExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ model_name: TODO.
+ onnx: TODO.
+ """
super().__init__()
# self.embed = model.embed #Embedding(model.embed, max_seq_len)
self.model = model
@@ -1116,6 +1291,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):
self.model_name = model_name
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -1133,6 +1313,14 @@ class ParaformerDecoderSANExport(torch.nn.Module):
ys_in_lens: torch.Tensor,
):
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = self.make_pad_mask(ys_in_lens)
tgt_mask, _ = self.prepare_mask(tgt_mask)
@@ -1151,6 +1339,11 @@ class ParaformerDecoderSANExport(torch.nn.Module):
return x, ys_in_lens
def get_dummy_inputs(self, enc_size):
+ """Get dummy inputs.
+
+ Args:
+ enc_size: Size/dimension parameter.
+ """
tgt = torch.LongTensor([0]).unsqueeze(0)
memory = torch.randn(1, 100, enc_size)
pre_acoustic_embeds = torch.randn(1, 1, enc_size)
@@ -1164,17 +1357,21 @@ class ParaformerDecoderSANExport(torch.nn.Module):
return (tgt, memory, pre_acoustic_embeds, cache)
def is_optimizable(self):
+ """Is optimizable."""
return True
def get_input_names(self):
+ """Get input names."""
cache_num = len(self.model.decoders) + len(self.model.decoders2)
return ["tgt", "memory", "pre_acoustic_embeds"] + ["cache_%d" % i for i in range(cache_num)]
def get_output_names(self):
+ """Get output names."""
cache_num = len(self.model.decoders) + len(self.model.decoders2)
return ["y"] + ["out_cache_%d" % i for i in range(cache_num)]
def get_dynamic_axes(self):
+ """Get dynamic axes."""
ret = {
"tgt": {0: "tgt_batch", 1: "tgt_length"},
"memory": {0: "memory_batch", 1: "memory_length"},
diff --git a/funasr/models/paraformer/export_meta.py b/funasr/models/paraformer/export_meta.py
index 8e086a212..0e8d194af 100644
--- a/funasr/models/paraformer/export_meta.py
+++ b/funasr/models/paraformer/export_meta.py
@@ -9,6 +9,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
model.device = kwargs.get("device")
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
@@ -41,6 +47,12 @@ def export_forward(
speech_lengths: torch.Tensor,
):
# a. To device
+ """Export forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
batch = {"speech": speech, "speech_lengths": speech_lengths}
# batch = to_device(batch, device=self.device)
@@ -57,20 +69,24 @@ def export_forward(
def export_dummy_inputs(self):
+ """Export dummy inputs."""
speech = torch.randn(2, 30, 560)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
return (speech, speech_lengths)
def export_input_names(self):
+ """Export input names."""
return ["speech", "speech_lengths"]
def export_output_names(self):
+ """Export output names."""
return ["logits", "token_num"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"speech": {0: "batch_size", 1: "feats_length"},
"speech_lengths": {
@@ -84,4 +100,5 @@ def export_dynamic_axes(self):
def export_name(
self,
):
+ """Export name."""
return "model.onnx"
diff --git a/funasr/models/paraformer/model.py b/funasr/models/paraformer/model.py
index fc20c13de..5060d8f20 100644
--- a/funasr/models/paraformer/model.py
+++ b/funasr/models/paraformer/model.py
@@ -89,6 +89,37 @@ class Paraformer(torch.nn.Module):
**kwargs,
):
+ """Initialize Paraformer.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ predictor: TODO.
+ predictor_conf: Configuration dict for predictor.
+ ctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ predictor_weight: TODO.
+ predictor_bias: TODO.
+ sampling_ratio: TODO.
+ share_embedding: TODO.
+ use_1st_decoder_loss: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -283,6 +314,12 @@ class Paraformer(torch.nn.Module):
def calc_predictor(self, encoder_out, encoder_out_lens):
+ """Calc predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -295,6 +332,14 @@ class Paraformer(torch.nn.Module):
self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
):
+ """Cal decoder with predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ sematic_embeds: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
decoder_outs = self.decoder(encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens)
decoder_out = decoder_outs[0]
decoder_out = torch.log_softmax(decoder_out, dim=-1)
@@ -307,6 +352,14 @@ class Paraformer(torch.nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -354,6 +407,15 @@ class Paraformer(torch.nn.Module):
def sampler(self, encoder_out, encoder_out_lens, ys_pad, ys_pad_lens, pre_acoustic_embeds):
+ """Sampler.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ pre_acoustic_embeds: TODO.
+ """
tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(
ys_pad.device
)
@@ -400,6 +462,14 @@ class Paraformer(torch.nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -413,6 +483,11 @@ class Paraformer(torch.nn.Module):
self,
**kwargs,
):
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.paraformer.search import BeamSearchPara
from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
from funasr.models.transformer.scorers.length_bonus import LengthBonus
@@ -466,6 +541,16 @@ class Paraformer(torch.nn.Module):
**kwargs,
):
# init beamsearch
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None
is_use_lm = (
kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
@@ -612,6 +697,11 @@ class Paraformer(torch.nn.Module):
return results, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
if "max_seq_len" not in kwargs:
diff --git a/funasr/models/paraformer_streaming/export_meta.py b/funasr/models/paraformer_streaming/export_meta.py
index 681568849..448a8266d 100644
--- a/funasr/models/paraformer_streaming/export_meta.py
+++ b/funasr/models/paraformer_streaming/export_meta.py
@@ -10,6 +10,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
# self.device = kwargs.get("device")
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
model.encoder = encoder_class(model.encoder, onnx=is_onnx)
@@ -57,6 +63,12 @@ def export_encoder_forward(
speech_lengths: torch.Tensor,
):
# a. To device
+ """Export encoder forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
batch = {"speech": speech, "speech_lengths": speech_lengths, "online": True}
# batch = to_device(batch, device=self.device)
@@ -68,20 +80,24 @@ def export_encoder_forward(
def export_encoder_dummy_inputs(self):
+ """Export encoder dummy inputs."""
speech = torch.randn(2, 30, 560)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
return (speech, speech_lengths)
def export_encoder_input_names(self):
+ """Export encoder input names."""
return ["speech", "speech_lengths"]
def export_encoder_output_names(self):
+ """Export encoder output names."""
return ["enc", "enc_len", "alphas"]
def export_encoder_dynamic_axes(self):
+ """Export encoder dynamic axes."""
return {
"speech": {0: "batch_size", 1: "feats_length"},
"speech_lengths": {
@@ -96,6 +112,7 @@ def export_encoder_dynamic_axes(self):
def export_encoder_name(self):
+ """Export encoder name."""
return "model.onnx"
@@ -107,6 +124,15 @@ def export_decoder_forward(
acoustic_embeds_len: torch.Tensor,
*args,
):
+ """Export decoder forward.
+
+ Args:
+ enc: TODO.
+ enc_len: TODO.
+ acoustic_embeds: TODO.
+ acoustic_embeds_len: TODO.
+ *args: Variable positional arguments.
+ """
decoder_out, out_caches = self.decoder(
enc, enc_len, acoustic_embeds, acoustic_embeds_len, *args
)
@@ -116,21 +142,26 @@ def export_decoder_forward(
def export_decoder_dummy_inputs(self):
+ """Export decoder dummy inputs."""
dummy_inputs = self.decoder.get_dummy_inputs(enc_size=self.encoder._output_size)
return dummy_inputs
def export_decoder_input_names(self):
+ """Export decoder input names."""
return self.decoder.get_input_names()
def export_decoder_output_names(self):
+ """Export decoder output names."""
return self.decoder.get_output_names()
def export_decoder_dynamic_axes(self):
+ """Export decoder dynamic axes."""
return self.decoder.get_dynamic_axes()
def export_decoder_name(self):
+ """Export decoder name."""
return "decoder.onnx"
diff --git a/funasr/models/paraformer_streaming/model.py b/funasr/models/paraformer_streaming/model.py
index 9b9e609ab..d463aa269 100644
--- a/funasr/models/paraformer_streaming/model.py
+++ b/funasr/models/paraformer_streaming/model.py
@@ -31,6 +31,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -48,6 +53,12 @@ class ParaformerStreaming(Paraformer):
**kwargs,
):
+ """Initialize ParaformerStreaming.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
self.sampling_ratio = kwargs.get("sampling_ratio", 0.2)
@@ -155,6 +166,14 @@ class ParaformerStreaming(Paraformer):
cache: dict = None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Encode chunk.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
"""Frontend + Encoder. Note that this method is used by asr_inference.py
@@ -189,6 +208,14 @@ class ParaformerStreaming(Paraformer):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att predictor loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -313,6 +340,16 @@ class ParaformerStreaming(Paraformer):
chunk_mask=None,
):
+ """Sampler.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ pre_acoustic_embeds: TODO.
+ chunk_mask: TODO.
+ """
tgt_mask = (~make_pad_mask(ys_pad_lens, maxlen=ys_pad_lens.max())[:, :, None]).to(
ys_pad.device
)
@@ -351,6 +388,12 @@ class ParaformerStreaming(Paraformer):
def calc_predictor(self, encoder_out, encoder_out_lens):
+ """Calc predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -412,6 +455,14 @@ class ParaformerStreaming(Paraformer):
return pre_acoustic_embeds, pre_token_length, pre_alphas, pre_peak_index
def calc_predictor_chunk(self, encoder_out, encoder_out_lens, cache=None, **kwargs):
+ """Calc predictor chunk.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
is_final = kwargs.get("is_final", False)
return self.predictor.forward_chunk(encoder_out, cache["encoder"], is_final=is_final)
@@ -419,6 +470,14 @@ class ParaformerStreaming(Paraformer):
def cal_decoder_with_predictor(
self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens
):
+ """Cal decoder with predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ sematic_embeds: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
decoder_outs = self.decoder(
encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, self.scama_mask
)
@@ -429,12 +488,27 @@ class ParaformerStreaming(Paraformer):
def cal_decoder_with_predictor_chunk(
self, encoder_out, encoder_out_lens, sematic_embeds, ys_pad_lens, cache=None
):
+ """Cal decoder with predictor chunk.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ sematic_embeds: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ cache: State cache dict for streaming inference.
+ """
decoder_outs = self.decoder.forward_chunk(encoder_out, sematic_embeds, cache["decoder"])
decoder_out = decoder_outs
decoder_out = torch.log_softmax(decoder_out, dim=-1)
return decoder_out, ys_pad_lens
def init_cache(self, cache: dict = None, **kwargs):
+ """Init cache.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
chunk_size = kwargs.get("chunk_size", [0, 10, 5])
@@ -478,6 +552,16 @@ class ParaformerStreaming(Paraformer):
frontend=None,
**kwargs,
):
+ """Generate chunk.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
cache = kwargs.get("cache", {})
speech = speech.to(device=kwargs["device"])
speech_lengths = speech_lengths.to(device=kwargs["device"])
@@ -567,6 +651,17 @@ class ParaformerStreaming(Paraformer):
cache: dict = None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
@@ -662,6 +757,11 @@ class ParaformerStreaming(Paraformer):
return result, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
models = export_rebuild_model(model=self, **kwargs)
diff --git a/funasr/models/paraformer_v2_community/decoder.py b/funasr/models/paraformer_v2_community/decoder.py
index 556873db5..1dcbaff92 100644
--- a/funasr/models/paraformer_v2_community/decoder.py
+++ b/funasr/models/paraformer_v2_community/decoder.py
@@ -121,6 +121,15 @@ class DecoderLayerSANM(torch.nn.Module):
return x, tgt_mask, memory, memory_mask, cache
def get_attn_mat(self, tgt, tgt_mask, memory, memory_mask=None, cache=None):
+ """Get attn mat.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt = self.feed_forward(tgt)
@@ -257,6 +266,35 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
tf2torch_tensor_name_prefix_torch: str = "decoder",
tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",
):
+ """Initialize ParaformerSANMDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ wo_input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ att_layer_num: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ lora_list: TODO.
+ lora_rank: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ chunk_multiply_factor: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -426,6 +464,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
ys_in_lens: torch.Tensor,
):
+ """Forward asf2.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
@@ -444,6 +490,14 @@ class ParaformerSANMDecoder(BaseTransformerDecoder):
ys_in_lens: torch.Tensor,
):
+ """Forward asf6.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ """
tgt = ys_in_pad
tgt_mask = myutils.sequence_mask(ys_in_lens, device=tgt.device)[:, :, None]
diff --git a/funasr/models/paraformer_v2_community/model.py b/funasr/models/paraformer_v2_community/model.py
index 2d3ac9070..eb11ede5b 100644
--- a/funasr/models/paraformer_v2_community/model.py
+++ b/funasr/models/paraformer_v2_community/model.py
@@ -69,6 +69,32 @@ class Paraformer(torch.nn.Module):
**kwargs,
):
+ """Initialize Paraformer.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ share_embedding: TODO.
+ use_1st_decoder_loss: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -259,6 +285,14 @@ class Paraformer(torch.nn.Module):
):
# 0. sampler
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
decoder_out_1st = None
batch_size = encoder_out.size(0)
@@ -312,6 +346,14 @@ class Paraformer(torch.nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -449,6 +491,16 @@ class Paraformer(torch.nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
if (
isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"
@@ -538,6 +590,11 @@ class Paraformer(torch.nn.Module):
return results, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
if "max_seq_len" not in kwargs:
diff --git a/funasr/models/qwen3_asr/model.py b/funasr/models/qwen3_asr/model.py
index 0054ebc42..4a494c13e 100644
--- a/funasr/models/qwen3_asr/model.py
+++ b/funasr/models/qwen3_asr/model.py
@@ -29,6 +29,11 @@ class Qwen3ASR(nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize Qwen3ASR.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
model_path = kwargs.get("model_path", kwargs.get("model", "Qwen/Qwen3-ASR-1.7B"))
device = kwargs.get("device", "cuda:0")
@@ -98,6 +103,11 @@ class Qwen3ASR(nn.Module):
return model_path
def forward(self, **kwargs):
+ """Forward pass for training.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
raise NotImplementedError("Qwen3ASR only supports inference mode")
def inference(
diff --git a/funasr/models/qwen_audio/audio.py b/funasr/models/qwen_audio/audio.py
index a1aad79ea..133ad86de 100644
--- a/funasr/models/qwen_audio/audio.py
+++ b/funasr/models/qwen_audio/audio.py
@@ -16,6 +16,12 @@ from funasr.register import tables
def exact_div(x, y):
+ """Exact div.
+
+ Args:
+ x: TODO.
+ y: TODO.
+ """
assert x % y == 0
return x // y
@@ -35,6 +41,12 @@ TOKENS_PER_SECOND = exact_div(SAMPLE_RATE, N_SAMPLES_PER_TOKEN) # 20ms per audi
def get_T_after_cnn(L_in, dilation=1):
+ """Get t after cnn.
+
+ Args:
+ L_in: TODO.
+ dilation: TODO.
+ """
for padding, kernel_size, stride in eval("[(1,3,1)] + [(1,3,2)] "):
L_out = L_in + 2 * padding - dilation * (kernel_size - 1) - 1
L_out = 1 + L_out // stride
@@ -43,6 +55,12 @@ def get_T_after_cnn(L_in, dilation=1):
def load_bytesio_audio(content, sr: int = SAMPLE_RATE):
+ """Load bytesio audio.
+
+ Args:
+ content: TODO.
+ sr: TODO.
+ """
cmd = [
"ffmpeg",
"-nostdin",
@@ -228,11 +246,21 @@ class ModelDimensions:
class LayerNorm(nn.LayerNorm):
def forward(self, x: Tensor) -> Tensor:
# return super().forward(x.float()).type(x.dtype)
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return super().forward(x).type(x.dtype)
class Linear(nn.Linear):
def forward(self, x: Tensor) -> Tensor:
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return F.linear(
x,
self.weight.to(x.dtype),
@@ -242,6 +270,13 @@ class Linear(nn.Linear):
class Conv1d(nn.Conv1d):
def _conv_forward(self, x: Tensor, weight: Tensor, bias: Optional[Tensor]) -> Tensor:
+ """Internal: conv forward.
+
+ Args:
+ x: TODO.
+ weight: TODO.
+ bias: TODO.
+ """
return super()._conv_forward(
x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype)
)
@@ -258,6 +293,12 @@ def sinusoids(length, channels, max_timescale=10000):
class MultiHeadAttention(nn.Module):
def __init__(self, n_state: int, n_head: int):
+ """Initialize MultiHeadAttention.
+
+ Args:
+ n_state: TODO.
+ n_head: TODO.
+ """
super().__init__()
self.n_head = n_head
self.query = Linear(n_state, n_state)
@@ -272,6 +313,14 @@ class MultiHeadAttention(nn.Module):
mask: Optional[Tensor] = None,
kv_cache: Optional[dict] = None,
):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ xa: TODO.
+ mask: TODO.
+ kv_cache: TODO.
+ """
q = self.query(x)
if kv_cache is None or xa is None or self.key not in kv_cache:
@@ -288,6 +337,14 @@ class MultiHeadAttention(nn.Module):
return self.out(wv), qk
def qkv_attention(self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None):
+ """Qkv attention.
+
+ Args:
+ q: TODO.
+ k: TODO.
+ v: TODO.
+ mask: TODO.
+ """
n_batch, n_ctx, n_state = q.shape
scale = (n_state // self.n_head) ** -0.25
q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale
@@ -304,6 +361,13 @@ class MultiHeadAttention(nn.Module):
class ResidualAttentionBlock(nn.Module):
def __init__(self, n_state: int, n_head: int, cross_attention: bool = False):
+ """Initialize ResidualAttentionBlock.
+
+ Args:
+ n_state: TODO.
+ n_head: TODO.
+ cross_attention: TODO.
+ """
super().__init__()
self.attn = MultiHeadAttention(n_state, n_head)
@@ -323,6 +387,14 @@ class ResidualAttentionBlock(nn.Module):
mask: Optional[Tensor] = None,
kv_cache: Optional[dict] = None,
):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ xa: TODO.
+ mask: TODO.
+ kv_cache: TODO.
+ """
x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0]
if self.cross_attn:
x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0]
@@ -344,6 +416,19 @@ class QwenAudioEncoder(nn.Module):
add_audio_bos_eos_token: bool = True,
**kwargs,
):
+ """Initialize QwenAudioEncoder.
+
+ Args:
+ n_mels: TODO.
+ n_ctx: TODO.
+ n_state: TODO.
+ n_head: TODO.
+ n_layer: TODO.
+ output_dim: Size/dimension parameter.
+ avg_pool: TODO.
+ add_audio_bos_eos_token: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1)
self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
@@ -423,6 +508,13 @@ class QwenAudioEncoder(nn.Module):
return x, bos, eos
def encode(self, input_audios: Tensor, input_audio_lengths: Tensor, audio_span_tokens: List):
+ """Encode.
+
+ Args:
+ input_audios: TODO.
+ input_audio_lengths: Lengths of input_audio.
+ audio_span_tokens: TODO.
+ """
real_input_audio_lens = input_audio_lengths[:, 0].tolist()
max_len_in_batch = max(real_input_audio_lens)
padding_mask = torch.ones([input_audios.size(0), max_len_in_batch]).to(
diff --git a/funasr/models/qwen_audio/model.py b/funasr/models/qwen_audio/model.py
index b0af4563c..f78d8914b 100644
--- a/funasr/models/qwen_audio/model.py
+++ b/funasr/models/qwen_audio/model.py
@@ -27,6 +27,12 @@ class QwenAudioWarp(nn.Module):
"""
def __init__(self, *args, **kwargs):
+ """Initialize QwenAudioWarp.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
from transformers import AutoModelForCausalLM, AutoTokenizer
from transformers.generation import GenerationConfig
@@ -43,6 +49,7 @@ class QwenAudioWarp(nn.Module):
def forward(
self,
):
+ """Forward pass for training."""
pass
def inference(
@@ -54,6 +61,16 @@ class QwenAudioWarp(nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
@@ -109,6 +126,7 @@ class QwenAudioChatWarp(nn.Module):
def forward(
self,
):
+ """Forward pass for training."""
pass
def inference(
@@ -120,6 +138,16 @@ class QwenAudioChatWarp(nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
diff --git a/funasr/models/rwkv_bat/rwkv_encoder.py b/funasr/models/rwkv_bat/rwkv_encoder.py
index a27088d69..6b33f2823 100644
--- a/funasr/models/rwkv_bat/rwkv_encoder.py
+++ b/funasr/models/rwkv_bat/rwkv_encoder.py
@@ -92,6 +92,7 @@ class RWKVEncoder(torch.nn.Module):
self.time_reduction_factor = time_reduction_factor
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(self, x: torch.Tensor, x_len) -> torch.Tensor:
@@ -133,6 +134,11 @@ class RWKVEncoder(torch.nn.Module):
def rwkv_infer(self, xs_pad):
+ """Rwkv infer.
+
+ Args:
+ xs_pad: TODO.
+ """
batch_size = xs_pad.shape[0]
hidden_sizes = [self._output_size for i in range(5)]
diff --git a/funasr/models/sa_asr/attention.py b/funasr/models/sa_asr/attention.py
index 74e687a52..109a98e4f 100644
--- a/funasr/models/sa_asr/attention.py
+++ b/funasr/models/sa_asr/attention.py
@@ -23,6 +23,7 @@ class CosineDistanceAttention(nn.Module):
"""
def __init__(self):
+ """Initialize CosineDistanceAttention."""
super().__init__()
self.softmax = nn.Softmax(dim=-1)
diff --git a/funasr/models/sa_asr/e2e_sa_asr.py b/funasr/models/sa_asr/e2e_sa_asr.py
index f4827b257..6d9422d2f 100644
--- a/funasr/models/sa_asr/e2e_sa_asr.py
+++ b/funasr/models/sa_asr/e2e_sa_asr.py
@@ -34,6 +34,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -64,6 +69,31 @@ class SAASRModel(FunASRModel):
sym_blank: str = "<blank>",
extract_feats_in_collect_stats: bool = True,
):
+ """Initialize SAASRModel.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ max_spk_num: TODO.
+ token_list: TODO.
+ frontend: Audio frontend for feature extraction.
+ specaug: TODO.
+ normalize: TODO.
+ asr_encoder: TODO.
+ spk_encoder: TODO.
+ decoder: TODO.
+ ctc: TODO.
+ spk_weight: TODO.
+ ctc_weight: TODO.
+ interctc_weight: TODO.
+ ignore_id: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ extract_feats_in_collect_stats: TODO.
+ """
assert 0.0 <= ctc_weight <= 1.0, ctc_weight
assert 0.0 <= interctc_weight < 1.0, interctc_weight
@@ -250,6 +280,14 @@ class SAASRModel(FunASRModel):
text: torch.Tensor,
text_lengths: torch.Tensor,
) -> Dict[str, torch.Tensor]:
+ """Collect feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ text: Text tensor or string input.
+ text_lengths: Length of each text sample.
+ """
if self.extract_feats_in_collect_stats:
feats, feats_lengths = self._extract_feats(speech, speech_lengths)
else:
@@ -326,6 +364,12 @@ class SAASRModel(FunASRModel):
def _extract_feats(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Internal: extract feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
assert speech_lengths.dim() == 1, speech_lengths.shape
# for data-parallel
@@ -439,6 +483,19 @@ class SAASRModel(FunASRModel):
text_id: torch.Tensor,
text_id_lengths: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ asr_encoder_out: TODO.
+ spk_encoder_out: TODO.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ profile: TODO.
+ profile_lens: Lengths of profile.
+ text_id: TODO.
+ text_id_lengths: Lengths of text_id.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -498,6 +555,14 @@ class SAASRModel(FunASRModel):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
diff --git a/funasr/models/sa_asr/transformer_decoder.py b/funasr/models/sa_asr/transformer_decoder.py
index be33fa527..32dffbb50 100644
--- a/funasr/models/sa_asr/transformer_decoder.py
+++ b/funasr/models/sa_asr/transformer_decoder.py
@@ -181,6 +181,18 @@ class BaseTransformerDecoder(nn.Module, BatchScorerInterface):
pos_enc_class=PositionalEncoding,
normalize_before: bool = True,
):
+ """Initialize BaseTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ """
super().__init__()
attention_dim = encoder_output_size
@@ -359,6 +371,24 @@ class TransformerDecoder(BaseTransformerDecoder):
normalize_before: bool = True,
concat_after: bool = False,
):
+ """Initialize TransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -411,6 +441,25 @@ class ParaformerDecoderSAN(BaseTransformerDecoder):
concat_after: bool = False,
embeds_id: int = -1,
):
+ """Initialize ParaformerDecoderSAN.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ embeds_id: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -513,6 +562,27 @@ class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize LightweightConvolutionTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
@@ -573,6 +643,27 @@ class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize LightweightConvolution2DTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
@@ -633,6 +724,27 @@ class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize DynamicConvolutionTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
@@ -693,6 +805,27 @@ class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize DynamicConvolution2DTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
diff --git a/funasr/models/sanm/attention.py b/funasr/models/sanm/attention.py
index 47d60cb67..34dfbbae2 100644
--- a/funasr/models/sanm/attention.py
+++ b/funasr/models/sanm/attention.py
@@ -19,6 +19,15 @@ import funasr.models.lora.layers as lora
def preprocess_for_attn(x, mask, cache, pad_fn, kernel_size):
+ """Preprocess for attn.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ cache: State cache dict for streaming inference.
+ pad_fn: TODO.
+ kernel_size: Size/dimension parameter.
+ """
x = x * mask
x = x.transpose(1, 2)
if cache is None:
@@ -205,6 +214,13 @@ class MultiHeadedAttentionSANM(nn.Module):
self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None):
+ """Forward fsmn.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ mask_shfit_chunk: TODO.
+ """
b, t, d = inputs.size()
if mask is not None:
mask = torch.reshape(mask, (b, -1, 1))
@@ -352,6 +368,11 @@ class MultiHeadedAttentionSANM(nn.Module):
class MultiHeadedAttentionSANMExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionSANMExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.d_k = model.d_k
self.h = model.h
@@ -364,6 +385,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):
self.all_head_size = self.h * self.d_k
def forward(self, x, mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ """
mask_3d_btd, mask_4d_bhlt = mask
q_h, k_h, v_h, v = self.forward_qkv(x)
fsmn_memory = self.forward_fsmn(v, mask_3d_btd)
@@ -373,11 +400,21 @@ class MultiHeadedAttentionSANMExport(nn.Module):
return att_outs + fsmn_memory
def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
+ """Transpose for scores.
+
+ Args:
+ x: TODO.
+ """
new_x_shape = x.size()[:-1] + (self.h, self.d_k)
x = x.view(new_x_shape)
return x.permute(0, 2, 1, 3)
def forward_qkv(self, x):
+ """Forward qkv.
+
+ Args:
+ x: TODO.
+ """
q_k_v = self.linear_q_k_v(x)
q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)
q_h = self.transpose_for_scores(q)
@@ -388,6 +425,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):
def forward_fsmn(self, inputs, mask):
# b, t, d = inputs.size()
# mask = torch.reshape(mask, (b, -1, 1))
+ """Forward fsmn.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ """
inputs = inputs * mask
x = inputs.transpose(1, 2)
x = self.pad_fn(x)
@@ -398,6 +441,13 @@ class MultiHeadedAttentionSANMExport(nn.Module):
return x
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
@@ -411,6 +461,11 @@ class MultiHeadedAttentionSANMExport(nn.Module):
class MultiHeadedAttentionSANMExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionSANMExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.d_k = model.d_k
self.h = model.h
@@ -423,6 +478,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):
self.all_head_size = self.h * self.d_k
def forward(self, x, mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ """
mask_3d_btd, mask_4d_bhlt = mask
q_h, k_h, v_h, v = self.forward_qkv(x)
fsmn_memory = self.forward_fsmn(v, mask_3d_btd)
@@ -432,11 +493,21 @@ class MultiHeadedAttentionSANMExport(nn.Module):
return att_outs + fsmn_memory
def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
+ """Transpose for scores.
+
+ Args:
+ x: TODO.
+ """
new_x_shape = x.size()[:-1] + (self.h, self.d_k)
x = x.view(new_x_shape)
return x.permute(0, 2, 1, 3)
def forward_qkv(self, x):
+ """Forward qkv.
+
+ Args:
+ x: TODO.
+ """
q_k_v = self.linear_q_k_v(x)
q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)
q_h = self.transpose_for_scores(q)
@@ -447,6 +518,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):
def forward_fsmn(self, inputs, mask):
# b, t, d = inputs.size()
# mask = torch.reshape(mask, (b, -1, 1))
+ """Forward fsmn.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ """
inputs = inputs * mask
x = inputs.transpose(1, 2)
x = self.pad_fn(x)
@@ -457,6 +534,13 @@ class MultiHeadedAttentionSANMExport(nn.Module):
return x
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
@@ -549,6 +633,11 @@ class MultiHeadedAttentionSANMDecoder(nn.Module):
class MultiHeadedAttentionSANMDecoderExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionSANMDecoderExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.fsmn_block = model.fsmn_block
self.pad_fn = model.pad_fn
@@ -556,6 +645,13 @@ class MultiHeadedAttentionSANMDecoderExport(nn.Module):
self.attn = None
def forward(self, inputs, mask, cache=None):
+ """Forward pass for training.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
x, cache = preprocess_for_attn(inputs, mask, cache, self.pad_fn, self.kernel_size)
x = self.fsmn_block(x)
x = x.transpose(1, 2)
@@ -750,6 +846,11 @@ class MultiHeadedAttentionCrossAtt(nn.Module):
class MultiHeadedAttentionCrossAttExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionCrossAttExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.d_k = model.d_k
self.h = model.h
@@ -760,16 +861,35 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):
self.all_head_size = self.h * self.d_k
def forward(self, x, memory, memory_mask, ret_attn=False):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ ret_attn: TODO.
+ """
q, k, v = self.forward_qkv(x, memory)
scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)
return self.forward_attention(v, scores, memory_mask, ret_attn)
def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
+ """Transpose for scores.
+
+ Args:
+ x: TODO.
+ """
new_x_shape = x.size()[:-1] + (self.h, self.d_k)
x = x.view(new_x_shape)
return x.permute(0, 2, 1, 3)
def forward_qkv(self, x, memory):
+ """Forward qkv.
+
+ Args:
+ x: TODO.
+ memory: TODO.
+ """
q = self.linear_q(x)
k_v = self.linear_k_v(memory)
@@ -780,6 +900,14 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):
return q, k, v
def forward_attention(self, value, scores, mask, ret_attn):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ ret_attn: TODO.
+ """
scores = scores + mask.to(scores.device)
attn = torch.softmax(scores, dim=-1)
diff --git a/funasr/models/sanm/decoder.py b/funasr/models/sanm/decoder.py
index 1a4fb26e2..1a0de72d5 100644
--- a/funasr/models/sanm/decoder.py
+++ b/funasr/models/sanm/decoder.py
@@ -233,6 +233,32 @@ class FsmnDecoder(BaseTransformerDecoder):
tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",
embed_tensor_name_prefix_tf: str = None,
):
+ """Initialize FsmnDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ att_layer_num: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ concat_embeds: TODO.
+ attention_dim: Size/dimension parameter.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ embed_tensor_name_prefix_tf: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
diff --git a/funasr/models/sanm/encoder.py b/funasr/models/sanm/encoder.py
index 735595971..d8a720b7a 100644
--- a/funasr/models/sanm/encoder.py
+++ b/funasr/models/sanm/encoder.py
@@ -221,6 +221,36 @@ class SANMEncoder(nn.Module):
tf2torch_tensor_name_prefix_torch: str = "encoder",
tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",
):
+ """Initialize SANMEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ padding_idx: TODO.
+ interctc_layer_idx: TODO.
+ interctc_use_conditioning: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ lora_list: TODO.
+ lora_rank: TODO.
+ lora_alpha: TODO.
+ lora_dropout: TODO.
+ selfattention_layer_type: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -356,6 +386,7 @@ class SANMEncoder(nn.Module):
self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
@@ -430,6 +461,12 @@ class SANMEncoder(nn.Module):
return xs_pad, olens, None
def _add_overlap_chunk(self, feats: np.ndarray, cache: dict = None):
+ """Internal: add overlap chunk.
+
+ Args:
+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
if len(cache) == 0:
@@ -446,6 +483,14 @@ class SANMEncoder(nn.Module):
cache: dict = None,
ctc: CTC = None,
):
+ """Forward chunk.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ cache: State cache dict for streaming inference.
+ ctc: TODO.
+ """
if cache is None:
cache = {}
xs_pad *= self.output_size() ** 0.5
@@ -504,6 +549,12 @@ class EncoderLayerSANMExport(nn.Module):
def forward(self, x, mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ """
residual = x
x = self.norm1(x)
x = self.self_attn(x, mask)
@@ -529,6 +580,16 @@ class SANMEncoderExport(nn.Module):
onnx: bool = True,
ctc_linear: nn.Module = None,
):
+ """Initialize SANMEncoderExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ feats_dim: Size/dimension parameter.
+ model_name: TODO.
+ onnx: TODO.
+ ctc_linear: TODO.
+ """
super().__init__()
self.embed = model.embed
if isinstance(self.embed, StreamSinusoidalPositionEncoder):
@@ -561,6 +622,11 @@ class SANMEncoderExport(nn.Module):
self.ctc_linear = ctc_linear
def prepare_mask(self, mask):
+ """Prepare mask.
+
+ Args:
+ mask: TODO.
+ """
mask_3d_btd = mask[:, :, None]
if len(mask.shape) == 2:
mask_4d_bhlt = 1 - mask[:, None, None, :]
@@ -571,6 +637,13 @@ class SANMEncoderExport(nn.Module):
return mask_3d_btd, mask_4d_bhlt
def forward(self, speech: torch.Tensor, speech_lengths: torch.Tensor, online: bool = False):
+ """Forward pass for training.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ online: TODO.
+ """
if not online:
speech = speech * self._output_size**0.5
@@ -596,19 +669,24 @@ class SANMEncoderExport(nn.Module):
return xs_pad, speech_lengths
def get_output_size(self):
+ """Get output size."""
return self.model.encoders[0].size
def get_dummy_inputs(self):
+ """Get dummy inputs."""
feats = torch.randn(1, 100, self.feats_dim)
return feats
def get_input_names(self):
+ """Get input names."""
return ["feats"]
def get_output_names(self):
+ """Get output names."""
return ["encoder_out", "encoder_out_lens", "predictor_weight"]
def get_dynamic_axes(self):
+ """Get dynamic axes."""
return {
"feats": {1: "feats_length"},
"encoder_out": {1: "enc_out_length"},
diff --git a/funasr/models/sanm/model.py b/funasr/models/sanm/model.py
index b3f84b2b9..f0d25b6d9 100644
--- a/funasr/models/sanm/model.py
+++ b/funasr/models/sanm/model.py
@@ -25,4 +25,10 @@ class SANM(Transformer):
**kwargs,
):
+ """Initialize SANM.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
diff --git a/funasr/models/sanm/multihead_att.py b/funasr/models/sanm/multihead_att.py
index 671d46011..3e9d18d82 100644
--- a/funasr/models/sanm/multihead_att.py
+++ b/funasr/models/sanm/multihead_att.py
@@ -7,6 +7,11 @@ import torch.nn as nn
class MultiHeadedAttentionSANMExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionSANMExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.d_k = model.d_k
self.h = model.h
@@ -19,6 +24,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):
self.all_head_size = self.h * self.d_k
def forward(self, x, mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ """
mask_3d_btd, mask_4d_bhlt = mask
q_h, k_h, v_h, v = self.forward_qkv(x)
fsmn_memory = self.forward_fsmn(v, mask_3d_btd)
@@ -28,11 +39,21 @@ class MultiHeadedAttentionSANMExport(nn.Module):
return att_outs + fsmn_memory
def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
+ """Transpose for scores.
+
+ Args:
+ x: TODO.
+ """
new_x_shape = x.size()[:-1] + (self.h, self.d_k)
x = x.view(new_x_shape)
return x.permute(0, 2, 1, 3)
def forward_qkv(self, x):
+ """Forward qkv.
+
+ Args:
+ x: TODO.
+ """
q_k_v = self.linear_q_k_v(x)
q, k, v = torch.split(q_k_v, int(self.h * self.d_k), dim=-1)
q_h = self.transpose_for_scores(q)
@@ -43,6 +64,12 @@ class MultiHeadedAttentionSANMExport(nn.Module):
def forward_fsmn(self, inputs, mask):
# b, t, d = inputs.size()
# mask = torch.reshape(mask, (b, -1, 1))
+ """Forward fsmn.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ """
inputs = inputs * mask
x = inputs.transpose(1, 2)
x = self.pad_fn(x)
@@ -53,6 +80,13 @@ class MultiHeadedAttentionSANMExport(nn.Module):
return x
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
@@ -65,6 +99,15 @@ class MultiHeadedAttentionSANMExport(nn.Module):
def preprocess_for_attn(x, mask, cache, pad_fn, kernel_size):
+ """Preprocess for attn.
+
+ Args:
+ x: TODO.
+ mask: TODO.
+ cache: State cache dict for streaming inference.
+ pad_fn: TODO.
+ kernel_size: Size/dimension parameter.
+ """
x = x * mask
x = x.transpose(1, 2)
if cache is None:
@@ -84,6 +127,11 @@ if torch_version >= (1, 8):
class MultiHeadedAttentionSANMDecoderExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionSANMDecoderExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.fsmn_block = model.fsmn_block
self.pad_fn = model.pad_fn
@@ -91,6 +139,13 @@ class MultiHeadedAttentionSANMDecoderExport(nn.Module):
self.attn = None
def forward(self, inputs, mask, cache=None):
+ """Forward pass for training.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
x, cache = preprocess_for_attn(inputs, mask, cache, self.pad_fn, self.kernel_size)
x = self.fsmn_block(x)
x = x.transpose(1, 2)
@@ -102,6 +157,11 @@ class MultiHeadedAttentionSANMDecoderExport(nn.Module):
class MultiHeadedAttentionCrossAttExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionCrossAttExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.d_k = model.d_k
self.h = model.h
@@ -112,16 +172,34 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):
self.all_head_size = self.h * self.d_k
def forward(self, x, memory, memory_mask):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ """
q, k, v = self.forward_qkv(x, memory)
scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)
return self.forward_attention(v, scores, memory_mask)
def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
+ """Transpose for scores.
+
+ Args:
+ x: TODO.
+ """
new_x_shape = x.size()[:-1] + (self.h, self.d_k)
x = x.view(new_x_shape)
return x.permute(0, 2, 1, 3)
def forward_qkv(self, x, memory):
+ """Forward qkv.
+
+ Args:
+ x: TODO.
+ memory: TODO.
+ """
q = self.linear_q(x)
k_v = self.linear_k_v(memory)
@@ -132,6 +210,13 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):
return q, k, v
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
@@ -145,6 +230,11 @@ class MultiHeadedAttentionCrossAttExport(nn.Module):
class OnnxMultiHeadedAttention(nn.Module):
def __init__(self, model):
+ """Initialize OnnxMultiHeadedAttention.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.d_k = model.d_k
self.h = model.h
@@ -156,16 +246,36 @@ class OnnxMultiHeadedAttention(nn.Module):
self.all_head_size = self.h * self.d_k
def forward(self, query, key, value, mask):
+ """Forward pass for training.
+
+ Args:
+ query: TODO.
+ key: Sample identifiers.
+ value: TODO.
+ mask: TODO.
+ """
q, k, v = self.forward_qkv(query, key, value)
scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)
return self.forward_attention(v, scores, mask)
def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
+ """Transpose for scores.
+
+ Args:
+ x: TODO.
+ """
new_x_shape = x.size()[:-1] + (self.h, self.d_k)
x = x.view(new_x_shape)
return x.permute(0, 2, 1, 3)
def forward_qkv(self, query, key, value):
+ """Forward qkv.
+
+ Args:
+ query: TODO.
+ key: Sample identifiers.
+ value: TODO.
+ """
q = self.linear_q(query)
k = self.linear_k(key)
v = self.linear_v(value)
@@ -175,6 +285,13 @@ class OnnxMultiHeadedAttention(nn.Module):
return q, k, v
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
@@ -188,12 +305,26 @@ class OnnxMultiHeadedAttention(nn.Module):
class OnnxRelPosMultiHeadedAttention(OnnxMultiHeadedAttention):
def __init__(self, model):
+ """Initialize OnnxRelPosMultiHeadedAttention.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__(model)
self.linear_pos = model.linear_pos
self.pos_bias_u = model.pos_bias_u
self.pos_bias_v = model.pos_bias_v
def forward(self, query, key, value, pos_emb, mask):
+ """Forward pass for training.
+
+ Args:
+ query: TODO.
+ key: Sample identifiers.
+ value: TODO.
+ pos_emb: TODO.
+ mask: TODO.
+ """
q, k, v = self.forward_qkv(query, key, value)
q = q.transpose(1, 2) # (batch, time1, head, d_k)
@@ -220,6 +351,11 @@ class OnnxRelPosMultiHeadedAttention(OnnxMultiHeadedAttention):
return self.forward_attention(v, scores, mask)
def rel_shift(self, x):
+ """Rel shift.
+
+ Args:
+ x: TODO.
+ """
zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype)
x_padded = torch.cat([zero_pad, x], dim=-1)
@@ -230,6 +366,13 @@ class OnnxRelPosMultiHeadedAttention(OnnxMultiHeadedAttention):
return x
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
diff --git a/funasr/models/sanm_kws/export_meta.py b/funasr/models/sanm_kws/export_meta.py
index a91a22cb0..2d55a2bf7 100644
--- a/funasr/models/sanm_kws/export_meta.py
+++ b/funasr/models/sanm_kws/export_meta.py
@@ -11,6 +11,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
# self.device = kwargs.get("device")
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
@@ -47,6 +53,12 @@ def export_encoder_forward(
speech_lengths: torch.Tensor,
):
# a. To device
+ """Export encoder forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
batch = {
"speech": speech,
"speech_lengths": speech_lengths,
@@ -63,21 +75,25 @@ def export_encoder_forward(
def export_encoder_dummy_inputs(self):
+ """Export encoder dummy inputs."""
speech = torch.randn(2, 30, 280)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
return (speech, speech_lengths)
def export_encoder_input_names(self):
+ """Export encoder input names."""
return ["speech", "speech_lengths"]
def export_encoder_output_names(self):
# return ["encoder_out", "encoder_out_len", "alphas"]
+ """Export encoder output names."""
return ["encoder_out", "encoder_out_len"]
def export_encoder_dynamic_axes(self):
+ """Export encoder dynamic axes."""
return {
"speech": {
0: "batch_size", 1: "feats_length"
@@ -95,4 +111,5 @@ def export_encoder_dynamic_axes(self):
def export_encoder_name(self):
+ """Export encoder name."""
return "encoder.onnx"
diff --git a/funasr/models/sanm_kws/model.py b/funasr/models/sanm_kws/model.py
index d8d8f955c..8577cf425 100644
--- a/funasr/models/sanm_kws/model.py
+++ b/funasr/models/sanm_kws/model.py
@@ -52,6 +52,26 @@ class SanmKWS(torch.nn.Module):
**kwargs,
):
+ """Initialize SanmKWS.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -167,6 +187,14 @@ class SanmKWS(torch.nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -185,6 +213,16 @@ class SanmKWS(torch.nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
keywords = kwargs.get("keywords")
from funasr.utils.kws_utils import KwsCtcPrefixDecoder
self.kws_decoder = KwsCtcPrefixDecoder(
@@ -258,6 +296,11 @@ class SanmKWS(torch.nn.Module):
return results, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
if "max_seq_len" not in kwargs:
diff --git a/funasr/models/sanm_kws_streaming/export_meta.py b/funasr/models/sanm_kws_streaming/export_meta.py
index a91a22cb0..2d55a2bf7 100644
--- a/funasr/models/sanm_kws_streaming/export_meta.py
+++ b/funasr/models/sanm_kws_streaming/export_meta.py
@@ -11,6 +11,12 @@ from funasr.register import tables
def export_rebuild_model(model, **kwargs):
# self.device = kwargs.get("device")
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
@@ -47,6 +53,12 @@ def export_encoder_forward(
speech_lengths: torch.Tensor,
):
# a. To device
+ """Export encoder forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
batch = {
"speech": speech,
"speech_lengths": speech_lengths,
@@ -63,21 +75,25 @@ def export_encoder_forward(
def export_encoder_dummy_inputs(self):
+ """Export encoder dummy inputs."""
speech = torch.randn(2, 30, 280)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
return (speech, speech_lengths)
def export_encoder_input_names(self):
+ """Export encoder input names."""
return ["speech", "speech_lengths"]
def export_encoder_output_names(self):
# return ["encoder_out", "encoder_out_len", "alphas"]
+ """Export encoder output names."""
return ["encoder_out", "encoder_out_len"]
def export_encoder_dynamic_axes(self):
+ """Export encoder dynamic axes."""
return {
"speech": {
0: "batch_size", 1: "feats_length"
@@ -95,4 +111,5 @@ def export_encoder_dynamic_axes(self):
def export_encoder_name(self):
+ """Export encoder name."""
return "encoder.onnx"
diff --git a/funasr/models/sanm_kws_streaming/model.py b/funasr/models/sanm_kws_streaming/model.py
index 184965955..3228a0643 100644
--- a/funasr/models/sanm_kws_streaming/model.py
+++ b/funasr/models/sanm_kws_streaming/model.py
@@ -31,6 +31,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -47,6 +52,12 @@ class SanmKWSStreaming(SanmKWS):
*args,
**kwargs,
):
+ """Initialize SanmKWSStreaming.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(
@@ -112,6 +123,14 @@ class SanmKWSStreaming(SanmKWS):
cache: dict = None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Encode chunk.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
"""Frontend + Encoder. Note that this method is used by asr_inference.py
@@ -140,6 +159,12 @@ class SanmKWSStreaming(SanmKWS):
return encoder_out, torch.tensor([encoder_out.size(1)])
def init_cache(self, cache: dict = None, **kwargs):
+ """Init cache.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
chunk_size = kwargs.get("chunk_size", [0, 10, 5])
@@ -185,6 +210,16 @@ class SanmKWSStreaming(SanmKWS):
frontend=None,
**kwargs,
):
+ """Generate chunk.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
cache = kwargs.get("cache", {})
speech = speech.to(device=kwargs["device"])
speech_lengths = speech_lengths.to(device=kwargs["device"])
@@ -259,6 +294,17 @@ class SanmKWSStreaming(SanmKWS):
cache: dict = None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
keywords = kwargs.get("keywords")
@@ -442,6 +488,11 @@ class SanmKWSStreaming(SanmKWS):
return result, meta_data
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
models = export_rebuild_model(model=self, **kwargs)
diff --git a/funasr/models/scama/beam_search.py b/funasr/models/scama/beam_search.py
index bf0bb5cf7..e114fb773 100644
--- a/funasr/models/scama/beam_search.py
+++ b/funasr/models/scama/beam_search.py
@@ -623,6 +623,15 @@ class BeamSearchScamaStreaming(torch.nn.Module):
pre_acoustic_embeds: torch.Tensor = None,
cache: dict = None,
) -> Tuple[Dict[str, torch.Tensor], Dict[str, Any]]:
+ """Score full.
+
+ Args:
+ hyp: TODO.
+ x: TODO.
+ x_mask: TODO.
+ pre_acoustic_embeds: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
"""Score new hypothesis by `self.full_scorers`.
@@ -766,6 +775,15 @@ class BeamSearchScamaStreaming(torch.nn.Module):
pre_acoustic_embeds: torch.Tensor = None,
cache: dict = None,
) -> List[Hypothesis]:
+ """Search.
+
+ Args:
+ running_hyps: TODO.
+ x: TODO.
+ x_mask: TODO.
+ pre_acoustic_embeds: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
"""Search new tokens for running hypotheses and encoded speech x.
@@ -831,6 +849,18 @@ class BeamSearchScamaStreaming(torch.nn.Module):
minlen: int = 0,
cache: dict = None,
) -> List[Hypothesis]:
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ scama_mask: TODO.
+ pre_acoustic_embeds: TODO.
+ maxlenratio: TODO.
+ minlenratio: TODO.
+ maxlen: TODO.
+ minlen: TODO.
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
"""Perform beam search.
diff --git a/funasr/models/scama/chunk_utilis.py b/funasr/models/scama/chunk_utilis.py
index 2fe3fa4e3..7d92ce5d9 100644
--- a/funasr/models/scama/chunk_utilis.py
+++ b/funasr/models/scama/chunk_utilis.py
@@ -25,6 +25,16 @@ class overlap_chunk:
decoder_att_look_back_factor: tuple = (1,),
):
+ """Initialize overlap_chunk.
+
+ Args:
+ chunk_size: Size/dimension parameter.
+ stride: TODO.
+ pad_left: TODO.
+ encoder_att_look_back_factor: TODO.
+ shfit_fsmn: TODO.
+ decoder_att_look_back_factor: TODO.
+ """
pad_left = self.check_chunk_size_args(chunk_size, pad_left)
encoder_att_look_back_factor = self.check_chunk_size_args(
chunk_size, encoder_att_look_back_factor
@@ -63,12 +73,23 @@ class overlap_chunk:
) = (None, None, None, None, None)
def check_chunk_size_args(self, chunk_size, x):
+ """Check chunk size args.
+
+ Args:
+ chunk_size: Size/dimension parameter.
+ x: TODO.
+ """
if len(x) < len(chunk_size):
x = [x[0] for i in chunk_size]
return x
def get_chunk_size(self, ind: int = 0):
# with torch.no_grad:
+ """Get chunk size.
+
+ Args:
+ ind: TODO.
+ """
chunk_size, stride, pad_left, encoder_att_look_back_factor, decoder_att_look_back_factor = (
self.chunk_size[ind],
self.stride[ind],
@@ -100,6 +121,12 @@ class overlap_chunk:
)
def random_choice(self, training=True, decoding_ind=None):
+ """Random choice.
+
+ Args:
+ training: TODO.
+ decoding_ind: TODO.
+ """
chunk_num = len(self.chunk_size)
ind = 0
if training and chunk_num > 1:
@@ -111,6 +138,14 @@ class overlap_chunk:
def gen_chunk_mask(self, x_len, ind=0, num_units=1, num_units_predictor=1):
+ """Gen chunk mask.
+
+ Args:
+ x_len: TODO.
+ ind: TODO.
+ num_units: TODO.
+ num_units_predictor: TODO.
+ """
with torch.no_grad():
x_len = x_len.cpu().numpy()
x_len_max = x_len.max()
@@ -271,6 +306,13 @@ class overlap_chunk:
return x_chunk, x_len_chunk
def remove_chunk(self, x_chunk, x_len_chunk, chunk_outs):
+ """Remove chunk.
+
+ Args:
+ x_chunk: TODO.
+ x_len_chunk: TODO.
+ chunk_outs: TODO.
+ """
x_chunk = x_chunk[:, : x_len_chunk.max(), :]
b, t, d = x_chunk.size()
x_len_chunk_mask = (~make_pad_mask(x_len_chunk, maxlen=t)).to(x_chunk.device)
@@ -286,24 +328,56 @@ class overlap_chunk:
return x, x_len
def get_x_add_mask(self, chunk_outs=None, device="cpu", idx=0, dtype=torch.float32):
+ """Get x add mask.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = torch.from_numpy(x).type(dtype).to(device)
return x
def get_x_len_chunk(self, chunk_outs=None, device="cpu", idx=1, dtype=torch.float32):
+ """Get x len chunk.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = torch.from_numpy(x).type(dtype).to(device)
return x
def get_x_rm_mask(self, chunk_outs=None, device="cpu", idx=2, dtype=torch.float32):
+ """Get x rm mask.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = torch.from_numpy(x).type(dtype).to(device)
return x
def get_x_len(self, chunk_outs=None, device="cpu", idx=3, dtype=torch.float32):
+ """Get x len.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = torch.from_numpy(x).type(dtype).to(device)
@@ -312,6 +386,16 @@ class overlap_chunk:
def get_mask_shfit_chunk(
self, chunk_outs=None, device="cpu", batch_size=1, num_units=1, idx=4, dtype=torch.float32
):
+ """Get mask shfit chunk.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ batch_size: Number of samples per batch.
+ num_units: TODO.
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = np.tile(
@@ -328,6 +412,16 @@ class overlap_chunk:
def get_mask_chunk_predictor(
self, chunk_outs=None, device="cpu", batch_size=1, num_units=1, idx=5, dtype=torch.float32
):
+ """Get mask chunk predictor.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ batch_size: Number of samples per batch.
+ num_units: TODO.
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = np.tile(
@@ -344,6 +438,15 @@ class overlap_chunk:
def get_mask_att_chunk_encoder(
self, chunk_outs=None, device="cpu", batch_size=1, idx=6, dtype=torch.float32
):
+ """Get mask att chunk encoder.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ batch_size: Number of samples per batch.
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = np.tile(
@@ -360,6 +463,15 @@ class overlap_chunk:
def get_mask_shift_att_chunk_decoder(
self, chunk_outs=None, device="cpu", batch_size=1, idx=7, dtype=torch.float32
):
+ """Get mask shift att chunk decoder.
+
+ Args:
+ chunk_outs: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ batch_size: Number of samples per batch.
+ idx: TODO.
+ dtype: TODO.
+ """
with torch.no_grad():
x = chunk_outs[idx] if chunk_outs is not None else self.chunk_outs[idx]
x = np.tile(x[None, None, :, 0], [batch_size, 1, 1])
@@ -383,6 +495,24 @@ def build_scama_mask_for_cross_attention_decoder(
is_training=True,
dtype: torch.dtype = torch.float32,
):
+ """Build scama mask for cross attention decoder.
+
+ Args:
+ predictor_alignments: TODO.
+ encoder_sequence_length: TODO.
+ chunk_size: Size/dimension parameter.
+ encoder_chunk_size: Size/dimension parameter.
+ attention_chunk_center_bias: TODO.
+ attention_chunk_size: Size/dimension parameter.
+ attention_chunk_type: TODO.
+ step: TODO.
+ predictor_mask_chunk_hopping: TODO.
+ decoder_att_look_back_factor: TODO.
+ mask_shift_att_chunk_decoder: TODO.
+ target_length: TODO.
+ is_training: Boolean flag for training.
+ dtype: TODO.
+ """
with torch.no_grad():
device = predictor_alignments.device
batch_size, chunk_num = predictor_alignments.size()
@@ -454,6 +584,7 @@ def build_scama_mask_for_cross_attention_decoder(
mask_mask_flip = mask_flip * mask
def _fn():
+ """Internal: fn."""
mask_sliced = mask[:b, :k, encoder_chunk_size:t]
zero_pad_right = torch.zeros(
[b, k, encoder_chunk_size], dtype=mask_sliced.dtype
diff --git a/funasr/models/scama/decoder.py b/funasr/models/scama/decoder.py
index 31b235778..e8495967d 100644
--- a/funasr/models/scama/decoder.py
+++ b/funasr/models/scama/decoder.py
@@ -233,6 +233,32 @@ class FsmnDecoderSCAMAOpt(BaseTransformerDecoder):
tf2torch_tensor_name_prefix_tf: str = "seq2seq/decoder",
embed_tensor_name_prefix_tf: str = None,
):
+ """Initialize FsmnDecoderSCAMAOpt.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ att_layer_num: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ concat_embeds: TODO.
+ attention_dim: Size/dimension parameter.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ embed_tensor_name_prefix_tf: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
diff --git a/funasr/models/scama/encoder.py b/funasr/models/scama/encoder.py
index 0cf48f526..88d3536d8 100644
--- a/funasr/models/scama/encoder.py
+++ b/funasr/models/scama/encoder.py
@@ -222,6 +222,37 @@ class SANMEncoderChunkOpt(nn.Module):
tf2torch_tensor_name_prefix_torch: str = "encoder",
tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",
):
+ """Initialize SANMEncoderChunkOpt.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ padding_idx: TODO.
+ interctc_layer_idx: TODO.
+ interctc_use_conditioning: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ selfattention_layer_type: TODO.
+ chunk_size: Size/dimension parameter.
+ stride: TODO.
+ pad_left: TODO.
+ encoder_att_look_back_factor: TODO.
+ decoder_att_look_back_factor: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -356,6 +387,7 @@ class SANMEncoderChunkOpt(nn.Module):
self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
@@ -446,6 +478,12 @@ class SANMEncoderChunkOpt(nn.Module):
return xs_pad, olens, None
def _add_overlap_chunk(self, feats: np.ndarray, cache: dict = None):
+ """Internal: add overlap chunk.
+
+ Args:
+ feats: Feature tensor (e.g., fbank), shape (batch, frames, dim).
+ cache: State cache dict for streaming inference.
+ """
if cache is None:
cache = {}
if len(cache) == 0:
@@ -462,6 +500,14 @@ class SANMEncoderChunkOpt(nn.Module):
cache: dict = None,
**kwargs,
):
+ """Forward chunk.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
is_final = kwargs.get("is_final", False)
diff --git a/funasr/models/scama/model.py b/funasr/models/scama/model.py
index bb207c88e..28be561ba 100644
--- a/funasr/models/scama/model.py
+++ b/funasr/models/scama/model.py
@@ -33,6 +33,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -73,6 +78,35 @@ class SCAMA(nn.Module):
**kwargs,
):
+ """Initialize SCAMA.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ predictor: TODO.
+ predictor_conf: Configuration dict for predictor.
+ predictor_bias: TODO.
+ predictor_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -270,6 +304,14 @@ class SCAMA(nn.Module):
cache: dict = None,
**kwargs,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Encode chunk.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
"""Frontend + Encoder. Note that this method is used by asr_inference.py
@@ -298,6 +340,14 @@ class SCAMA(nn.Module):
return encoder_out, torch.tensor([encoder_out.size(1)])
def calc_predictor_chunk(self, encoder_out, encoder_out_lens, cache=None, **kwargs):
+ """Calc predictor chunk.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
is_final = kwargs.get("is_final", False)
return self.predictor.forward_chunk(encoder_out, cache["encoder"], is_final=is_final)
@@ -309,6 +359,14 @@ class SCAMA(nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att predictor loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -404,6 +462,14 @@ class SCAMA(nn.Module):
):
# ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
# ys_in_lens = ys_pad_lens + 1
+ """Calc predictor mask.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_out_pad, ys_in_lens = None, None
encoder_out_mask = sequence_mask(
@@ -473,6 +539,11 @@ class SCAMA(nn.Module):
**kwargs,
):
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.scama.beam_search import BeamSearchScamaStreaming
from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
@@ -528,6 +599,16 @@ class SCAMA(nn.Module):
frontend=None,
**kwargs,
):
+ """Generate chunk.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
cache = kwargs.get("cache", {})
speech = speech.to(device=kwargs["device"])
speech_lengths = speech_lengths.to(device=kwargs["device"])
@@ -605,6 +686,12 @@ class SCAMA(nn.Module):
return results
def init_cache(self, cache: dict = None, **kwargs):
+ """Init cache.
+
+ Args:
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
device = kwargs.get("device", "cuda")
@@ -655,6 +742,17 @@ class SCAMA(nn.Module):
cache: dict = None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ cache: State cache dict for streaming inference.
+ **kwargs: Additional keyword arguments.
+ """
if cache is None:
cache = {}
diff --git a/funasr/models/scama/utils.py b/funasr/models/scama/utils.py
index c3f7bc37c..7c7c59683 100644
--- a/funasr/models/scama/utils.py
+++ b/funasr/models/scama/utils.py
@@ -6,6 +6,14 @@ from torch.nn import functional as F
def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):
+ """Sequence mask.
+
+ Args:
+ lengths: TODO.
+ maxlen: TODO.
+ dtype: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
if maxlen is None:
maxlen = lengths.max()
row_vector = torch.arange(0, maxlen, 1).to(lengths.device)
@@ -17,6 +25,12 @@ def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):
def apply_cmvn(inputs, mvn):
+ """Apply cmvn.
+
+ Args:
+ inputs: TODO.
+ mvn: TODO.
+ """
device = inputs.device
dtype = inputs.dtype
frame, dim = inputs.shape
@@ -36,6 +50,15 @@ def drop_and_add(
stoch_layer_coeff: float = 1.0,
):
+ """Drop and add.
+
+ Args:
+ inputs: TODO.
+ outputs: TODO.
+ training: TODO.
+ dropout_rate: TODO.
+ stoch_layer_coeff: TODO.
+ """
outputs = F.dropout(outputs, p=dropout_rate, training=training, inplace=True)
outputs *= stoch_layer_coeff
@@ -48,6 +71,11 @@ def drop_and_add(
def proc_tf_vocab(vocab_path):
+ """Proc tf vocab.
+
+ Args:
+ vocab_path: TODO.
+ """
with open(vocab_path, encoding="utf-8") as f:
token_list = [line.rstrip() for line in f]
if "<unk>" not in token_list:
@@ -56,6 +84,13 @@ def proc_tf_vocab(vocab_path):
def gen_config_for_tfmodel(config_path, vocab_path, output_dir):
+ """Gen config for tfmodel.
+
+ Args:
+ config_path: TODO.
+ vocab_path: TODO.
+ output_dir: Directory for saving output files.
+ """
token_list = proc_tf_vocab(vocab_path)
with open(config_path, encoding="utf-8") as f:
config = yaml.safe_load(f)
@@ -72,6 +107,11 @@ def gen_config_for_tfmodel(config_path, vocab_path, output_dir):
class NoAliasSafeDumper(yaml.SafeDumper):
# Disable anchor/alias in yaml because looks ugly
def ignore_aliases(self, data):
+ """Ignore aliases.
+
+ Args:
+ data: TODO.
+ """
return True
diff --git a/funasr/models/seaco_paraformer/export_meta.py b/funasr/models/seaco_paraformer/export_meta.py
index 947f4d956..980e062ab 100644
--- a/funasr/models/seaco_paraformer/export_meta.py
+++ b/funasr/models/seaco_paraformer/export_meta.py
@@ -16,17 +16,31 @@ class ContextualEmbedderExport(torch.nn.Module):
feats_dim=560,
**kwargs,
):
+ """Initialize ContextualEmbedderExport.
+
+ Args:
+ model: Model instance or model name.
+ max_seq_len: TODO.
+ feats_dim: Size/dimension parameter.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.embedding = model.decoder.embed # model.bias_embed
model.bias_encoder.batch_first = False
self.bias_encoder = model.bias_encoder
def forward(self, hotword):
+ """Forward pass for training.
+
+ Args:
+ hotword: Hotword/keyword for boosting recognition.
+ """
hotword = self.embedding(hotword).transpose(0, 1) # batch second
hw_embed, (_, _) = self.bias_encoder(hotword)
return hw_embed
def export_dummy_inputs(self):
+ """Export dummy inputs."""
hotword = torch.tensor(
[
[10, 11, 12, 13, 14, 10, 11, 12, 13, 14],
@@ -42,12 +56,15 @@ class ContextualEmbedderExport(torch.nn.Module):
return hotword
def export_input_names(self):
+ """Export input names."""
return ["hotword"]
def export_output_names(self):
+ """Export output names."""
return ["hw_embed"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"hotword": {
0: "num_hotwords",
@@ -58,10 +75,17 @@ class ContextualEmbedderExport(torch.nn.Module):
}
def export_name(self):
+ """Export name."""
return "model_eb.onnx"
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
model.device = kwargs.get("device")
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
@@ -124,6 +148,13 @@ def export_backbone_forward(
# lmbd: float,
):
# a. To device
+ """Export backbone forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ bias_embed: TODO.
+ """
batch = {"speech": speech, "speech_lengths": speech_lengths}
enc, enc_len = self.encoder(**batch)
@@ -173,6 +204,7 @@ def export_backbone_forward(
def export_backbone_dummy_inputs(self):
+ """Export backbone dummy inputs."""
speech = torch.randn(2, 30, self.feats_dim)
speech_lengths = torch.tensor([15, 30], dtype=torch.int32)
bias_embed = torch.randn(2, 1, 512)
@@ -180,14 +212,17 @@ def export_backbone_dummy_inputs(self):
def export_backbone_input_names(self):
+ """Export backbone input names."""
return ["speech", "speech_lengths", "bias_embed"]
def export_backbone_output_names(self):
+ """Export backbone output names."""
return ["logits", "token_num", "us_alphas", "us_cif_peak"]
def export_backbone_dynamic_axes(self):
+ """Export backbone dynamic axes."""
return {
"speech": {0: "batch_size", 1: "feats_length"},
"speech_lengths": {
diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.py
index b30c5cb98..d6ae4bee9 100644
--- a/funasr/models/seaco_paraformer/model.py
+++ b/funasr/models/seaco_paraformer/model.py
@@ -38,6 +38,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -54,6 +59,12 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
*args,
**kwargs,
):
+ """Initialize SeacoParaformer.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
self.inner_dim = kwargs.get("inner_dim", 256)
@@ -177,9 +188,21 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
return loss, stats, weight
def _merge(self, cif_attended, dec_attended):
+ """Internal: merge.
+
+ Args:
+ cif_attended: TODO.
+ dec_attended: TODO.
+ """
return cif_attended + dec_attended
def calc_predictor(self, encoder_out, encoder_out_lens):
+ """Calc predictor.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -199,6 +222,17 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
seaco_label_pad: torch.Tensor,
):
# predictor forward
+ """Internal: calc seaco loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_lengths: Lengths of ys.
+ hotword_pad: TODO.
+ hotword_lengths: Lengths of hotword.
+ seaco_label_pad: TODO.
+ """
encoder_out_mask = (
~make_pad_mask(encoder_out_lens, maxlen=encoder_out.size(1))[:, None, :]
).to(encoder_out.device)
@@ -243,6 +277,17 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
):
# decoder forward
+ """Internal: seaco decode with ASF.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ sematic_embeds: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ hw_list: TODO.
+ nfilter: TODO.
+ seaco_weight: TODO.
+ """
decoder_out, decoder_hidden, _ = self.decoder(
encoder_out,
encoder_out_lens,
@@ -311,6 +356,12 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
dha_pred = torch.log_softmax(dha_output, dim=-1)
def _merge_res(dec_output, dha_output):
+ """Internal: merge res.
+
+ Args:
+ dec_output: TODO.
+ dha_output: TODO.
+ """
lmbd = torch.Tensor([seaco_weight] * dha_output.shape[0])
dha_ids = dha_output.max(-1)[-1] # [0]
dha_mask = (dha_ids == self.NO_BIAS).int().unsqueeze(-1)
@@ -328,6 +379,12 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
return decoder_pred
def _hotword_representation(self, hotword_pad, hotword_lengths):
+ """Internal: hotword representation.
+
+ Args:
+ hotword_pad: TODO.
+ hotword_lengths: Lengths of hotword.
+ """
if self.bias_encoder_type != "lstm":
logging.error("Unsupported bias encoder type")
@@ -370,6 +427,16 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
):
# init beamsearch
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
is_use_ctc = kwargs.get("decoding_ctc_weight", 0.0) > 0.00001 and self.ctc != None
is_use_lm = (
kwargs.get("lm_weight", 0.0) > 0.00001 and kwargs.get("lm_file", None) is not None
@@ -512,7 +579,20 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
def generate_hotwords_list(self, hotword_list_or_file, tokenizer=None, frontend=None):
+ """Generate hotwords list.
+
+ Args:
+ hotword_list_or_file: TODO.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ """
def seg_tokenize(txt, seg_dict):
+ """Seg tokenize.
+
+ Args:
+ txt: TODO.
+ seg_dict: TODO.
+ """
pattern = re.compile(r"^[\u4E00-\u9FA50-9]+$")
out_txt = ""
for word in txt:
@@ -610,6 +690,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
self,
**kwargs,
):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
if "max_seq_len" not in kwargs:
kwargs["max_seq_len"] = 512
from .export_meta import export_rebuild_model
@@ -620,6 +705,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
@lru_cache(maxsize=1)
def load_seg_dict(seg_dict_file):
+ """Load seg dict.
+
+ Args:
+ seg_dict_file: TODO.
+ """
seg_dict = {}
assert isinstance(seg_dict_file, str)
with open(seg_dict_file, "r", encoding="utf8") as f:
diff --git a/funasr/models/sense_voice/export_meta.py b/funasr/models/sense_voice/export_meta.py
index 449388ef7..58bccd3ba 100644
--- a/funasr/models/sense_voice/export_meta.py
+++ b/funasr/models/sense_voice/export_meta.py
@@ -9,6 +9,12 @@ from funasr.utils.torch_function import sequence_mask
def export_rebuild_model(model, **kwargs):
+ """Export rebuild model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
model.device = kwargs.get("device")
model.make_pad_mask = sequence_mask(kwargs["max_seq_len"], flip=False)
model.forward = types.MethodType(export_forward, model)
@@ -29,6 +35,15 @@ def export_forward(
):
# speech = speech.to(device="cuda")
# speech_lengths = speech_lengths.to(device="cuda")
+ """Export forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ language: Language identifier.
+ textnorm: TODO.
+ **kwargs: Additional keyword arguments.
+ """
language_query = self.embed(language.to(speech.device)).unsqueeze(1)
textnorm_query = self.embed(textnorm.to(speech.device)).unsqueeze(1)
@@ -51,6 +66,7 @@ def export_forward(
return ctc_logits, encoder_out_lens
def export_dummy_inputs(self):
+ """Export dummy inputs."""
speech = torch.randn(2, 30, 560)
speech_lengths = torch.tensor([6, 30], dtype=torch.int32)
language = torch.tensor([0, 0], dtype=torch.int32)
@@ -58,12 +74,15 @@ def export_dummy_inputs(self):
return (speech, speech_lengths, language, textnorm)
def export_input_names(self):
+ """Export input names."""
return ["speech", "speech_lengths", "language", "textnorm"]
def export_output_names(self):
+ """Export output names."""
return ["ctc_logits", "encoder_out_lens"]
def export_dynamic_axes(self):
+ """Export dynamic axes."""
return {
"speech": {0: "batch_size", 1: "feats_length"},
"speech_lengths": {0: "batch_size"},
@@ -74,4 +93,5 @@ def export_dynamic_axes(self):
}
def export_name(self):
+ """Export name."""
return "model.onnx"
diff --git a/funasr/models/sense_voice/model.py b/funasr/models/sense_voice/model.py
index 30a9e42b9..3ec936c54 100644
--- a/funasr/models/sense_voice/model.py
+++ b/funasr/models/sense_voice/model.py
@@ -26,11 +26,24 @@ class SinusoidalPositionEncoder(torch.nn.Module):
""" """
def __init__(self, d_model=80, dropout_rate=0.1):
+ """Initialize SinusoidalPositionEncoder.
+
+ Args:
+ d_model: D Model instance.
+ dropout_rate: TODO.
+ """
super().__init__()
def encode(
self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32
):
+ """Encode.
+
+ Args:
+ positions: TODO.
+ depth: TODO.
+ dtype: TODO.
+ """
batch_size = positions.size(0)
positions = positions.type(dtype)
device = positions.device
@@ -48,6 +61,11 @@ class SinusoidalPositionEncoder(torch.nn.Module):
return encoding.type(dtype)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
batch_size, timesteps, input_dim = x.size()
positions = torch.arange(1, timesteps + 1, device=x.device)[None, :]
position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device)
@@ -127,6 +145,13 @@ class MultiHeadedAttentionSANM(nn.Module):
self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
def forward_fsmn(self, inputs, mask, mask_shfit_chunk=None):
+ """Forward fsmn.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ mask_shfit_chunk: TODO.
+ """
b, t, d = inputs.size()
if mask is not None:
mask = torch.reshape(mask, (b, -1, 1))
@@ -274,9 +299,20 @@ class MultiHeadedAttentionSANM(nn.Module):
class LayerNorm(nn.LayerNorm):
def __init__(self, *args, **kwargs):
+ """Initialize LayerNorm.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = F.layer_norm(
input.float(),
self.normalized_shape,
@@ -288,6 +324,14 @@ class LayerNorm(nn.LayerNorm):
def sequence_mask(lengths, maxlen=None, dtype=torch.float32, device=None):
+ """Sequence mask.
+
+ Args:
+ lengths: TODO.
+ maxlen: TODO.
+ dtype: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
if maxlen is None:
maxlen = lengths.max()
row_vector = torch.arange(0, maxlen, 1).to(lengths.device)
@@ -473,6 +517,31 @@ class SenseVoiceEncoderSmall(nn.Module):
selfattention_layer_type: str = "sanm",
**kwargs,
):
+ """Initialize SenseVoiceEncoderSmall.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ tp_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ stochastic_depth_rate: TODO.
+ input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ padding_idx: TODO.
+ kernel_size: Size/dimension parameter.
+ sanm_shfit: TODO.
+ selfattention_layer_type: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self._output_size = output_size
@@ -548,6 +617,7 @@ class SenseVoiceEncoderSmall(nn.Module):
self.tp_norm = LayerNorm(output_size)
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
@@ -608,6 +678,25 @@ class SenseVoiceSmall(nn.Module):
**kwargs,
):
+ """Initialize SenseVoiceSmall.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ ctc_conf: Configuration dict for ctc.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ length_normalized_loss: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -663,6 +752,12 @@ class SenseVoiceSmall(nn.Module):
@staticmethod
def from_pretrained(model: str = None, **kwargs):
+ """From pretrained.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
from funasr import AutoModel
model, kwargs = AutoModel.build_model(model=model, trust_remote_code=True, **kwargs)
@@ -781,6 +876,14 @@ class SenseVoiceSmall(nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -795,6 +898,12 @@ class SenseVoiceSmall(nn.Module):
encoder_out: torch.Tensor,
ys_pad: torch.Tensor,
):
+ """Internal: calc rich ce loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ ys_pad: TODO.
+ """
decoder_out = self.ctc.ctc_lo(encoder_out)
# 2. Compute attention loss
loss_rich = self.criterion_att(decoder_out, ys_pad.contiguous())
@@ -816,6 +925,16 @@ class SenseVoiceSmall(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
meta_data = {}
if (
isinstance(data_in, torch.Tensor) and kwargs.get("data_type", "sound") == "fbank"
@@ -961,6 +1080,11 @@ class SenseVoiceSmall(nn.Module):
return results, meta_data
def post(self, timestamp):
+ """Post.
+
+ Args:
+ timestamp: TODO.
+ """
timestamp_new = []
words_new = []
prev_word = None
@@ -990,6 +1114,11 @@ class SenseVoiceSmall(nn.Module):
return timestamp_new, words_new
def export(self, **kwargs):
+ """Export.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from .export_meta import export_rebuild_model
if "max_seq_len" not in kwargs:
diff --git a/funasr/models/sense_voice/whisper_lib/__init__.py b/funasr/models/sense_voice/whisper_lib/__init__.py
index 855b8cf26..2cd10869b 100644
--- a/funasr/models/sense_voice/whisper_lib/__init__.py
+++ b/funasr/models/sense_voice/whisper_lib/__init__.py
@@ -48,6 +48,13 @@ _ALIGNMENT_HEADS = {
def _download(url: str, root: str, in_memory: bool) -> Union[bytes, str]:
+ """Internal: download.
+
+ Args:
+ url: TODO.
+ root: TODO.
+ in_memory: TODO.
+ """
os.makedirs(root, exist_ok=True)
expected_sha256 = url.split("/")[-2]
diff --git a/funasr/models/sense_voice/whisper_lib/decoding.py b/funasr/models/sense_voice/whisper_lib/decoding.py
index 644755f5f..73bb0a71c 100644
--- a/funasr/models/sense_voice/whisper_lib/decoding.py
+++ b/funasr/models/sense_voice/whisper_lib/decoding.py
@@ -175,6 +175,12 @@ class Inference:
class PyTorchInference(Inference):
def __init__(self, model: "Whisper", initial_token_length: int):
+ """Initialize PyTorchInference.
+
+ Args:
+ model: Model instance or model name.
+ initial_token_length: TODO.
+ """
self.model: "Whisper" = model
self.initial_token_length = initial_token_length
self.kv_cache = {}
@@ -185,6 +191,12 @@ class PyTorchInference(Inference):
self.kv_modules = key_modules + value_modules
def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:
+ """Logits.
+
+ Args:
+ tokens: TODO.
+ audio_features: TODO.
+ """
if not self.kv_cache:
self.kv_cache, self.hooks = self.model.install_kv_cache_hooks()
@@ -195,6 +207,7 @@ class PyTorchInference(Inference):
return self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache)
def cleanup_caching(self):
+ """Cleanup caching."""
for hook in self.hooks:
hook.remove()
@@ -202,6 +215,11 @@ class PyTorchInference(Inference):
self.hooks = []
def rearrange_kv_cache(self, source_indices):
+ """Rearrange kv cache.
+
+ Args:
+ source_indices: TODO.
+ """
if source_indices != list(range(len(source_indices))):
for module in self.kv_modules:
# update the key/value cache to contain the selected sequences
@@ -224,10 +242,27 @@ class MaximumLikelihoodRanker(SequenceRanker):
"""
def __init__(self, length_penalty: Optional[float]):
+ """Initialize MaximumLikelihoodRanker.
+
+ Args:
+ length_penalty: TODO.
+ """
self.length_penalty = length_penalty
def rank(self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]):
+ """Rank.
+
+ Args:
+ tokens: TODO.
+ sum_logprobs: TODO.
+ """
def scores(logprobs, lengths):
+ """Scores.
+
+ Args:
+ logprobs: TODO.
+ lengths: TODO.
+ """
result = []
for logprob, length in zip(logprobs, lengths):
if self.length_penalty is None:
@@ -299,10 +334,23 @@ class TokenDecoder:
class GreedyDecoder(TokenDecoder):
def __init__(self, temperature: float, eot: int):
+ """Initialize GreedyDecoder.
+
+ Args:
+ temperature: TODO.
+ eot: TODO.
+ """
self.temperature = temperature
self.eot = eot
def update(self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor) -> Tuple[Tensor, bool]:
+ """Update.
+
+ Args:
+ tokens: TODO.
+ logits: TODO.
+ sum_logprobs: TODO.
+ """
if self.temperature == 0:
next_tokens = logits.argmax(dim=-1)
else:
@@ -320,6 +368,12 @@ class GreedyDecoder(TokenDecoder):
def finalize(self, tokens: Tensor, sum_logprobs: Tensor):
# make sure each sequence has at least one EOT token at the end
+ """Finalize.
+
+ Args:
+ tokens: TODO.
+ sum_logprobs: TODO.
+ """
tokens = F.pad(tokens, (0, 1), value=self.eot)
return tokens, sum_logprobs.tolist()
@@ -332,6 +386,14 @@ class BeamSearchDecoder(TokenDecoder):
inference: Inference,
patience: Optional[float] = None,
):
+ """Initialize BeamSearchDecoder.
+
+ Args:
+ beam_size: Size/dimension parameter.
+ eot: TODO.
+ inference: TODO.
+ patience: TODO.
+ """
self.beam_size = beam_size
self.eot = eot
self.inference = inference
@@ -342,9 +404,17 @@ class BeamSearchDecoder(TokenDecoder):
assert self.max_candidates > 0, f"Invalid beam size ({beam_size}) or patience ({patience})"
def reset(self):
+ """Reset."""
self.finished_sequences = None
def update(self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor) -> Tuple[Tensor, bool]:
+ """Update.
+
+ Args:
+ tokens: TODO.
+ logits: TODO.
+ sum_logprobs: TODO.
+ """
if tokens.shape[0] % self.beam_size != 0:
raise ValueError(f"{tokens.shape}[0] % {self.beam_size} != 0")
@@ -402,6 +472,12 @@ class BeamSearchDecoder(TokenDecoder):
def finalize(self, preceding_tokens: Tensor, sum_logprobs: Tensor):
# collect all finished sequences, including patience, and add unfinished ones if not enough
+ """Finalize.
+
+ Args:
+ preceding_tokens: TODO.
+ sum_logprobs: TODO.
+ """
sum_logprobs = sum_logprobs.cpu()
for i, sequences in enumerate(self.finished_sequences):
if len(sequences) < self.beam_size: # when not enough sequences are finished
@@ -438,19 +514,42 @@ class LogitFilter:
class SuppressBlank(LogitFilter):
def __init__(self, tokenizer: Tokenizer, sample_begin: int):
+ """Initialize SuppressBlank.
+
+ Args:
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ sample_begin: TODO.
+ """
self.tokenizer = tokenizer
self.sample_begin = sample_begin
def apply(self, logits: Tensor, tokens: Tensor):
+ """Apply.
+
+ Args:
+ logits: TODO.
+ tokens: TODO.
+ """
if tokens.shape[1] == self.sample_begin:
logits[:, self.tokenizer.encode(" ") + [self.tokenizer.eot]] = -np.inf
class SuppressTokens(LogitFilter):
def __init__(self, suppress_tokens: Sequence[int]):
+ """Initialize SuppressTokens.
+
+ Args:
+ suppress_tokens: TODO.
+ """
self.suppress_tokens = list(suppress_tokens)
def apply(self, logits: Tensor, tokens: Tensor):
+ """Apply.
+
+ Args:
+ logits: TODO.
+ tokens: TODO.
+ """
logits[:, self.suppress_tokens] = -np.inf
@@ -458,6 +557,13 @@ class GainEventToken(LogitFilter):
def __init__(
self, bg_tokens: Sequence[int], ed_tokens: Sequence[int], gain_values: Sequence[float]
):
+ """Initialize GainEventToken.
+
+ Args:
+ bg_tokens: TODO.
+ ed_tokens: TODO.
+ gain_values: TODO.
+ """
self.bg_tokens = list(bg_tokens)
self.ed_tokens = list(ed_tokens)
self.gain_value = [np.log(max(ga, 1e-9)) for ga in gain_values]
@@ -465,6 +571,12 @@ class GainEventToken(LogitFilter):
assert len(self.bg_tokens) == len(self.gain_value)
def apply(self, logits: Tensor, tokens: Tensor):
+ """Apply.
+
+ Args:
+ logits: TODO.
+ tokens: TODO.
+ """
for i in range(len(tokens)):
for bg, ed, ga in zip(self.bg_tokens, self.ed_tokens, self.gain_value):
sum_bg = sum([1 if x == bg else 0 for x in tokens[i]])
@@ -480,12 +592,25 @@ class ThresholdEmoToken(LogitFilter):
def __init__(
self, unk_tokens: Sequence[int], emo_tokens: Sequence[int], th_values: Sequence[float]
):
+ """Initialize ThresholdEmoToken.
+
+ Args:
+ unk_tokens: TODO.
+ emo_tokens: TODO.
+ th_values: TODO.
+ """
self.unk_token = list(unk_tokens)[0]
self.emo_tokens = list(emo_tokens)
self.th_values = list(th_values)
assert len(self.emo_tokens) == len(self.th_values)
def apply(self, logits: Tensor, tokens: Tensor):
+ """Apply.
+
+ Args:
+ logits: TODO.
+ tokens: TODO.
+ """
for i in range(len(tokens)):
for emo, th in zip(self.emo_tokens, self.th_values):
if logits[i].argmax() == emo and logits[i].softmax(dim=-1)[emo] < th:
@@ -509,12 +634,25 @@ class ApplyTimestampRules(LogitFilter):
sample_begin: int,
max_initial_timestamp_index: Optional[int],
):
+ """Initialize ApplyTimestampRules.
+
+ Args:
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ sample_begin: TODO.
+ max_initial_timestamp_index: TODO.
+ """
self.tokenizer = tokenizer
self.sample_begin = sample_begin
self.max_initial_timestamp_index = max_initial_timestamp_index
def apply(self, logits: Tensor, tokens: Tensor):
# suppress <|notimestamps|> which is handled by without_timestamps
+ """Apply.
+
+ Args:
+ logits: TODO.
+ tokens: TODO.
+ """
if self.tokenizer.no_timestamps is not None:
logits[:, self.tokenizer.no_timestamps] = -np.inf
@@ -566,6 +704,12 @@ class DecodingTask:
logit_filters: List[LogitFilter]
def __init__(self, model: "Whisper", options: DecodingOptions):
+ """Initialize DecodingTask.
+
+ Args:
+ model: Model instance or model name.
+ options: TODO.
+ """
self.model = model
language = options.language or "en"
@@ -637,6 +781,11 @@ class DecodingTask:
)
def _verify_options(self, options: DecodingOptions) -> DecodingOptions:
+ """Internal: verify options.
+
+ Args:
+ options: TODO.
+ """
if options.beam_size is not None and options.best_of is not None:
raise ValueError("beam_size and best_of can't be given together")
if options.temperature == 0:
@@ -650,6 +799,7 @@ class DecodingTask:
return options
def _get_initial_tokens(self) -> Tuple[int]:
+ """Internal: get initial tokens."""
tokens = list(self.sot_sequence)
if prefix := self.options.prefix:
@@ -678,6 +828,7 @@ class DecodingTask:
return tuple(tokens)
def _get_suppress_tokens(self) -> Tuple[int]:
+ """Internal: get suppress tokens."""
suppress_tokens = self.options.suppress_tokens
if isinstance(suppress_tokens, str):
@@ -707,6 +858,11 @@ class DecodingTask:
return tuple(sorted(set(suppress_tokens)))
def _get_audio_features(self, mel: Tensor):
+ """Internal: get audio features.
+
+ Args:
+ mel: TODO.
+ """
if self.options.fp16:
mel = mel.half()
@@ -727,6 +883,12 @@ class DecodingTask:
return audio_features
def _detect_language(self, audio_features: Tensor, tokens: Tensor):
+ """Internal: detect language.
+
+ Args:
+ audio_features: TODO.
+ tokens: TODO.
+ """
languages = [self.options.language] * audio_features.shape[0]
lang_probs = None
@@ -755,6 +917,12 @@ class DecodingTask:
return languages, lang_probs
def _main_loop(self, audio_features: Tensor, tokens: Tensor):
+ """Internal: main loop.
+
+ Args:
+ audio_features: TODO.
+ tokens: TODO.
+ """
n_batch = tokens.shape[0]
sum_logprobs: Tensor = torch.zeros(n_batch, device=audio_features.device)
no_speech_probs = [np.nan] * n_batch
@@ -786,6 +954,11 @@ class DecodingTask:
@torch.no_grad()
def run(self, mel: Tensor) -> List[DecodingResult]:
+ """Run.
+
+ Args:
+ mel: TODO.
+ """
self.decoder.reset()
tokenizer: Tokenizer = self.tokenizer
n_audio: int = mel.shape[0]
diff --git a/funasr/models/sense_voice/whisper_lib/model.py b/funasr/models/sense_voice/whisper_lib/model.py
index eb1380d11..d4d293a0e 100644
--- a/funasr/models/sense_voice/whisper_lib/model.py
+++ b/funasr/models/sense_voice/whisper_lib/model.py
@@ -35,9 +35,20 @@ class ModelDimensions:
class LayerNorm(nn.LayerNorm):
def __init__(self, *args, **kwargs):
+ """Initialize LayerNorm.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(*args, **kwargs)
def forward(self, input):
+ """Forward pass for training.
+
+ Args:
+ input: Input audio/text data.
+ """
output = F.layer_norm(
input.float(),
self.normalized_shape,
@@ -50,6 +61,11 @@ class LayerNorm(nn.LayerNorm):
class Linear(nn.Linear):
def forward(self, x: Tensor) -> Tensor:
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return F.linear(
x,
self.weight.to(x.dtype),
@@ -59,6 +75,13 @@ class Linear(nn.Linear):
class Conv1d(nn.Conv1d):
def _conv_forward(self, x: Tensor, weight: Tensor, bias: Optional[Tensor]) -> Tensor:
+ """Internal: conv forward.
+
+ Args:
+ x: TODO.
+ weight: TODO.
+ bias: TODO.
+ """
return super()._conv_forward(
x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype)
)
@@ -75,6 +98,12 @@ def sinusoids(length, channels, max_timescale=10000):
class MultiHeadAttention(nn.Module):
def __init__(self, n_state: int, n_head: int):
+ """Initialize MultiHeadAttention.
+
+ Args:
+ n_state: TODO.
+ n_head: TODO.
+ """
super().__init__()
self.n_head = n_head
self.query = Linear(n_state, n_state)
@@ -90,6 +119,15 @@ class MultiHeadAttention(nn.Module):
kv_cache: Optional[dict] = None,
**kwargs,
):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ xa: TODO.
+ mask: TODO.
+ kv_cache: TODO.
+ **kwargs: Additional keyword arguments.
+ """
is_pad_mask = kwargs.get("is_pad_mask", False)
q = self.query(x)
@@ -115,6 +153,15 @@ class MultiHeadAttention(nn.Module):
mask: Optional[Tensor] = None,
**kwargs,
):
+ """Qkv attention.
+
+ Args:
+ q: TODO.
+ k: TODO.
+ v: TODO.
+ mask: TODO.
+ **kwargs: Additional keyword arguments.
+ """
is_pad_mask = kwargs.get("is_pad_mask", False)
n_batch, n_ctx, n_state = q.shape
scale = (n_state // self.n_head) ** -0.25
@@ -143,6 +190,12 @@ class MultiHeadAttention(nn.Module):
class MultiHeadAttentionSdpa(nn.Module):
def __init__(self, n_state: int, n_head: int):
+ """Initialize MultiHeadAttentionSdpa.
+
+ Args:
+ n_state: TODO.
+ n_head: TODO.
+ """
super().__init__()
self.n_head = n_head
self.query = Linear(n_state, n_state)
@@ -158,6 +211,15 @@ class MultiHeadAttentionSdpa(nn.Module):
kv_cache: Optional[dict] = None,
**kwargs,
):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ xa: TODO.
+ mask: TODO.
+ kv_cache: TODO.
+ **kwargs: Additional keyword arguments.
+ """
is_pad_mask = kwargs.get("is_pad_mask", False)
q = self.query(x)
@@ -183,6 +245,15 @@ class MultiHeadAttentionSdpa(nn.Module):
mask: Optional[Tensor] = None,
**kwargs,
):
+ """Qkv attention.
+
+ Args:
+ q: TODO.
+ k: TODO.
+ v: TODO.
+ mask: TODO.
+ **kwargs: Additional keyword arguments.
+ """
is_pad_mask = kwargs.get("is_pad_mask", False)
is_causal = kwargs.get("is_causal", False)
n_batch, n_ctx, n_state = q.shape
@@ -222,6 +293,14 @@ att_type_dict = {
class ResidualAttentionBlock(nn.Module):
def __init__(self, n_state: int, n_head: int, cross_attention: bool = False, **kwargs):
+ """Initialize ResidualAttentionBlock.
+
+ Args:
+ n_state: TODO.
+ n_head: TODO.
+ cross_attention: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
att_type = kwargs.get("att_type", "default")
@@ -245,6 +324,15 @@ class ResidualAttentionBlock(nn.Module):
kv_cache: Optional[dict] = None,
**kwargs,
):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ xa: TODO.
+ mask: TODO.
+ kv_cache: TODO.
+ **kwargs: Additional keyword arguments.
+ """
is_pad_mask = kwargs.get("is_pad_mask", False)
is_pad_memory_mask = kwargs.get("is_pad_memory_mask", False)
memory_mask = kwargs.get("memory_mask", None)
@@ -266,6 +354,16 @@ class ResidualAttentionBlock(nn.Module):
class AudioEncoder(nn.Module):
def __init__(self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int, **kwargs):
+ """Initialize AudioEncoder.
+
+ Args:
+ n_mels: TODO.
+ n_ctx: TODO.
+ n_state: TODO.
+ n_head: TODO.
+ n_layer: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, stride=2, padding=1)
self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)
@@ -301,6 +399,16 @@ class AudioEncoder(nn.Module):
class TextDecoder(nn.Module):
def __init__(self, n_vocab: int, n_ctx: int, n_state: int, n_head: int, n_layer: int, **kwargs):
+ """Initialize TextDecoder.
+
+ Args:
+ n_vocab: TODO.
+ n_ctx: TODO.
+ n_state: TODO.
+ n_head: TODO.
+ n_layer: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
self.token_embedding = nn.Embedding(n_vocab, n_state)
@@ -342,6 +450,11 @@ class TextDecoder(nn.Module):
return logits
def init_state(self, x):
+ """Init state.
+
+ Args:
+ x: TODO.
+ """
state = {}
return state
@@ -369,6 +482,11 @@ class TextDecoder(nn.Module):
class Whisper(nn.Module):
def __init__(self, dims: ModelDimensions):
+ """Initialize Whisper.
+
+ Args:
+ dims: TODO.
+ """
super().__init__()
self.dims = dims
self.encoder = AudioEncoder(
@@ -396,29 +514,54 @@ class Whisper(nn.Module):
# model.register_buffer("alignment_heads", alignment_heads_dense, persistent=False)
def set_alignment_heads(self, dump: bytes):
+ """Set alignment heads.
+
+ Args:
+ dump: TODO.
+ """
array = np.frombuffer(gzip.decompress(base64.b85decode(dump)), dtype=bool).copy()
mask = torch.from_numpy(array).reshape(self.dims.n_text_layer, self.dims.n_text_head)
self.register_buffer("alignment_heads", mask.to_sparse(), persistent=False)
def embed_audio(self, mel: torch.Tensor):
+ """Embed audio.
+
+ Args:
+ mel: TODO.
+ """
return self.encoder(mel)
def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor):
+ """Logits.
+
+ Args:
+ tokens: TODO.
+ audio_features: TODO.
+ """
return self.decoder(tokens, audio_features)
def forward(self, mel: torch.Tensor, tokens: torch.Tensor) -> Dict[str, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ mel: TODO.
+ tokens: TODO.
+ """
return self.decoder(tokens, self.encoder(mel))
@property
def device(self):
+ """Device."""
return next(self.parameters()).device
@property
def is_multilingual(self):
+ """Is multilingual."""
return self.dims.n_vocab >= 51865
@property
def num_languages(self):
+ """Num languages."""
return self.dims.n_vocab - 51765 - int(self.is_multilingual)
def install_kv_cache_hooks(self, cache: Optional[dict] = None):
@@ -439,6 +582,13 @@ class Whisper(nn.Module):
hooks = []
def save_to_cache(module, _, output):
+ """Save to cache.
+
+ Args:
+ module: TODO.
+ _: TODO.
+ output: TODO.
+ """
if module not in cache or output.shape[1] > self.dims.n_text_ctx:
# save as-is, for the first token or cross attention
cache[module] = output
@@ -447,6 +597,11 @@ class Whisper(nn.Module):
return cache[module]
def install_hooks(layer: nn.Module):
+ """Install hooks.
+
+ Args:
+ layer: TODO.
+ """
if isinstance(layer, MultiHeadAttention):
hooks.append(layer.key.register_forward_hook(save_to_cache))
hooks.append(layer.value.register_forward_hook(save_to_cache))
diff --git a/funasr/models/sense_voice/whisper_lib/normalizers/basic.py b/funasr/models/sense_voice/whisper_lib/normalizers/basic.py
index 76addc520..021fddc24 100644
--- a/funasr/models/sense_voice/whisper_lib/normalizers/basic.py
+++ b/funasr/models/sense_voice/whisper_lib/normalizers/basic.py
@@ -58,10 +58,21 @@ def remove_symbols(s: str):
class BasicTextNormalizer:
def __init__(self, remove_diacritics: bool = False, split_letters: bool = False):
+ """Initialize BasicTextNormalizer.
+
+ Args:
+ remove_diacritics: TODO.
+ split_letters: TODO.
+ """
self.clean = remove_symbols_and_diacritics if remove_diacritics else remove_symbols
self.split_letters = split_letters
def __call__(self, s: str):
+ """Internal: call .
+
+ Args:
+ s: TODO.
+ """
s = s.lower()
s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets
s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis
diff --git a/funasr/models/sense_voice/whisper_lib/normalizers/english.py b/funasr/models/sense_voice/whisper_lib/normalizers/english.py
index cb4653098..651e53eb6 100644
--- a/funasr/models/sense_voice/whisper_lib/normalizers/english.py
+++ b/funasr/models/sense_voice/whisper_lib/normalizers/english.py
@@ -21,6 +21,7 @@ class EnglishNumberNormalizer:
"""
def __init__(self):
+ """Initialize EnglishNumberNormalizer."""
super().__init__()
self.zeros = {"o", "oh", "zero"}
@@ -161,17 +162,32 @@ class EnglishNumberNormalizer:
self.literal_words = {"one", "ones"}
def process_words(self, words: List[str]) -> Iterator[str]:
+ """Process words.
+
+ Args:
+ words: TODO.
+ """
prefix: Optional[str] = None
value: Optional[Union[str, int]] = None
skip = False
def to_fraction(s: str):
+ """To fraction.
+
+ Args:
+ s: TODO.
+ """
try:
return Fraction(s)
except ValueError:
return None
def output(result: Union[str, int]):
+ """Output.
+
+ Args:
+ result: TODO.
+ """
nonlocal prefix, value
result = str(result)
if prefix is not None:
@@ -383,6 +399,11 @@ class EnglishNumberNormalizer:
def preprocess(self, s: str):
# replace "<number> and a half" with "<number> point five"
+ """Preprocess.
+
+ Args:
+ s: TODO.
+ """
results = []
segments = re.split(r"\band\s+a\s+half\b", s)
@@ -411,7 +432,17 @@ class EnglishNumberNormalizer:
return s
def postprocess(self, s: str):
+ """Postprocess.
+
+ Args:
+ s: TODO.
+ """
def combine_cents(m: Match):
+ """Combine cents.
+
+ Args:
+ m: TODO.
+ """
try:
currency = m.group(1)
integer = m.group(2)
@@ -421,6 +452,11 @@ class EnglishNumberNormalizer:
return m.string
def extract_cents(m: Match):
+ """Extract cents.
+
+ Args:
+ m: TODO.
+ """
try:
return f"¢{int(m.group(1))}"
except ValueError:
@@ -436,6 +472,11 @@ class EnglishNumberNormalizer:
return s
def __call__(self, s: str):
+ """Internal: call .
+
+ Args:
+ s: TODO.
+ """
s = self.preprocess(s)
s = " ".join(word for word in self.process_words(s.split()) if word is not None)
s = self.postprocess(s)
@@ -451,15 +492,22 @@ class EnglishSpellingNormalizer:
"""
def __init__(self):
+ """Initialize EnglishSpellingNormalizer."""
mapping_path = os.path.join(os.path.dirname(__file__), "english.json")
self.mapping = json.load(open(mapping_path))
def __call__(self, s: str):
+ """Internal: call .
+
+ Args:
+ s: TODO.
+ """
return " ".join(self.mapping.get(word, word) for word in s.split())
class EnglishTextNormalizer:
def __init__(self):
+ """Initialize EnglishTextNormalizer."""
self.ignore_patterns = r"\b(hmm|mm|mhm|mmm|uh|um)\b"
self.replacers = {
# common contractions
@@ -520,6 +568,11 @@ class EnglishTextNormalizer:
self.standardize_spellings = EnglishSpellingNormalizer()
def __call__(self, s: str):
+ """Internal: call .
+
+ Args:
+ s: TODO.
+ """
s = s.lower()
s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets
diff --git a/funasr/models/sense_voice/whisper_lib/timing.py b/funasr/models/sense_voice/whisper_lib/timing.py
index ba9cb1399..4e28fbc59 100644
--- a/funasr/models/sense_voice/whisper_lib/timing.py
+++ b/funasr/models/sense_voice/whisper_lib/timing.py
@@ -54,6 +54,11 @@ def median_filter(x: torch.Tensor, filter_width: int):
@numba.jit(nopython=True)
def backtrace(trace: np.ndarray):
+ """Backtrace.
+
+ Args:
+ trace: TODO.
+ """
i = trace.shape[0] - 1
j = trace.shape[1] - 1
trace[0, :] = 2
@@ -79,6 +84,11 @@ def backtrace(trace: np.ndarray):
@numba.jit(nopython=True, parallel=True)
def dtw_cpu(x: np.ndarray):
+ """Dtw cpu.
+
+ Args:
+ x: TODO.
+ """
N, M = x.shape
cost = np.ones((N + 1, M + 1), dtype=np.float32) * np.inf
trace = -np.ones((N + 1, M + 1), dtype=np.float32)
@@ -104,6 +114,12 @@ def dtw_cpu(x: np.ndarray):
def dtw_cuda(x, BLOCK_SIZE=1024):
+ """Dtw cuda.
+
+ Args:
+ x: TODO.
+ BLOCK_SIZE: TODO.
+ """
from .triton_ops import dtw_kernel
M, N = x.shape
@@ -133,6 +149,11 @@ def dtw_cuda(x, BLOCK_SIZE=1024):
def dtw(x: torch.Tensor) -> np.ndarray:
+ """Dtw.
+
+ Args:
+ x: TODO.
+ """
if x.is_cuda:
try:
return dtw_cuda(x)
@@ -164,6 +185,15 @@ def find_alignment(
medfilt_width: int = 7,
qk_scale: float = 1.0,
) -> List[WordTiming]:
+ """Find alignment.
+
+ Args:
+ model: Model instance or model name.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ text_tokens: TODO.
+ mel: TODO.
+ num_frames: TODO.
+ """
if len(text_tokens) == 0:
return []
@@ -235,6 +265,13 @@ def find_alignment(
def merge_punctuations(alignment: List[WordTiming], prepended: str, appended: str):
# merge prepended punctuations
+ """Merge punctuations.
+
+ Args:
+ alignment: TODO.
+ prepended: TODO.
+ appended: TODO.
+ """
i = len(alignment) - 2
j = len(alignment) - 1
while i >= 0:
@@ -279,6 +316,11 @@ def add_word_timestamps(
last_speech_timestamp: float,
**kwargs,
):
+ """Add word timestamps.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
if len(segments) == 0:
return
diff --git a/funasr/models/sense_voice/whisper_lib/tokenizer.py b/funasr/models/sense_voice/whisper_lib/tokenizer.py
index 5b276c2a0..4a7520519 100644
--- a/funasr/models/sense_voice/whisper_lib/tokenizer.py
+++ b/funasr/models/sense_voice/whisper_lib/tokenizer.py
@@ -168,6 +168,7 @@ class Tokenizer:
special_tokens: Dict[str, int] = field(default_factory=dict)
def __post_init__(self):
+ """Internal: post init ."""
for special in self.encoding.special_tokens_set:
special_token = self.encoding.encode_single_token(special)
self.special_tokens[special] = special_token
@@ -192,9 +193,21 @@ class Tokenizer:
self.sot_sequence = tuple(sot_sequence)
def encode(self, text, **kwargs):
+ """Encode.
+
+ Args:
+ text: Text tensor or string input.
+ **kwargs: Additional keyword arguments.
+ """
return self.encoding.encode(text, **kwargs)
def decode(self, token_ids: List[int], **kwargs) -> str:
+ """Decode.
+
+ Args:
+ token_ids: TODO.
+ **kwargs: Additional keyword arguments.
+ """
token_ids = [t for t in token_ids if t < self.timestamp_begin]
return self.encoding.decode(token_ids, **kwargs)
@@ -206,46 +219,57 @@ class Tokenizer:
return self.encoding.decode(token_ids, **kwargs)
def get_vocab_size(self) -> int:
+ """Get vocab size."""
return self.encoding.n_vocab
@cached_property
def eot(self) -> int:
+ """Eot."""
return self.encoding.eot_token
@cached_property
def transcribe(self) -> int:
+ """Transcribe."""
return self.special_tokens["<|transcribe|>"]
@cached_property
def translate(self) -> int:
+ """Translate."""
return self.special_tokens["<|translate|>"]
@cached_property
def sot(self) -> int:
+ """Sot."""
return self.special_tokens["<|startoftranscript|>"]
@cached_property
def sot_sense(self) -> int:
+ """Sot sense."""
return self.special_tokens["<|startoftranscript|>"]
@cached_property
def sot_lm(self) -> int:
+ """Sot lm."""
return self.special_tokens["<|startoflm|>"]
@cached_property
def sot_prev(self) -> int:
+ """Sot prev."""
return self.special_tokens["<|startofprev|>"]
@cached_property
def no_speech(self) -> int:
+ """No speech."""
return self.special_tokens["<|nospeech|>"]
@cached_property
def no_timestamps(self) -> int:
+ """No timestamps."""
return self.special_tokens["<|notimestamps|>"]
@cached_property
def timestamp_begin(self) -> int:
+ """Timestamp begin."""
return self.special_tokens["<|0.00|>"]
@cached_property
@@ -257,6 +281,11 @@ class Tokenizer:
return self.to_language_token(self.language)
def to_language_token(self, language):
+ """To language token.
+
+ Args:
+ language: Language identifier.
+ """
if token := self.special_tokens.get(f"<|{language}|>", None):
return token
@@ -264,6 +293,7 @@ class Tokenizer:
@cached_property
def all_language_tokens(self) -> Tuple[int]:
+ """All language tokens."""
result = []
for token, token_id in self.special_tokens.items():
if token.strip("<|>") in LANGUAGES:
@@ -272,10 +302,12 @@ class Tokenizer:
@cached_property
def all_language_codes(self) -> Tuple[str]:
+ """All language codes."""
return tuple(self.decode([_l]).strip("<|>") for _l in self.all_language_tokens)
@cached_property
def sot_sequence_including_notimestamps(self) -> Tuple[int]:
+ """Sot sequence including notimestamps."""
return tuple(list(self.sot_sequence) + [self.no_timestamps])
@cached_property
@@ -313,6 +345,11 @@ class Tokenizer:
return tuple(sorted(result))
def split_to_word_tokens(self, tokens: List[int]):
+ """Split to word tokens.
+
+ Args:
+ tokens: TODO.
+ """
if self.language in {"zh", "ja", "th", "lo", "my", "yue"}:
# These languages don't typically use spaces, so it is difficult to split words
# without morpheme analysis. Here, we instead split words at any
@@ -322,6 +359,11 @@ class Tokenizer:
return self.split_tokens_on_spaces(tokens)
def split_tokens_on_unicode(self, tokens: List[int]):
+ """Split tokens on unicode.
+
+ Args:
+ tokens: TODO.
+ """
decoded_full = self.decode_with_timestamps(tokens)
replacement_char = "\ufffd"
@@ -347,6 +389,11 @@ class Tokenizer:
return words, word_tokens
def split_tokens_on_spaces(self, tokens: List[int]):
+ """Split tokens on spaces.
+
+ Args:
+ tokens: TODO.
+ """
subwords, subword_tokens_list = self.split_tokens_on_unicode(tokens)
words = []
word_tokens = []
@@ -367,6 +414,13 @@ class Tokenizer:
@lru_cache(maxsize=None)
def get_encoding(name: str = "gpt2", num_languages: int = 99, vocab_path: str = None):
+ """Get encoding.
+
+ Args:
+ name: TODO.
+ num_languages: TODO.
+ vocab_path: TODO.
+ """
if vocab_path is None:
vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken")
@@ -430,6 +484,11 @@ def get_tokenizer(
encoding_path: Optional[str] = None,
vocab_path: Optional[str] = None,
) -> Tokenizer:
+ """Get tokenizer.
+
+ Args:
+ multilingual: TODO.
+ """
if language is not None:
language = language.lower()
if language not in LANGUAGES:
diff --git a/funasr/models/sense_voice/whisper_lib/transcribe.py b/funasr/models/sense_voice/whisper_lib/transcribe.py
index 5c5f49d80..3124f8245 100644
--- a/funasr/models/sense_voice/whisper_lib/transcribe.py
+++ b/funasr/models/sense_voice/whisper_lib/transcribe.py
@@ -174,6 +174,11 @@ def transcribe(
warnings.warn("Word-level timestamps on translations may not be reliable.")
def decode_with_fallback(segment: torch.Tensor) -> DecodingResult:
+ """Decode with fallback.
+
+ Args:
+ segment: TODO.
+ """
temperatures = [temperature] if isinstance(temperature, (int, float)) else temperature
decode_result = None
@@ -225,6 +230,7 @@ def transcribe(
initial_prompt_tokens = []
def new_segment(*, start: float, end: float, tokens: torch.Tensor, result: DecodingResult):
+ """New segment."""
tokens = tokens.tolist()
text_tokens = [token for token in tokens if token < tokenizer.eot]
return {
@@ -282,6 +288,11 @@ def transcribe(
# anomalous words are very long/short/improbable
def word_anomaly_score(word: dict) -> float:
+ """Word anomaly score.
+
+ Args:
+ word: TODO.
+ """
probability = word.get("probability", 0.0)
duration = word["end"] - word["start"]
score = 0.0
@@ -294,6 +305,11 @@ def transcribe(
return score
def is_segment_anomaly(segment: Optional[dict]) -> bool:
+ """Is segment anomaly.
+
+ Args:
+ segment: TODO.
+ """
if segment is None or not segment["words"]:
return False
words = [w for w in segment["words"] if w["word"] not in punctuation]
@@ -302,6 +318,11 @@ def transcribe(
return score >= 3 or score + 0.01 >= len(words)
def next_words_segment(segments: List[dict]) -> Optional[dict]:
+ """Next words segment.
+
+ Args:
+ segments: TODO.
+ """
return next((s for s in segments if s["words"]), None)
timestamp_tokens: torch.Tensor = tokens.ge(tokenizer.timestamp_begin)
@@ -467,9 +488,15 @@ def transcribe(
def cli():
+ """Cli."""
from . import available_models
def valid_model_name(name):
+ """Valid model name.
+
+ Args:
+ name: TODO.
+ """
if name in available_models() or os.path.exists(name):
return name
raise ValueError(
diff --git a/funasr/models/sense_voice/whisper_lib/triton_ops.py b/funasr/models/sense_voice/whisper_lib/triton_ops.py
index 9919595ce..ecb27b6ac 100644
--- a/funasr/models/sense_voice/whisper_lib/triton_ops.py
+++ b/funasr/models/sense_voice/whisper_lib/triton_ops.py
@@ -12,6 +12,19 @@ except ImportError:
@triton.jit
def dtw_kernel(cost, trace, x, x_stride, cost_stride, trace_stride, N, M, BLOCK_SIZE: tl.constexpr):
+ """Dtw kernel.
+
+ Args:
+ cost: TODO.
+ trace: TODO.
+ x: TODO.
+ x_stride: TODO.
+ cost_stride: TODO.
+ trace_stride: TODO.
+ N: TODO.
+ M: TODO.
+ BLOCK_SIZE: TODO.
+ """
offsets = tl.arange(0, BLOCK_SIZE)
mask = offsets < M
@@ -41,7 +54,21 @@ def dtw_kernel(cost, trace, x, x_stride, cost_stride, trace_stride, N, M, BLOCK_
@lru_cache(maxsize=None)
def median_kernel(filter_width: int):
@triton.jit
+ """Median kernel.
+
+ Args:
+ filter_width: TODO.
+ """
def kernel(y, x, x_stride, y_stride, BLOCK_SIZE: tl.constexpr): # x.shape[-1] == filter_width
+ """Kernel.
+
+ Args:
+ y: TODO.
+ x: TODO.
+ x_stride: TODO.
+ y_stride: TODO.
+ BLOCK_SIZE: TODO.
+ """
row_idx = tl.program_id(0)
offsets = tl.arange(0, BLOCK_SIZE)
mask = offsets < y_stride
diff --git a/funasr/models/sense_voice/whisper_lib/utils.py b/funasr/models/sense_voice/whisper_lib/utils.py
index 5fc61256f..bbd3f6cff 100644
--- a/funasr/models/sense_voice/whisper_lib/utils.py
+++ b/funasr/models/sense_voice/whisper_lib/utils.py
@@ -12,21 +12,42 @@ if system_encoding != "utf-8":
def make_safe(string):
# replaces any character not representable using the system default encoding with an '?',
# avoiding UnicodeEncodeError (https://github.com/openai/whisper/discussions/729).
+ """Make safe.
+
+ Args:
+ string: TODO.
+ """
return string.encode(system_encoding, errors="replace").decode(system_encoding)
else:
def make_safe(string):
# utf-8 can encode any Unicode code point, so no need to do the round-trip encoding
+ """Make safe.
+
+ Args:
+ string: TODO.
+ """
return string
def exact_div(x, y):
+ """Exact div.
+
+ Args:
+ x: TODO.
+ y: TODO.
+ """
assert x % y == 0
return x // y
def str2bool(string):
+ """Str2bool.
+
+ Args:
+ string: TODO.
+ """
str2val = {"True": True, "False": False}
if string in str2val:
return str2val[string]
@@ -35,19 +56,41 @@ def str2bool(string):
def optional_int(string):
+ """Optional int.
+
+ Args:
+ string: TODO.
+ """
return None if string == "None" else int(string)
def optional_float(string):
+ """Optional float.
+
+ Args:
+ string: TODO.
+ """
return None if string == "None" else float(string)
def compression_ratio(text) -> float:
+ """Compression ratio.
+
+ Args:
+ text: Text tensor or string input.
+ """
text_bytes = text.encode("utf-8")
return len(text_bytes) / len(zlib.compress(text_bytes))
def format_timestamp(seconds: float, always_include_hours: bool = False, decimal_marker: str = "."):
+ """Format timestamp.
+
+ Args:
+ seconds: TODO.
+ always_include_hours: TODO.
+ decimal_marker: TODO.
+ """
assert seconds >= 0, "non-negative timestamp expected"
milliseconds = round(seconds * 1000.0)
@@ -65,6 +108,11 @@ def format_timestamp(seconds: float, always_include_hours: bool = False, decimal
def get_start(segments: List[dict]) -> Optional[float]:
+ """Get start.
+
+ Args:
+ segments: TODO.
+ """
return next(
(w["start"] for s in segments for w in s["words"]),
segments[0]["start"] if segments else None,
@@ -72,6 +120,11 @@ def get_start(segments: List[dict]) -> Optional[float]:
def get_end(segments: List[dict]) -> Optional[float]:
+ """Get end.
+
+ Args:
+ segments: TODO.
+ """
return next(
(w["end"] for s in reversed(segments) for w in reversed(s["words"])),
segments[-1]["end"] if segments else None,
@@ -82,9 +135,22 @@ class ResultWriter:
extension: str
def __init__(self, output_dir: str):
+ """Initialize ResultWriter.
+
+ Args:
+ output_dir: Directory for saving output files.
+ """
self.output_dir = output_dir
def __call__(self, result: dict, audio_path: str, options: Optional[dict] = None, **kwargs):
+ """Internal: call .
+
+ Args:
+ result: TODO.
+ audio_path: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
audio_basename = os.path.basename(audio_path)
audio_basename = os.path.splitext(audio_basename)[0]
output_path = os.path.join(self.output_dir, audio_basename + "." + self.extension)
@@ -93,6 +159,14 @@ class ResultWriter:
self.write_result(result, file=f, options=options, **kwargs)
def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):
+ """Write result.
+
+ Args:
+ result: TODO.
+ file: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
raise NotImplementedError
@@ -100,6 +174,14 @@ class WriteTXT(ResultWriter):
extension: str = "txt"
def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):
+ """Write result.
+
+ Args:
+ result: TODO.
+ file: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
for segment in result["segments"]:
print(segment["text"].strip(), file=file, flush=True)
@@ -118,6 +200,12 @@ class SubtitlesWriter(ResultWriter):
highlight_words: bool = False,
max_words_per_line: Optional[int] = None,
):
+ """Iterate result.
+
+ Args:
+ result: TODO.
+ options: TODO.
+ """
options = options or {}
max_line_width = max_line_width or options.get("max_line_width")
max_line_count = max_line_count or options.get("max_line_count")
@@ -128,6 +216,7 @@ class SubtitlesWriter(ResultWriter):
max_words_per_line = max_words_per_line or 1000
def iterate_subtitles():
+ """Iterate subtitles."""
line_len = 0
line_count = 1
# the next subtitle to yield (a list of word timings with whitespace)
@@ -205,6 +294,11 @@ class SubtitlesWriter(ResultWriter):
yield segment_start, segment_end, segment_text
def format_timestamp(self, seconds: float):
+ """Format timestamp.
+
+ Args:
+ seconds: TODO.
+ """
return format_timestamp(
seconds=seconds,
always_include_hours=self.always_include_hours,
@@ -218,6 +312,14 @@ class WriteVTT(SubtitlesWriter):
decimal_marker: str = "."
def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):
+ """Write result.
+
+ Args:
+ result: TODO.
+ file: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
print("WEBVTT\n", file=file)
for start, end, text in self.iterate_result(result, options, **kwargs):
print(f"{start} --> {end}\n{text}\n", file=file, flush=True)
@@ -229,6 +331,14 @@ class WriteSRT(SubtitlesWriter):
decimal_marker: str = ","
def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):
+ """Write result.
+
+ Args:
+ result: TODO.
+ file: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
for i, (start, end, text) in enumerate(
self.iterate_result(result, options, **kwargs), start=1
):
@@ -248,6 +358,14 @@ class WriteTSV(ResultWriter):
extension: str = "tsv"
def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):
+ """Write result.
+
+ Args:
+ result: TODO.
+ file: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
print("start", "end", "text", sep="\t", file=file)
for segment in result["segments"]:
print(round(1000 * segment["start"]), file=file, end="\t")
@@ -259,10 +377,24 @@ class WriteJSON(ResultWriter):
extension: str = "json"
def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):
+ """Write result.
+
+ Args:
+ result: TODO.
+ file: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
json.dump(result, file)
def get_writer(output_format: str, output_dir: str) -> Callable[[dict, TextIO, dict], None]:
+ """Get writer.
+
+ Args:
+ output_format: TODO.
+ output_dir: Directory for saving output files.
+ """
writers = {
"txt": WriteTXT,
"vtt": WriteVTT,
@@ -275,6 +407,14 @@ def get_writer(output_format: str, output_dir: str) -> Callable[[dict, TextIO, d
all_writers = [writer(output_dir) for writer in writers.values()]
def write_all(result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):
+ """Write all.
+
+ Args:
+ result: TODO.
+ file: TODO.
+ options: TODO.
+ **kwargs: Additional keyword arguments.
+ """
for writer in all_writers:
writer(result, file, options, **kwargs)
diff --git a/funasr/models/sond/e2e_diar_sond.py b/funasr/models/sond/e2e_diar_sond.py
index 0f5a7c27c..44cf6d361 100644
--- a/funasr/models/sond/e2e_diar_sond.py
+++ b/funasr/models/sond/e2e_diar_sond.py
@@ -34,6 +34,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -69,6 +74,33 @@ class DiarSondModel(FunASRModel):
onfly_shuffle_speaker: bool = True,
):
+ """Initialize DiarSondModel.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ frontend: Audio frontend for feature extraction.
+ specaug: TODO.
+ profileaug: TODO.
+ normalize: TODO.
+ encoder: TODO.
+ speaker_encoder: TODO.
+ ci_scorer: TODO.
+ cd_scorer: TODO.
+ decoder: TODO.
+ token_list: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ max_spk_num: TODO.
+ label_aggregator: TODO.
+ normalize_speech_speaker: TODO.
+ ignore_id: TODO.
+ speaker_discrimination_loss_weight: TODO.
+ inter_score_loss_weight: TODO.
+ inputs_type: TODO.
+ model_regularizer_weight: TODO.
+ freeze_encoder: TODO.
+ onfly_shuffle_speaker: TODO.
+ """
super().__init__()
self.encoder = encoder
@@ -109,6 +141,7 @@ class DiarSondModel(FunASRModel):
self.to_regularize_parameters = None
def get_regularize_parameters(self):
+ """Get regularize parameters."""
to_regularize_parameters, normal_parameters = [], []
for name, param in self.named_parameters():
if (
@@ -124,6 +157,7 @@ class DiarSondModel(FunASRModel):
return to_regularize_parameters, normal_parameters
def generate_pse_embedding(self):
+ """Generate pse embedding."""
embedding = np.zeros((len(self.token_list), self.max_spk_num), dtype=np.float32)
for idx, pse_label in enumerate(self.token_list):
emb = int2vec(int(pse_label), vec_dim=self.max_spk_num, dtype=np.float32)
@@ -299,6 +333,7 @@ class DiarSondModel(FunASRModel):
return loss, stats, weight
def calculate_regularizer_loss(self):
+ """Calculate regularizer loss."""
regularizer_loss = 0.0
for name, param in self.to_regularize_parameters:
regularizer_loss = regularizer_loss + torch.norm(param, p=2)
@@ -307,6 +342,13 @@ class DiarSondModel(FunASRModel):
def classification_loss(
self, predictions: torch.Tensor, labels: torch.Tensor, prediction_lengths: torch.Tensor
) -> torch.Tensor:
+ """Classification loss.
+
+ Args:
+ predictions: TODO.
+ labels: TODO.
+ prediction_lengths: Lengths of prediction.
+ """
mask = make_pad_mask(prediction_lengths, maxlen=labels.shape[1])
pad_labels = labels.masked_fill(mask.to(predictions.device), value=self.ignore_id)
loss = self.criterion_diar(predictions.contiguous(), pad_labels)
@@ -316,6 +358,12 @@ class DiarSondModel(FunASRModel):
def speaker_discrimination_loss(
self, profile: torch.Tensor, profile_lengths: torch.Tensor
) -> torch.Tensor:
+ """Speaker discrimination loss.
+
+ Args:
+ profile: TODO.
+ profile_lengths: Lengths of profile.
+ """
profile_mask = (
torch.linalg.norm(profile, ord=2, dim=2, keepdim=True) > 0
).float() # (B, N, 1)
@@ -339,6 +387,12 @@ class DiarSondModel(FunASRModel):
return loss
def calculate_multi_labels(self, pse_labels, pse_labels_lengths):
+ """Calculate multi labels.
+
+ Args:
+ pse_labels: TODO.
+ pse_labels_lengths: Lengths of pse_labels.
+ """
mask = make_pad_mask(pse_labels_lengths, maxlen=pse_labels.shape[1])
padding_labels = pse_labels.masked_fill(mask.to(pse_labels.device), value=0).to(pse_labels)
multi_labels = F.embedding(padding_labels, self.pse_embedding)
@@ -352,6 +406,14 @@ class DiarSondModel(FunASRModel):
pse_labels: torch.Tensor,
pse_labels_lengths: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Internal score loss.
+
+ Args:
+ cd_score: TODO.
+ ci_score: TODO.
+ pse_labels: TODO.
+ pse_labels_lengths: Lengths of pse_labels.
+ """
multi_labels = self.calculate_multi_labels(pse_labels, pse_labels_lengths)
ci_loss = self.criterion_bce(ci_score, multi_labels, pse_labels_lengths)
cd_loss = self.criterion_bce(cd_score, multi_labels, pse_labels_lengths)
@@ -366,6 +428,16 @@ class DiarSondModel(FunASRModel):
binary_labels: torch.Tensor = None,
binary_labels_lengths: torch.Tensor = None,
) -> Dict[str, torch.Tensor]:
+ """Collect feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ profile: TODO.
+ profile_lengths: Lengths of profile.
+ binary_labels: TODO.
+ binary_labels_lengths: Lengths of binary_labels.
+ """
feats, feats_lengths = self._extract_feats(speech, speech_lengths)
return {"feats": feats, "feats_lengths": feats_lengths}
@@ -374,6 +446,12 @@ class DiarSondModel(FunASRModel):
profile: torch.Tensor,
profile_lengths: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Encode speaker.
+
+ Args:
+ profile: TODO.
+ profile_lengths: Lengths of profile.
+ """
with autocast(False):
if profile.shape[1] < self.max_spk_num:
profile = F.pad(
@@ -392,6 +470,12 @@ class DiarSondModel(FunASRModel):
speech: torch.Tensor,
speech_lengths: torch.Tensor,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Encode speech.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
if self.encoder is not None and self.inputs_type == "raw":
speech, speech_lengths = self.encode(speech, speech_lengths)
speech_mask = ~make_pad_mask(speech_lengths, maxlen=speech.shape[1])
@@ -402,6 +486,12 @@ class DiarSondModel(FunASRModel):
@staticmethod
def concate_speech_ivc(speech: torch.Tensor, ivc: torch.Tensor) -> torch.Tensor:
+ """Concate speech ivc.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ ivc: TODO.
+ """
nn, tt = ivc.shape[1], speech.shape[1]
speech = speech.unsqueeze(dim=1) # B x 1 x T x D
speech = speech.expand(-1, nn, -1, -1) # B x N x T x D
@@ -417,6 +507,14 @@ class DiarSondModel(FunASRModel):
seq_len: torch.Tensor = None,
spk_len: torch.Tensor = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Calc similarity.
+
+ Args:
+ speech_encoder_outputs: TODO.
+ speaker_encoder_outputs: TODO.
+ seq_len: TODO.
+ spk_len: TODO.
+ """
bb, tt = speech_encoder_outputs.shape[0], speech_encoder_outputs.shape[1]
d_sph, d_spk = speech_encoder_outputs.shape[2], speaker_encoder_outputs.shape[2]
if self.normalize_speech_speaker:
@@ -439,6 +537,12 @@ class DiarSondModel(FunASRModel):
return ci_simi, cd_simi
def post_net_forward(self, simi, seq_len):
+ """Post net forward.
+
+ Args:
+ simi: TODO.
+ seq_len: TODO.
+ """
logits = self.decoder(simi, seq_len)[0]
return logits
@@ -452,6 +556,15 @@ class DiarSondModel(FunASRModel):
return_inter_outputs: bool = False,
) -> [torch.Tensor, Optional[list]]:
# speech encoding
+ """Prediction forward.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ profile: TODO.
+ profile_lengths: Lengths of profile.
+ return_inter_outputs: TODO.
+ """
speech, speech_lengths = self.encode_speech(speech, speech_lengths)
# speaker encoding
profile, profile_lengths = self.encode_speaker(profile, profile_lengths)
@@ -510,6 +623,12 @@ class DiarSondModel(FunASRModel):
def _extract_feats(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Internal: extract feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
batch_size = speech.shape[0]
speech_lengths = (
speech_lengths
@@ -537,6 +656,13 @@ class DiarSondModel(FunASRModel):
def calc_diarization_error(pred, label, length):
# Note (jiatong): Credit to https://github.com/hitachi-speech/EEND
+ """Calc diarization error.
+
+ Args:
+ pred: TODO.
+ label: TODO.
+ length: TODO.
+ """
(batch_size, max_len, num_output) = label.size()
# mask the padding part
mask = ~make_pad_mask(length, maxlen=label.shape[1]).unsqueeze(-1).numpy()
diff --git a/funasr/models/sond/encoder/ci_scorers.py b/funasr/models/sond/encoder/ci_scorers.py
index a60560b7c..043fa30d2 100644
--- a/funasr/models/sond/encoder/ci_scorers.py
+++ b/funasr/models/sond/encoder/ci_scorers.py
@@ -4,6 +4,7 @@ from torch.nn import functional as F
class DotScorer(torch.nn.Module):
def __init__(self):
+ """Initialize DotScorer."""
super().__init__()
def forward(
@@ -13,12 +14,19 @@ class DotScorer(torch.nn.Module):
):
# xs_pad: B, T, D
# spk_emb: B, N, D
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ spk_emb: TODO.
+ """
scores = torch.matmul(xs_pad, spk_emb.transpose(1, 2))
return scores
class CosScorer(torch.nn.Module):
def __init__(self):
+ """Initialize CosScorer."""
super().__init__()
def forward(
@@ -28,5 +36,11 @@ class CosScorer(torch.nn.Module):
):
# xs_pad: B, T, D
# spk_emb: B, N, D
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ spk_emb: TODO.
+ """
scores = F.cosine_similarity(xs_pad.unsqueeze(2), spk_emb.unsqueeze(1), dim=-1)
return scores
diff --git a/funasr/models/sond/encoder/conv_encoder.py b/funasr/models/sond/encoder/conv_encoder.py
index c4f7098c7..44023ef56 100644
--- a/funasr/models/sond/encoder/conv_encoder.py
+++ b/funasr/models/sond/encoder/conv_encoder.py
@@ -26,6 +26,17 @@ class EncoderLayer(nn.Module):
include_batch_norm=False,
residual=False,
):
+ """Initialize EncoderLayer.
+
+ Args:
+ input_units: TODO.
+ num_units: TODO.
+ kernel_size: Size/dimension parameter.
+ activation: TODO.
+ stride: TODO.
+ include_batch_norm: TODO.
+ residual: TODO.
+ """
super().__init__()
left_padding = math.ceil((kernel_size - stride) / 2)
right_padding = kernel_size - stride - left_padding
@@ -47,12 +58,23 @@ class EncoderLayer(nn.Module):
@staticmethod
def get_activation(activation):
+ """Get activation.
+
+ Args:
+ activation: TODO.
+ """
if activation == "tanh":
return nn.Tanh()
else:
return nn.ReLU()
def forward(self, xs_pad, ilens=None):
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ """
outputs = self.conv1d(self.conv_padding(xs_pad))
if self.residual and self.stride == 1 and self.input_units == self.num_units:
outputs = outputs + xs_pad
@@ -89,6 +111,26 @@ class ConvEncoder(AbsEncoder):
tf2torch_tensor_name_prefix_torch: str = "speaker_encoder",
tf2torch_tensor_name_prefix_tf: str = "EAND/speaker_encoder",
):
+ """Initialize ConvEncoder.
+
+ Args:
+ num_layers: TODO.
+ input_units: TODO.
+ num_units: TODO.
+ kernel_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ position_encoder: TODO.
+ activation: TODO.
+ auxiliary_states: TODO.
+ out_units: TODO.
+ out_norm: TODO.
+ out_residual: TODO.
+ include_batchnorm: TODO.
+ regularization_weight: TODO.
+ stride: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ """
super().__init__()
self._output_size = num_units
@@ -143,6 +185,7 @@ class ConvEncoder(AbsEncoder):
self.after_norm = LayerNorm(out_units)
def output_size(self) -> int:
+ """Output size."""
return self.num_units
def forward(
@@ -152,6 +195,13 @@ class ConvEncoder(AbsEncoder):
prev_states: torch.Tensor = None,
) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ prev_states: TODO.
+ """
inputs = xs_pad
if self.position_encoder is not None:
inputs = self.position_encoder(inputs)
diff --git a/funasr/models/sond/encoder/ecapa_tdnn_encoder.py b/funasr/models/sond/encoder/ecapa_tdnn_encoder.py
index 1af8b7033..a66b5db47 100644
--- a/funasr/models/sond/encoder/ecapa_tdnn_encoder.py
+++ b/funasr/models/sond/encoder/ecapa_tdnn_encoder.py
@@ -16,6 +16,18 @@ class _BatchNorm1d(nn.Module):
combine_batch_time=False,
skip_transpose=False,
):
+ """Initialize _BatchNorm1d.
+
+ Args:
+ input_shape: TODO.
+ input_size: Size/dimension parameter.
+ eps: TODO.
+ momentum: TODO.
+ affine: TODO.
+ track_running_stats: TODO.
+ combine_batch_time: TODO.
+ skip_transpose: TODO.
+ """
super().__init__()
self.combine_batch_time = combine_batch_time
self.skip_transpose = skip_transpose
@@ -34,6 +46,11 @@ class _BatchNorm1d(nn.Module):
)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
shape_or = x.shape
if self.combine_batch_time:
if x.ndim == 3:
@@ -69,6 +86,21 @@ class _Conv1d(nn.Module):
padding_mode="reflect",
skip_transpose=False,
):
+ """Initialize _Conv1d.
+
+ Args:
+ out_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ input_shape: TODO.
+ in_channels: TODO.
+ stride: TODO.
+ dilation: TODO.
+ padding: TODO.
+ groups: TODO.
+ bias: TODO.
+ padding_mode: TODO.
+ skip_transpose: TODO.
+ """
super().__init__()
self.kernel_size = kernel_size
self.stride = stride
@@ -96,6 +128,11 @@ class _Conv1d(nn.Module):
)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
if not self.skip_transpose:
x = x.transpose(1, -1)
@@ -133,6 +170,14 @@ class _Conv1d(nn.Module):
stride: int,
):
# Detecting input shape
+ """Internal: manage padding.
+
+ Args:
+ x: TODO.
+ kernel_size: Size/dimension parameter.
+ dilation: TODO.
+ stride: TODO.
+ """
L_in = x.shape[-1]
# Time padding
@@ -165,6 +210,14 @@ class _Conv1d(nn.Module):
def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int):
+ """Get padding elem.
+
+ Args:
+ L_in: TODO.
+ stride: TODO.
+ kernel_size: Size/dimension parameter.
+ dilation: TODO.
+ """
if stride > 1:
n_steps = math.ceil(((L_in - kernel_size * dilation) / stride) + 1)
L_out = stride * (n_steps - 1) + kernel_size * dilation
@@ -180,15 +233,35 @@ def get_padding_elem(L_in: int, stride: int, kernel_size: int, dilation: int):
# Skip transpose as much as possible for efficiency
class Conv1d(_Conv1d):
def __init__(self, *args, **kwargs):
+ """Initialize Conv1d.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(skip_transpose=True, *args, **kwargs)
class BatchNorm1d(_BatchNorm1d):
def __init__(self, *args, **kwargs):
+ """Initialize BatchNorm1d.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(skip_transpose=True, *args, **kwargs)
def length_to_mask(length, max_len=None, dtype=None, device=None):
+ """Length to mask.
+
+ Args:
+ length: TODO.
+ max_len: TODO.
+ dtype: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
assert len(length.shape) == 1
if max_len is None:
@@ -217,6 +290,16 @@ class TDNNBlock(nn.Module):
activation=nn.ReLU,
groups=1,
):
+ """Initialize TDNNBlock.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ dilation: TODO.
+ activation: TODO.
+ groups: TODO.
+ """
super(TDNNBlock, self).__init__()
self.conv = Conv1d(
in_channels=in_channels,
@@ -229,6 +312,11 @@ class TDNNBlock(nn.Module):
self.norm = BatchNorm1d(input_size=out_channels)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
return self.norm(self.activation(self.conv(x)))
@@ -258,6 +346,15 @@ class Res2NetBlock(torch.nn.Module):
"""
def __init__(self, in_channels, out_channels, scale=8, kernel_size=3, dilation=1):
+ """Initialize Res2NetBlock.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ scale: TODO.
+ kernel_size: Size/dimension parameter.
+ dilation: TODO.
+ """
super(Res2NetBlock, self).__init__()
assert in_channels % scale == 0
assert out_channels % scale == 0
@@ -279,6 +376,11 @@ class Res2NetBlock(torch.nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
y = []
for i, x_i in enumerate(torch.chunk(x, self.scale, dim=1)):
if i == 0:
@@ -315,6 +417,13 @@ class SEBlock(nn.Module):
"""
def __init__(self, in_channels, se_channels, out_channels):
+ """Initialize SEBlock.
+
+ Args:
+ in_channels: TODO.
+ se_channels: TODO.
+ out_channels: TODO.
+ """
super(SEBlock, self).__init__()
self.conv1 = Conv1d(in_channels=in_channels, out_channels=se_channels, kernel_size=1)
@@ -323,6 +432,12 @@ class SEBlock(nn.Module):
self.sigmoid = torch.nn.Sigmoid()
def forward(self, x, lengths=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ lengths: TODO.
+ """
L = x.shape[-1]
if lengths is not None:
mask = length_to_mask(lengths * L, max_len=L, device=x.device)
@@ -360,6 +475,13 @@ class AttentiveStatisticsPooling(nn.Module):
"""
def __init__(self, channels, attention_channels=128, global_context=True):
+ """Initialize AttentiveStatisticsPooling.
+
+ Args:
+ channels: TODO.
+ attention_channels: TODO.
+ global_context: TODO.
+ """
super().__init__()
self.eps = 1e-12
@@ -382,6 +504,14 @@ class AttentiveStatisticsPooling(nn.Module):
L = x.shape[-1]
def _compute_statistics(x, m, dim=2, eps=self.eps):
+ """Internal: compute statistics.
+
+ Args:
+ x: TODO.
+ m: TODO.
+ dim: TODO.
+ eps: TODO.
+ """
mean = (m * x).sum(dim)
std = torch.sqrt((m * (x - mean.unsqueeze(dim)).pow(2)).sum(dim).clamp(eps))
return mean, std
@@ -460,6 +590,18 @@ class SERes2NetBlock(nn.Module):
activation=torch.nn.ReLU,
groups=1,
):
+ """Initialize SERes2NetBlock.
+
+ Args:
+ in_channels: TODO.
+ out_channels: TODO.
+ res2net_scale: TODO.
+ se_channels: TODO.
+ kernel_size: Size/dimension parameter.
+ dilation: TODO.
+ activation: TODO.
+ groups: TODO.
+ """
super().__init__()
self.out_channels = out_channels
self.tdnn1 = TDNNBlock(
@@ -492,6 +634,12 @@ class SERes2NetBlock(nn.Module):
)
def forward(self, x, lengths=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ lengths: TODO.
+ """
residual = x
if self.shortcut:
residual = self.shortcut(x)
@@ -550,6 +698,23 @@ class ECAPA_TDNN(torch.nn.Module):
window_shift=1,
):
+ """Initialize ECAPA_TDNN.
+
+ Args:
+ input_size: Size/dimension parameter.
+ lin_neurons: TODO.
+ activation: TODO.
+ channels: TODO.
+ kernel_sizes: TODO.
+ dilations: TODO.
+ attention_channels: TODO.
+ res2net_scale: TODO.
+ se_channels: TODO.
+ global_context: TODO.
+ groups: TODO.
+ window_size: Size/dimension parameter.
+ window_shift: TODO.
+ """
super().__init__()
assert len(channels) == len(kernel_sizes)
assert len(channels) == len(dilations)
@@ -612,6 +777,12 @@ class ECAPA_TDNN(torch.nn.Module):
def windowed_pooling(self, x, lengths=None):
# x: Batch, Channel, Time
+ """Windowed pooling.
+
+ Args:
+ x: TODO.
+ lengths: TODO.
+ """
tt = x.shape[2]
num_chunk = int(math.ceil(tt / self.window_shift))
pad = self.window_size // 2
diff --git a/funasr/models/sond/encoder/fsmn_encoder.py b/funasr/models/sond/encoder/fsmn_encoder.py
index 9ec9912a4..b1c785198 100644
--- a/funasr/models/sond/encoder/fsmn_encoder.py
+++ b/funasr/models/sond/encoder/fsmn_encoder.py
@@ -24,6 +24,14 @@ class FsmnBlock(torch.nn.Module):
kernel_size,
fsmn_shift=0,
):
+ """Initialize FsmnBlock.
+
+ Args:
+ n_feat: TODO.
+ dropout_rate: TODO.
+ kernel_size: Size/dimension parameter.
+ fsmn_shift: TODO.
+ """
super().__init__()
self.dropout = nn.Dropout(p=dropout_rate)
self.fsmn_block = nn.Conv1d(
@@ -37,6 +45,13 @@ class FsmnBlock(torch.nn.Module):
self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
def forward(self, inputs, mask, mask_shfit_chunk=None):
+ """Forward pass for training.
+
+ Args:
+ inputs: TODO.
+ mask: TODO.
+ mask_shfit_chunk: TODO.
+ """
b, t, d = inputs.size()
if mask is not None:
mask = torch.reshape(mask, (b, -1, 1))
@@ -55,6 +70,15 @@ class FsmnBlock(torch.nn.Module):
class EncoderLayer(torch.nn.Module):
def __init__(self, in_size, size, feed_forward, fsmn_block, dropout_rate=0.0):
+ """Initialize EncoderLayer.
+
+ Args:
+ in_size: Size/dimension parameter.
+ size: TODO.
+ feed_forward: TODO.
+ fsmn_block: TODO.
+ dropout_rate: TODO.
+ """
super().__init__()
self.in_size = in_size
self.size = size
@@ -67,6 +91,12 @@ class EncoderLayer(torch.nn.Module):
) -> Tuple[torch.Tensor, torch.Tensor]:
# xs_pad in Batch, Time, Dim
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ mask: TODO.
+ """
context = self.ffn(xs_pad)[0]
memory = self.memory(context, mask)
@@ -160,11 +190,19 @@ class FsmnEncoder(AbsEncoder):
self.conv1d = nn.Conv1d(num_memory_units, out_units, 1, 1)
def output_size(self) -> int:
+ """Output size."""
return self.num_memory_units
def forward(
self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None
) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ prev_states: TODO.
+ """
inputs = xs_pad
if self.position_encoder is not None:
inputs = self.position_encoder(inputs)
diff --git a/funasr/models/sond/encoder/resnet34_encoder.py b/funasr/models/sond/encoder/resnet34_encoder.py
index ba577a3a4..ddbdc3571 100644
--- a/funasr/models/sond/encoder/resnet34_encoder.py
+++ b/funasr/models/sond/encoder/resnet34_encoder.py
@@ -12,6 +12,14 @@ class BasicLayer(torch.nn.Module):
def __init__(self, in_filters: int, filters: int, stride: int, bn_momentum: float = 0.5):
+ """Initialize BasicLayer.
+
+ Args:
+ in_filters: TODO.
+ filters: TODO.
+ stride: TODO.
+ bn_momentum: TODO.
+ """
super().__init__()
self.stride = stride
self.in_filters = in_filters
@@ -31,6 +39,12 @@ class BasicLayer(torch.nn.Module):
def proper_padding(self, x, stride):
# align padding mode to tf.layers.conv2d with padding_mod="same"
+ """Proper padding.
+
+ Args:
+ x: TODO.
+ stride: TODO.
+ """
if stride == 1:
return F.pad(x, (1, 1, 1, 1), "constant", 0)
elif stride == 2:
@@ -39,6 +53,12 @@ class BasicLayer(torch.nn.Module):
return F.pad(x, (w % 2, 1, h % 2, 1), "constant", 0)
def forward(self, xs_pad, ilens):
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ """
identity = xs_pad
if self.in_filters != self.filters or self.stride > 1:
identity = self.conv_sc(identity)
@@ -60,6 +80,15 @@ class BasicLayer(torch.nn.Module):
class BasicBlock(torch.nn.Module):
def __init__(self, in_filters, filters, num_layer, stride, bn_momentum=0.5):
+ """Initialize BasicBlock.
+
+ Args:
+ in_filters: TODO.
+ filters: TODO.
+ num_layer: TODO.
+ stride: TODO.
+ bn_momentum: TODO.
+ """
super().__init__()
self.num_layer = num_layer
@@ -71,6 +100,12 @@ class BasicBlock(torch.nn.Module):
def forward(self, xs_pad, ilens):
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ """
for i in range(self.num_layer):
xs_pad, ilens = self._modules["layer_{}".format(i)](xs_pad, ilens)
@@ -88,6 +123,17 @@ class ResNet34(AbsEncoder):
layers_in_block=(3, 4, 6, 3),
filters_in_block=(32, 64, 128, 256),
):
+ """Initialize ResNet34.
+
+ Args:
+ input_size: Size/dimension parameter.
+ use_head_conv: TODO.
+ batchnorm_momentum: TODO.
+ use_head_maxpool: TODO.
+ num_nodes_pooling_layer: TODO.
+ layers_in_block: TODO.
+ filters_in_block: TODO.
+ """
super(ResNet34, self).__init__()
self.use_head_conv = use_head_conv
@@ -132,12 +178,20 @@ class ResNet34(AbsEncoder):
self.time_ds_ratio = 8
def output_size(self) -> int:
+ """Output size."""
return self.num_nodes_pooling_layer
def forward(
self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ prev_states: TODO.
+ """
features = xs_pad
assert (
features.size(-1) == self.input_size
@@ -181,6 +235,20 @@ class ResNet34_SP_L2Reg(AbsEncoder):
tf2torch_tensor_name_prefix_tf="EAND/speech_encoder",
tf_train_steps=720000,
):
+ """Initialize ResNet34_SP_L2Reg.
+
+ Args:
+ input_size: Size/dimension parameter.
+ use_head_conv: TODO.
+ batchnorm_momentum: TODO.
+ use_head_maxpool: TODO.
+ num_nodes_pooling_layer: TODO.
+ layers_in_block: TODO.
+ filters_in_block: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ tf_train_steps: TODO.
+ """
super(ResNet34_SP_L2Reg, self).__init__()
self.use_head_conv = use_head_conv
@@ -230,12 +298,20 @@ class ResNet34_SP_L2Reg(AbsEncoder):
self.time_ds_ratio = 8
def output_size(self) -> int:
+ """Output size."""
return self.num_nodes_pooling_layer
def forward(
self, xs_pad: torch.Tensor, ilens: torch.Tensor, prev_states: torch.Tensor = None
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ prev_states: TODO.
+ """
features = xs_pad
assert (
features.size(-1) == self.input_size
@@ -321,6 +397,7 @@ class ResNet34Diar(ResNet34):
)
def output_size(self) -> int:
+ """Output size."""
if self.embedding_node.startswith("resnet1"):
return self.num_nodes_resnet1
elif self.embedding_node.startswith("resnet2"):
@@ -335,6 +412,13 @@ class ResNet34Diar(ResNet34):
prev_states: torch.Tensor = None,
) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ prev_states: TODO.
+ """
endpoints = OrderedDict()
res_out, ilens = super().forward(xs_pad, ilens)
endpoints["resnet0_bn"] = res_out
@@ -419,6 +503,7 @@ class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg):
)
def output_size(self) -> int:
+ """Output size."""
if self.embedding_node.startswith("resnet1"):
return self.num_nodes_resnet1
elif self.embedding_node.startswith("resnet2"):
@@ -433,6 +518,13 @@ class ResNet34SpL2RegDiar(ResNet34_SP_L2Reg):
prev_states: torch.Tensor = None,
) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ prev_states: TODO.
+ """
endpoints = OrderedDict()
res_out, ilens = super().forward(xs_pad, ilens)
endpoints["resnet0_bn"] = res_out
diff --git a/funasr/models/sond/encoder/self_attention_encoder.py b/funasr/models/sond/encoder/self_attention_encoder.py
index 2e979b1be..5796649f9 100644
--- a/funasr/models/sond/encoder/self_attention_encoder.py
+++ b/funasr/models/sond/encoder/self_attention_encoder.py
@@ -145,6 +145,30 @@ class SelfAttentionEncoder(AbsEncoder):
tf2torch_tensor_name_prefix_tf: str = "seq2seq/encoder",
out_units=None,
):
+ """Initialize SelfAttentionEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ padding_idx: TODO.
+ interctc_layer_idx: TODO.
+ interctc_use_conditioning: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ out_units: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -257,6 +281,7 @@ class SelfAttentionEncoder(AbsEncoder):
self.output_linear = nn.Linear(output_size, out_units)
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
diff --git a/funasr/models/sond/label_aggregation.py b/funasr/models/sond/label_aggregation.py
index dddc4c4bf..f3fdb7d0a 100644
--- a/funasr/models/sond/label_aggregation.py
+++ b/funasr/models/sond/label_aggregation.py
@@ -12,6 +12,13 @@ class LabelAggregate(torch.nn.Module):
hop_length: int = 128,
center: bool = True,
):
+ """Initialize LabelAggregate.
+
+ Args:
+ win_length: TODO.
+ hop_length: TODO.
+ center: TODO.
+ """
super().__init__()
self.win_length = win_length
@@ -19,6 +26,7 @@ class LabelAggregate(torch.nn.Module):
self.center = center
def extra_repr(self):
+ """Extra repr."""
return (
f"win_length={self.win_length}, "
f"hop_length={self.hop_length}, "
@@ -85,11 +93,17 @@ class LabelAggregateMaxPooling(torch.nn.Module):
self,
hop_length: int = 8,
):
+ """Initialize LabelAggregateMaxPooling.
+
+ Args:
+ hop_length: TODO.
+ """
super().__init__()
self.hop_length = hop_length
def extra_repr(self):
+ """Extra repr."""
return f"hop_length={self.hop_length}, "
def forward(
diff --git a/funasr/models/sond/pooling/pooling_layers.py b/funasr/models/sond/pooling/pooling_layers.py
index 0a426a94f..4b8d69d64 100644
--- a/funasr/models/sond/pooling/pooling_layers.py
+++ b/funasr/models/sond/pooling/pooling_layers.py
@@ -13,9 +13,19 @@ class TAP(nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize TAP.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super(TAP, self).__init__()
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
pooling_mean = x.mean(dim=-1)
# To be compatable with 2D input
pooling_mean = pooling_mean.flatten(start_dim=1)
@@ -28,10 +38,20 @@ class TSDP(nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize TSDP.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super(TSDP, self).__init__()
def forward(self, x):
# The last dimension is the temporal axis
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8)
pooling_std = pooling_std.flatten(start_dim=1)
return pooling_std
@@ -45,10 +65,20 @@ class TSTP(nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize TSTP.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super(TSTP, self).__init__()
def forward(self, x):
# The last dimension is the temporal axis
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
pooling_mean = x.mean(dim=-1)
pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8)
pooling_mean = pooling_mean.flatten(start_dim=1)
@@ -64,6 +94,13 @@ class ASTP(nn.Module):
"""
def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False):
+ """Initialize ASTP.
+
+ Args:
+ in_dim: Size/dimension parameter.
+ bottleneck_dim: Size/dimension parameter.
+ global_context_att: TODO.
+ """
super(ASTP, self).__init__()
self.global_context_att = global_context_att
diff --git a/funasr/models/sond/pooling/statistic_pooling.py b/funasr/models/sond/pooling/statistic_pooling.py
index 91db2393e..e35b74551 100644
--- a/funasr/models/sond/pooling/statistic_pooling.py
+++ b/funasr/models/sond/pooling/statistic_pooling.py
@@ -10,6 +10,12 @@ VAR2STD_EPSILON = 1e-12
class StatisticPooling(torch.nn.Module):
def __init__(self, pooling_dim: Union[int, Tuple] = 2, eps=1e-12):
+ """Initialize StatisticPooling.
+
+ Args:
+ pooling_dim: Size/dimension parameter.
+ eps: TODO.
+ """
super(StatisticPooling, self).__init__()
if isinstance(pooling_dim, int):
pooling_dim = (pooling_dim,)
@@ -19,6 +25,12 @@ class StatisticPooling(torch.nn.Module):
def forward(self, xs_pad, ilens=None):
# xs_pad in (Batch, Channel, Time, Frequency)
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ """
if ilens is None:
masks = torch.ones_like(xs_pad).to(xs_pad)
else:
@@ -47,6 +59,13 @@ def statistic_pooling(
) -> torch.Tensor:
# xs_pad in (Batch, Channel, Time, Frequency)
+ """Statistic pooling.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ pooling_dim: Size/dimension parameter.
+ """
if ilens is None:
seq_mask = torch.ones_like(xs_pad).to(xs_pad)
else:
@@ -79,6 +98,15 @@ def windowed_statistic_pooling(
) -> Tuple[torch.Tensor, int]:
# xs_pad in (Batch, Channel, Time, Frequency)
+ """Windowed statistic pooling.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ pooling_dim: Size/dimension parameter.
+ pooling_size: Size/dimension parameter.
+ pooling_stride: TODO.
+ """
tt = xs_pad.shape[2]
num_chunk = int(math.ceil(tt / pooling_stride))
pad = pooling_size // 2
diff --git a/funasr/models/sond/sv_decoder.py b/funasr/models/sond/sv_decoder.py
index 7d8484db9..2051c530c 100644
--- a/funasr/models/sond/sv_decoder.py
+++ b/funasr/models/sond/sv_decoder.py
@@ -12,6 +12,15 @@ class DenseDecoder(AbsDecoder):
num_nodes_last_layer: int = 256,
batchnorm_momentum: float = 0.5,
):
+ """Initialize DenseDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ num_nodes_resnet1: TODO.
+ num_nodes_last_layer: TODO.
+ batchnorm_momentum: TODO.
+ """
super(DenseDecoder, self).__init__()
self.resnet1_dense = torch.nn.Linear(encoder_output_size, num_nodes_resnet1)
self.resnet1_bn = torch.nn.BatchNorm1d(
@@ -26,6 +35,11 @@ class DenseDecoder(AbsDecoder):
self.output_dense = torch.nn.Linear(num_nodes_last_layer, vocab_size, bias=False)
def forward(self, features):
+ """Forward pass for training.
+
+ Args:
+ features: TODO.
+ """
embeddings = {}
features = self.resnet1_dense(features)
embeddings["resnet1_dense"] = features
diff --git a/funasr/models/specaug/mask_along_axis.py b/funasr/models/specaug/mask_along_axis.py
index d4f237b3d..72e131a8f 100644
--- a/funasr/models/specaug/mask_along_axis.py
+++ b/funasr/models/specaug/mask_along_axis.py
@@ -147,6 +147,14 @@ class MaskAlongAxis(torch.nn.Module):
dim: Union[int, str] = "time",
replace_with_zero: bool = True,
):
+ """Initialize MaskAlongAxis.
+
+ Args:
+ mask_width_range: TODO.
+ num_mask: TODO.
+ dim: TODO.
+ replace_with_zero: TODO.
+ """
if isinstance(mask_width_range, int):
mask_width_range = (0, mask_width_range)
if len(mask_width_range) != 2:
@@ -176,6 +184,7 @@ class MaskAlongAxis(torch.nn.Module):
self.replace_with_zero = replace_with_zero
def extra_repr(self):
+ """Extra repr."""
return (
f"mask_width_range={self.mask_width_range}, "
f"num_mask={self.num_mask}, axis={self.mask_axis}"
@@ -212,6 +221,14 @@ class MaskAlongAxisVariableMaxWidth(torch.nn.Module):
dim: Union[int, str] = "time",
replace_with_zero: bool = True,
):
+ """Initialize MaskAlongAxisVariableMaxWidth.
+
+ Args:
+ mask_width_ratio_range: TODO.
+ num_mask: TODO.
+ dim: TODO.
+ replace_with_zero: TODO.
+ """
if isinstance(mask_width_ratio_range, float):
mask_width_ratio_range = (0.0, mask_width_ratio_range)
if len(mask_width_ratio_range) != 2:
@@ -242,6 +259,7 @@ class MaskAlongAxisVariableMaxWidth(torch.nn.Module):
self.replace_with_zero = replace_with_zero
def extra_repr(self):
+ """Extra repr."""
return (
f"mask_width_ratio_range={self.mask_width_ratio_range}, "
f"num_mask={self.num_mask}, axis={self.mask_axis}"
@@ -281,6 +299,15 @@ class MaskAlongAxisLFR(torch.nn.Module):
replace_with_zero: bool = True,
lfr_rate: int = 1,
):
+ """Initialize MaskAlongAxisLFR.
+
+ Args:
+ mask_width_range: TODO.
+ num_mask: TODO.
+ dim: TODO.
+ replace_with_zero: TODO.
+ lfr_rate: TODO.
+ """
if isinstance(mask_width_range, int):
mask_width_range = (0, mask_width_range)
if len(mask_width_range) != 2:
@@ -313,6 +340,7 @@ class MaskAlongAxisLFR(torch.nn.Module):
self.lfr_rate = lfr_rate
def extra_repr(self):
+ """Extra repr."""
return (
f"mask_width_range={self.mask_width_range}, "
f"num_mask={self.num_mask}, axis={self.mask_axis}"
diff --git a/funasr/models/specaug/profileaug.py b/funasr/models/specaug/profileaug.py
index 669d323f8..05a5c63b5 100644
--- a/funasr/models/specaug/profileaug.py
+++ b/funasr/models/specaug/profileaug.py
@@ -23,6 +23,17 @@ class ProfileAug(nn.Module):
disturb_aug_prob: float = 0.4,
disturb_alpha: float = 0.2,
) -> None:
+ """Initialize ProfileAug.
+
+ Args:
+ apply_split_aug: TODO.
+ split_aug_prob: TODO.
+ apply_merge_aug: TODO.
+ merge_aug_prob: TODO.
+ apply_disturb_aug: TODO.
+ disturb_aug_prob: TODO.
+ disturb_alpha: TODO.
+ """
super().__init__()
self.apply_split_aug = apply_split_aug
self.split_aug_prob = split_aug_prob
@@ -34,6 +45,13 @@ class ProfileAug(nn.Module):
def split_aug(self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor):
# B, N
+ """Split aug.
+
+ Args:
+ profile: TODO.
+ binary_labels: TODO.
+ mask: TODO.
+ """
bsz, dim = profile.shape[0], profile.shape[-1]
profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False)
spk_count = binary_labels.sum(dim=1)
@@ -56,6 +74,13 @@ class ProfileAug(nn.Module):
return profile, binary_labels, mask
def merge_aug(self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor):
+ """Merge aug.
+
+ Args:
+ profile: TODO.
+ binary_labels: TODO.
+ mask: TODO.
+ """
bsz, dim = profile.shape[0], profile.shape[-1]
profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False)
spk_count = binary_labels.sum(dim=1)
@@ -86,6 +111,13 @@ class ProfileAug(nn.Module):
return profile, binary_labels, mask
def disturb_aug(self, profile: torch.Tensor, binary_labels: torch.Tensor, mask: torch.Tensor):
+ """Disturb aug.
+
+ Args:
+ profile: TODO.
+ binary_labels: TODO.
+ mask: TODO.
+ """
bsz, dim = profile.shape[0], profile.shape[-1]
profile_norm = torch.linalg.norm(profile, dim=-1, keepdim=False)
spk_count = binary_labels.sum(dim=1)
@@ -118,6 +150,16 @@ class ProfileAug(nn.Module):
) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]:
# copy inputs to avoid inplace-operation
+ """Forward pass for training.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ profile: TODO.
+ profile_lengths: Lengths of profile.
+ binary_labels: TODO.
+ labels_length: TODO.
+ """
speech, profile, binary_labels = (
torch.clone(speech),
torch.clone(profile),
diff --git a/funasr/models/specaug/specaug.py b/funasr/models/specaug/specaug.py
index 75a4b0b28..5a23502d7 100644
--- a/funasr/models/specaug/specaug.py
+++ b/funasr/models/specaug/specaug.py
@@ -41,6 +41,20 @@ class SpecAug(nn.Module):
time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None,
num_time_mask: int = 2,
):
+ """Initialize SpecAug.
+
+ Args:
+ apply_time_warp: TODO.
+ time_warp_window: TODO.
+ time_warp_mode: TODO.
+ apply_freq_mask: TODO.
+ freq_mask_width_range: TODO.
+ num_freq_mask: TODO.
+ apply_time_mask: TODO.
+ time_mask_width_range: TODO.
+ time_mask_width_ratio_range: TODO.
+ num_time_mask: TODO.
+ """
if not apply_time_warp and not apply_time_mask and not apply_freq_mask:
raise ValueError("Either one of time_warp, time_mask, or freq_mask should be applied")
if (
@@ -93,6 +107,12 @@ class SpecAug(nn.Module):
self.time_mask = None
def forward(self, x, x_lengths=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ x_lengths: Lengths of x.
+ """
if self.time_warp is not None:
x, x_lengths = self.time_warp(x, x_lengths)
if self.freq_mask is not None:
@@ -122,6 +142,21 @@ class SpecAugLFR(nn.Module):
time_mask_width_ratio_range: Optional[Union[float, Sequence[float]]] = None,
num_time_mask: int = 2,
):
+ """Initialize SpecAugLFR.
+
+ Args:
+ apply_time_warp: TODO.
+ time_warp_window: TODO.
+ time_warp_mode: TODO.
+ apply_freq_mask: TODO.
+ freq_mask_width_range: TODO.
+ num_freq_mask: TODO.
+ lfr_rate: TODO.
+ apply_time_mask: TODO.
+ time_mask_width_range: TODO.
+ time_mask_width_ratio_range: TODO.
+ num_time_mask: TODO.
+ """
if not apply_time_warp and not apply_time_mask and not apply_freq_mask:
raise ValueError("Either one of time_warp, time_mask, or freq_mask should be applied")
if (
@@ -177,6 +212,12 @@ class SpecAugLFR(nn.Module):
self.time_mask = None
def forward(self, x, x_lengths=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ x_lengths: Lengths of x.
+ """
if self.time_warp is not None:
x, x_lengths = self.time_warp(x, x_lengths)
if self.freq_mask is not None:
diff --git a/funasr/models/specaug/time_warp.py b/funasr/models/specaug/time_warp.py
index ef3532f6a..afdffbffc 100644
--- a/funasr/models/specaug/time_warp.py
+++ b/funasr/models/specaug/time_warp.py
@@ -56,11 +56,18 @@ class TimeWarp(torch.nn.Module):
"""
def __init__(self, window: int = 80, mode: str = DEFAULT_TIME_WARP_MODE):
+ """Initialize TimeWarp.
+
+ Args:
+ window: TODO.
+ mode: TODO.
+ """
super().__init__()
self.window = window
self.mode = mode
def extra_repr(self):
+ """Extra repr."""
return f"window={self.window}, mode={self.mode}"
def forward(self, x: torch.Tensor, x_lengths: torch.Tensor = None):
diff --git a/funasr/models/transducer/model.py b/funasr/models/transducer/model.py
index 37925fcf6..e72200083 100644
--- a/funasr/models/transducer/model.py
+++ b/funasr/models/transducer/model.py
@@ -28,6 +28,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -72,6 +77,38 @@ class Transducer(torch.nn.Module):
**kwargs,
):
+ """Initialize Transducer.
+
+ Args:
+ frontend: Audio frontend for feature extraction.
+ frontend_conf: Configuration dict for frontend.
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ joint_network: TODO.
+ joint_network_conf: Configuration dict for joint_network.
+ transducer_weight: TODO.
+ fastemit_lambda: TODO.
+ auxiliary_ctc_weight: TODO.
+ auxiliary_ctc_dropout_rate: TODO.
+ auxiliary_lm_loss_weight: TODO.
+ auxiliary_lm_loss_smoothing: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -413,6 +450,11 @@ class Transducer(torch.nn.Module):
):
# 1. Build ASR model
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
scorers = {}
if self.ctc != None:
@@ -449,6 +491,15 @@ class Transducer(torch.nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
diff --git a/funasr/models/transducer/rnn_decoder.py b/funasr/models/transducer/rnn_decoder.py
index 4a0ffa3c3..10a56b295 100644
--- a/funasr/models/transducer/rnn_decoder.py
+++ b/funasr/models/transducer/rnn_decoder.py
@@ -35,6 +35,27 @@ def build_attention_list(
han_win: int = 5,
):
+ """Build attention list.
+
+ Args:
+ eprojs: TODO.
+ dunits: TODO.
+ atype: TODO.
+ num_att: TODO.
+ num_encs: TODO.
+ aheads: TODO.
+ adim: TODO.
+ awin: TODO.
+ aconv_chans: TODO.
+ aconv_filts: TODO.
+ han_mode: TODO.
+ han_type: TODO.
+ han_heads: TODO.
+ han_dim: Size/dimension parameter.
+ han_conv_chans: TODO.
+ han_conv_filts: TODO.
+ han_win: TODO.
+ """
att_list = torch.nn.ModuleList()
if num_encs == 1:
for i in range(num_att):
@@ -99,6 +120,21 @@ class RNNDecoder(nn.Module):
att_conf: dict = None,
):
# FIXME(kamo): The parts of num_spk should be refactored more more more
+ """Initialize RNNDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ rnn_type: TODO.
+ num_layers: TODO.
+ hidden_size: Size/dimension parameter.
+ sampling_probability: TODO.
+ dropout: TODO.
+ context_residual: TODO.
+ replace_sos: TODO.
+ num_encs: TODO.
+ att_conf: Configuration dict for att.
+ """
if rnn_type not in {"lstm", "gru"}:
raise ValueError(f"Not supported: rnn_type={rnn_type}")
@@ -151,9 +187,23 @@ class RNNDecoder(nn.Module):
self.att_list = build_attention_list(eprojs=eprojs, dunits=hidden_size, **att_conf)
def zero_state(self, hs_pad):
+ """Zero state.
+
+ Args:
+ hs_pad: TODO.
+ """
return hs_pad.new_zeros(hs_pad.size(0), self.dunits)
def rnn_forward(self, ey, z_list, c_list, z_prev, c_prev):
+ """Rnn forward.
+
+ Args:
+ ey: TODO.
+ z_list: TODO.
+ c_list: TODO.
+ z_prev: TODO.
+ c_prev: TODO.
+ """
if self.dtype == "lstm":
z_list[0], c_list[0] = self.decoder[0](ey, (z_prev[0], c_prev[0]))
for i in range(1, self.dlayers):
@@ -170,6 +220,15 @@ class RNNDecoder(nn.Module):
def forward(self, hs_pad, hlens, ys_in_pad, ys_in_lens, strm_idx=0):
# to support mutiple encoder asr mode, in single encoder mode,
# convert torch.Tensor to List of torch.Tensor
+ """Forward pass for training.
+
+ Args:
+ hs_pad: TODO.
+ hlens: TODO.
+ ys_in_pad: TODO.
+ ys_in_lens: Lengths of ys_in.
+ strm_idx: TODO.
+ """
if self.num_encs == 1:
hs_pad = [hs_pad]
hlens = [hlens]
@@ -254,6 +313,11 @@ class RNNDecoder(nn.Module):
def init_state(self, x):
# to support mutiple encoder asr mode, in single encoder mode,
# convert torch.Tensor to List of torch.Tensor
+ """Init state.
+
+ Args:
+ x: TODO.
+ """
if self.num_encs == 1:
x = [x]
@@ -283,6 +347,13 @@ class RNNDecoder(nn.Module):
def score(self, yseq, state, x):
# to support mutiple encoder asr mode, in single encoder mode,
# convert torch.Tensor to List of torch.Tensor
+ """Score.
+
+ Args:
+ yseq: TODO.
+ state: TODO.
+ x: TODO.
+ """
if self.num_encs == 1:
x = [x]
diff --git a/funasr/models/transformer/attention.py b/funasr/models/transformer/attention.py
index 2844333a1..f99eaff16 100644
--- a/funasr/models/transformer/attention.py
+++ b/funasr/models/transformer/attention.py
@@ -122,6 +122,11 @@ class MultiHeadedAttention(nn.Module):
class MultiHeadedAttentionExport(nn.Module):
def __init__(self, model):
+ """Initialize MultiHeadedAttentionExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.d_k = model.d_k
self.h = model.h
@@ -133,16 +138,36 @@ class MultiHeadedAttentionExport(nn.Module):
self.all_head_size = self.h * self.d_k
def forward(self, query, key, value, mask):
+ """Forward pass for training.
+
+ Args:
+ query: TODO.
+ key: Sample identifiers.
+ value: TODO.
+ mask: TODO.
+ """
q, k, v = self.forward_qkv(query, key, value)
scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_k)
return self.forward_attention(v, scores, mask)
def transpose_for_scores(self, x: torch.Tensor) -> torch.Tensor:
+ """Transpose for scores.
+
+ Args:
+ x: TODO.
+ """
new_x_shape = x.size()[:-1] + (self.h, self.d_k)
x = x.view(new_x_shape)
return x.permute(0, 2, 1, 3)
def forward_qkv(self, query, key, value):
+ """Forward qkv.
+
+ Args:
+ query: TODO.
+ key: Sample identifiers.
+ value: TODO.
+ """
q = self.linear_q(query)
k = self.linear_k(key)
v = self.linear_v(value)
@@ -152,6 +177,13 @@ class MultiHeadedAttentionExport(nn.Module):
return q, k, v
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
@@ -165,12 +197,26 @@ class MultiHeadedAttentionExport(nn.Module):
class RelPosMultiHeadedAttentionExport(MultiHeadedAttentionExport):
def __init__(self, model):
+ """Initialize RelPosMultiHeadedAttentionExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__(model)
self.linear_pos = model.linear_pos
self.pos_bias_u = model.pos_bias_u
self.pos_bias_v = model.pos_bias_v
def forward(self, query, key, value, pos_emb, mask):
+ """Forward pass for training.
+
+ Args:
+ query: TODO.
+ key: Sample identifiers.
+ value: TODO.
+ pos_emb: TODO.
+ mask: TODO.
+ """
q, k, v = self.forward_qkv(query, key, value)
q = q.transpose(1, 2) # (batch, time1, head, d_k)
@@ -197,6 +243,11 @@ class RelPosMultiHeadedAttentionExport(MultiHeadedAttentionExport):
return self.forward_attention(v, scores, mask)
def rel_shift(self, x):
+ """Rel shift.
+
+ Args:
+ x: TODO.
+ """
zero_pad = torch.zeros((*x.size()[:3], 1), device=x.device, dtype=x.dtype)
x_padded = torch.cat([zero_pad, x], dim=-1)
@@ -207,6 +258,13 @@ class RelPosMultiHeadedAttentionExport(MultiHeadedAttentionExport):
return x
def forward_attention(self, value, scores, mask):
+ """Forward attention.
+
+ Args:
+ value: TODO.
+ scores: TODO.
+ mask: TODO.
+ """
scores = scores + mask
attn = torch.softmax(scores, dim=-1)
diff --git a/funasr/models/transformer/decoder.py b/funasr/models/transformer/decoder.py
index 7d849f349..307d5365b 100644
--- a/funasr/models/transformer/decoder.py
+++ b/funasr/models/transformer/decoder.py
@@ -149,6 +149,11 @@ class DecoderLayer(nn.Module):
class DecoderLayerExport(nn.Module):
def __init__(self, model):
+ """Initialize DecoderLayerExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.self_attn = model.self_attn
self.src_attn = model.src_attn
@@ -158,6 +163,15 @@ class DecoderLayerExport(nn.Module):
self.norm3 = model.norm3
def forward(self, tgt, tgt_mask, memory, memory_mask, cache=None):
+ """Forward pass for training.
+
+ Args:
+ tgt: TODO.
+ tgt_mask: TODO.
+ memory: TODO.
+ memory_mask: TODO.
+ cache: State cache dict for streaming inference.
+ """
residual = tgt
tgt = self.norm1(tgt)
tgt_q = tgt
@@ -209,6 +223,18 @@ class BaseTransformerDecoder(nn.Module, BatchScorerInterface):
pos_enc_class=PositionalEncoding,
normalize_before: bool = True,
):
+ """Initialize BaseTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ """
super().__init__()
attention_dim = encoder_output_size
@@ -387,6 +413,24 @@ class TransformerDecoder(BaseTransformerDecoder):
normalize_before: bool = True,
concat_after: bool = False,
):
+ """Initialize TransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ """
super().__init__(
vocab_size=vocab_size,
encoder_output_size=encoder_output_size,
@@ -435,6 +479,27 @@ class LightweightConvolutionTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize LightweightConvolutionTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
@@ -495,6 +560,27 @@ class LightweightConvolution2DTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize LightweightConvolution2DTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
@@ -555,6 +641,27 @@ class DynamicConvolutionTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize DynamicConvolutionTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
@@ -615,6 +722,27 @@ class DynamicConvolution2DTransformerDecoder(BaseTransformerDecoder):
conv_kernel_length: Sequence[int] = (11, 11, 11, 11, 11, 11),
conv_usebias: int = False,
):
+ """Initialize DynamicConvolution2DTransformerDecoder.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ encoder_output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ self_attention_dropout_rate: TODO.
+ src_attention_dropout_rate: TODO.
+ input_layer: TODO.
+ use_output_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ conv_wshare: TODO.
+ conv_kernel_length: TODO.
+ conv_usebias: TODO.
+ """
if len(conv_kernel_length) != num_blocks:
raise ValueError(
"conv_kernel_length must have equal number of values to num_blocks: "
diff --git a/funasr/models/transformer/embedding.py b/funasr/models/transformer/embedding.py
index 9ef2f4b0e..93863e623 100644
--- a/funasr/models/transformer/embedding.py
+++ b/funasr/models/transformer/embedding.py
@@ -182,6 +182,7 @@ class LearnableFourierPosEnc(torch.nn.Module):
)
def _reset(self):
+ """Internal: reset."""
self.w_r.data = torch.normal(0, (1 / math.sqrt(self.gamma)), (1, self.d_model // 2))
def extend_pe(self, x):
@@ -384,11 +385,24 @@ class SinusoidalPositionEncoder(torch.nn.Module):
""" """
def __init__(self, d_model=80, dropout_rate=0.1):
+ """Initialize SinusoidalPositionEncoder.
+
+ Args:
+ d_model: D Model instance.
+ dropout_rate: TODO.
+ """
super().__init__()
def encode(
self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32
):
+ """Encode.
+
+ Args:
+ positions: TODO.
+ depth: TODO.
+ dtype: TODO.
+ """
batch_size = positions.size(0)
positions = positions.type(dtype)
device = positions.device
@@ -406,6 +420,11 @@ class SinusoidalPositionEncoder(torch.nn.Module):
return encoding.type(dtype)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
batch_size, timesteps, input_dim = x.size()
positions = torch.arange(1, timesteps + 1, device=x.device)[None, :]
position_encoding = self.encode(positions, input_dim, x.dtype).to(x.device)
@@ -417,11 +436,24 @@ class StreamSinusoidalPositionEncoder(torch.nn.Module):
""" """
def __init__(self, d_model=80, dropout_rate=0.1):
+ """Initialize StreamSinusoidalPositionEncoder.
+
+ Args:
+ d_model: D Model instance.
+ dropout_rate: TODO.
+ """
super().__init__()
def encode(
self, positions: torch.Tensor = None, depth: int = None, dtype: torch.dtype = torch.float32
):
+ """Encode.
+
+ Args:
+ positions: TODO.
+ depth: TODO.
+ dtype: TODO.
+ """
batch_size = positions.size(0)
positions = positions.type(dtype)
log_timescale_increment = torch.log(torch.tensor([10000], dtype=dtype)) / (depth / 2 - 1)
@@ -434,6 +466,12 @@ class StreamSinusoidalPositionEncoder(torch.nn.Module):
return encoding.type(dtype)
def forward(self, x, cache=None):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ cache: State cache dict for streaming inference.
+ """
batch_size, timesteps, input_dim = x.size()
start_idx = 0
if cache is not None:
@@ -516,6 +554,11 @@ class StreamingRelPositionalEncoding(torch.nn.Module):
class ScaledSinuEmbedding(torch.nn.Module):
def __init__(self, dim):
+ """Initialize ScaledSinuEmbedding.
+
+ Args:
+ dim: TODO.
+ """
super().__init__()
self.scale = torch.nn.Parameter(
torch.ones(
@@ -526,6 +569,11 @@ class ScaledSinuEmbedding(torch.nn.Module):
self.register_buffer("inv_freq", inv_freq)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
n, device = x.shape[1], x.device
t = torch.arange(n, device=device).type_as(self.inv_freq)
sinu = einsum("i , j -> i j", t, self.inv_freq)
diff --git a/funasr/models/transformer/encoder.py b/funasr/models/transformer/encoder.py
index 987924f0c..78c6b8c7b 100644
--- a/funasr/models/transformer/encoder.py
+++ b/funasr/models/transformer/encoder.py
@@ -182,6 +182,27 @@ class TransformerEncoder(nn.Module):
interctc_layer_idx: List[int] = [],
interctc_use_conditioning: bool = False,
):
+ """Initialize TransformerEncoder.
+
+ Args:
+ input_size: Size/dimension parameter.
+ output_size: Size/dimension parameter.
+ attention_heads: TODO.
+ linear_units: TODO.
+ num_blocks: TODO.
+ dropout_rate: TODO.
+ positional_dropout_rate: TODO.
+ attention_dropout_rate: TODO.
+ input_layer: TODO.
+ pos_enc_class: TODO.
+ normalize_before: TODO.
+ concat_after: TODO.
+ positionwise_layer_type: TODO.
+ positionwise_conv_kernel_size: Size/dimension parameter.
+ padding_idx: TODO.
+ interctc_layer_idx: TODO.
+ interctc_use_conditioning: TODO.
+ """
super().__init__()
self._output_size = output_size
@@ -260,6 +281,7 @@ class TransformerEncoder(nn.Module):
self.conditioning_layer = None
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(
diff --git a/funasr/models/transformer/layer_norm.py b/funasr/models/transformer/layer_norm.py
index ce4f8877f..d73552409 100644
--- a/funasr/models/transformer/layer_norm.py
+++ b/funasr/models/transformer/layer_norm.py
@@ -61,6 +61,14 @@ class GlobalLayerNorm(nn.Module):
"""
def __init__(self, dim, shape, eps=1e-8, elementwise_affine=True):
+ """Initialize GlobalLayerNorm.
+
+ Args:
+ dim: TODO.
+ shape: TODO.
+ eps: TODO.
+ elementwise_affine: TODO.
+ """
super(GlobalLayerNorm, self).__init__()
self.dim = dim
self.eps = eps
@@ -125,6 +133,12 @@ class CumulativeLayerNorm(nn.LayerNorm):
"""
def __init__(self, dim, elementwise_affine=True):
+ """Initialize CumulativeLayerNorm.
+
+ Args:
+ dim: TODO.
+ elementwise_affine: TODO.
+ """
super(CumulativeLayerNorm, self).__init__(
dim, elementwise_affine=elementwise_affine, eps=1e-8
)
@@ -156,11 +170,22 @@ class CumulativeLayerNorm(nn.LayerNorm):
class ScaleNorm(nn.Module):
def __init__(self, dim, eps=1e-5):
+ """Initialize ScaleNorm.
+
+ Args:
+ dim: TODO.
+ eps: TODO.
+ """
super().__init__()
self.scale = dim**-0.5
self.eps = eps
self.g = nn.Parameter(torch.ones(1))
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
norm = torch.norm(x, dim=-1, keepdim=True) * self.scale
return x / norm.clamp(min=self.eps) * self.g
diff --git a/funasr/models/transformer/model.py b/funasr/models/transformer/model.py
index adfd52520..a6c1c7ade 100644
--- a/funasr/models/transformer/model.py
+++ b/funasr/models/transformer/model.py
@@ -56,6 +56,36 @@ class Transformer(nn.Module):
**kwargs,
):
+ """Initialize Transformer.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ interctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -269,6 +299,14 @@ class Transformer(nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -300,6 +338,14 @@ class Transformer(nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -313,6 +359,11 @@ class Transformer(nn.Module):
self,
**kwargs,
):
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.transformer.search import BeamSearch
from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
from funasr.models.transformer.scorers.length_bonus import LengthBonus
@@ -364,6 +415,16 @@ class Transformer(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
diff --git a/funasr/models/transformer/positionwise_feed_forward.py b/funasr/models/transformer/positionwise_feed_forward.py
index 7cfa5f9d5..3cf7887a2 100644
--- a/funasr/models/transformer/positionwise_feed_forward.py
+++ b/funasr/models/transformer/positionwise_feed_forward.py
@@ -36,6 +36,11 @@ class PositionwiseFeedForward(torch.nn.Module):
class PositionwiseFeedForwardDecoderSANMExport(torch.nn.Module):
def __init__(self, model):
+ """Initialize PositionwiseFeedForwardDecoderSANMExport.
+
+ Args:
+ model: Model instance or model name.
+ """
super().__init__()
self.w_1 = model.w_1
self.w_2 = model.w_2
@@ -43,6 +48,11 @@ class PositionwiseFeedForwardDecoderSANMExport(torch.nn.Module):
self.norm = model.norm
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = self.activation(self.w_1(x))
x = self.w_2(self.norm(x))
return x
diff --git a/funasr/models/transformer/scorers/ctc_prefix_score.py b/funasr/models/transformer/scorers/ctc_prefix_score.py
index ce0703e73..df5bcfd55 100644
--- a/funasr/models/transformer/scorers/ctc_prefix_score.py
+++ b/funasr/models/transformer/scorers/ctc_prefix_score.py
@@ -262,6 +262,14 @@ class CTCPrefixScore(object):
"""
def __init__(self, x, blank, eos, xp):
+ """Initialize CTCPrefixScore.
+
+ Args:
+ x: TODO.
+ blank: TODO.
+ eos: TODO.
+ xp: TODO.
+ """
self.xp = xp
self.logzero = -10000000000.0
self.blank = blank
diff --git a/funasr/models/transformer/utils/nets_utils.py b/funasr/models/transformer/utils/nets_utils.py
index 29d23ee59..856788828 100644
--- a/funasr/models/transformer/utils/nets_utils.py
+++ b/funasr/models/transformer/utils/nets_utils.py
@@ -511,6 +511,12 @@ class Swish(torch.nn.Module):
"""
def __init__(self, beta: float = 1.0, use_builtin: bool = False) -> None:
+ """Initialize Swish.
+
+ Args:
+ beta: TODO.
+ use_builtin: TODO.
+ """
super().__init__()
self.beta = beta
diff --git a/funasr/models/transformer/utils/subsampling.py b/funasr/models/transformer/utils/subsampling.py
index b18bd3132..12eff29ca 100644
--- a/funasr/models/transformer/utils/subsampling.py
+++ b/funasr/models/transformer/utils/subsampling.py
@@ -350,6 +350,17 @@ class Conv1dSubsampling(torch.nn.Module):
tf2torch_tensor_name_prefix_torch: str = "stride_conv",
tf2torch_tensor_name_prefix_tf: str = "seq2seq/proj_encoder/downsampling",
):
+ """Initialize Conv1dSubsampling.
+
+ Args:
+ idim: TODO.
+ odim: TODO.
+ kernel_size: Size/dimension parameter.
+ stride: TODO.
+ pad: TODO.
+ tf2torch_tensor_name_prefix_torch: TODO.
+ tf2torch_tensor_name_prefix_tf: TODO.
+ """
super(Conv1dSubsampling, self).__init__()
self.conv = torch.nn.Conv1d(idim, odim, kernel_size, stride)
self.pad_fn = torch.nn.ConstantPad1d(pad, 0.0)
@@ -359,6 +370,7 @@ class Conv1dSubsampling(torch.nn.Module):
self.tf2torch_tensor_name_prefix_tf = tf2torch_tensor_name_prefix_tf
def output_size(self) -> int:
+ """Output size."""
return self.odim
def forward(self, x, x_len):
diff --git a/funasr/models/uniasr/model.py b/funasr/models/uniasr/model.py
index bde637777..efe1f18c0 100644
--- a/funasr/models/uniasr/model.py
+++ b/funasr/models/uniasr/model.py
@@ -73,6 +73,51 @@ class UniASR(torch.nn.Module):
share_embedding: bool = False,
**kwargs,
):
+ """Initialize UniASR.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ encoder2: TODO.
+ encoder2_conf: Configuration dict for encoder2.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ decoder2: TODO.
+ decoder2_conf: Configuration dict for decoder2.
+ predictor: TODO.
+ predictor_conf: Configuration dict for predictor.
+ predictor_bias: TODO.
+ predictor_weight: TODO.
+ predictor2: TODO.
+ predictor2_conf: Configuration dict for predictor2.
+ predictor2_bias: TODO.
+ predictor2_weight: TODO.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ ctc2: TODO.
+ ctc2_conf: Configuration dict for ctc2.
+ ctc2_weight: TODO.
+ decoder_attention_chunk_type: TODO.
+ decoder_attention_chunk_type2: TODO.
+ stride_conv: TODO.
+ stride_conv_conf: Configuration dict for stride_conv.
+ loss_weight_model1: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -308,6 +353,14 @@ class UniASR(torch.nn.Module):
text: torch.Tensor,
text_lengths: torch.Tensor,
) -> Dict[str, torch.Tensor]:
+ """Collect feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ text: Text tensor or string input.
+ text_lengths: Length of each text sample.
+ """
if self.extract_feats_in_collect_stats:
feats, feats_lengths = self._extract_feats(speech, speech_lengths)
else:
@@ -477,6 +530,14 @@ class UniASR(torch.nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -507,6 +568,14 @@ class UniASR(torch.nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att predictor loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -608,6 +677,14 @@ class UniASR(torch.nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att predictor loss2.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -711,6 +788,14 @@ class UniASR(torch.nn.Module):
):
# ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
# ys_in_lens = ys_pad_lens + 1
+ """Calc predictor mask.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_out_pad, ys_in_lens = None, None
encoder_out_mask = sequence_mask(
@@ -793,6 +878,14 @@ class UniASR(torch.nn.Module):
):
# ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
# ys_in_lens = ys_pad_lens + 1
+ """Calc predictor mask2.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_out_pad, ys_in_lens = None, None
encoder_out_mask = sequence_mask(
@@ -870,6 +963,11 @@ class UniASR(torch.nn.Module):
self,
**kwargs,
):
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.uniasr.beam_search import BeamSearchScama
from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
from funasr.models.transformer.scorers.length_bonus import LengthBonus
@@ -926,6 +1024,16 @@ class UniASR(torch.nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
decoding_model = kwargs.get("decoding_model", "normal")
token_num_relax = kwargs.get("token_num_relax", 5)
if decoding_model == "fast":
diff --git a/funasr/models/whisper/model.py b/funasr/models/whisper/model.py
index 398eea3fd..d2656dcce 100644
--- a/funasr/models/whisper/model.py
+++ b/funasr/models/whisper/model.py
@@ -32,6 +32,12 @@ from funasr.register import tables
@tables.register("model_classes", "WhisperWarp")
class WhisperWarp(nn.Module):
def __init__(self, *args, **kwargs):
+ """Initialize WhisperWarp.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
hub = kwargs.get("hub", "funasr")
if hub == "openai":
@@ -51,6 +57,7 @@ class WhisperWarp(nn.Module):
def forward(
self,
):
+ """Forward pass for training."""
pass
def inference(
@@ -62,6 +69,16 @@ class WhisperWarp(nn.Module):
frontend=None,
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
diff --git a/funasr/models/whisper_lid/decoder.py b/funasr/models/whisper_lid/decoder.py
index 1c8ab47f8..a355e508e 100644
--- a/funasr/models/whisper_lid/decoder.py
+++ b/funasr/models/whisper_lid/decoder.py
@@ -26,6 +26,14 @@ class OpenAIWhisperDecoderWarp(nn.Module):
download_dir: str = None,
use_padmask: bool = False,
):
+ """Initialize OpenAIWhisperDecoderWarp.
+
+ Args:
+ dropout_rate: TODO.
+ whisper_model: Whisper Model instance.
+ download_dir: TODO.
+ use_padmask: TODO.
+ """
super().__init__()
assert whisper_model in whisper.available_models()
diff --git a/funasr/models/whisper_lid/encoder.py b/funasr/models/whisper_lid/encoder.py
index b6fb28b50..73de72e77 100644
--- a/funasr/models/whisper_lid/encoder.py
+++ b/funasr/models/whisper_lid/encoder.py
@@ -30,6 +30,16 @@ class OpenAIWhisperEncoderWarp(nn.Module):
use_padmask: bool = False,
specaug_conf: Union[dict, None] = None,
):
+ """Initialize OpenAIWhisperEncoderWarp.
+
+ Args:
+ dropout_rate: TODO.
+ whisper_model: Whisper Model instance.
+ download_dir: TODO.
+ use_specaug: TODO.
+ use_padmask: TODO.
+ specaug_conf: Configuration dict for specaug.
+ """
super().__init__()
# note that originally Whisper doesn't use dropouts
@@ -53,6 +63,12 @@ class OpenAIWhisperEncoderWarp(nn.Module):
input: torch.Tensor,
ilens: torch.Tensor = None,
) -> torch.Tensor:
+ """Whisper encode.
+
+ Args:
+ input: Input audio/text data.
+ ilens: TODO.
+ """
x = F.gelu(self.encoders.conv1(input))
x = F.gelu(self.encoders.conv2(x))
x = x.permute(0, 2, 1)
@@ -93,6 +109,7 @@ class OpenAIWhisperEncoderWarp(nn.Module):
def output_size(self) -> int:
# dummy output size
+ """Output size."""
return self.encoders.conv2.weight.shape[0]
def forward(
@@ -101,6 +118,13 @@ class OpenAIWhisperEncoderWarp(nn.Module):
ilens: torch.Tensor,
prev_states: torch.Tensor = None,
) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+ """Forward pass for training.
+
+ Args:
+ xs_pad: TODO.
+ ilens: TODO.
+ prev_states: TODO.
+ """
feats, feats_lens = xs_pad, ilens
if self.specaug is not None and self.encoders.training:
diff --git a/funasr/models/whisper_lid/eres2net/ResNet.py b/funasr/models/whisper_lid/eres2net/ResNet.py
index df3b96d24..554ca8ec2 100644
--- a/funasr/models/whisper_lid/eres2net/ResNet.py
+++ b/funasr/models/whisper_lid/eres2net/ResNet.py
@@ -20,9 +20,15 @@ from funasr.models.whisper_lid.eres2net.fusion import AFF
class ReLU(nn.Hardtanh):
def __init__(self, inplace=False):
+ """Initialize ReLU.
+
+ Args:
+ inplace: TODO.
+ """
super(ReLU, self).__init__(0, 20, inplace)
def __repr__(self):
+ """Internal: repr ."""
inplace_str = "inplace" if self.inplace else ""
return self.__class__.__name__ + " (" + inplace_str + ")"
@@ -41,6 +47,15 @@ class BasicBlockERes2Net(nn.Module):
expansion = 2
def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):
+ """Initialize BasicBlockERes2Net.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockERes2Net, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -71,6 +86,11 @@ class BasicBlockERes2Net(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -103,6 +123,15 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):
expansion = 2
def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):
+ """Initialize BasicBlockERes2Net_diff_AFF.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockERes2Net_diff_AFF, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -138,6 +167,11 @@ class BasicBlockERes2Net_diff_AFF(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -179,6 +213,18 @@ class ERes2Net(nn.Module):
pooling_func="TSTP",
two_emb_layer=False,
):
+ """Initialize ERes2Net.
+
+ Args:
+ block: TODO.
+ block_fuse: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ """
super(ERes2Net, self).__init__()
self.in_planes = m_channels
self.feat_dim = feat_dim
@@ -221,6 +267,14 @@ class ERes2Net(nn.Module):
self.seg_2 = nn.Identity()
def _make_layer(self, block, planes, num_blocks, stride):
+ """Internal: make layer.
+
+ Args:
+ block: TODO.
+ planes: TODO.
+ num_blocks: TODO.
+ stride: TODO.
+ """
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
@@ -229,10 +283,17 @@ class ERes2Net(nn.Module):
return nn.Sequential(*layers)
def output_size(self) -> int:
+ """Output size."""
return self._output_size
def forward(self, x, ilens):
# assert x.shape[1] == ilens.max()
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)
x = x.unsqueeze_(1)
out = F.relu(self.bn1(self.conv1(x)))
@@ -263,6 +324,15 @@ class BasicBlockRes2Net(nn.Module):
expansion = 2
def __init__(self, in_planes, planes, stride=1, baseWidth=32, scale=2):
+ """Initialize BasicBlockRes2Net.
+
+ Args:
+ in_planes: TODO.
+ planes: TODO.
+ stride: TODO.
+ baseWidth: TODO.
+ scale: TODO.
+ """
super(BasicBlockRes2Net, self).__init__()
width = int(math.floor(planes * (baseWidth / 64.0)))
self.conv1 = conv1x1(in_planes, width * scale, stride)
@@ -292,6 +362,11 @@ class BasicBlockRes2Net(nn.Module):
self.scale = scale
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
residual = x
out = self.conv1(x)
@@ -333,6 +408,17 @@ class Res2Net(nn.Module):
pooling_func="TSTP",
two_emb_layer=False,
):
+ """Initialize Res2Net.
+
+ Args:
+ block: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ """
super(Res2Net, self).__init__()
self.in_planes = m_channels
self.feat_dim = feat_dim
@@ -358,6 +444,14 @@ class Res2Net(nn.Module):
self.seg_2 = nn.Identity()
def _make_layer(self, block, planes, num_blocks, stride):
+ """Internal: make layer.
+
+ Args:
+ block: TODO.
+ planes: TODO.
+ num_blocks: TODO.
+ stride: TODO.
+ """
strides = [stride] + [1] * (num_blocks - 1)
layers = []
for stride in strides:
@@ -366,6 +460,11 @@ class Res2Net(nn.Module):
return nn.Sequential(*layers)
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
x = x.permute(0, 2, 1) # (B,T,F) => (B,F,T)
x = x.unsqueeze_(1)
diff --git a/funasr/models/whisper_lid/eres2net/fusion.py b/funasr/models/whisper_lid/eres2net/fusion.py
index fbe699efb..74886e293 100644
--- a/funasr/models/whisper_lid/eres2net/fusion.py
+++ b/funasr/models/whisper_lid/eres2net/fusion.py
@@ -8,6 +8,12 @@ import torch.nn as nn
class AFF(nn.Module):
def __init__(self, channels=64, r=4):
+ """Initialize AFF.
+
+ Args:
+ channels: TODO.
+ r: TODO.
+ """
super(AFF, self).__init__()
inter_channels = int(channels // r)
@@ -20,6 +26,12 @@ class AFF(nn.Module):
)
def forward(self, x, ds_y):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ds_y: TODO.
+ """
xa = torch.cat((x, ds_y), dim=1)
x_att = self.local_att(xa)
x_att = 1.0 + torch.tanh(x_att)
diff --git a/funasr/models/whisper_lid/eres2net/pooling_layers.py b/funasr/models/whisper_lid/eres2net/pooling_layers.py
index da23d7b4c..6128e1c5f 100644
--- a/funasr/models/whisper_lid/eres2net/pooling_layers.py
+++ b/funasr/models/whisper_lid/eres2net/pooling_layers.py
@@ -15,9 +15,19 @@ class TAP(nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize TAP.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super(TAP, self).__init__()
def forward(self, x):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
pooling_mean = x.mean(dim=-1)
# To be compatable with 2D input
pooling_mean = pooling_mean.flatten(start_dim=1)
@@ -30,10 +40,20 @@ class TSDP(nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize TSDP.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super(TSDP, self).__init__()
def forward(self, x):
# The last dimension is the temporal axis
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ """
pooling_std = torch.sqrt(torch.var(x, dim=-1) + 1e-8)
pooling_std = pooling_std.flatten(start_dim=1)
return pooling_std
@@ -47,10 +67,21 @@ class TSTP(nn.Module):
"""
def __init__(self, **kwargs):
+ """Initialize TSTP.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
super(TSTP, self).__init__()
def forward(self, x, olens):
# The last dimension is the temporal axis
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ olens: TODO.
+ """
masks = (~make_pad_mask(olens, maxlen=x.shape[-1])[:, None, None, :]).to(x.device)
x_masked = x * masks
sum_without_padding = torch.sum(x_masked, axis=-1)
@@ -76,6 +107,13 @@ class ASTP(nn.Module):
"""
def __init__(self, in_dim, bottleneck_dim=128, global_context_att=False):
+ """Initialize ASTP.
+
+ Args:
+ in_dim: Size/dimension parameter.
+ bottleneck_dim: Size/dimension parameter.
+ global_context_att: TODO.
+ """
super(ASTP, self).__init__()
self.global_context_att = global_context_att
diff --git a/funasr/models/whisper_lid/eres2net/simple_avg.py b/funasr/models/whisper_lid/eres2net/simple_avg.py
index 9dbe7b398..fcf7c6ff3 100644
--- a/funasr/models/whisper_lid/eres2net/simple_avg.py
+++ b/funasr/models/whisper_lid/eres2net/simple_avg.py
@@ -6,13 +6,25 @@ from funasr.modules.nets_utils import make_pad_mask
class SimpleAvg(AbsEncoder):
def __init__(self, feat_dim):
+ """Initialize SimpleAvg.
+
+ Args:
+ feat_dim: Size/dimension parameter.
+ """
super(SimpleAvg, self).__init__()
self.feat_dim = feat_dim
def forward(self, x, ilens):
+ """Forward pass for training.
+
+ Args:
+ x: TODO.
+ ilens: TODO.
+ """
mask = ~make_pad_mask(ilens, maxlen=x.shape[1]).to(x.device)
avg_x = (x * mask[:, :, None]).sum(1) / mask.sum(-1)[:, None]
return avg_x
def output_size(self) -> int:
+ """Output size."""
return self.feat_dim
diff --git a/funasr/models/whisper_lid/lid_predictor.py b/funasr/models/whisper_lid/lid_predictor.py
index d5b0d37ac..b3ccf11a5 100644
--- a/funasr/models/whisper_lid/lid_predictor.py
+++ b/funasr/models/whisper_lid/lid_predictor.py
@@ -19,6 +19,18 @@ class LidPredictor(ERes2Net):
pooling_func="TSTP",
two_emb_layer=False,
):
+ """Initialize LidPredictor.
+
+ Args:
+ block: TODO.
+ block_fuse: TODO.
+ num_blocks: TODO.
+ m_channels: TODO.
+ feat_dim: Size/dimension parameter.
+ embedding_size: Size/dimension parameter.
+ pooling_func: TODO.
+ two_emb_layer: TODO.
+ """
super(LidPredictor, self).__init__(
block=block,
block_fuse=block_fuse,
diff --git a/funasr/models/whisper_lid/model.py b/funasr/models/whisper_lid/model.py
index 02cd3730a..baca008b0 100644
--- a/funasr/models/whisper_lid/model.py
+++ b/funasr/models/whisper_lid/model.py
@@ -55,6 +55,36 @@ class OpenAIWhisperModel(nn.Module):
**kwargs,
):
+ """Initialize OpenAIWhisperModel.
+
+ Args:
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ normalize: TODO.
+ normalize_conf: Configuration dict for normalize.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ decoder: TODO.
+ decoder_conf: Configuration dict for decoder.
+ ctc: TODO.
+ ctc_conf: Configuration dict for ctc.
+ ctc_weight: TODO.
+ interctc_weight: TODO.
+ input_size: Size/dimension parameter.
+ vocab_size: Size/dimension parameter.
+ ignore_id: TODO.
+ blank_id: TODO.
+ sos: TODO.
+ eos: TODO.
+ lsm_weight: TODO.
+ length_normalized_loss: TODO.
+ report_cer: TODO.
+ report_wer: TODO.
+ sym_space: TODO.
+ sym_blank: TODO.
+ share_embedding: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
@@ -264,6 +294,14 @@ class OpenAIWhisperModel(nn.Module):
ys_pad: torch.Tensor,
ys_pad_lens: torch.Tensor,
):
+ """Internal: calc att loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
ys_in_pad, ys_out_pad = add_sos_eos(ys_pad, self.sos, self.eos, self.ignore_id)
ys_in_lens = ys_pad_lens + 1
@@ -295,6 +333,14 @@ class OpenAIWhisperModel(nn.Module):
ys_pad_lens: torch.Tensor,
):
# Calc CTC loss
+ """Internal: calc ctc loss.
+
+ Args:
+ encoder_out: Encoder output tensor.
+ encoder_out_lens: Encoder output lengths.
+ ys_pad: TODO.
+ ys_pad_lens: Lengths of ys_pad.
+ """
loss_ctc = self.ctc(encoder_out, encoder_out_lens, ys_pad, ys_pad_lens)
# Calc CER using CTC
@@ -308,6 +354,11 @@ class OpenAIWhisperModel(nn.Module):
self,
**kwargs,
):
+ """Init beam search.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.transformer.search import BeamSearch
from funasr.models.transformer.scorers.ctc import CTCPrefixScorer
from funasr.models.transformer.scorers.length_bonus import LengthBonus
@@ -359,6 +410,16 @@ class OpenAIWhisperModel(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
@@ -472,6 +533,21 @@ class OpenAIWhisperLIDModel(nn.Module):
random_clip: bool = False,
**kwargs,
):
+ """Initialize OpenAIWhisperLIDModel.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ specaug: TODO.
+ specaug_conf: Configuration dict for specaug.
+ encoder: TODO.
+ encoder_conf: Configuration dict for encoder.
+ lid_predictor: TODO.
+ lid_predictor_conf: Configuration dict for lid_predictor.
+ proj_dim: Size/dimension parameter.
+ clip_frames: TODO.
+ random_clip: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__()
if specaug is not None:
specaug_class = tables.specaug_classes.get(specaug)
@@ -509,6 +585,14 @@ class OpenAIWhisperLIDModel(nn.Module):
lid: torch.Tensor, # lid label, (batch_size, 1)
lid_lengths: torch.Tensor,
):
+ """Forward pass for training.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ lid: TODO.
+ lid_lengths: Lengths of lid.
+ """
assert lid.shape[1] == 1
batch_size = speech.shape[0]
encoder_out, encoder_out_lens = self.encode(speech, speech_lengths)
@@ -611,6 +695,16 @@ class OpenAIWhisperLIDModel(nn.Module):
**kwargs,
):
+ """Run inference on input data.
+
+ Args:
+ data_in: Input data (audio samples, file paths, or text).
+ data_lengths: Lengths of each input sample in the batch.
+ key: Sample identifiers.
+ tokenizer: Tokenizer instance for text encoding/decoding.
+ frontend: Audio frontend for feature extraction.
+ **kwargs: Additional keyword arguments.
+ """
if kwargs.get("batch_size", 1) > 1:
raise NotImplementedError("batch decoding is not implemented")
diff --git a/funasr/models/xvector/e2e_sv.py b/funasr/models/xvector/e2e_sv.py
index 56c9e3b10..c6062b59c 100644
--- a/funasr/models/xvector/e2e_sv.py
+++ b/funasr/models/xvector/e2e_sv.py
@@ -36,6 +36,11 @@ else:
# Nothing to do if torch<1.6.0
@contextmanager
def autocast(enabled=True):
+ """Autocast.
+
+ Args:
+ enabled: TODO.
+ """
yield
@@ -56,6 +61,20 @@ class ESPnetSVModel(FunASRModel):
decoder: AbsDecoder,
):
+ """Initialize ESPnetSVModel.
+
+ Args:
+ vocab_size: Size/dimension parameter.
+ token_list: TODO.
+ frontend: Audio frontend for feature extraction.
+ specaug: TODO.
+ normalize: TODO.
+ preencoder: TODO.
+ encoder: TODO.
+ postencoder: TODO.
+ pooling_layer: TODO.
+ decoder: TODO.
+ """
super().__init__()
# note that eos is the same as sos (equivalent ID)
self.vocab_size = vocab_size
@@ -197,6 +216,14 @@ class ESPnetSVModel(FunASRModel):
text: torch.Tensor,
text_lengths: torch.Tensor,
) -> Dict[str, torch.Tensor]:
+ """Collect feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ text: Text tensor or string input.
+ text_lengths: Length of each text sample.
+ """
if self.extract_feats_in_collect_stats:
feats, feats_lengths = self._extract_feats(speech, speech_lengths)
else:
@@ -246,6 +273,12 @@ class ESPnetSVModel(FunASRModel):
def _extract_feats(
self, speech: torch.Tensor, speech_lengths: torch.Tensor
) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Internal: extract feats.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ """
assert speech_lengths.dim() == 1, speech_lengths.shape
# for data-parallel
diff --git a/funasr/optimizers/fairseq_adam.py b/funasr/optimizers/fairseq_adam.py
index 32fae6a25..c140c62f0 100644
--- a/funasr/optimizers/fairseq_adam.py
+++ b/funasr/optimizers/fairseq_adam.py
@@ -46,6 +46,16 @@ class FairseqAdam(torch.optim.Optimizer):
weight_decay=0,
amsgrad=False,
):
+ """Initialize FairseqAdam.
+
+ Args:
+ params: TODO.
+ lr: TODO.
+ adam_betas: TODO.
+ adam_eps: TODO.
+ weight_decay: TODO.
+ amsgrad: TODO.
+ """
defaults = dict(
lr=lr, betas=adam_betas, eps=adam_eps, weight_decay=weight_decay, amsgrad=amsgrad
)
@@ -54,10 +64,12 @@ class FairseqAdam(torch.optim.Optimizer):
@property
def supports_memory_efficient_fp16(self):
+ """Supports memory efficient fp16."""
return True
@property
def supports_flat_params(self):
+ """Supports flat params."""
return True
def step(self, closure=None):
diff --git a/funasr/optimizers/sgd.py b/funasr/optimizers/sgd.py
index fb7a3df01..04993916f 100644
--- a/funasr/optimizers/sgd.py
+++ b/funasr/optimizers/sgd.py
@@ -20,6 +20,16 @@ class SGD(torch.optim.SGD):
weight_decay: float = 0.0,
nesterov: bool = False,
):
+ """Initialize SGD.
+
+ Args:
+ params: TODO.
+ lr: TODO.
+ momentum: TODO.
+ dampening: TODO.
+ weight_decay: TODO.
+ nesterov: TODO.
+ """
super().__init__(
params,
lr=lr,
diff --git a/funasr/register.py b/funasr/register.py
index a8b1fd333..b2ef75a3d 100644
--- a/funasr/register.py
+++ b/funasr/register.py
@@ -50,6 +50,11 @@ class RegisterTables:
"""Decorator to register a class."""
def decorator(target_class):
+ """Decorator.
+
+ Args:
+ target_class: TODO.
+ """
if not hasattr(self, register_tables_key):
setattr(self, register_tables_key, {})
logging.debug(f"New registry table added: {register_tables_key}")
diff --git a/funasr/schedulers/abs_scheduler.py b/funasr/schedulers/abs_scheduler.py
index 7395f259c..7181a8756 100644
--- a/funasr/schedulers/abs_scheduler.py
+++ b/funasr/schedulers/abs_scheduler.py
@@ -7,14 +7,25 @@ import torch.optim.lr_scheduler as L
class AbsScheduler(ABC):
@abstractmethod
def step(self, epoch: int = None):
+ """Step.
+
+ Args:
+ epoch: TODO.
+ """
pass
@abstractmethod
def state_dict(self):
+ """State dict."""
pass
@abstractmethod
def load_state_dict(self, state):
+ """Load state dict.
+
+ Args:
+ state: TODO.
+ """
pass
@@ -22,42 +33,76 @@ class AbsScheduler(ABC):
class AbsBatchStepScheduler(AbsScheduler):
@abstractmethod
def step(self, epoch: int = None):
+ """Step.
+
+ Args:
+ epoch: TODO.
+ """
pass
@abstractmethod
def state_dict(self):
+ """State dict."""
pass
@abstractmethod
def load_state_dict(self, state):
+ """Load state dict.
+
+ Args:
+ state: TODO.
+ """
pass
class AbsEpochStepScheduler(AbsScheduler):
@abstractmethod
def step(self, epoch: int = None):
+ """Step.
+
+ Args:
+ epoch: TODO.
+ """
pass
@abstractmethod
def state_dict(self):
+ """State dict."""
pass
@abstractmethod
def load_state_dict(self, state):
+ """Load state dict.
+
+ Args:
+ state: TODO.
+ """
pass
class AbsValEpochStepScheduler(AbsEpochStepScheduler):
@abstractmethod
def step(self, val, epoch: int = None):
+ """Step.
+
+ Args:
+ val: TODO.
+ epoch: TODO.
+ """
pass
@abstractmethod
def state_dict(self):
+ """State dict."""
pass
@abstractmethod
def load_state_dict(self, state):
+ """Load state dict.
+
+ Args:
+ state: TODO.
+ """
pass
diff --git a/funasr/schedulers/lambdalr_cus.py b/funasr/schedulers/lambdalr_cus.py
index e3bb1fb47..899cceb3c 100644
--- a/funasr/schedulers/lambdalr_cus.py
+++ b/funasr/schedulers/lambdalr_cus.py
@@ -25,12 +25,22 @@ class CustomLambdaLR(_LRScheduler):
last_epoch=-1,
verbose=False,
):
+ """Initialize CustomLambdaLR.
+
+ Args:
+ optimizer: TODO.
+ warmup_steps: TODO.
+ total_steps: TODO.
+ last_epoch: TODO.
+ verbose: TODO.
+ """
self.warmup_steps = warmup_steps
self.total_steps = total_steps
super().__init__(optimizer, last_epoch, verbose)
def get_lr(self):
+ """Get lr."""
step = self.last_epoch + 1
if step < self.warmup_steps:
lr_scale = step / self.warmup_steps
diff --git a/funasr/schedulers/noam_lr.py b/funasr/schedulers/noam_lr.py
index b33b4bde6..8cec25ddf 100644
--- a/funasr/schedulers/noam_lr.py
+++ b/funasr/schedulers/noam_lr.py
@@ -31,6 +31,14 @@ class NoamLR(_LRScheduler, AbsBatchStepScheduler):
warmup_steps: Union[int, float] = 25000,
last_epoch: int = -1,
):
+ """Initialize NoamLR.
+
+ Args:
+ optimizer: TODO.
+ model_size: Size/dimension parameter.
+ warmup_steps: TODO.
+ last_epoch: TODO.
+ """
self.model_size = model_size
self.warmup_steps = warmup_steps
@@ -46,15 +54,22 @@ class NoamLR(_LRScheduler, AbsBatchStepScheduler):
super().__init__(optimizer, last_epoch)
def lr_for_WarmupLR(self, lr: float) -> float:
+ """Lr for warmuplr.
+
+ Args:
+ lr: TODO.
+ """
return lr / self.model_size**0.5 / self.warmup_steps**0.5
def __repr__(self):
+ """Internal: repr ."""
return (
f"{self.__class__.__name__}(model_size={self.model_size}, "
f"warmup_steps={self.warmup_steps})"
)
def get_lr(self):
+ """Get lr."""
step_num = self.last_epoch + 1
return [
lr * self.model_size**-0.5 * min(step_num**-0.5, step_num * self.warmup_steps**-1.5)
diff --git a/funasr/schedulers/tri_stage_scheduler.py b/funasr/schedulers/tri_stage_scheduler.py
index 7fc13b2ff..110477803 100644
--- a/funasr/schedulers/tri_stage_scheduler.py
+++ b/funasr/schedulers/tri_stage_scheduler.py
@@ -21,6 +21,15 @@ class TriStageLR(_LRScheduler, AbsBatchStepScheduler):
init_lr_scale: float = 0.01,
final_lr_scale: float = 0.01,
):
+ """Initialize TriStageLR.
+
+ Args:
+ optimizer: TODO.
+ last_epoch: TODO.
+ phase_ratio: TODO.
+ init_lr_scale: TODO.
+ final_lr_scale: TODO.
+ """
self.optimizer = optimizer
self.last_epoch = last_epoch
self.phase_ratio = phase_ratio
@@ -29,6 +38,11 @@ class TriStageLR(_LRScheduler, AbsBatchStepScheduler):
self.optimizer_lr = self.optimizer.defaults["lr"]
def init_tri_stage_scheudler(self, max_update):
+ """Init tri stage scheudler.
+
+ Args:
+ max_update: TODO.
+ """
self.max_update = max_update
self.peak_lr = self.optimizer_lr
self.init_lr = self.init_lr_scale * self.optimizer_lr
@@ -95,10 +109,16 @@ class TriStageLR(_LRScheduler, AbsBatchStepScheduler):
self.set_optimizer_lr(self.lr)
def set_optimizer_lr(self, lr):
+ """Set optimizer lr.
+
+ Args:
+ lr: TODO.
+ """
for param_group in self.optimizer.param_groups:
param_group["lr"] = lr
def get_lr(self):
+ """Get lr."""
step_num = self.last_epoch + 1
self.step_update(step_num)
return [self.lr]
diff --git a/funasr/schedulers/warmup_lr.py b/funasr/schedulers/warmup_lr.py
index 847b53ad7..13af0226f 100644
--- a/funasr/schedulers/warmup_lr.py
+++ b/funasr/schedulers/warmup_lr.py
@@ -30,6 +30,13 @@ class WarmupLR(_LRScheduler, AbsBatchStepScheduler):
warmup_steps: Union[int, float] = 25000,
last_epoch: int = -1,
):
+ """Initialize WarmupLR.
+
+ Args:
+ optimizer: TODO.
+ warmup_steps: TODO.
+ last_epoch: TODO.
+ """
self.warmup_steps = warmup_steps
# __init__() must be invoked before setting field
@@ -37,9 +44,11 @@ class WarmupLR(_LRScheduler, AbsBatchStepScheduler):
super().__init__(optimizer, last_epoch)
def __repr__(self):
+ """Internal: repr ."""
return f"{self.__class__.__name__}(warmup_steps={self.warmup_steps})"
def get_lr(self):
+ """Get lr."""
step_num = self.last_epoch + 1
return [
lr * self.warmup_steps**0.5 * min(step_num**-0.5, step_num * self.warmup_steps**-1.5)
diff --git a/funasr/tokenizer/abs_tokenizer.py b/funasr/tokenizer/abs_tokenizer.py
index e125d292b..a1d51b228 100644
--- a/funasr/tokenizer/abs_tokenizer.py
+++ b/funasr/tokenizer/abs_tokenizer.py
@@ -9,10 +9,20 @@ from typing import Union, Iterable, List, Dict
class AbsTokenizer(ABC):
@abstractmethod
def text2tokens(self, line: str) -> List[str]:
+ """Text2tokens.
+
+ Args:
+ line: TODO.
+ """
raise NotImplementedError
@abstractmethod
def tokens2text(self, tokens: Iterable[str]) -> str:
+ """Tokens2text.
+
+ Args:
+ tokens: TODO.
+ """
raise NotImplementedError
@@ -24,6 +34,13 @@ class BaseTokenizer(ABC):
**kwargs,
):
+ """Initialize BaseTokenizer.
+
+ Args:
+ token_list: TODO.
+ unk_symbol: TODO.
+ **kwargs: Additional keyword arguments.
+ """
if token_list is not None:
if isinstance(token_list, (Path, str)) and token_list.endswith(".txt"):
token_list = Path(token_list)
@@ -63,31 +80,63 @@ class BaseTokenizer(ABC):
self.unk_id = self.token2id[self.unk_symbol]
def encode(self, text, **kwargs):
+ """Encode.
+
+ Args:
+ text: Text tensor or string input.
+ **kwargs: Additional keyword arguments.
+ """
tokens = self.text2tokens(text)
text_ints = self.tokens2ids(tokens)
return text_ints
def decode(self, text_ints):
+ """Decode.
+
+ Args:
+ text_ints: TODO.
+ """
token = self.ids2tokens(text_ints)
text = self.tokens2text(token)
return text
def get_num_vocabulary_size(self) -> int:
+ """Get num vocabulary size."""
return len(self.token_list)
def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]:
+ """Ids2tokens.
+
+ Args:
+ integers: TODO.
+ """
if isinstance(integers, np.ndarray) and integers.ndim != 1:
raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}")
return [self.token_list[i] for i in integers]
def tokens2ids(self, tokens: Iterable[str]) -> List[int]:
+ """Tokens2ids.
+
+ Args:
+ tokens: TODO.
+ """
return [self.token2id.get(i, self.unk_id) for i in tokens]
@abstractmethod
def text2tokens(self, line: str) -> List[str]:
+ """Text2tokens.
+
+ Args:
+ line: TODO.
+ """
raise NotImplementedError
@abstractmethod
def tokens2text(self, tokens: Iterable[str]) -> str:
+ """Tokens2text.
+
+ Args:
+ tokens: TODO.
+ """
raise NotImplementedError
diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py
index 7b517dacb..d25c98926 100644
--- a/funasr/tokenizer/char_tokenizer.py
+++ b/funasr/tokenizer/char_tokenizer.py
@@ -20,6 +20,16 @@ class CharTokenizer(BaseTokenizer):
seg_dict: str = None,
**kwargs,
):
+ """Initialize CharTokenizer.
+
+ Args:
+ non_linguistic_symbols: TODO.
+ space_symbol: TODO.
+ remove_non_linguistic_symbols: TODO.
+ split_with_space: TODO.
+ seg_dict: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(**kwargs)
self.space_symbol = space_symbol
if non_linguistic_symbols is None:
@@ -42,6 +52,7 @@ class CharTokenizer(BaseTokenizer):
self.seg_dict = load_seg_dict(seg_dict)
def __repr__(self):
+ """Internal: repr ."""
return (
f"{self.__class__.__name__}("
f'space_symbol="{self.space_symbol}"'
@@ -51,6 +62,11 @@ class CharTokenizer(BaseTokenizer):
def text2tokens(self, line: Union[str, list]) -> List[str]:
# if self.split_with_space:
+ """Text2tokens.
+
+ Args:
+ line: TODO.
+ """
if self.seg_dict is not None:
tokens = line.strip().split(" ")
tokens = seg_tokenize(tokens, self.seg_dict)
@@ -74,11 +90,21 @@ class CharTokenizer(BaseTokenizer):
return tokens
def tokens2text(self, tokens: Iterable[str]) -> str:
+ """Tokens2text.
+
+ Args:
+ tokens: TODO.
+ """
tokens = [t if t != self.space_symbol else " " for t in tokens]
return "".join(tokens)
def load_seg_dict(seg_dict_file):
+ """Load seg dict.
+
+ Args:
+ seg_dict_file: TODO.
+ """
seg_dict = {}
assert isinstance(seg_dict_file, str)
with open(seg_dict_file, "r", encoding="utf8") as f:
@@ -93,6 +119,12 @@ def load_seg_dict(seg_dict_file):
def seg_tokenize(txt, seg_dict):
# pattern = re.compile(r'^[\u4E00-\u9FA50-9]+$')
+ """Seg tokenize.
+
+ Args:
+ txt: TODO.
+ seg_dict: TODO.
+ """
pattern = re.compile(r"([\u4E00-\u9FA5A-Za-z0-9])")
out_txt = ""
for word in txt:
diff --git a/funasr/tokenizer/cleaner.py b/funasr/tokenizer/cleaner.py
index 882db5c97..1c609caf0 100644
--- a/funasr/tokenizer/cleaner.py
+++ b/funasr/tokenizer/cleaner.py
@@ -22,6 +22,11 @@ class TextCleaner:
def __init__(self, cleaner_types: Collection[str] = None):
+ """Initialize TextCleaner.
+
+ Args:
+ cleaner_types: TODO.
+ """
if cleaner_types is None:
self.cleaner_types = []
elif isinstance(cleaner_types, str):
@@ -30,6 +35,11 @@ class TextCleaner:
self.cleaner_types = list(cleaner_types)
def __call__(self, text: str) -> str:
+ """Internal: call .
+
+ Args:
+ text: Text tensor or string input.
+ """
for t in self.cleaner_types:
if t == "tacotron":
# text = tacotron_cleaner.cleaners.custom_english_cleaners(text)
diff --git a/funasr/tokenizer/hf_tokenizer.py b/funasr/tokenizer/hf_tokenizer.py
index 2b782179c..57c23cb53 100644
--- a/funasr/tokenizer/hf_tokenizer.py
+++ b/funasr/tokenizer/hf_tokenizer.py
@@ -3,6 +3,12 @@ from funasr.register import tables
@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):
+ """Huggingfacetokenizer.
+
+ Args:
+ init_param_path: TODO.
+ **kwargs: Additional keyword arguments.
+ """
try:
from transformers import AutoTokenizer
except Exception as e:
diff --git a/funasr/tokenizer/korean_cleaner.py b/funasr/tokenizer/korean_cleaner.py
index eceebe216..ff0beeb60 100644
--- a/funasr/tokenizer/korean_cleaner.py
+++ b/funasr/tokenizer/korean_cleaner.py
@@ -6,6 +6,11 @@ import re
class KoreanCleaner:
@classmethod
def _normalize_numbers(cls, text):
+ """Internal: normalize numbers.
+
+ Args:
+ text: Text tensor or string input.
+ """
number_to_kor = {
"0": "영",
"1": "일",
@@ -25,6 +30,11 @@ class KoreanCleaner:
@classmethod
def _normalize_english_text(cls, text):
+ """Internal: normalize english text.
+
+ Args:
+ text: Text tensor or string input.
+ """
upper_alphabet_to_kor = {
"A": "에이",
"B": "비",
@@ -64,6 +74,11 @@ class KoreanCleaner:
@classmethod
def normalize_text(cls, text):
# stage 0 : text strip
+ """Normalize text.
+
+ Args:
+ text: Text tensor or string input.
+ """
text = text.strip()
# stage 1 : normalize numbers
diff --git a/funasr/tokenizer/phoneme_tokenizer.py b/funasr/tokenizer/phoneme_tokenizer.py
index 12b29ec22..ab479242a 100644
--- a/funasr/tokenizer/phoneme_tokenizer.py
+++ b/funasr/tokenizer/phoneme_tokenizer.py
@@ -43,6 +43,11 @@ g2p_classes = [
def split_by_space(text) -> List[str]:
+ """Split by space.
+
+ Args:
+ text: Text tensor or string input.
+ """
if " " in text:
text = text.replace(" ", " <space> ")
return [c.replace("<space>", " ") for c in text.split(" ")]
@@ -51,6 +56,11 @@ def split_by_space(text) -> List[str]:
def pyopenjtalk_g2p(text) -> List[str]:
+ """Pyopenjtalk g2p.
+
+ Args:
+ text: Text tensor or string input.
+ """
import pyopenjtalk
# phones is a str object separated by space
@@ -60,6 +70,11 @@ def pyopenjtalk_g2p(text) -> List[str]:
def pyopenjtalk_g2p_accent(text) -> List[str]:
+ """Pyopenjtalk g2p accent.
+
+ Args:
+ text: Text tensor or string input.
+ """
import pyopenjtalk
import re
@@ -72,6 +87,11 @@ def pyopenjtalk_g2p_accent(text) -> List[str]:
def pyopenjtalk_g2p_accent_with_pause(text) -> List[str]:
+ """Pyopenjtalk g2p accent with pause.
+
+ Args:
+ text: Text tensor or string input.
+ """
import pyopenjtalk
import re
@@ -87,6 +107,11 @@ def pyopenjtalk_g2p_accent_with_pause(text) -> List[str]:
def pyopenjtalk_g2p_kana(text) -> List[str]:
+ """Pyopenjtalk g2p kana.
+
+ Args:
+ text: Text tensor or string input.
+ """
import pyopenjtalk
kanas = pyopenjtalk.g2p(text, kana=True)
@@ -173,6 +198,12 @@ def pyopenjtalk_g2p_prosody(text: str, drop_unvoiced_vowels: bool = True) -> Lis
def _numeric_feature_by_regex(regex, s):
+ """Internal: numeric feature by regex.
+
+ Args:
+ regex: TODO.
+ s: TODO.
+ """
match = re.search(regex, s)
if match is None:
return -50
@@ -180,6 +211,11 @@ def _numeric_feature_by_regex(regex, s):
def pypinyin_g2p(text) -> List[str]:
+ """Pypinyin g2p.
+
+ Args:
+ text: Text tensor or string input.
+ """
from pypinyin import pinyin
from pypinyin import Style
@@ -188,6 +224,11 @@ def pypinyin_g2p(text) -> List[str]:
def pypinyin_g2p_phone(text) -> List[str]:
+ """Pypinyin g2p phone.
+
+ Args:
+ text: Text tensor or string input.
+ """
from pypinyin import pinyin
from pypinyin import Style
from pypinyin.style._utils import get_finals
@@ -215,10 +256,20 @@ class G2p_en:
"""
def __init__(self, no_space: bool = False):
+ """Initialize G2p_en.
+
+ Args:
+ no_space: TODO.
+ """
self.no_space = no_space
self.g2p = None
def __call__(self, text) -> List[str]:
+ """Internal: call .
+
+ Args:
+ text: Text tensor or string input.
+ """
if self.g2p is None:
self.g2p = g2p_en.G2p()
@@ -239,6 +290,14 @@ class G2pk:
"""
def __init__(self, descritive=False, group_vowels=False, to_syl=False, no_space=False):
+ """Initialize G2pk.
+
+ Args:
+ descritive: TODO.
+ group_vowels: TODO.
+ to_syl: TODO.
+ no_space: TODO.
+ """
self.descritive = descritive
self.group_vowels = group_vowels
self.to_syl = to_syl
@@ -246,6 +305,11 @@ class G2pk:
self.g2p = None
def __call__(self, text) -> List[str]:
+ """Internal: call .
+
+ Args:
+ text: Text tensor or string input.
+ """
if self.g2p is None:
import g2pk
@@ -276,18 +340,39 @@ class Jaso:
VALID_CHARS = JAMO_LEADS + JAMO_VOWELS + JAMO_TAILS + PUNC + SPACE
def __init__(self, space_symbol=" ", no_space=False):
+ """Initialize Jaso.
+
+ Args:
+ space_symbol: TODO.
+ no_space: TODO.
+ """
self.space_symbol = space_symbol
self.no_space = no_space
def _text_to_jaso(self, line: str) -> List[str]:
+ """Internal: text to jaso.
+
+ Args:
+ line: TODO.
+ """
jasos = list(jamo.hangul_to_jamo(line))
return jasos
def _remove_non_korean_characters(self, tokens):
+ """Internal: remove non korean characters.
+
+ Args:
+ tokens: TODO.
+ """
new_tokens = [token for token in tokens if token in self.VALID_CHARS]
return new_tokens
def __call__(self, text) -> List[str]:
+ """Internal: call .
+
+ Args:
+ text: Text tensor or string input.
+ """
graphemes = [x for x in self._text_to_jaso(text)]
graphemes = self._remove_non_korean_characters(graphemes)
@@ -320,6 +405,17 @@ class Phonemizer:
**phonemizer_kwargs,
):
# delayed import
+ """Initialize Phonemizer.
+
+ Args:
+ backend: TODO.
+ word_separator: TODO.
+ syllable_separator: TODO.
+ phone_separator: TODO.
+ strip: TODO.
+ split_by_single_token: TODO.
+ **phonemizer_kwargs: Additional keyword arguments.
+ """
from phonemizer.backend import BACKENDS
from phonemizer.separator import Separator
@@ -340,6 +436,11 @@ class Phonemizer:
self.split_by_single_token = split_by_single_token
def __call__(self, text) -> List[str]:
+ """Internal: call .
+
+ Args:
+ text: Text tensor or string input.
+ """
tokens = self.phonemizer.phonemize(
[text],
separator=self.separator,
@@ -362,6 +463,14 @@ class PhonemeTokenizer(AbsTokenizer):
space_symbol: str = "<space>",
remove_non_linguistic_symbols: bool = False,
):
+ """Initialize PhonemeTokenizer.
+
+ Args:
+ g2p_type: TODO.
+ non_linguistic_symbols: TODO.
+ space_symbol: TODO.
+ remove_non_linguistic_symbols: TODO.
+ """
if g2p_type is None:
self.g2p = split_by_space
elif g2p_type == "g2p_en":
@@ -493,6 +602,7 @@ class PhonemeTokenizer(AbsTokenizer):
self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
def __repr__(self):
+ """Internal: repr ."""
return (
f"{self.__class__.__name__}("
f'g2p_type="{self.g2p_type}", '
@@ -502,6 +612,11 @@ class PhonemeTokenizer(AbsTokenizer):
)
def text2tokens(self, line: str) -> List[str]:
+ """Text2tokens.
+
+ Args:
+ line: TODO.
+ """
tokens = []
while len(line) != 0:
for w in self.non_linguistic_symbols:
@@ -521,4 +636,9 @@ class PhonemeTokenizer(AbsTokenizer):
def tokens2text(self, tokens: Iterable[str]) -> str:
# phoneme type is not invertible
+ """Tokens2text.
+
+ Args:
+ tokens: TODO.
+ """
return "".join(tokens)
diff --git a/funasr/tokenizer/sentencepiece_tokenizer.py b/funasr/tokenizer/sentencepiece_tokenizer.py
index 0b47a9f6a..acba22c2a 100644
--- a/funasr/tokenizer/sentencepiece_tokenizer.py
+++ b/funasr/tokenizer/sentencepiece_tokenizer.py
@@ -12,6 +12,12 @@ from funasr.register import tables
@tables.register("tokenizer_classes", "SentencepiecesTokenizer")
class SentencepiecesTokenizer(BaseTokenizer):
def __init__(self, bpemodel: Union[Path, str], **kwargs):
+ """Initialize SentencepiecesTokenizer.
+
+ Args:
+ bpemodel: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(**kwargs)
self.bpemodel = str(bpemodel)
# NOTE(kamo):
@@ -23,35 +29,72 @@ class SentencepiecesTokenizer(BaseTokenizer):
self._build_sentence_piece_processor()
def __repr__(self):
+ """Internal: repr ."""
return f'{self.__class__.__name__}(model="{self.bpemodel}")'
def _build_sentence_piece_processor(self):
# Build SentencePieceProcessor lazily.
+ """Internal: build sentence piece processor."""
if self.sp is None:
self.sp = spm.SentencePieceProcessor()
self.sp.load(self.bpemodel)
def text2tokens(self, line: str) -> List[str]:
+ """Text2tokens.
+
+ Args:
+ line: TODO.
+ """
self._build_sentence_piece_processor()
return self.sp.EncodeAsPieces(line)
def tokens2text(self, tokens: Iterable[str]) -> str:
+ """Tokens2text.
+
+ Args:
+ tokens: TODO.
+ """
self._build_sentence_piece_processor()
return self.sp.DecodePieces(list(tokens))
def encode(self, line: str, **kwargs) -> List[int]:
+ """Encode.
+
+ Args:
+ line: TODO.
+ **kwargs: Additional keyword arguments.
+ """
self._build_sentence_piece_processor()
return self.sp.EncodeAsIds(line)
def decode(self, line: List[int], **kwargs):
+ """Decode.
+
+ Args:
+ line: TODO.
+ **kwargs: Additional keyword arguments.
+ """
self._build_sentence_piece_processor()
return self.sp.DecodeIds(line)
def get_vocab_size(self):
+ """Get vocab size."""
return self.sp.GetPieceSize()
def ids2tokens(self, *args, **kwargs):
+ """Ids2tokens.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
return self.decode(*args, **kwargs)
def tokens2ids(self, *args, **kwargs):
+ """Tokens2ids.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
return self.encode(*args, **kwargs)
diff --git a/funasr/tokenizer/token_id_converter.py b/funasr/tokenizer/token_id_converter.py
index c3fc71d5a..ff60d4992 100644
--- a/funasr/tokenizer/token_id_converter.py
+++ b/funasr/tokenizer/token_id_converter.py
@@ -14,6 +14,12 @@ class TokenIDConverter:
unk_symbol: str = "<unk>",
):
+ """Initialize TokenIDConverter.
+
+ Args:
+ token_list: TODO.
+ unk_symbol: TODO.
+ """
if isinstance(token_list, (Path, str)):
token_list = Path(token_list)
self.token_list_repr = str(token_list)
@@ -45,12 +51,23 @@ class TokenIDConverter:
self.unk_id = self.token2id[self.unk_symbol]
def get_num_vocabulary_size(self) -> int:
+ """Get num vocabulary size."""
return len(self.token_list)
def ids2tokens(self, integers: Union[np.ndarray, Iterable[int]]) -> List[str]:
+ """Ids2tokens.
+
+ Args:
+ integers: TODO.
+ """
if isinstance(integers, np.ndarray) and integers.ndim != 1:
raise ValueError(f"Must be 1 dim ndarray, but got {integers.ndim}")
return [self.token_list[i] for i in integers]
def tokens2ids(self, tokens: Iterable[str]) -> List[int]:
+ """Tokens2ids.
+
+ Args:
+ tokens: TODO.
+ """
return [self.token2id.get(i, self.unk_id) for i in tokens]
diff --git a/funasr/tokenizer/whisper_tokenizer.py b/funasr/tokenizer/whisper_tokenizer.py
index 3fd029c57..c085b4811 100644
--- a/funasr/tokenizer/whisper_tokenizer.py
+++ b/funasr/tokenizer/whisper_tokenizer.py
@@ -3,6 +3,11 @@ from funasr.register import tables
@tables.register("tokenizer_classes", "WhisperTokenizer")
def WhisperTokenizer(**kwargs):
+ """Whispertokenizer.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
try:
from whisper.tokenizer import get_tokenizer
except:
@@ -24,6 +29,11 @@ def WhisperTokenizer(**kwargs):
@tables.register("tokenizer_classes", "SenseVoiceTokenizer")
def SenseVoiceTokenizer(**kwargs):
+ """Sensevoicetokenizer.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
from funasr.models.sense_voice.whisper_lib.tokenizer import get_tokenizer
language = kwargs.get("language", None)
diff --git a/funasr/tokenizer/word_tokenizer.py b/funasr/tokenizer/word_tokenizer.py
index c1fd6e039..ded95f444 100644
--- a/funasr/tokenizer/word_tokenizer.py
+++ b/funasr/tokenizer/word_tokenizer.py
@@ -15,6 +15,13 @@ class WordTokenizer(AbsTokenizer):
non_linguistic_symbols: Union[Path, str, Iterable[str]] = None,
remove_non_linguistic_symbols: bool = False,
):
+ """Initialize WordTokenizer.
+
+ Args:
+ delimiter: TODO.
+ non_linguistic_symbols: TODO.
+ remove_non_linguistic_symbols: TODO.
+ """
self.delimiter = delimiter
if not remove_non_linguistic_symbols and non_linguistic_symbols is not None:
@@ -37,9 +44,15 @@ class WordTokenizer(AbsTokenizer):
self.remove_non_linguistic_symbols = remove_non_linguistic_symbols
def __repr__(self):
+ """Internal: repr ."""
return f'{self.__class__.__name__}(delimiter="{self.delimiter}")'
def text2tokens(self, line: str) -> List[str]:
+ """Text2tokens.
+
+ Args:
+ line: TODO.
+ """
tokens = []
for t in line.split(self.delimiter):
if self.remove_non_linguistic_symbols and t in self.non_linguistic_symbols:
@@ -48,6 +61,11 @@ class WordTokenizer(AbsTokenizer):
return tokens
def tokens2text(self, tokens: Iterable[str]) -> str:
+ """Tokens2text.
+
+ Args:
+ tokens: TODO.
+ """
if self.delimiter is None:
delimiter = " "
else:
diff --git a/funasr/train_utils/forward_adaptor.py b/funasr/train_utils/forward_adaptor.py
index eb6da2bb1..b66c13fe2 100644
--- a/funasr/train_utils/forward_adaptor.py
+++ b/funasr/train_utils/forward_adaptor.py
@@ -20,6 +20,12 @@ class ForwardAdaptor(torch.nn.Module):
"""
def __init__(self, module: torch.nn.Module, name: str):
+ """Initialize ForwardAdaptor.
+
+ Args:
+ module: TODO.
+ name: TODO.
+ """
super().__init__()
self.module = module
self.name = name
@@ -27,5 +33,11 @@ class ForwardAdaptor(torch.nn.Module):
raise ValueError(f"{module} doesn't have {name}")
def forward(self, *args, **kwargs):
+ """Forward pass for training.
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
func = getattr(self.module, self.name)
return func(*args, **kwargs)
diff --git a/funasr/train_utils/model_summary.py b/funasr/train_utils/model_summary.py
index 842cd21fa..c9215fffa 100644
--- a/funasr/train_utils/model_summary.py
+++ b/funasr/train_utils/model_summary.py
@@ -41,10 +41,20 @@ def get_human_readable_count(number: int) -> str:
def to_bytes(dtype) -> int:
# torch.float16 -> 16
+ """To bytes.
+
+ Args:
+ dtype: TODO.
+ """
return int(str(dtype)[-2:]) // 8
def model_summary(model: torch.nn.Module) -> str:
+ """Model summary.
+
+ Args:
+ model: Model instance or model name.
+ """
message = "Model structure:\n"
message += str(model)
diff --git a/funasr/train_utils/recursive_op.py b/funasr/train_utils/recursive_op.py
index 1c2a92fd8..1e471012e 100644
--- a/funasr/train_utils/recursive_op.py
+++ b/funasr/train_utils/recursive_op.py
@@ -7,6 +7,13 @@ if torch.distributed.is_available():
def recursive_sum(obj, weight: torch.Tensor, distributed: bool = False):
+ """Recursive sum.
+
+ Args:
+ obj: TODO.
+ weight: TODO.
+ distributed: TODO.
+ """
assert weight.dim() == 1, weight.size()
if isinstance(obj, (tuple, list)):
return type(obj)(recursive_sum(v, weight, distributed) for v in obj)
@@ -25,6 +32,12 @@ def recursive_sum(obj, weight: torch.Tensor, distributed: bool = False):
def recursive_divide(a, b: torch.Tensor):
+ """Recursive divide.
+
+ Args:
+ a: TODO.
+ b: TODO.
+ """
if isinstance(a, (tuple, list)):
return type(a)(recursive_divide(v, b) for v in a)
elif isinstance(a, dict):
@@ -39,6 +52,13 @@ def recursive_divide(a, b: torch.Tensor):
def recursive_average(obj, weight: torch.Tensor, distributed: bool = False):
+ """Recursive average.
+
+ Args:
+ obj: TODO.
+ weight: TODO.
+ distributed: TODO.
+ """
obj = recursive_sum(obj, weight, distributed)
weight = weight.sum()
if distributed:
diff --git a/funasr/train_utils/set_all_random_seed.py b/funasr/train_utils/set_all_random_seed.py
index ebdca3f53..f23b0a89b 100644
--- a/funasr/train_utils/set_all_random_seed.py
+++ b/funasr/train_utils/set_all_random_seed.py
@@ -5,6 +5,11 @@ import torch
def set_all_random_seed(seed: int):
+ """Set all random seed.
+
+ Args:
+ seed: TODO.
+ """
random.seed(seed)
np.random.seed(seed)
torch.random.manual_seed(seed)
diff --git a/funasr/train_utils/trainer.py b/funasr/train_utils/trainer.py
index badf8fa2c..c95acd658 100644
--- a/funasr/train_utils/trainer.py
+++ b/funasr/train_utils/trainer.py
@@ -23,6 +23,11 @@ except:
@contextmanager
def maybe_autocast(enabled):
+ """Maybe autocast.
+
+ Args:
+ enabled: TODO.
+ """
if enabled:
with autocast():
yield
@@ -661,6 +666,24 @@ class Trainer:
**kwargs,
):
+ """Log.
+
+ Args:
+ epoch: TODO.
+ batch_idx: TODO.
+ step_in_epoch: TODO.
+ batch_num_epoch: TODO.
+ lr: TODO.
+ loss: TODO.
+ speed_stats: TODO.
+ stats: TODO.
+ writer: TODO.
+ tag: TODO.
+ data_split_i: TODO.
+ data_split_num: TODO.
+ log_step: TODO.
+ **kwargs: Additional keyword arguments.
+ """
if (batch_idx + 1) % self.log_interval == 0:
batch_idx = log_step if log_step is not None else batch_idx
gpu_info = (
@@ -720,6 +743,11 @@ class Trainer:
def close(self, writer=None):
+ """Close.
+
+ Args:
+ writer: TODO.
+ """
if self.use_ddp or self.use_fsdp:
dist.barrier()
diff --git a/funasr/train_utils/trainer_ds.py b/funasr/train_utils/trainer_ds.py
index b1f2f6857..3e5f68eee 100644
--- a/funasr/train_utils/trainer_ds.py
+++ b/funasr/train_utils/trainer_ds.py
@@ -25,6 +25,12 @@ except:
@contextmanager
def maybe_autocast(dtype=None, use_deepspeed=False):
+ """Maybe autocast.
+
+ Args:
+ dtype: TODO.
+ use_deepspeed: TODO.
+ """
if use_deepspeed:
with torch.cuda.amp.autocast(enabled=True, dtype=dtype, cache_enabled=False):
yield
@@ -668,6 +674,13 @@ class Trainer:
self.train_acc_avg = train_acc_avg.detach().cpu().item() / self.world_size
def forward_step(self, model, batch, loss_dict={}):
+ """Forward step.
+
+ Args:
+ model: Model instance or model name.
+ batch: TODO.
+ loss_dict: TODO.
+ """
with maybe_autocast(dtype=self.dtype, use_deepspeed=self.use_deepspeed):
retval = model(**batch)
@@ -679,6 +692,13 @@ class Trainer:
loss_dict["weight"] = weight
def backward_step(self, model, scaler, loss_dict={}):
+ """Backward step.
+
+ Args:
+ model: Model instance or model name.
+ scaler: TODO.
+ loss_dict: TODO.
+ """
loss = loss_dict["loss"]
if self.use_deepspeed:
@@ -691,6 +711,15 @@ class Trainer:
loss.backward()
def update_step(self, model, optim, scheduler, scaler, loss_dict=None):
+ """Update step.
+
+ Args:
+ model: Model instance or model name.
+ optim: TODO.
+ scheduler: TODO.
+ scaler: TODO.
+ loss_dict: TODO.
+ """
batch_idx = loss_dict["batch_idx"]
if self.use_deepspeed:
model.step()
@@ -822,6 +851,13 @@ class Trainer:
tag="train",
**kwargs,
):
+ """Log.
+
+ Args:
+ loss_dict: TODO.
+ tag: TODO.
+ **kwargs: Additional keyword arguments.
+ """
loss = loss_dict["loss"].detach().cpu().item()
epoch = loss_dict["epoch"]
batch_idx = loss_dict["batch_idx"]
@@ -892,6 +928,11 @@ class Trainer:
def close(self, writer=None):
+ """Close.
+
+ Args:
+ writer: TODO.
+ """
if self.use_ddp or self.use_fsdp:
dist.barrier()
@@ -903,6 +944,12 @@ class Trainer:
def warp_model(self, model, **kwargs):
+ """Warp model.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
if self.use_deepspeed:
from deepspeed.runtime.zero.stage_1_and_2 import (
estimate_zero2_model_states_mem_needs_all_live,
@@ -948,6 +995,12 @@ class Trainer:
return model
def warp_optim_scheduler(self, model, **kwargs):
+ """Warp optim scheduler.
+
+ Args:
+ model: Model instance or model name.
+ **kwargs: Additional keyword arguments.
+ """
from funasr.optimizers import optim_classes
from funasr.schedulers import scheduler_classes
from omegaconf import OmegaConf, DictConfig
@@ -989,6 +1042,11 @@ class Trainer:
else:
def scheduler(opt):
+ """Scheduler.
+
+ Args:
+ opt: TODO.
+ """
return scheduler_class(opt, **kwargs.get("scheduler_conf"))
model, optimizer, _, scheduler = deepspeed.initialize(
diff --git a/funasr/utils/compute_det_ctc.py b/funasr/utils/compute_det_ctc.py
index 47c81821b..6bd8e5fef 100644
--- a/funasr/utils/compute_det_ctc.py
+++ b/funasr/utils/compute_det_ctc.py
@@ -13,15 +13,23 @@ from funasr.utils.kws_utils import split_mixed_label
class thread_wrapper(threading.Thread):
def __init__(self, func, args=()):
+ """Initialize thread_wrapper.
+
+ Args:
+ func: TODO.
+ args: TODO.
+ """
super(thread_wrapper, self).__init__()
self.func = func
self.args = args
self.result = []
def run(self):
+ """Run."""
self.result = self.func(*self.args)
def get_result(self):
+ """Get result."""
try:
return self.result
except Exception:
@@ -29,12 +37,22 @@ class thread_wrapper(threading.Thread):
def space_mixed_label(input_str):
+ """Space mixed label.
+
+ Args:
+ input_str: TODO.
+ """
splits = split_mixed_label(input_str)
space_str = ''.join(f'{sub} ' for sub in splits)
return space_str.strip()
def read_lists(list_file):
+ """Read lists.
+
+ Args:
+ list_file: TODO.
+ """
lists = []
with open(list_file, 'r', encoding='utf8') as fin:
for line in fin:
@@ -44,6 +62,12 @@ def read_lists(list_file):
def make_pair(wav_lists, trans_lists):
+ """Make pair.
+
+ Args:
+ wav_lists: TODO.
+ trans_lists: TODO.
+ """
logging.info('make pair for wav-trans list')
trans_table = {}
@@ -74,6 +98,12 @@ def make_pair(wav_lists, trans_lists):
def count_duration(tid, data_lists):
+ """Count duration.
+
+ Args:
+ tid: TODO.
+ data_lists: TODO.
+ """
results = []
for obj in data_lists:
@@ -102,6 +132,14 @@ def count_duration(tid, data_lists):
def load_data_and_score(keywords_list, data_file, trans_file, score_file):
# score_table: {uttid: [keywordlist]}
+ """Load data and score.
+
+ Args:
+ keywords_list: TODO.
+ data_file: TODO.
+ trans_file: TODO.
+ score_file: TODO.
+ """
score_table = {}
with open(score_file, 'r', encoding='utf8') as fin:
# read score file and store in table
diff --git a/funasr/utils/datadir_writer.py b/funasr/utils/datadir_writer.py
index 882db7708..628ee27ea 100644
--- a/funasr/utils/datadir_writer.py
+++ b/funasr/utils/datadir_writer.py
@@ -17,6 +17,11 @@ class DatadirWriter:
"""
def __init__(self, p: Union[Path, str]):
+ """Initialize DatadirWriter.
+
+ Args:
+ p: TODO.
+ """
self.path = Path(p)
self.chilidren = {}
self.fd = None
@@ -24,9 +29,15 @@ class DatadirWriter:
self.keys = set()
def __enter__(self):
+ """Internal: enter ."""
return self
def __getitem__(self, key: str) -> "DatadirWriter":
+ """Internal: getitem .
+
+ Args:
+ key: Sample identifiers.
+ """
if self.fd is not None:
raise RuntimeError("This writer points out a file")
@@ -39,6 +50,12 @@ class DatadirWriter:
return retval
def __setitem__(self, key: str, value: str):
+ """Internal: setitem .
+
+ Args:
+ key: Sample identifiers.
+ value: TODO.
+ """
if self.has_children:
raise RuntimeError("This writer points out a directory")
if key in self.keys:
@@ -53,9 +70,17 @@ class DatadirWriter:
self.fd.flush()
def __exit__(self, exc_type, exc_val, exc_tb):
+ """Internal: exit .
+
+ Args:
+ exc_type: TODO.
+ exc_val: TODO.
+ exc_tb: TODO.
+ """
self.close()
def close(self):
+ """Close."""
if self.has_children:
prev_child = None
for child in self.chilidren.values():
diff --git a/funasr/utils/dynamic_import.py b/funasr/utils/dynamic_import.py
index e0a056bd3..cdd6b3106 100644
--- a/funasr/utils/dynamic_import.py
+++ b/funasr/utils/dynamic_import.py
@@ -22,6 +22,11 @@ def load_module_from_path(file_path):
def import_module_from_path(file_path: str):
+ """Import module from path.
+
+ Args:
+ file_path: TODO.
+ """
if file_path.startswith("http"):
from funasr.download.file import download_from_url
diff --git a/funasr/utils/export_utils.py b/funasr/utils/export_utils.py
index b03b05272..b48417f39 100644
--- a/funasr/utils/export_utils.py
+++ b/funasr/utils/export_utils.py
@@ -10,6 +10,16 @@ warnings.filterwarnings("ignore")
def export(
model, data_in=None, quantize: bool = False, opset_version: int = 14, type="onnx", **kwargs
):
+ """Export.
+
+ Args:
+ model: Model instance or model name.
+ data_in: Input data (audio samples, file paths, or text).
+ quantize: TODO.
+ opset_version: TODO.
+ type: TODO.
+ **kwargs: Additional keyword arguments.
+ """
model_scripts = model.export(**kwargs)
export_dir = kwargs.get("output_dir", os.path.dirname(kwargs.get("init_param")))
os.makedirs(export_dir, exist_ok=True)
@@ -62,6 +72,16 @@ def _onnx(
**kwargs,
):
+ """Internal: onnx.
+
+ Args:
+ model: Model instance or model name.
+ data_in: Input data (audio samples, file paths, or text).
+ quantize: TODO.
+ opset_version: TODO.
+ export_dir: TODO.
+ **kwargs: Additional keyword arguments.
+ """
device = kwargs.get("device", "cpu")
dummy_input = model.export_dummy_inputs()
@@ -117,6 +137,13 @@ def _onnx(
def _torchscripts(model, path, device="cuda"):
+ """Internal: torchscripts.
+
+ Args:
+ model: Model instance or model name.
+ path: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
dummy_input = model.export_dummy_inputs()
if device == "cuda":
@@ -136,6 +163,13 @@ def _torchscripts(model, path, device="cuda"):
def _bladedisc_opt(model, model_inputs, enable_fp16=True):
+ """Internal: bladedisc opt.
+
+ Args:
+ model: Model instance or model name.
+ model_inputs: TODO.
+ enable_fp16: TODO.
+ """
model = model.eval()
try:
import torch_blade
@@ -155,6 +189,13 @@ def _bladedisc_opt(model, model_inputs, enable_fp16=True):
def _rescale_input_hook(m, x, scale):
+ """Internal: rescale input hook.
+
+ Args:
+ m: TODO.
+ x: TODO.
+ scale: TODO.
+ """
if len(x) > 1:
return (x[0] / scale, *x[1:])
else:
@@ -162,6 +203,14 @@ def _rescale_input_hook(m, x, scale):
def _rescale_output_hook(m, x, y, scale):
+ """Internal: rescale output hook.
+
+ Args:
+ m: TODO.
+ x: TODO.
+ y: TODO.
+ scale: TODO.
+ """
if isinstance(y, tuple):
return (y[0] / scale, *y[1:])
else:
@@ -170,9 +219,22 @@ def _rescale_output_hook(m, x, y, scale):
def _rescale_encoder_model(model, input_data):
# Calculate absmax
+ """Internal: rescale encoder model.
+
+ Args:
+ model: Model instance or model name.
+ input_data: TODO.
+ """
absmax = torch.tensor(0).cuda()
def stat_input_hook(m, x, y):
+ """Stat input hook.
+
+ Args:
+ m: TODO.
+ x: TODO.
+ y: TODO.
+ """
val = x[0] if isinstance(x, tuple) else x
absmax.copy_(torch.max(absmax, val.detach().abs().max()))
@@ -200,6 +262,13 @@ def _rescale_encoder_model(model, input_data):
def _bladedisc_opt_for_encdec(model, path, enable_fp16):
# Get input data
# TODO: better to use real data
+ """Internal: bladedisc opt for encdec.
+
+ Args:
+ model: Model instance or model name.
+ path: TODO.
+ enable_fp16: TODO.
+ """
input_data = model.export_dummy_inputs()
if isinstance(input_data, torch.Tensor):
input_data = input_data.cuda()
@@ -210,6 +279,12 @@ def _bladedisc_opt_for_encdec(model, path, enable_fp16):
decoder_inputs = list()
def get_input_hook(m, x):
+ """Get input hook.
+
+ Args:
+ m: TODO.
+ x: TODO.
+ """
decoder_inputs.extend(list(x))
hook = model.decoder.register_forward_pre_hook(get_input_hook)
@@ -232,6 +307,13 @@ def _onnx_opt_for_encdec(model, path, enable_fp16):
# Get input data
# TODO: better to use real data
+ """Internal: onnx opt for encdec.
+
+ Args:
+ model: Model instance or model name.
+ path: TODO.
+ enable_fp16: TODO.
+ """
input_data = model.export_dummy_inputs()
if isinstance(input_data, torch.Tensor):
@@ -243,6 +325,12 @@ def _onnx_opt_for_encdec(model, path, enable_fp16):
decoder_inputs = list()
def get_input_hook(m, x):
+ """Get input hook.
+
+ Args:
+ m: TODO.
+ x: TODO.
+ """
decoder_inputs.extend(list(x))
hook = model.decoder.register_forward_pre_hook(get_input_hook)
diff --git a/funasr/utils/install_model_requirements.py b/funasr/utils/install_model_requirements.py
index 73a2dcb39..56a1f5424 100644
--- a/funasr/utils/install_model_requirements.py
+++ b/funasr/utils/install_model_requirements.py
@@ -3,6 +3,11 @@ import subprocess
def install_requirements(requirements_path):
+ """Install requirements.
+
+ Args:
+ requirements_path: TODO.
+ """
try:
result = pip_install_r(requirements_path)
# check status
@@ -26,6 +31,11 @@ def install_requirements(requirements_path):
def pip_install_r(requirements_path):
+ """Pip install r.
+
+ Args:
+ requirements_path: TODO.
+ """
cmd = []
if shutil.which("pip") is not None:
cmd = ["pip"]
diff --git a/funasr/utils/kws_utils.py b/funasr/utils/kws_utils.py
index f133fe659..1086050e4 100644
--- a/funasr/utils/kws_utils.py
+++ b/funasr/utils/kws_utils.py
@@ -11,6 +11,11 @@ symbol_str = '[’!"#$%&\'()*+,-./:;<>=?@,。?★、…【】《》?“”
def split_mixed_label(input_str):
+ """Split mixed label.
+
+ Args:
+ input_str: TODO.
+ """
tokens = []
s = input_str.lower()
while len(s) > 0:
@@ -25,6 +30,13 @@ def split_mixed_label(input_str):
def query_token_set(txt, symbol_table, lexicon_table):
+ """Query token set.
+
+ Args:
+ txt: TODO.
+ symbol_table: TODO.
+ lexicon_table: TODO.
+ """
tokens_str = tuple()
tokens_idx = tuple()
@@ -218,6 +230,12 @@ class KwsCtcPrefixDecoder():
def is_sublist(self, main_list, check_list):
+ """Is sublist.
+
+ Args:
+ main_list: TODO.
+ check_list: TODO.
+ """
if len(main_list) < len(check_list):
return -1
@@ -240,6 +258,12 @@ class KwsCtcPrefixDecoder():
logits: torch.Tensor,
logits_lengths: torch.Tensor,
):
+ """Internal: decode inside.
+
+ Args:
+ logits: TODO.
+ logits_lengths: Lengths of logits.
+ """
hyps = self.beam_search(logits, logits_lengths, self.keywords_idxset)
hit_keyword = None
diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py
index e287d0c7a..ce8525b74 100644
--- a/funasr/utils/load_utils.py
+++ b/funasr/utils/load_utils.py
@@ -25,6 +25,7 @@ except:
def is_ffmpeg_installed():
+ """Is ffmpeg installed."""
try:
output = subprocess.check_output(["ffmpeg", "-version"], stderr=subprocess.STDOUT)
return "ffmpeg version" in output.decode("utf-8")
@@ -222,6 +223,12 @@ def validate_frame_rate(
):
# 将文件读取为字节流
+ """Validate frame rate.
+
+ Args:
+ input: Input audio/text data.
+ fs: TODO.
+ """
byte_data = BytesIO(input)
# 使用 pydub 加载音频
diff --git a/funasr/utils/misc.py b/funasr/utils/misc.py
index ba16e05c1..f7ff937d2 100644
--- a/funasr/utils/misc.py
+++ b/funasr/utils/misc.py
@@ -8,6 +8,12 @@ from omegaconf import DictConfig, OmegaConf
def statistic_model_parameters(model, prefix=None):
+ """Statistic model parameters.
+
+ Args:
+ model: Model instance or model name.
+ prefix: TODO.
+ """
var_dict = model.state_dict()
numel = 0
for i, key in enumerate(
@@ -19,16 +25,36 @@ def statistic_model_parameters(model, prefix=None):
def int2vec(x, vec_dim=8, dtype=np.int32):
+ """Int2vec.
+
+ Args:
+ x: TODO.
+ vec_dim: Size/dimension parameter.
+ dtype: TODO.
+ """
b = ("{:0" + str(vec_dim) + "b}").format(x)
# little-endian order: lower bit first
return (np.array(list(b)[::-1]) == "1").astype(dtype)
def seq2arr(seq, vec_dim=8):
+ """Seq2arr.
+
+ Args:
+ seq: TODO.
+ vec_dim: Size/dimension parameter.
+ """
return np.row_stack([int2vec(int(x), vec_dim) for x in seq])
def load_scp_as_dict(scp_path, value_type="str", kv_sep=" "):
+ """Load scp as dict.
+
+ Args:
+ scp_path: TODO.
+ value_type: TODO.
+ kv_sep: TODO.
+ """
with io.open(scp_path, "r", encoding="utf-8") as f:
ret_dict = OrderedDict()
for one_line in f.readlines():
@@ -42,6 +68,13 @@ def load_scp_as_dict(scp_path, value_type="str", kv_sep=" "):
def load_scp_as_list(scp_path, value_type="str", kv_sep=" "):
+ """Load scp as list.
+
+ Args:
+ scp_path: TODO.
+ value_type: TODO.
+ kv_sep: TODO.
+ """
with io.open(scp_path, "r", encoding="utf8") as f:
ret_dict = []
for one_line in f.readlines():
@@ -75,6 +108,11 @@ def deep_update(original, update):
def prepare_model_dir(**kwargs):
+ """Prepare model dir.
+
+ Args:
+ **kwargs: Additional keyword arguments.
+ """
os.makedirs(kwargs.get("output_dir", "./"), exist_ok=True)
yaml_file = os.path.join(kwargs.get("output_dir", "./"), "config.yaml")
diff --git a/funasr/utils/postprocess_utils.py b/funasr/utils/postprocess_utils.py
index 39a8a10dd..beaa87cb4 100644
--- a/funasr/utils/postprocess_utils.py
+++ b/funasr/utils/postprocess_utils.py
@@ -6,12 +6,22 @@ from typing import Any, List, Union
def isChinese(ch: str):
+ """Ischinese.
+
+ Args:
+ ch: TODO.
+ """
if "\u4e00" <= ch <= "\u9fff" or "\u0030" <= ch <= "\u0039" or ch == "@":
return True
return False
def isAllChinese(word: Union[List[Any], str]):
+ """Isallchinese.
+
+ Args:
+ word: TODO.
+ """
word_lists = []
for i in word:
cur = i.replace(" ", "")
@@ -31,6 +41,11 @@ def isAllChinese(word: Union[List[Any], str]):
def isAllAlpha(word: Union[List[Any], str]):
+ """Isallalpha.
+
+ Args:
+ word: TODO.
+ """
word_lists = []
for i in word:
cur = i.replace(" ", "")
@@ -54,6 +69,12 @@ def isAllAlpha(word: Union[List[Any], str]):
# def abbr_dispose(words: List[Any]) -> List[Any]:
def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]:
+ """Abbr dispose.
+
+ Args:
+ words: TODO.
+ time_stamp: TODO.
+ """
words_size = len(words)
word_lists = []
abbr_begin = []
@@ -142,6 +163,12 @@ def abbr_dispose(words: List[Any], time_stamp: List[List] = None) -> List[Any]:
def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):
+ """Sentence postprocess.
+
+ Args:
+ words: TODO.
+ time_stamp: TODO.
+ """
middle_lists = []
word_lists = []
word_item = ""
@@ -252,6 +279,11 @@ def sentence_postprocess(words: List[Any], time_stamp: List[List] = None):
def sentence_postprocess_sentencepiece(words):
+ """Sentence postprocess sentencepiece.
+
+ Args:
+ words: TODO.
+ """
middle_lists = []
word_lists = []
word_item = ""
@@ -377,6 +409,11 @@ event_set = {
def format_str_v2(s):
+ """Format str v2.
+
+ Args:
+ s: TODO.
+ """
sptk_dict = {}
for sptk in emoji_dict:
sptk_dict[sptk] = s.count(sptk)
@@ -397,10 +434,25 @@ def format_str_v2(s):
def rich_transcription_postprocess(s):
+ """Rich transcription postprocess.
+
+ Args:
+ s: TODO.
+ """
def get_emo(s):
+ """Get emo.
+
+ Args:
+ s: TODO.
+ """
return s[-1] if s[-1] in emo_set else None
def get_event(s):
+ """Get event.
+
+ Args:
+ s: TODO.
+ """
return s[0] if s[0] in event_set else None
s = s.replace("<|nospeech|><|Event_UNK|>", "❓")
diff --git a/funasr/utils/speaker_utils.py b/funasr/utils/speaker_utils.py
index e470849f8..2b811e948 100644
--- a/funasr/utils/speaker_utils.py
+++ b/funasr/utils/speaker_utils.py
@@ -16,6 +16,11 @@ from funasr.utils.modelscope_file import File
def check_audio_list(audio: list):
+ """Check audio list.
+
+ Args:
+ audio: TODO.
+ """
audio_dur = 0
for i in range(len(audio)):
seg = audio[i]
@@ -32,6 +37,11 @@ def check_audio_list(audio: list):
def sv_preprocess(inputs: Union[np.ndarray, list]):
+ """Sv preprocess.
+
+ Args:
+ inputs: TODO.
+ """
output = []
for i in range(len(inputs)):
if isinstance(inputs[i], str):
@@ -58,12 +68,23 @@ def sv_preprocess(inputs: Union[np.ndarray, list]):
def sv_chunk(vad_segments: list, fs=16000) -> list:
+ """Sv chunk.
+
+ Args:
+ vad_segments: TODO.
+ fs: TODO.
+ """
config = {
"seg_dur": 1.5,
"seg_shift": 0.75,
}
def seg_chunk(seg_data):
+ """Seg chunk.
+
+ Args:
+ seg_data: TODO.
+ """
seg_st = seg_data[0]
data = seg_data[2]
chunk_len = int(config["seg_dur"] * fs)
@@ -90,6 +111,11 @@ def sv_chunk(vad_segments: list, fs=16000) -> list:
def extract_feature(audio):
+ """Extract feature.
+
+ Args:
+ audio: TODO.
+ """
features = []
for au in audio:
feature = Kaldi.fbank(au.unsqueeze(0), num_mel_bins=80)
@@ -102,6 +128,14 @@ def extract_feature(audio):
def postprocess(
segments: list, vad_segments: list, labels: np.ndarray, embeddings: np.ndarray
) -> list:
+ """Postprocess.
+
+ Args:
+ segments: TODO.
+ vad_segments: TODO.
+ labels: TODO.
+ embeddings: TODO.
+ """
assert len(segments) == len(labels)
labels = correct_labels(labels)
distribute_res = []
@@ -118,6 +152,12 @@ def postprocess(
spk_embs = np.stack(spk_embs)
def is_overlapped(t1, t2):
+ """Is overlapped.
+
+ Args:
+ t1: TODO.
+ t2: TODO.
+ """
if t1 > t2 + 1e-4:
return True
return False
@@ -136,6 +176,11 @@ def postprocess(
def correct_labels(labels):
+ """Correct labels.
+
+ Args:
+ labels: TODO.
+ """
labels_id = 0
id2id = {}
new_labels = []
@@ -148,6 +193,11 @@ def correct_labels(labels):
def merge_seque(distribute_res):
+ """Merge seque.
+
+ Args:
+ distribute_res: TODO.
+ """
res = [distribute_res[0]]
for i in range(1, len(distribute_res)):
if distribute_res[i][2] != res[-1][2] or distribute_res[i][0] > res[-1][1]:
@@ -159,6 +209,12 @@ def merge_seque(distribute_res):
def smooth(res, mindur=1):
# short segments are assigned to nearest speakers.
+ """Smooth.
+
+ Args:
+ res: TODO.
+ mindur: TODO.
+ """
for i in range(len(res)):
res[i][0] = round(res[i][0], 2)
res[i][1] = round(res[i][1], 2)
@@ -178,6 +234,12 @@ def smooth(res, mindur=1):
def distribute_spk(sentence_list, sd_time_list):
+ """Distribute spk.
+
+ Args:
+ sentence_list: TODO.
+ sd_time_list: TODO.
+ """
sd_sentence_list = []
for d in sentence_list:
sentence_start = d["ts_list"][0][0]
diff --git a/funasr/utils/timestamp_tools.py b/funasr/utils/timestamp_tools.py
index ccfc1a91d..ef5af8466 100644
--- a/funasr/utils/timestamp_tools.py
+++ b/funasr/utils/timestamp_tools.py
@@ -9,6 +9,12 @@ from itertools import zip_longest
def cif_wo_hidden(alphas, threshold):
+ """Cif wo hidden.
+
+ Args:
+ alphas: TODO.
+ threshold: TODO.
+ """
batch_size, len_time = alphas.size()
# loop varss
integrate = torch.zeros([batch_size], device=alphas.device)
@@ -31,6 +37,17 @@ def cif_wo_hidden(alphas, threshold):
def ts_prediction_lfr6_standard(
us_alphas, us_peaks, char_list, vad_offset=0.0, force_time_shift=-1.5, sil_in_str=True, upsample_rate=3,
):
+ """Ts prediction lfr6 standard.
+
+ Args:
+ us_alphas: TODO.
+ us_peaks: TODO.
+ char_list: TODO.
+ vad_offset: TODO.
+ force_time_shift: TODO.
+ sil_in_str: TODO.
+ upsample_rate: TODO.
+ """
if not len(char_list):
return "", []
START_END_THRESHOLD = 5
@@ -206,6 +223,14 @@ def timestamp_sentence(
def timestamp_sentence_en(
punc_id_list, timestamp_postprocessed, text_postprocessed, return_raw_text=False
):
+ """Timestamp sentence en.
+
+ Args:
+ punc_id_list: TODO.
+ timestamp_postprocessed: TODO.
+ text_postprocessed: TODO.
+ return_raw_text: TODO.
+ """
punc_list = [",", ".", "?", ","]
res = []
if text_postprocessed is None:
diff --git a/funasr/utils/torch_function.py b/funasr/utils/torch_function.py
index f637bbf82..cc24e373d 100644
--- a/funasr/utils/torch_function.py
+++ b/funasr/utils/torch_function.py
@@ -8,6 +8,12 @@ import numpy as np
class MakePadMask(nn.Module):
def __init__(self, max_seq_len=512, flip=True):
+ """Initialize MakePadMask.
+
+ Args:
+ max_seq_len: TODO.
+ flip: TODO.
+ """
super().__init__()
if flip:
self.mask_pad = torch.Tensor(1 - np.tri(max_seq_len)).type(torch.bool)
@@ -46,9 +52,23 @@ class MakePadMask(nn.Module):
class sequence_mask(nn.Module):
def __init__(self, max_seq_len=512, flip=True):
+ """Initialize sequence_mask.
+
+ Args:
+ max_seq_len: TODO.
+ flip: TODO.
+ """
super().__init__()
def forward(self, lengths, max_seq_len=None, dtype=torch.float32, device=None):
+ """Forward pass for training.
+
+ Args:
+ lengths: TODO.
+ max_seq_len: TODO.
+ dtype: TODO.
+ device: Target device ("cuda:0", "cpu", etc.).
+ """
if max_seq_len is None:
max_seq_len = lengths.max()
row_vector = torch.arange(0, max_seq_len, 1).to(lengths.device)
@@ -61,6 +81,14 @@ class sequence_mask(nn.Module):
def normalize(
input: torch.Tensor, p: float = 2.0, dim: int = 1, out: Optional[torch.Tensor] = None
) -> torch.Tensor:
+ """Normalize.
+
+ Args:
+ input: Input audio/text data.
+ p: TODO.
+ dim: TODO.
+ out: TODO.
+ """
if out is None:
denom = input.norm(p, dim, keepdim=True).expand_as(input)
return input / denom
@@ -70,10 +98,16 @@ def normalize(
def subsequent_mask(size: torch.Tensor):
+ """Subsequent mask.
+
+ Args:
+ size: TODO.
+ """
return torch.ones(size, size).tril()
def MakePadMask_test():
+ """Makepadmask test."""
feats_length = torch.tensor([10]).type(torch.long)
mask_fn = MakePadMask()
mask = mask_fn(feats_length)
diff --git a/funasr/utils/type_utils.py b/funasr/utils/type_utils.py
index 6b36f9c4b..fe99013be 100644
--- a/funasr/utils/type_utils.py
+++ b/funasr/utils/type_utils.py
@@ -7,10 +7,20 @@ import humanfriendly
def str2bool(value: str) -> bool:
+ """Str2bool.
+
+ Args:
+ value: TODO.
+ """
return bool(strtobool(value))
def remove_parenthesis(value: str):
+ """Remove parenthesis.
+
+ Args:
+ value: TODO.
+ """
value = value.strip()
if value.startswith("(") and value.endswith(")"):
value = value[1:-1]
@@ -20,6 +30,11 @@ def remove_parenthesis(value: str):
def remove_quotes(value: str):
+ """Remove quotes.
+
+ Args:
+ value: TODO.
+ """
value = value.strip()
if value.startswith('"') and value.endswith('"'):
value = value[1:-1]
@@ -73,12 +88,22 @@ def float_or_none(value: str) -> Optional[float]:
def humanfriendly_parse_size_or_none(value) -> Optional[float]:
+ """Humanfriendly parse size or none.
+
+ Args:
+ value: TODO.
+ """
if value.strip().lower() in ("none", "null", "nil"):
return None
return humanfriendly.parse_size(value)
def str_or_int(value: str) -> Union[str, int]:
+ """Str or int.
+
+ Args:
+ value: TODO.
+ """
try:
return int(value)
except ValueError:
diff --git a/funasr/utils/vad_utils.py b/funasr/utils/vad_utils.py
index 8400e60b6..2622dd552 100644
--- a/funasr/utils/vad_utils.py
+++ b/funasr/utils/vad_utils.py
@@ -3,6 +3,13 @@ from torch.nn.utils.rnn import pad_sequence
def slice_padding_fbank(speech, speech_lengths, vad_segments):
+ """Slice padding fbank.
+
+ Args:
+ speech: Speech audio tensor, shape (batch, time).
+ speech_lengths: Length of each speech sample.
+ vad_segments: TODO.
+ """
speech_list = []
speech_lengths_list = []
for i, segment in enumerate(vad_segments):
diff --git a/funasr/utils/version_checker.py b/funasr/utils/version_checker.py
index 9bc11deb9..156881bab 100644
--- a/funasr/utils/version_checker.py
+++ b/funasr/utils/version_checker.py
@@ -3,6 +3,11 @@ from funasr import __version__ # Ensure that __version__ is defined in your pac
def get_pypi_version(package_name):
+ """Get pypi version.
+
+ Args:
+ package_name: TODO.
+ """
import requests
url = f"https://pypi.org/pypi/{package_name}/json"
@@ -15,6 +20,11 @@ def get_pypi_version(package_name):
def check_for_update(disable=False):
+ """Check for update.
+
+ Args:
+ disable: TODO.
+ """
current_version = version.parse(__version__)
print(f"funasr version: {current_version}.")
if disable: