Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
train finetune demo
游雁0587592632a3
diff --git a/examples/aishell/branchformer/demo_infer.sh b/examples/aishell/branchformer/demo_infer.shnew file mode 120000index 000000000..9d0a7a9e3--- /dev/null+++ b/examples/aishell/branchformer/demo_infer.sh@@ -0,0 +1 @@+../paraformer/demo_infer.sh\ No newline at end of filediff --git a/examples/aishell/branchformer/demo_train_or_finetune.sh b/examples/aishell/branchformer/demo_train_or_finetune.shnew file mode 120000index 000000000..bbabdbe84--- /dev/null+++ b/examples/aishell/branchformer/demo_train_or_finetune.sh@@ -0,0 +1 @@+../paraformer/demo_train_or_finetune.sh\ No newline at end of filediff --git a/examples/aishell/conformer/demo_infer.sh b/examples/aishell/conformer/demo_infer.shnew file mode 120000index 000000000..9d0a7a9e3--- /dev/null+++ b/examples/aishell/conformer/demo_infer.sh@@ -0,0 +1 @@+../paraformer/demo_infer.sh\ No newline at end of filediff --git a/examples/aishell/conformer/demo_train_or_finetune.sh b/examples/aishell/conformer/demo_train_or_finetune.shnew file mode 120000index 000000000..bbabdbe84--- /dev/null+++ b/examples/aishell/conformer/demo_train_or_finetune.sh@@ -0,0 +1 @@+../paraformer/demo_train_or_finetune.sh\ No newline at end of filediff --git a/examples/aishell/e_branchformer/demo_infer.sh b/examples/aishell/e_branchformer/demo_infer.shnew file mode 120000index 000000000..9d0a7a9e3--- /dev/null+++ b/examples/aishell/e_branchformer/demo_infer.sh@@ -0,0 +1 @@+../paraformer/demo_infer.sh\ No newline at end of filediff --git a/examples/aishell/e_branchformer/demo_train_or_finetune.sh b/examples/aishell/e_branchformer/demo_train_or_finetune.shnew file mode 120000index 000000000..bbabdbe84--- /dev/null+++ b/examples/aishell/e_branchformer/demo_train_or_finetune.sh@@ -0,0 +1 @@+../paraformer/demo_train_or_finetune.sh\ No newline at end of filediff --git a/examples/aishell/e_branchformer/infer.sh b/examples/aishell/e_branchformer/infer.shdeleted file mode 100644index bcd4305f6..000000000--- a/examples/aishell/e_branchformer/infer.sh+++ /dev/null@@ -1,12 +0,0 @@---python -m funasr.bin.inference \---config-path="/mnt/workspace/FunASR/examples/aishell/paraformer/exp/baseline_paraformer_conformer_12e_6d_2048_256_zh_char_exp3" \---config-name="config.yaml" \-++init_param="/mnt/workspace/FunASR/examples/aishell/paraformer/exp/baseline_paraformer_conformer_12e_6d_2048_256_zh_char_exp3/model.pt.ep38" \-++tokenizer_conf.token_list="/mnt/nfs/zhifu.gzf/data/AISHELL-1-feats/DATA/data/zh_token_list/char/tokens.txt" \-++frontend_conf.cmvn_file="/mnt/nfs/zhifu.gzf/data/AISHELL-1-feats/DATA/data/train/am.mvn" \-++input="/mnt/nfs/zhifu.gzf/data/AISHELL-1/data_aishell/wav/train/S0002/BAC009S0002W0122.wav" \-++output_dir="./outputs/debug" \-++device="cuda:0" \-diff --git a/examples/aishell/conformer/infer.sh b/examples/aishell/paraformer/demo_infer.shsimilarity index 82%rename from examples/aishell/conformer/infer.shrename to examples/aishell/paraformer/demo_infer.shindex bcd4305f6..b3b989f7e 100644--- a/examples/aishell/conformer/infer.sh+++ b/examples/aishell/paraformer/demo_infer.sh@@ -1,3 +1,6 @@+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)+python -m funasr.bin.inference \diff --git a/examples/aishell/paraformer/demo_train_or_finetune.sh b/examples/aishell/paraformer/demo_train_or_finetune.shnew file mode 100644index 000000000..4c95dc763--- /dev/null+++ b/examples/aishell/paraformer/demo_train_or_finetune.sh@@ -0,0 +1,51 @@+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)+++# which gpu to train or finetune+export CUDA_VISIBLE_DEVICES="0,1"+gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')++# data dir, which contains: train.json, val.json, tokens.jsonl/tokens.txt, am.mvn+data_dir="/Users/zhifu/funasr1.0/data/list"++## generate jsonl from wav.scp and text.txt+#python -m funasr.datasets.audio_datasets.scp2jsonl \+#++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \+#++data_type_list='["source", "target"]' \+#++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl++train_data="${data_dir}/train.jsonl"+val_data="${data_dir}/val.jsonl"+tokens="${data_dir}/tokens.jsonl"+cmvn_file="${data_dir}/am.mvn"++# exp output dir+output_dir="/Users/zhifu/exp"+log_file="${output_dir}/log.txt"++workspace=`pwd`+config="paraformer_conformer_12e_6d_2048_256.yaml"++init_param="${output_dir}/model.pt"++mkdir -p ${output_dir}+echo "log_file: ${log_file}"++torchrun \+--nnodes 1 \+--nproc_per_node ${gpu_num} \+../../../funasr/bin/train.py \+--config-path "${workspace}/conf" \+--config-name "${config}" \+++train_data_set_list="${train_data}" \+++valid_data_set_list="${val_data}" \+++tokenizer_conf.token_list="${tokens}" \+++frontend_conf.cmvn_file="${cmvn_file}" \+++dataset_conf.batch_size=32 \+++dataset_conf.batch_type="example" \+++dataset_conf.num_workers=4 \+++train_conf.max_epoch=20 \+++optim_conf.lr=0.0002 \+++init_param="${init_param}" \+++output_dir="${output_dir}" &> ${log_file}diff --git a/examples/aishell/paraformer/infer.sh b/examples/aishell/paraformer/infer.shdeleted file mode 100644index bcd4305f6..000000000--- a/examples/aishell/paraformer/infer.sh+++ /dev/null@@ -1,12 +0,0 @@---python -m funasr.bin.inference \---config-path="/mnt/workspace/FunASR/examples/aishell/paraformer/exp/baseline_paraformer_conformer_12e_6d_2048_256_zh_char_exp3" \---config-name="config.yaml" \-++init_param="/mnt/workspace/FunASR/examples/aishell/paraformer/exp/baseline_paraformer_conformer_12e_6d_2048_256_zh_char_exp3/model.pt.ep38" \-++tokenizer_conf.token_list="/mnt/nfs/zhifu.gzf/data/AISHELL-1-feats/DATA/data/zh_token_list/char/tokens.txt" \-++frontend_conf.cmvn_file="/mnt/nfs/zhifu.gzf/data/AISHELL-1-feats/DATA/data/train/am.mvn" \-++input="/mnt/nfs/zhifu.gzf/data/AISHELL-1/data_aishell/wav/train/S0002/BAC009S0002W0122.wav" \-++output_dir="./outputs/debug" \-++device="cuda:0" \-diff --git a/examples/aishell/transformer/demo_infer.sh b/examples/aishell/transformer/demo_infer.shnew file mode 120000index 000000000..9d0a7a9e3--- /dev/null+++ b/examples/aishell/transformer/demo_infer.sh@@ -0,0 +1 @@+../paraformer/demo_infer.sh\ No newline at end of filediff --git a/examples/aishell/transformer/demo_train_or_finetune.sh b/examples/aishell/transformer/demo_train_or_finetune.shnew file mode 120000index 000000000..bbabdbe84--- /dev/null+++ b/examples/aishell/transformer/demo_train_or_finetune.sh@@ -0,0 +1 @@+../paraformer/demo_train_or_finetune.sh\ No newline at end of filediff --git a/examples/aishell/transformer/infer.sh b/examples/aishell/transformer/infer.shdeleted file mode 100644index bcd4305f6..000000000--- a/examples/aishell/transformer/infer.sh+++ /dev/null@@ -1,12 +0,0 @@---python -m funasr.bin.inference \---config-path="/mnt/workspace/FunASR/examples/aishell/paraformer/exp/baseline_paraformer_conformer_12e_6d_2048_256_zh_char_exp3" \---config-name="config.yaml" \-++init_param="/mnt/workspace/FunASR/examples/aishell/paraformer/exp/baseline_paraformer_conformer_12e_6d_2048_256_zh_char_exp3/model.pt.ep38" \-++tokenizer_conf.token_list="/mnt/nfs/zhifu.gzf/data/AISHELL-1-feats/DATA/data/zh_token_list/char/tokens.txt" \-++frontend_conf.cmvn_file="/mnt/nfs/zhifu.gzf/data/AISHELL-1-feats/DATA/data/train/am.mvn" \-++input="/mnt/nfs/zhifu.gzf/data/AISHELL-1/data_aishell/wav/train/S0002/BAC009S0002W0122.wav" \-++output_dir="./outputs/debug" \-++device="cuda:0" \-diff --git a/examples/industrial_data_pretraining/paraformer/demo.sh b/examples/industrial_data_pretraining/paraformer/demo.shindex f9a03f9ea..477aee6d2 100644--- a/examples/industrial_data_pretraining/paraformer/demo.sh+++ b/examples/industrial_data_pretraining/paraformer/demo.sh@@ -1,8 +1,10 @@+# method1, inference from model hub+model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"model_revision="v2.0.4"-python funasr/bin/inference.py \+python -m funasr.bin.inference \+model=${model} \+model_revision=${model_revision} \+input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav" \@@ -10,5 +12,18 @@ python funasr/bin/inference.py \+device="cpu" \+# method2, inference from local model++#python -m funasr.bin.inference \+#--config-path="/Users/zhifu/funasr_github/test_local/funasr_cli_egs" \+#--config-name="config.yaml" \+#++init_param="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/model.pt" \+#++tokenizer_conf.token_list="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/tokens.txt" \+#++frontend_conf.cmvn_file="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/am.mvn" \+#++input="data/wav.scp" \+#++output_dir="./outputs/debug" \+#++device="cuda" \++diff --git a/examples/industrial_data_pretraining/paraformer/finetune.sh b/examples/industrial_data_pretraining/paraformer/finetune.shdeleted file mode 100644index 266346cca..000000000--- a/examples/industrial_data_pretraining/paraformer/finetune.sh+++ /dev/null@@ -1,27 +0,0 @@--## download model-#local_path_root=../modelscope_models-#mkdir -p ${local_path_root}-#local_path=${local_path_root}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch-#git clone https://www.modelscope.cn/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch.git ${local_path}--## generate jsonl from wav.scp and text.txt-#python -m funasr.datasets.audio_datasets.scp2jsonl \-#++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \-#++data_type_list='["source", "target"]' \-#++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl---# torchrun \-# --nnodes 1 \-# --nproc_per_node 1 \-python funasr/bin/train.py \-+model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \-+model_revision="v2.0.4" \-+train_data_set_list="/Users/zhifu/funasr_github/test_local/aishell2_dev_ios/asr_task_debug_len_10.jsonl" \-+valid_data_set_list="/Users/zhifu/funasr_github/test_local/aishell2_dev_ios/asr_task_debug_len_10.jsonl" \-++dataset_conf.batch_size=64 \-++dataset_conf.batch_type="example" \-++train_conf.max_epoch=2 \-++dataset_conf.num_workers=4 \-+output_dir="outputs/debug/ckpt/funasr2/exp2"\ No newline at end of filediff --git a/examples/industrial_data_pretraining/paraformer/finetune_from_local_model.sh b/examples/industrial_data_pretraining/paraformer/finetune_from_local_model.shnew file mode 100644index 000000000..2e0b8eeea--- /dev/null+++ b/examples/industrial_data_pretraining/paraformer/finetune_from_local_model.sh@@ -0,0 +1,58 @@+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++# download model+local_path_root=../modelscope_models+mkdir -p ${local_path_root}+local_path=${local_path_root}/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch+git clone https://www.modelscope.cn/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch.git ${local_path}+++# which gpu to train or finetune+export CUDA_VISIBLE_DEVICES="0,1"+gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')++# data dir, which contains: train.json, val.json+data_dir="/Users/zhifu/funasr1.0/data/list"++## generate jsonl from wav.scp and text.txt+#python -m funasr.datasets.audio_datasets.scp2jsonl \+#++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \+#++data_type_list='["source", "target"]' \+#++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl++train_data="${data_dir}/train.jsonl"+val_data="${data_dir}/val.jsonl"++tokens="${local_path}/tokens.jsonl"+cmvn_file="${local_path}/am.mvn"++# exp output dir+output_dir="/Users/zhifu/exp"+log_file="${output_dir}/log.txt"++workspace=`pwd`+config="${local_path}/config.yaml"++init_param="${local_path}/model.pt"++mkdir -p ${output_dir}+echo "log_file: ${log_file}"++torchrun \+--nnodes 1 \+--nproc_per_node ${gpu_num} \+../../../funasr/bin/train.py \+--config-path "${workspace}/conf" \+--config-name "${config}" \+++train_data_set_list="${train_data}" \+++valid_data_set_list="${val_data}" \+++tokenizer_conf.token_list="${tokens}" \+++frontend_conf.cmvn_file="${cmvn_file}" \+++dataset_conf.batch_size=32 \+++dataset_conf.batch_type="example" \+++dataset_conf.num_workers=4 \+++train_conf.max_epoch=20 \+++optim_conf.lr=0.0002 \+++init_param="${init_param}" \+++output_dir="${output_dir}" &> ${log_file}diff --git a/examples/industrial_data_pretraining/paraformer/finetune_from_model_hub.sh b/examples/industrial_data_pretraining/paraformer/finetune_from_model_hub.shnew file mode 100644index 000000000..0ccb83a12--- /dev/null+++ b/examples/industrial_data_pretraining/paraformer/finetune_from_model_hub.sh@@ -0,0 +1,42 @@+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++# which gpu to train or finetune+export CUDA_VISIBLE_DEVICES="0,1"+gpu_num=$(echo $CUDA_VISIBLE_DEVICES | awk -F "," '{print NF}')++# data dir, which contains: train.json, val.json+data_dir="/Users/zhifu/funasr1.0/data/list"++## generate jsonl from wav.scp and text.txt+#python -m funasr.datasets.audio_datasets.scp2jsonl \+#++scp_file_list='["/Users/zhifu/funasr1.0/test_local/wav.scp", "/Users/zhifu/funasr1.0/test_local/text.txt"]' \+#++data_type_list='["source", "target"]' \+#++jsonl_file_out=/Users/zhifu/funasr1.0/test_local/audio_datasets.jsonl++train_data="${data_dir}/train.jsonl"+val_data="${data_dir}/val.jsonl"+++# exp output dir+output_dir="/Users/zhifu/exp"+log_file="${output_dir}/log.txt"+++mkdir -p ${output_dir}+echo "log_file: ${log_file}"++torchrun \+--nnodes 1 \+--nproc_per_node ${gpu_num} \+funasr/bin/train.py \++model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \++model_revision="v2.0.4" \+++train_data_set_list="${train_data}" \+++valid_data_set_list="${val_data}" \+++dataset_conf.batch_size=32 \+++dataset_conf.batch_type="example" \+++dataset_conf.num_workers=4 \+++train_conf.max_epoch=20 \+++optim_conf.lr=0.0002 \+++output_dir="${output_dir}" &> ${log_file}\ No newline at end of filediff --git a/examples/industrial_data_pretraining/paraformer/infer_after_finetune.sh b/examples/industrial_data_pretraining/paraformer/infer_after_finetune.shdeleted file mode 100644index 94797bf40..000000000--- a/examples/industrial_data_pretraining/paraformer/infer_after_finetune.sh+++ /dev/null@@ -1,12 +0,0 @@---python funasr/bin/inference.py \---config-path="/Users/zhifu/funasr_github/test_local/funasr_cli_egs" \---config-name="config.yaml" \-++init_param="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/model.pt" \-++tokenizer_conf.token_list="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/tokens.txt" \-++frontend_conf.cmvn_file="/Users/zhifu/funasr_github/test_local/funasr_cli_egs/am.mvn" \-++input="data/wav.scp" \-++output_dir="./outputs/debug" \-++device="cuda" \-diff --git a/funasr/bin/train.py b/funasr/bin/train.pyindex 066145247..4538224f6 100644--- a/funasr/bin/train.py+++ b/funasr/bin/train.py@@ -96,15 +96,18 @@ def main(**kwargs):init_param = (init_param,)logging.info("init_param is not None: %s", init_param)for p in init_param:- logging.info(f"Loading pretrained params from {p}")- load_pretrained_model(- model=model,- path=p,- ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True),- oss_bucket=kwargs.get("oss_bucket", None),- scope_map=kwargs.get("scope_map", None),- excludes=kwargs.get("excludes", None),- )+ if os.path.exists(p):+ logging.info(f"Loading pretrained params from {p}")+ load_pretrained_model(+ model=model,+ path=p,+ ignore_init_mismatch=kwargs.get("ignore_init_mismatch", True),+ oss_bucket=kwargs.get("oss_bucket", None),+ scope_map=kwargs.get("scope_map", None),+ excludes=kwargs.get("excludes", None),+ )+ else:+ logging.info(f"Checkpoint does not exist, init randomly: {p}")else:initialize(model, kwargs.get("init", "kaiming_normal"))