Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
Dev gzf llm (#1506)
zhifu gao675b4605e8d1
diff --git a/README.md b/README.mdindex 38fd6868d..e23596c16 100644--- a/README.md+++ b/README.md@@ -112,7 +112,7 @@ Notes: Support recognition of single audio file, as well as file list in Kaldi-sfrom funasr import AutoModel# paraformer-zh is a multi-functional asr model# use vad, punc, spk or not as you need-model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc-c",+model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc",# spk_model="cam++",)res = model.generate(input=f"{model.model_path}/example/asr_example.wav",diff --git a/README_zh.md b/README_zh.mdindex 725b21056..3e58aa77a 100644--- a/README_zh.md+++ b/README_zh.md@@ -106,7 +106,7 @@ funasr ++model=paraformer-zh ++vad_model="fsmn-vad" ++punc_model="ct-punc" ++inpfrom funasr import AutoModel# paraformer-zh is a multi-functional asr model# use vad, punc, spk or not as you need-model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc-c",+model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc",# spk_model="cam++")res = model.generate(input=f"{model.model_path}/example/asr_example.wav",diff --git a/examples/industrial_data_pretraining/bicif_paraformer/demo.py b/examples/industrial_data_pretraining/bicif_paraformer/demo.pyindex bb63872d2..c8c4fa784 100644--- a/examples/industrial_data_pretraining/bicif_paraformer/demo.py+++ b/examples/industrial_data_pretraining/bicif_paraformer/demo.py@@ -5,13 +5,13 @@from funasr import AutoModel-model = AutoModel(model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",+model = AutoModel(model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",model_revision="v2.0.4",- vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",+ vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",vad_model_revision="v2.0.4",- punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",+ punc_model="iic/punc_ct-transformer_cn-en-common-vocab471067-large",punc_model_revision="v2.0.4",- # spk_model="damo/speech_campplus_sv_zh-cn_16k-common",+ # spk_model="iic/speech_campplus_sv_zh-cn_16k-common",# spk_model_revision="v2.0.2",)diff --git a/examples/industrial_data_pretraining/bicif_paraformer/demo.sh b/examples/industrial_data_pretraining/bicif_paraformer/demo.shindex 97701388f..b6c1d1723 100644--- a/examples/industrial_data_pretraining/bicif_paraformer/demo.sh+++ b/examples/industrial_data_pretraining/bicif_paraformer/demo.sh@@ -1,11 +1,12 @@-model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch"+model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch"model_revision="v2.0.4"-vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch"+vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"vad_model_revision="v2.0.4"-punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"-punc_model_revision="v2.0.3"-spk_model="damo/speech_campplus_sv_zh-cn_16k-common"+#punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"+punc_model="iic/punc_ct-transformer_cn-en-common-vocab471067-large"+punc_model_revision="v2.0.4"+spk_model="iic/speech_campplus_sv_zh-cn_16k-common"spk_model_revision="v2.0.2"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/bicif_paraformer/export.py b/examples/industrial_data_pretraining/bicif_paraformer/export.pyindex c819f7a28..28a9e7b0e 100644--- a/examples/industrial_data_pretraining/bicif_paraformer/export.py+++ b/examples/industrial_data_pretraining/bicif_paraformer/export.py@@ -7,7 +7,7 @@from funasr import AutoModel-model = AutoModel(model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",+model = AutoModel(model="iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",model_revision="v2.0.4", device="cpu")res = model.export(type="onnx", quantize=False)diff --git a/examples/industrial_data_pretraining/campplus_sv/demo.py b/examples/industrial_data_pretraining/campplus_sv/demo.pyindex 16d629b29..e7c4cc90b 100644--- a/examples/industrial_data_pretraining/campplus_sv/demo.py+++ b/examples/industrial_data_pretraining/campplus_sv/demo.py@@ -5,7 +5,7 @@from funasr import AutoModel-model = AutoModel(model="damo/speech_campplus_sv_zh-cn_16k-common",+model = AutoModel(model="iic/speech_campplus_sv_zh-cn_16k-common",model_revision="v2.0.2",)diff --git a/examples/industrial_data_pretraining/contextual_paraformer/demo.py b/examples/industrial_data_pretraining/contextual_paraformer/demo.pyindex 38d7fbfd0..4168a7d73 100755--- a/examples/industrial_data_pretraining/contextual_paraformer/demo.py+++ b/examples/industrial_data_pretraining/contextual_paraformer/demo.py@@ -5,7 +5,7 @@from funasr import AutoModel-model = AutoModel(model="damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404", model_revision="v2.0.4")+model = AutoModel(model="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404", model_revision="v2.0.4")res = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav",hotword='达摩院 魔搭')diff --git a/examples/industrial_data_pretraining/contextual_paraformer/demo.sh b/examples/industrial_data_pretraining/contextual_paraformer/demo.shindex 1bd4f7f5b..987f2ff29 100755--- a/examples/industrial_data_pretraining/contextual_paraformer/demo.sh+++ b/examples/industrial_data_pretraining/contextual_paraformer/demo.sh@@ -1,5 +1,5 @@-model="damo/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404"+model="iic/speech_paraformer-large-contextual_asr_nat-zh-cn-16k-common-vocab8404"model_revision="v2.0.4"python ../../../funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/ct_transformer/demo.py b/examples/industrial_data_pretraining/ct_transformer/demo.pyindex 1aed7860f..31755dcc1 100644--- a/examples/industrial_data_pretraining/ct_transformer/demo.py+++ b/examples/industrial_data_pretraining/ct_transformer/demo.py@@ -5,7 +5,7 @@from funasr import AutoModel-model = AutoModel(model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch", model_revision="v2.0.4")+model = AutoModel(model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch", model_revision="v2.0.4")res = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt")print(res)@@ -13,7 +13,7 @@ print(res)from funasr import AutoModel-model = AutoModel(model="damo/punc_ct-transformer_cn-en-common-vocab471067-large", model_revision="v2.0.4")+model = AutoModel(model="iic/punc_ct-transformer_cn-en-common-vocab471067-large", model_revision="v2.0.4")res = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_text/punc_example.txt")print(res)\ No newline at end of filediff --git a/examples/industrial_data_pretraining/ct_transformer/demo.sh b/examples/industrial_data_pretraining/ct_transformer/demo.shindex fcfa07ce3..56d0bd378 100644--- a/examples/industrial_data_pretraining/ct_transformer/demo.sh+++ b/examples/industrial_data_pretraining/ct_transformer/demo.sh@@ -1,8 +1,8 @@-#model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"+#model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"#model_revision="v2.0.4"-model="damo/punc_ct-transformer_cn-en-common-vocab471067-large"+model="iic/punc_ct-transformer_cn-en-common-vocab471067-large"model_revision="v2.0.4"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/ct_transformer_streaming/demo.sh b/examples/industrial_data_pretraining/ct_transformer_streaming/demo.shindex b07085250..7124fd565 100644--- a/examples/industrial_data_pretraining/ct_transformer_streaming/demo.sh+++ b/examples/industrial_data_pretraining/ct_transformer_streaming/demo.sh@@ -1,5 +1,5 @@-model="damo/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727"+model="iic/punc_ct-transformer_zh-cn-common-vad_realtime-vocab272727"model_revision="v2.0.4"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/emotion2vec/demo.py b/examples/industrial_data_pretraining/emotion2vec/demo.pyindex 66c25cf9c..227f504ea 100644--- a/examples/industrial_data_pretraining/emotion2vec/demo.py+++ b/examples/industrial_data_pretraining/emotion2vec/demo.py@@ -5,7 +5,7 @@from funasr import AutoModel-# model="damo/emotion2vec_base"+# model="iic/emotion2vec_base"model = AutoModel(model="iic/emotion2vec_base_finetuned", model_revision="v2.0.4")wav_file = f"{model.model_path}/example/test.wav"diff --git a/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.py b/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.pyindex c28db7a07..03c01ea06 100644--- a/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.py+++ b/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.py@@ -6,7 +6,7 @@from funasr import AutoModelwav_file = "https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/vad_example.wav"-model = AutoModel(model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch", model_revision="v2.0.4")+model = AutoModel(model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch", model_revision="v2.0.4")res = model.generate(input=wav_file)print(res)diff --git a/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.sh b/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.shindex a003fa5ee..8cdf3edf6 100644--- a/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.sh+++ b/examples/industrial_data_pretraining/fsmn_vad_streaming/demo.sh@@ -1,6 +1,6 @@-model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch"+model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"model_revision="v2.0.4"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/monotonic_aligner/demo.py b/examples/industrial_data_pretraining/monotonic_aligner/demo.pyindex 8e71c90ab..2e76f649f 100644--- a/examples/industrial_data_pretraining/monotonic_aligner/demo.py+++ b/examples/industrial_data_pretraining/monotonic_aligner/demo.py@@ -5,7 +5,7 @@from funasr import AutoModel-model = AutoModel(model="damo/speech_timestamp_prediction-v1-16k-offline", model_revision="v2.0.4")+model = AutoModel(model="iic/speech_timestamp_prediction-v1-16k-offline", model_revision="v2.0.4")res = model.generate(input=("https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav","欢迎大家来到魔搭社区进行体验"),diff --git a/examples/industrial_data_pretraining/monotonic_aligner/demo.sh b/examples/industrial_data_pretraining/monotonic_aligner/demo.shindex 032092259..ece97ff76 100644--- a/examples/industrial_data_pretraining/monotonic_aligner/demo.sh+++ b/examples/industrial_data_pretraining/monotonic_aligner/demo.sh@@ -1,5 +1,5 @@-model="damo/speech_timestamp_prediction-v1-16k-offline"+model="iic/speech_timestamp_prediction-v1-16k-offline"model_revision="v2.0.4"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/paraformer-zh-spk/demo.py b/examples/industrial_data_pretraining/paraformer-zh-spk/demo.pyindex 1149aad6c..64b5c0b2b 100644--- a/examples/industrial_data_pretraining/paraformer-zh-spk/demo.py+++ b/examples/industrial_data_pretraining/paraformer-zh-spk/demo.py@@ -5,13 +5,13 @@from funasr import AutoModel-model = AutoModel(model="damo/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",+model = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",model_revision="v2.0.4",- vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",+ vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",vad_model_revision="v2.0.4",- punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",+ punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",punc_model_revision="v2.0.4",- spk_model="damo/speech_campplus_sv_zh-cn_16k-common",+ spk_model="iic/speech_campplus_sv_zh-cn_16k-common",spk_model_revision="v2.0.2")diff --git a/examples/industrial_data_pretraining/paraformer-zh-spk/demo.sh b/examples/industrial_data_pretraining/paraformer-zh-spk/demo.shindex 319a7c9e2..dae1aca6f 100644--- a/examples/industrial_data_pretraining/paraformer-zh-spk/demo.sh+++ b/examples/industrial_data_pretraining/paraformer-zh-spk/demo.sh@@ -1,11 +1,11 @@-model="damo/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"+model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"model_revision="v2.0.4"-vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch"+vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"vad_model_revision="v2.0.4"-punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"+punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"punc_model_revision="v2.0.4"-spk_model="damo/speech_campplus_sv_zh-cn_16k-common"+spk_model="iic/speech_campplus_sv_zh-cn_16k-common"spk_model_revision="v2.0.2"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/paraformer/demo.py b/examples/industrial_data_pretraining/paraformer/demo.pyindex 651df1e54..965f8f3fc 100644--- a/examples/industrial_data_pretraining/paraformer/demo.py+++ b/examples/industrial_data_pretraining/paraformer/demo.py@@ -22,7 +22,7 @@ print(res)''' can not use currentlyfrom funasr import AutoFrontend-frontend = AutoFrontend(model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", model_revision="v2.0.4")+frontend = AutoFrontend(model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", model_revision="v2.0.4")fbanks = frontend(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", batch_size=2)diff --git a/examples/industrial_data_pretraining/paraformer/finetune.sh b/examples/industrial_data_pretraining/paraformer/finetune.shindex 21b29b6b1..e1273dabd 100644--- a/examples/industrial_data_pretraining/paraformer/finetune.sh+++ b/examples/industrial_data_pretraining/paraformer/finetune.sh@@ -32,7 +32,7 @@ torchrun \--nnodes 1 \--nproc_per_node ${gpu_num} \funasr/bin/train.py \-++model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \+++model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \++model_revision="v2.0.4" \++train_data_set_list="${train_data}" \++valid_data_set_list="${val_data}" \diff --git a/examples/industrial_data_pretraining/paraformer/infer.sh b/examples/industrial_data_pretraining/paraformer/infer.shindex a1975eac8..aa523e5a5 100644--- a/examples/industrial_data_pretraining/paraformer/infer.sh+++ b/examples/industrial_data_pretraining/paraformer/infer.sh@@ -8,7 +8,7 @@ input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asroutput_dir="./outputs/debug"-model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"+model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"model_revision="v2.0.4"device="cuda:0" # "cuda:0" for gpu0, "cuda:1" for gpu1, "cpu"diff --git a/examples/industrial_data_pretraining/scama/demo.sh b/examples/industrial_data_pretraining/scama/demo.shindex edb7196e3..c3f7bb488 100644--- a/examples/industrial_data_pretraining/scama/demo.sh+++ b/examples/industrial_data_pretraining/scama/demo.sh@@ -1,5 +1,5 @@-model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online"+model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-online"model_revision="v2.0.4"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/seaco_paraformer/demo.py b/examples/industrial_data_pretraining/seaco_paraformer/demo.pyindex 551dd8bf8..69e9020f4 100644--- a/examples/industrial_data_pretraining/seaco_paraformer/demo.py+++ b/examples/industrial_data_pretraining/seaco_paraformer/demo.py@@ -7,11 +7,11 @@ from funasr import AutoModelmodel = AutoModel(model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch",model_revision="v2.0.4",- # vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch",+ # vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",# vad_model_revision="v2.0.4",- # punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",+ # punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",# punc_model_revision="v2.0.4",- # spk_model="damo/speech_campplus_sv_zh-cn_16k-common",+ # spk_model="iic/speech_campplus_sv_zh-cn_16k-common",# spk_model_revision="v2.0.2",)diff --git a/examples/industrial_data_pretraining/seaco_paraformer/demo.sh b/examples/industrial_data_pretraining/seaco_paraformer/demo.shindex 76f52ef7b..9a02572fa 100644--- a/examples/industrial_data_pretraining/seaco_paraformer/demo.sh+++ b/examples/industrial_data_pretraining/seaco_paraformer/demo.sh@@ -1,9 +1,9 @@-model="damo/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"+model="iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"model_revision="v2.0.4"-vad_model="damo/speech_fsmn_vad_zh-cn-16k-common-pytorch"+vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch"vad_model_revision="v2.0.4"-punc_model="damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"+punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch"punc_model_revision="v2.0.4"python funasr/bin/inference.py \diff --git a/examples/industrial_data_pretraining/uniasr/demo.py b/examples/industrial_data_pretraining/uniasr/demo.pyindex c6973aacf..3006d5eae 100644--- a/examples/industrial_data_pretraining/uniasr/demo.py+++ b/examples/industrial_data_pretraining/uniasr/demo.py@@ -16,7 +16,7 @@ print(res)''' can not use currentlyfrom funasr import AutoFrontend-frontend = AutoFrontend(model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", model_revision="v2.0.4")+frontend = AutoFrontend(model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", model_revision="v2.0.4")fbanks = frontend(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", batch_size=2)diff --git a/examples/industrial_data_pretraining/uniasr/demo.sh b/examples/industrial_data_pretraining/uniasr/demo.shindex 7491e98b5..71f249ac3 100644--- a/examples/industrial_data_pretraining/uniasr/demo.sh+++ b/examples/industrial_data_pretraining/uniasr/demo.sh@@ -1,5 +1,5 @@-model="damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"+model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"model_revision="v2.0.4"python funasr/bin/inference.py \diff --git a/funasr/models/ct_transformer/export_meta.py b/funasr/models/ct_transformer/export_meta.pynew file mode 100644index 000000000..691f21124--- /dev/null+++ b/funasr/models/ct_transformer/export_meta.py@@ -0,0 +1,67 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++import types+import torch+from funasr.register import tables+++def export_rebuild_model(model, **kwargs):++ is_onnx = kwargs.get("type", "onnx") == "onnx"+ encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")+ model.encoder = encoder_class(model.encoder, onnx=is_onnx)++ model.forward = types.MethodType(export_forward, model)+ model.export_dummy_inputs = types.MethodType(export_dummy_inputs, model)+ model.export_input_names = types.MethodType(export_input_names, model)+ model.export_output_names = types.MethodType(export_output_names, model)+ model.export_dynamic_axes = types.MethodType(export_dynamic_axes, model)+ model.export_name = types.MethodType(export_name, model)++ return model+++def export_forward(self, inputs: torch.Tensor, text_lengths: torch.Tensor):+ """Compute loss value from buffer sequences.++ Args:+ input (torch.Tensor): Input ids. (batch, len)+ hidden (torch.Tensor): Target ids. (batch, len)++ """+ x = self.embed(inputs)+ h, _ = self.encoder(x, text_lengths)+ y = self.decoder(h)+ return y++def export_dummy_inputs(self):+ length = 120+ text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)).type(torch.int32)+ text_lengths = torch.tensor([length-20, length], dtype=torch.int32)+ return (text_indexes, text_lengths)++def export_input_names(self):+ return ['inputs', 'text_lengths']++def export_output_names(self):+ return ['logits']++def export_dynamic_axes(self):+ return {+ 'inputs': {+ 0: 'batch_size',+ 1: 'feats_length'+ },+ 'text_lengths': {+ 0: 'batch_size',+ },+ 'logits': {+ 0: 'batch_size',+ 1: 'logits_length'+ },+ }+def export_name(self):+ return "model.onnx"\ No newline at end of filediff --git a/funasr/models/ct_transformer/model.py b/funasr/models/ct_transformer/model.pyindex 9f680fdc8..57a23ccb2 100644--- a/funasr/models/ct_transformer/model.py+++ b/funasr/models/ct_transformer/model.py@@ -17,7 +17,10 @@ from funasr.train_utils.device_funcs import force_gatherablefrom funasr.utils.load_utils import load_audio_text_image_videofrom funasr.models.transformer.utils.nets_utils import make_pad_maskfrom funasr.models.ct_transformer.utils import split_to_mini_sentence, split_words-+try:+ import jieba+except:+ passif LooseVersion(torch.__version__) >= LooseVersion("1.6.0"):from torch.cuda.amp import autocastelse:@@ -69,6 +72,10 @@ class CTTransformer(torch.nn.Module):self.sos = sosself.eos = eosself.sentence_end_id = sentence_end_id+ self.jieba_usr_dict = None+ if kwargs.get("jieba_usr_dict", None) is not None:+ jieba.load_userdict(kwargs["jieba_usr_dict"])+ self.jieba_usr_dict = jieba@@ -237,14 +244,8 @@ class CTTransformer(torch.nn.Module):# text = data_in[0]# text_lengths = data_lengths[0] if data_lengths is not None else Nonesplit_size = kwargs.get("split_size", 20)-- jieba_usr_dict = kwargs.get("jieba_usr_dict", None)- if jieba_usr_dict and isinstance(jieba_usr_dict, str):- import jieba- jieba.load_userdict(jieba_usr_dict)- jieba_usr_dict = jieba- kwargs["jieba_usr_dict"] = "jieba_usr_dict"- tokens = split_words(text, jieba_usr_dict=jieba_usr_dict)++ tokens = split_words(text, jieba_usr_dict=self.jieba_usr_dict)tokens_int = tokenizer.encode(tokens)mini_sentences = split_to_mini_sentence(tokens, split_size)@@ -347,7 +348,7 @@ class CTTransformer(torch.nn.Module):else:punc_array = torch.cat([punc_array, punctuations], dim=0)# post processing when using word level punc model- if jieba_usr_dict:+ if self.jieba_usr_dict is not None:len_tokens = len(tokens)new_punc_array = copy.copy(punc_array).tolist()# for i, (token, punc_id) in enumerate(zip(tokens[::-1], punc_array.tolist()[::-1])):@@ -364,57 +365,10 @@ class CTTransformer(torch.nn.Module):results.append(result_i)return results, meta_data- def export(- self,- **kwargs,- ):-- is_onnx = kwargs.get("type", "onnx") == "onnx"- encoder_class = tables.encoder_classes.get(kwargs["encoder"]+"Export")- self.encoder = encoder_class(self.encoder, onnx=is_onnx)-- self.forward = self.export_forward-- return self-- def export_forward(self, inputs: torch.Tensor, text_lengths: torch.Tensor):- """Compute loss value from buffer sequences.+ def export(self, **kwargs):++ from .export_meta import export_rebuild_model+ models = export_rebuild_model(model=self, **kwargs)+ return models- Args:- input (torch.Tensor): Input ids. (batch, len)- hidden (torch.Tensor): Target ids. (batch, len)- """- x = self.embed(inputs)- h, _ = self.encoder(x, text_lengths)- y = self.decoder(h)- return y-- def export_dummy_inputs(self):- length = 120- text_indexes = torch.randint(0, self.embed.num_embeddings, (2, length)).type(torch.int32)- text_lengths = torch.tensor([length-20, length], dtype=torch.int32)- return (text_indexes, text_lengths)-- def export_input_names(self):- return ['inputs', 'text_lengths']-- def export_output_names(self):- return ['logits']-- def export_dynamic_axes(self):- return {- 'inputs': {- 0: 'batch_size',- 1: 'feats_length'- },- 'text_lengths': {- 0: 'batch_size',- },- 'logits': {- 0: 'batch_size',- 1: 'logits_length'- },- }- def export_name(self):- return "model.onnx"\ No newline at end of filediff --git a/funasr/models/ct_transformer_streaming/export_meta.py b/funasr/models/ct_transformer_streaming/export_meta.pynew file mode 100644index 000000000..e4745d6fb--- /dev/null+++ b/funasr/models/ct_transformer_streaming/export_meta.py@@ -0,0 +1,77 @@+#!/usr/bin/env python3+# -*- encoding: utf-8 -*-+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++import types+import torch+from funasr.register import tables+++def export_rebuild_model(model, **kwargs):++ is_onnx = kwargs.get("type", "onnx") == "onnx"+ encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")+ model.encoder = encoder_class(model.encoder, onnx=is_onnx)++ model.forward = types.MethodType(export_forward, model)+ model.export_dummy_inputs = types.MethodType(export_dummy_inputs, model)+ model.export_input_names = types.MethodType(export_input_names, model)+ model.export_output_names = types.MethodType(export_output_names, model)+ model.export_dynamic_axes = types.MethodType(export_dynamic_axes, model)+ model.export_name = types.MethodType(export_name, model)++ return model++def export_forward(self, inputs: torch.Tensor,+ text_lengths: torch.Tensor,+ vad_indexes: torch.Tensor,+ sub_masks: torch.Tensor,+ ):+ """Compute loss value from buffer sequences.++ Args:+ input (torch.Tensor): Input ids. (batch, len)+ hidden (torch.Tensor): Target ids. (batch, len)++ """+ x = self.embed(inputs)+ # mask = self._target_mask(input)+ h, _ = self.encoder(x, text_lengths, vad_indexes, sub_masks)+ y = self.decoder(h)+ return y++def export_dummy_inputs(self):+ length = 120+ text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)).type(torch.int32)+ text_lengths = torch.tensor([length], dtype=torch.int32)+ vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :]+ sub_masks = torch.ones(length, length, dtype=torch.float32)+ sub_masks = torch.tril(sub_masks).type(torch.float32)+ return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :])++def export_input_names(self):+ return ['inputs', 'text_lengths', 'vad_masks', 'sub_masks']++def export_output_names(self):+ return ['logits']++def export_dynamic_axes(self):+ return {+ 'inputs': {+ 1: 'feats_length'+ },+ 'vad_masks': {+ 2: 'feats_length1',+ 3: 'feats_length2'+ },+ 'sub_masks': {+ 2: 'feats_length1',+ 3: 'feats_length2'+ },+ 'logits': {+ 1: 'logits_length'+ },+ }+def export_name(self):+ return "model.onnx"diff --git a/funasr/models/ct_transformer_streaming/model.py b/funasr/models/ct_transformer_streaming/model.pyindex 129cc95cc..e6977ad19 100644--- a/funasr/models/ct_transformer_streaming/model.py+++ b/funasr/models/ct_transformer_streaming/model.py@@ -173,68 +173,9 @@ class CTTransformerStreaming(CTTransformer):return results, meta_data- def export(- self,- **kwargs,- ):-- is_onnx = kwargs.get("type", "onnx") == "onnx"- encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")- self.encoder = encoder_class(self.encoder, onnx=is_onnx)+ def export(self, **kwargs):- self.forward = self.export_forward-- return self-- def export_forward(self, inputs: torch.Tensor,- text_lengths: torch.Tensor,- vad_indexes: torch.Tensor,- sub_masks: torch.Tensor,- ):- """Compute loss value from buffer sequences.+ from .export_meta import export_rebuild_model+ models = export_rebuild_model(model=self, **kwargs)+ return models- Args:- input (torch.Tensor): Input ids. (batch, len)- hidden (torch.Tensor): Target ids. (batch, len)-- """- x = self.embed(inputs)- # mask = self._target_mask(input)- h, _ = self.encoder(x, text_lengths, vad_indexes, sub_masks)- y = self.decoder(h)- return y-- def export_dummy_inputs(self):- length = 120- text_indexes = torch.randint(0, self.embed.num_embeddings, (1, length)).type(torch.int32)- text_lengths = torch.tensor([length], dtype=torch.int32)- vad_mask = torch.ones(length, length, dtype=torch.float32)[None, None, :, :]- sub_masks = torch.ones(length, length, dtype=torch.float32)- sub_masks = torch.tril(sub_masks).type(torch.float32)- return (text_indexes, text_lengths, vad_mask, sub_masks[None, None, :, :])-- def export_input_names(self):- return ['inputs', 'text_lengths', 'vad_masks', 'sub_masks']-- def export_output_names(self):- return ['logits']-- def export_dynamic_axes(self):- return {- 'inputs': {- 1: 'feats_length'- },- 'vad_masks': {- 2: 'feats_length1',- 3: 'feats_length2'- },- 'sub_masks': {- 2: 'feats_length1',- 3: 'feats_length2'- },- 'logits': {- 1: 'logits_length'- },- }- def export_name(self):- return "model.onnx"diff --git a/setup.py b/setup.pyindex b41148c16..5cb56658f 100644--- a/setup.py+++ b/setup.py@@ -14,16 +14,14 @@ requirements = {"librosa","jamo", # For kss"PyYAML>=5.1.2",- # "soundfile>=0.12.1",+ "soundfile>=0.12.1","kaldiio>=2.17.0","torch_complex",# "nltk>=3.4.5",- # ASR"sentencepiece", # train"jieba",- # "rotary_embedding_torch",+ "rotary_embedding_torch",# "ffmpeg-python",- # TTS# "pypinyin>=0.44.0",# "espnet_tts_frontend",# ENH@@ -54,6 +52,7 @@ requirements = {"torch_optimizer","fairscale","transformers",+ "openai-whisper"],"setup": ["numpy",@@ -96,6 +95,7 @@ requirements = {],}requirements["all"].extend(requirements["train"])+requirements["all"].extend(requirements["llm"])requirements["test"].extend(requirements["train"])install_requires = requirements["install"]