Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

Dev gzf (#1467)

zhifu gaoa7d7a0f3a2e7
diff --git a/README.md b/README.md
index 9841fe1e1..d15905025 100644
--- a/README.md
+++ b/README.md
@@ -215,14 +215,14 @@ More examples ref to [docs](https://github.com/alibaba-damo-academy/FunASR/tree/
### Command-line usage
```shell
-funasr-export ++model=paraformer ++quantize=false
+funasr-export ++model=paraformer ++quantize=false ++device=cpu
```
-### python
+### Python
```python
from funasr import AutoModel
-model = AutoModel(model="paraformer")
+model = AutoModel(model="paraformer", device="cpu")
res = model.export(quantize=False)
```
diff --git a/examples/industrial_data_pretraining/bicif_paraformer/export.py b/examples/industrial_data_pretraining/bicif_paraformer/export.py
index 138f23a69..c819f7a28 100644
--- a/examples/industrial_data_pretraining/bicif_paraformer/export.py
+++ b/examples/industrial_data_pretraining/bicif_paraformer/export.py
@@ -8,7 +8,7 @@
from funasr import AutoModel
model = AutoModel(model="damo/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch",
- model_revision="v2.0.4")
+ model_revision="v2.0.4", device="cpu")
res = model.export(type="onnx", quantize=False)
print(res)
@@ -17,7 +17,7 @@ print(res)
# method2, inference from local path
from funasr import AutoModel
-model = AutoModel(model="/Users/zhifu/.cache/modelscope/hub/iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch")
+model = AutoModel(model="/Users/zhifu/.cache/modelscope/hub/iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch", device="cpu")
res = model.export(type="onnx", quantize=False)
print(res)
\ No newline at end of file
diff --git a/examples/industrial_data_pretraining/bicif_paraformer/export.sh b/examples/industrial_data_pretraining/bicif_paraformer/export.sh
index 42b63484b..b6883b7c9 100644
--- a/examples/industrial_data_pretraining/bicif_paraformer/export.sh
+++ b/examples/industrial_data_pretraining/bicif_paraformer/export.sh
@@ -12,7 +12,8 @@ python -m funasr.bin.export \
++model=${model} \
++model_revision=${model_revision} \
++type="onnx" \
-++quantize=false
+++quantize=false \
+++device="cpu"
# method2, inference from local path
model="/Users/zhifu/.cache/modelscope/hub/iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch"
@@ -20,4 +21,5 @@ model="/Users/zhifu/.cache/modelscope/hub/iic/speech_paraformer-large_asr_nat-zh
python -m funasr.bin.export \
++model=${model} \
++type="onnx" \
-++quantize=false
\ No newline at end of file
+++quantize=false \
+++device="cpu"
\ No newline at end of file
diff --git a/examples/industrial_data_pretraining/ct_transformer/export.sh b/examples/industrial_data_pretraining/ct_transformer/export.sh
index 755645855..a11cda5d4 100644
--- a/examples/industrial_data_pretraining/ct_transformer/export.sh
+++ b/examples/industrial_data_pretraining/ct_transformer/export.sh
@@ -12,7 +12,8 @@ python -m funasr.bin.export \
++model=${model} \
++model_revision=${model_revision} \
++type="onnx" \
-++quantize=false
+++quantize=false \
+++device="cpu"
# method2, inference from local path
@@ -21,4 +22,5 @@ model="/Users/zhifu/.cache/modelscope/hub/iic/punc_ct-transformer_zh-cn-common-v
python -m funasr.bin.export \
++model=${model} \
++type="onnx" \
-++quantize=false
\ No newline at end of file
+++quantize=false \
+++device="cpu"
\ No newline at end of file
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index c4bab03e2..28b9e94fc 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -156,7 +156,7 @@ class AutoModel:
kwargs["batch_size"] = 1
kwargs["device"] = device
- if kwargs.get("ncpu", None):
+ if kwargs.get("ncpu", 4):
torch.set_num_threads(kwargs.get("ncpu"))
# build tokenizer
@@ -476,11 +476,13 @@ class AutoModel:
calib_num: int = 100,
opset_version: int = 14,
**cfg):
- os.environ['EXPORTING_MODEL'] = 'TRUE'
+
+ device = cfg.get("device", "cpu")
+ model = self.model.to(device=device)
kwargs = self.kwargs
deep_update(kwargs, cfg)
+ kwargs["device"] = device
del kwargs["model"]
- model = self.model
model.eval()
batch_size = 1
diff --git a/funasr/bin/export.py b/funasr/bin/export.py
index 7d476644a..cb160e9c5 100644
--- a/funasr/bin/export.py
+++ b/funasr/bin/export.py
@@ -24,7 +24,8 @@ def main_hydra(cfg: DictConfig):
if kwargs.get("debug", False):
import pdb; pdb.set_trace()
-
+ if "device" not in kwargs:
+ kwargs["device"] = "cpu"
model = AutoModel(**kwargs)
res = model.export(input=kwargs.get("input", None),
diff --git a/funasr/models/bicif_paraformer/model.py b/funasr/models/bicif_paraformer/model.py
index eb7318b20..4802da07d 100644
--- a/funasr/models/bicif_paraformer/model.py
+++ b/funasr/models/bicif_paraformer/model.py
@@ -23,7 +23,7 @@ from funasr.models.transformer.utils.add_sos_eos import add_sos_eos
from funasr.utils.timestamp_tools import ts_prediction_lfr6_standard
from funasr.models.transformer.utils.nets_utils import make_pad_mask, pad_list
from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
-
+from funasr.train_utils.device_funcs import to_device
if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"):
from torch.cuda.amp import autocast
@@ -348,6 +348,7 @@ class BiCifParaformer(Paraformer):
max_seq_len=512,
**kwargs,
):
+ self.device = kwargs.get("device")
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
self.encoder = encoder_class(self.encoder, onnx=is_onnx)
@@ -370,14 +371,14 @@ class BiCifParaformer(Paraformer):
return self
- def _export_forward(
+ def export_forward(
self,
speech: torch.Tensor,
speech_lengths: torch.Tensor,
):
# a. To device
batch = {"speech": speech, "speech_lengths": speech_lengths}
- # batch = to_device(batch, device=self.device)
+ batch = to_device(batch, device=self.device)
enc, enc_len = self.encoder(**batch)
mask = self.make_pad_mask(enc_len)[:, None, :]
diff --git a/funasr/models/ct_transformer/model.py b/funasr/models/ct_transformer/model.py
index 31b8c27d5..88ee86734 100644
--- a/funasr/models/ct_transformer/model.py
+++ b/funasr/models/ct_transformer/model.py
@@ -18,7 +18,6 @@ from funasr.utils.load_utils import load_audio_text_image_video
from funasr.models.transformer.utils.nets_utils import make_pad_mask
from funasr.models.ct_transformer.utils import split_to_mini_sentence, split_words
-
if LooseVersion(torch.__version__) >= LooseVersion("1.6.0"):
from torch.cuda.amp import autocast
else:
@@ -378,7 +377,7 @@ class CTTransformer(torch.nn.Module):
return self
- def _export_forward(self, inputs: torch.Tensor, text_lengths: torch.Tensor):
+ def export_forward(self, inputs: torch.Tensor, text_lengths: torch.Tensor):
"""Compute loss value from buffer sequences.
Args:
diff --git a/funasr/models/ct_transformer_streaming/model.py b/funasr/models/ct_transformer_streaming/model.py
index a9b2efb82..4752c4b5a 100644
--- a/funasr/models/ct_transformer_streaming/model.py
+++ b/funasr/models/ct_transformer_streaming/model.py
@@ -186,7 +186,7 @@ class CTTransformerStreaming(CTTransformer):
return self
- def _export_forward(self, inputs: torch.Tensor,
+ def export_forward(self, inputs: torch.Tensor,
text_lengths: torch.Tensor,
vad_indexes: torch.Tensor,
sub_masks: torch.Tensor,
diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.py
index c3063b0a8..d06db2093 100644
--- a/funasr/models/fsmn_vad_streaming/model.py
+++ b/funasr/models/fsmn_vad_streaming/model.py
@@ -651,7 +651,7 @@ class FsmnVADStreaming(nn.Module):
return self
- def _export_forward(self, feats: torch.Tensor, *args, **kwargs):
+ def export_forward(self, feats: torch.Tensor, *args, **kwargs):
scores, out_caches = self.encoder(feats, *args)
diff --git a/funasr/models/paraformer/model.py b/funasr/models/paraformer/model.py
index 586d72de0..f5f0e4e22 100644
--- a/funasr/models/paraformer/model.py
+++ b/funasr/models/paraformer/model.py
@@ -21,7 +21,7 @@ from funasr.losses.label_smoothing_loss import LabelSmoothingLoss
from funasr.models.transformer.utils.add_sos_eos import add_sos_eos
from funasr.models.transformer.utils.nets_utils import make_pad_mask, pad_list
from funasr.utils.load_utils import load_audio_text_image_video, extract_fbank
-
+from funasr.train_utils.device_funcs import to_device
@tables.register("model_classes", "Paraformer")
class Paraformer(torch.nn.Module):
@@ -554,7 +554,7 @@ class Paraformer(torch.nn.Module):
max_seq_len=512,
**kwargs,
):
-
+ self.device = kwargs.get("device")
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"]+"Export")
self.encoder = encoder_class(self.encoder, onnx=is_onnx)
@@ -579,14 +579,14 @@ class Paraformer(torch.nn.Module):
return self
- def _export_forward(
+ def export_forward(
self,
speech: torch.Tensor,
speech_lengths: torch.Tensor,
):
# a. To device
batch = {"speech": speech, "speech_lengths": speech_lengths}
- # batch = to_device(batch, device=self.device)
+ batch = to_device(batch, device=self.device)
enc, enc_len = self.encoder(**batch)
mask = self.make_pad_mask(enc_len)[:, None, :]
diff --git a/funasr/models/paraformer_streaming/model.py b/funasr/models/paraformer_streaming/model.py
index cebbfc16d..63dba5d88 100644
--- a/funasr/models/paraformer_streaming/model.py
+++ b/funasr/models/paraformer_streaming/model.py
@@ -566,7 +566,7 @@ class ParaformerStreaming(Paraformer):
max_seq_len=512,
**kwargs,
):
-
+ self.device = kwargs.get("device")
is_onnx = kwargs.get("type", "onnx") == "onnx"
encoder_class = tables.encoder_classes.get(kwargs["encoder"] + "Export")
self.encoder = encoder_class(self.encoder, onnx=is_onnx)
@@ -612,7 +612,7 @@ class ParaformerStreaming(Paraformer):
return encoder_model, decoder_model
- def _export_encoder_forward(
+ def export_encoder_forward(
self,
speech: torch.Tensor,
speech_lengths: torch.Tensor,
@@ -663,7 +663,7 @@ class ParaformerStreaming(Paraformer):
def export_encoder_name(self):
return "model.onnx"
- def _export_decoder_forward(
+ def export_decoder_forward(
self,
enc: torch.Tensor,
enc_len: torch.Tensor,
diff --git a/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py b/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py
index e047db936..82548ad9c 100644
--- a/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py
+++ b/runtime/python/onnxruntime/funasr_onnx/paraformer_bin.py
@@ -63,8 +63,8 @@ class Paraformer():
"For the users in China, you could install with the command:\n" \
"\npip3 install -U funasr -i https://mirror.sjtu.edu.cn/pypi/web/simple"
- model = AutoModel(model=cache_dir)
- model_dir = model.export(type="onnx", quantize=quantize, device="cpu")
+ model = AutoModel(model=model_dir)
+ model_dir = model.export(type="onnx", quantize=quantize)
config_file = os.path.join(model_dir, 'config.yaml')
cmvn_file = os.path.join(model_dir, 'am.mvn')
diff --git a/runtime/python/onnxruntime/funasr_onnx/paraformer_online_bin.py b/runtime/python/onnxruntime/funasr_onnx/paraformer_online_bin.py
index 7da5afc6a..692596079 100644
--- a/runtime/python/onnxruntime/funasr_onnx/paraformer_online_bin.py
+++ b/runtime/python/onnxruntime/funasr_onnx/paraformer_online_bin.py
@@ -55,8 +55,8 @@ class Paraformer():
"For the users in China, you could install with the command:\n" \
"\npip3 install -U funasr -i https://mirror.sjtu.edu.cn/pypi/web/simple"
- model = AutoModel(model=cache_dir)
- model_dir = model.export(type="onnx", quantize=quantize, device="cpu")
+ model = AutoModel(model=model_dir)
+ model_dir = model.export(type="onnx", quantize=quantize)
config_file = os.path.join(model_dir, 'config.yaml')
cmvn_file = os.path.join(model_dir, 'am.mvn')
diff --git a/runtime/python/onnxruntime/funasr_onnx/punc_bin.py b/runtime/python/onnxruntime/funasr_onnx/punc_bin.py
index 4e1014f07..b1aca6ea2 100644
--- a/runtime/python/onnxruntime/funasr_onnx/punc_bin.py
+++ b/runtime/python/onnxruntime/funasr_onnx/punc_bin.py
@@ -55,8 +55,8 @@ class CT_Transformer():
"For the users in China, you could install with the command:\n" \
"\npip3 install -U funasr -i https://mirror.sjtu.edu.cn/pypi/web/simple"
- model = AutoModel(model=cache_dir)
- model_dir = model.export(type="onnx", quantize=quantize, device="cpu")
+ model = AutoModel(model=model_dir)
+ model_dir = model.export(type="onnx", quantize=quantize)
config_file = os.path.join(model_dir, 'punc.yaml')
config = read_yaml(config_file)
diff --git a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
index af32b1d53..384f3779e 100644
--- a/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
+++ b/runtime/python/onnxruntime/funasr_onnx/vad_bin.py
@@ -61,8 +61,8 @@ class Fsmn_vad():
"For the users in China, you could install with the command:\n" \
"\npip3 install -U funasr -i https://mirror.sjtu.edu.cn/pypi/web/simple"
- model = AutoModel(model=cache_dir)
- model_dir = model.export(type="onnx", quantize=quantize, device="cpu")
+ model = AutoModel(model=model_dir)
+ model_dir = model.export(type="onnx", quantize=quantize)
config_file = os.path.join(model_dir, 'vad.yaml')
cmvn_file = os.path.join(model_dir, 'vad.mvn')
config = read_yaml(config_file)
@@ -225,8 +225,8 @@ class Fsmn_vad_online():
"For the users in China, you could install with the command:\n" \
"\npip3 install -U funasr -i https://mirror.sjtu.edu.cn/pypi/web/simple"
- model = AutoModel(model=cache_dir)
- model_dir = model.export(type="onnx", quantize=quantize, device="cpu")
+ model = AutoModel(model=model_dir)
+ model_dir = model.export(type="onnx", quantize=quantize)
config_file = os.path.join(model_dir, 'vad.yaml')
cmvn_file = os.path.join(model_dir, 'vad.mvn')