Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat(runtime): add SenseVoice TensorRT deployment (#3463)
zhifu gao6408aaa96d54
diff --git a/runtime/triton_gpu/README.md b/runtime/triton_gpu/README.mdindex 36bb8f6d7..b257424d4 100644--- a/runtime/triton_gpu/README.md+++ b/runtime/triton_gpu/README.md@@ -20,7 +20,72 @@ docker run -it --name "sensevoice-server" --gpus all --net host -v $your_mount_d```### Export SenseVoice Model to Onnx-Please follow the official guide of FunASR to export the sensevoice onnx file. Also, you need to download the tokenizer file by yourself.+Please follow the official FunASR guide to export the SenseVoice ONNX file. Also,+download the tokenizer file used by the scoring model.++The default deployment uses Triton's ONNX Runtime backend. Export an unquantized+graph when you plan to build a native TensorRT engine:++```python+from funasr import AutoModel++model = AutoModel(model="iic/SenseVoiceSmall", device="cuda:0")+model.export(+ type="onnx",+ quantize=False,+ device="cuda:0",+ output_dir="./sensevoice_onnx",+ max_seq_len=4096,+)+```++Do not use `model_quant.onnx` for native TensorRT. Dynamic ONNX quantization adds+`DynamicQuantizeLinear` and `MatMulInteger`, which are not supported by this+TensorRT path. The builder detects these operators and exits with an actionable+error before starting an expensive engine build.++### Build a Native TensorRT Engine++Build the plan on the same GPU architecture and TensorRT version used by the+target Triton server. TensorRT plans are not portable across GPU compute+capabilities or arbitrary TensorRT versions.++Inside the target Triton environment, install ONNX if needed and run:++```sh+pip install "onnx>=1.16"++python runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py \+ ./sensevoice_onnx/model.onnx \+ runtime/triton_gpu/model_repo_sense_voice_small/encoder/1/model.plan \+ --precision fp16 \+ --min-batch 1 --opt-batch 8 --max-batch 16 \+ --min-frames 1 --opt-frames 512 --max-frames 4096 \+ --workspace-gb 8++cp runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt \+ runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt+```++The frame bounds apply after the SenseVoice LFR frontend. With the default+frontend, one feature frame represents approximately 60 ms of audio. Tune the+optimization profile to production traffic; larger maximum batch and frame+bounds increase build time and may require more GPU memory. The script validates+the ONNX checker result, exact SenseVoice tensor contract, profile ordering, GPU+FP16 capability, TensorRT parser result, and atomic plan output.++The maintained baseline was verified with TensorRT 10.0.1 on an NVIDIA H100:++| Check | Result |+|---|---|+| FP32 ONNX parser | 0 TensorRT errors |+| FP16 plan, batch 1-16, frames 1-4096 | 527,504,916 bytes; 113.9 s build |+| Random features, 30 and 64 frames | 100% CTC top-1 agreement with PyTorch |+| Bundled Chinese example | Exact transcript: `开饭时间早上九点至下午五点` |++Keep `config.pbtxt` unchanged to continue using ONNX Runtime, or replace it with+the provided `config.pbtxt.tensorrt` after placing `model.plan` in `encoder/1`.+### Launch ServerLog of directory tree:```shdiff --git a/runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt b/runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrtnew file mode 100644index 000000000..2d12292ce--- /dev/null+++ b/runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt@@ -0,0 +1,68 @@+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++name: "encoder"+platform: "tensorrt_plan"+default_model_filename: "model.plan"++max_batch_size: 16++input [+ {+ name: "speech"+ data_type: TYPE_FP32+ dims: [-1, 560]+ },+ {+ name: "speech_lengths"+ data_type: TYPE_INT32+ dims: [1]+ reshape: { shape: [] }+ },+ {+ name: "language"+ data_type: TYPE_INT32+ dims: [1]+ reshape: { shape: [] }+ },+ {+ name: "textnorm"+ data_type: TYPE_INT32+ dims: [1]+ reshape: { shape: [] }+ }+]++output [+ {+ name: "ctc_logits"+ data_type: TYPE_FP32+ dims: [-1, 25055]+ },+ {+ name: "encoder_out_lens"+ data_type: TYPE_INT32+ dims: [1]+ reshape: { shape: [] }+ }+]++dynamic_batching {}++instance_group [+ {+ count: 1+ kind: KIND_GPU+ }+]diff --git a/runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py b/runtime/triton_gpu/scripts/build_sensevoice_tensorrt.pynew file mode 100755index 000000000..b58471b22--- /dev/null+++ b/runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py@@ -0,0 +1,290 @@+#!/usr/bin/env python3+# Copyright FunASR (https://github.com/modelscope/FunASR). All Rights Reserved.+# MIT License (https://opensource.org/licenses/MIT)++import argparse+from collections import Counter+import hashlib+import json+import os+from pathlib import Path+import sys+import tempfile+import time+++FEATURE_WIDTH = 560+VOCAB_SIZE = 25055+CONTROL_INPUTS = ("speech_lengths", "language", "textnorm")+UNSUPPORTED_QUANTIZED_OPS = frozenset({"DynamicQuantizeLinear", "MatMulInteger"})+++class ShapeProfile:+ def __init__(+ self,+ *,+ min_batch,+ opt_batch,+ max_batch,+ min_frames,+ opt_frames,+ max_frames,+ ):+ self.min_batch = min_batch+ self.opt_batch = opt_batch+ self.max_batch = max_batch+ self.min_frames = min_frames+ self.opt_frames = opt_frames+ self.max_frames = max_frames+ self._validate()++ def _validate(self):+ batch = (self.min_batch, self.opt_batch, self.max_batch)+ frames = (self.min_frames, self.opt_frames, self.max_frames)+ if not (0 < batch[0] <= batch[1] <= batch[2]):+ raise ValueError("batch bounds must satisfy 0 < min <= opt <= max")+ if not (0 < frames[0] <= frames[1] <= frames[2]):+ raise ValueError("frame bounds must satisfy 0 < min <= opt <= max")++ def tensor_shapes(self):+ batch = ((self.min_batch,), (self.opt_batch,), (self.max_batch,))+ shapes = {+ "speech": (+ (self.min_batch, self.min_frames, FEATURE_WIDTH),+ (self.opt_batch, self.opt_frames, FEATURE_WIDTH),+ (self.max_batch, self.max_frames, FEATURE_WIDTH),+ )+ }+ shapes.update({name: batch for name in CONTROL_INPUTS})+ return shapes++ def as_dict(self):+ return {+ "batch": [self.min_batch, self.opt_batch, self.max_batch],+ "frames": [self.min_frames, self.opt_frames, self.max_frames],+ }+++def reject_unsupported_quantization(op_types):+ counts = Counter(op_types)+ blocked = {name: counts[name] for name in sorted(UNSUPPORTED_QUANTIZED_OPS) if counts[name]}+ if not blocked:+ return+ details = ", ".join(f"{name}={count}" for name, count in blocked.items())+ raise ValueError(+ "The ONNX graph is dynamically quantized and cannot be parsed by the "+ f"supported TensorRT path ({details}). Export the FP32 graph with "+ "model.export(type='onnx', quantize=False), then build FP16 or FP32 "+ "inside TensorRT."+ )+++def validate_tensor_contract(inputs, outputs):+ expected_inputs = {"speech", *CONTROL_INPUTS}+ expected_outputs = {"ctc_logits", "encoder_out_lens"}+ if set(inputs) != expected_inputs:+ raise ValueError(+ f"SenseVoice inputs must be {sorted(expected_inputs)}, got {sorted(inputs)}"+ )+ if set(outputs) != expected_outputs:+ raise ValueError(+ f"SenseVoice outputs must be {sorted(expected_outputs)}, got {sorted(outputs)}"+ )++ speech_dtype, speech_shape = inputs["speech"]+ if speech_dtype != "FLOAT" or len(speech_shape) != 3 or speech_shape[-1] != FEATURE_WIDTH:+ raise ValueError(+ f"speech must be FLOAT [batch, frames, {FEATURE_WIDTH}], got "+ f"{speech_dtype} {speech_shape}"+ )+ for name in CONTROL_INPUTS:+ dtype, shape = inputs[name]+ if dtype != "INT32" or len(shape) != 1:+ raise ValueError(f"{name} must be INT32 [batch], got {dtype} {shape}")++ logits_dtype, logits_shape = outputs["ctc_logits"]+ if logits_dtype != "FLOAT" or len(logits_shape) != 3 or logits_shape[-1] != VOCAB_SIZE:+ raise ValueError(+ f"ctc_logits must be FLOAT [batch, frames, {VOCAB_SIZE}], got "+ f"{logits_dtype} {logits_shape}"+ )+ lens_dtype, lens_shape = outputs["encoder_out_lens"]+ if lens_dtype != "INT32" or len(lens_shape) != 1:+ raise ValueError(+ "encoder_out_lens must be INT32 [batch], got " f"{lens_dtype} {lens_shape}"+ )+++def _onnx_tensor_specs(values, onnx):+ specs = {}+ for value in values:+ tensor_type = value.type.tensor_type+ dtype = onnx.TensorProto.DataType.Name(tensor_type.elem_type)+ shape = tuple(dim.dim_param or dim.dim_value for dim in tensor_type.shape.dim)+ specs[value.name] = (dtype, shape)+ return specs+++def inspect_onnx(onnx_path):+ try:+ import onnx+ except ImportError as exc:+ raise RuntimeError("Install the ONNX dependency with `pip install onnx`.") from exc++ onnx.checker.check_model(str(onnx_path))+ model = onnx.load(str(onnx_path), load_external_data=False)+ reject_unsupported_quantization(node.op_type for node in model.graph.node)+ inputs = _onnx_tensor_specs(model.graph.input, onnx)+ outputs = _onnx_tensor_specs(model.graph.output, onnx)+ validate_tensor_contract(inputs, outputs)+ return {+ "ir_version": model.ir_version,+ "opsets": {item.domain or "ai.onnx": item.version for item in model.opset_import},+ "node_count": len(model.graph.node),+ "inputs": inputs,+ "outputs": outputs,+ }+++def _sha256(path):+ digest = hashlib.sha256()+ with path.open("rb") as handle:+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):+ digest.update(chunk)+ return digest.hexdigest()+++def _atomic_write(path, data):+ path.parent.mkdir(parents=True, exist_ok=True)+ fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)+ try:+ with os.fdopen(fd, "wb") as handle:+ handle.write(data)+ handle.flush()+ os.fsync(handle.fileno())+ os.replace(temporary, path)+ except Exception:+ try:+ os.unlink(temporary)+ except FileNotFoundError:+ pass+ raise+++def build_engine(+ onnx_path,+ engine_path,+ profile,+ *,+ precision="fp16",+ workspace_gb=8.0,+ verbose=False,+):+ try:+ import tensorrt as trt+ except ImportError as exc:+ raise RuntimeError(+ "Install TensorRT 10 or newer with `pip install tensorrt-cu12`."+ ) from exc++ severity = trt.Logger.VERBOSE if verbose else trt.Logger.WARNING+ logger = trt.Logger(severity)+ builder = trt.Builder(logger)+ network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))+ parser = trt.OnnxParser(network, logger)+ if not parser.parse_from_file(str(onnx_path)):+ errors = "\n".join(str(parser.get_error(i)) for i in range(parser.num_errors))+ raise RuntimeError(f"TensorRT failed to parse {onnx_path}:\n{errors}")++ config = builder.create_builder_config()+ config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, int(workspace_gb * (1 << 30)))+ if precision == "fp16":+ if not builder.platform_has_fast_fp16:+ raise RuntimeError("This GPU does not report fast FP16 support.")+ config.set_flag(trt.BuilderFlag.FP16)++ optimization_profile = builder.create_optimization_profile()+ for name, shapes in profile.tensor_shapes().items():+ optimization_profile.set_shape(name, *shapes)+ if not optimization_profile:+ raise RuntimeError("TensorRT rejected the optimization profile.")+ config.add_optimization_profile(optimization_profile)++ started = time.monotonic()+ serialized = builder.build_serialized_network(network, config)+ if serialized is None:+ raise RuntimeError("TensorRT could not build a serialized engine.")+ _atomic_write(engine_path, serialized)+ return {+ "engine": str(engine_path),+ "engine_bytes": engine_path.stat().st_size,+ "engine_sha256": _sha256(engine_path),+ "onnx": str(onnx_path),+ "onnx_sha256": _sha256(onnx_path),+ "precision": precision,+ "profile": profile.as_dict(),+ "tensorrt_version": trt.__version__,+ "build_seconds": round(time.monotonic() - started, 3),+ }+++def parse_args(argv=None):+ parser = argparse.ArgumentParser(+ description="Build a TensorRT plan for the FunASR SenseVoice encoder."+ )+ parser.add_argument("onnx_model", type=Path)+ parser.add_argument("engine", type=Path)+ parser.add_argument("--precision", choices=("fp16", "fp32"), default="fp16")+ parser.add_argument("--workspace-gb", type=float, default=8.0)+ parser.add_argument("--min-batch", type=int, default=1)+ parser.add_argument("--opt-batch", type=int, default=8)+ parser.add_argument("--max-batch", type=int, default=16)+ parser.add_argument("--min-frames", type=int, default=1)+ parser.add_argument("--opt-frames", type=int, default=512)+ parser.add_argument("--max-frames", type=int, default=4096)+ parser.add_argument("--force", action="store_true")+ parser.add_argument("--verbose", action="store_true")+ return parser.parse_args(argv)+++def main(argv=None):+ args = parse_args(argv)+ if not args.onnx_model.is_file():+ raise FileNotFoundError(f"ONNX model does not exist: {args.onnx_model}")+ if args.engine.exists() and not args.force:+ raise FileExistsError(f"Engine already exists (pass --force): {args.engine}")+ if args.workspace_gb <= 0:+ raise ValueError("--workspace-gb must be greater than zero")++ profile = ShapeProfile(+ min_batch=args.min_batch,+ opt_batch=args.opt_batch,+ max_batch=args.max_batch,+ min_frames=args.min_frames,+ opt_frames=args.opt_frames,+ max_frames=args.max_frames,+ )+ graph = inspect_onnx(args.onnx_model)+ print(json.dumps({"onnx_validation": graph}, default=str, sort_keys=True))+ result = build_engine(+ args.onnx_model,+ args.engine,+ profile,+ precision=args.precision,+ workspace_gb=args.workspace_gb,+ verbose=args.verbose,+ )+ print(json.dumps(result, sort_keys=True))+++def cli(argv=None):+ try:+ main(argv)+ except (FileNotFoundError, FileExistsError, RuntimeError, ValueError) as exc:+ print(f"error: {exc}", file=sys.stderr)+ return 2+ return 0+++if __name__ == "__main__":+ raise SystemExit(cli())diff --git a/tests/test_sensevoice_tensorrt_builder.py b/tests/test_sensevoice_tensorrt_builder.pynew file mode 100644index 000000000..bca6eb12f--- /dev/null+++ b/tests/test_sensevoice_tensorrt_builder.py@@ -0,0 +1,120 @@+import importlib.util+from pathlib import Path+import subprocess+import sys++import pytest+++SCRIPT = (+ Path(__file__).parents[1]+ / "runtime"+ / "triton_gpu"+ / "scripts"+ / "build_sensevoice_tensorrt.py"+)+++def load_builder_module():+ spec = importlib.util.spec_from_file_location("build_sensevoice_tensorrt", SCRIPT)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def test_profile_expands_batch_and_frame_bounds_for_every_input():+ builder = load_builder_module()++ profile = builder.ShapeProfile(+ min_batch=1,+ opt_batch=8,+ max_batch=16,+ min_frames=1,+ opt_frames=512,+ max_frames=4096,+ )++ assert profile.tensor_shapes() == {+ "speech": ((1, 1, 560), (8, 512, 560), (16, 4096, 560)),+ "speech_lengths": ((1,), (8,), (16,)),+ "language": ((1,), (8,), (16,)),+ "textnorm": ((1,), (8,), (16,)),+ }+++ "overrides",+ [+ {"min_batch": 2, "opt_batch": 1},+ {"opt_batch": 17, "max_batch": 16},+ {"min_frames": 0},+ {"min_frames": 32, "opt_frames": 16},+ {"opt_frames": 4097, "max_frames": 4096},+ ],+)+def test_profile_rejects_invalid_or_non_monotonic_bounds(overrides):+ builder = load_builder_module()+ values = {+ "min_batch": 1,+ "opt_batch": 8,+ "max_batch": 16,+ "min_frames": 1,+ "opt_frames": 512,+ "max_frames": 4096,+ }+ values.update(overrides)++ with pytest.raises(ValueError, match="min <= opt <= max"):+ builder.ShapeProfile(**values)+++def test_quantized_onnx_is_rejected_before_tensorrt_parsing():+ builder = load_builder_module()++ with pytest.raises(ValueError, match=r"quantized.*DynamicQuantizeLinear.*281.*quantize=False"):+ builder.reject_unsupported_quantization(+ ["MatMul", "DynamicQuantizeLinear"] * 281 + ["MatMulInteger"] * 281+ )+++def test_fp32_onnx_operator_set_passes_quantization_gate():+ builder = load_builder_module()++ builder.reject_unsupported_quantization(["MatMul", "LayerNormalization", "Conv", "Softmax"])+++def test_tensor_contract_requires_sensevoice_names_dtypes_and_feature_width():+ builder = load_builder_module()+ valid_inputs = {+ "speech": ("FLOAT", ("batch_size", "feats_length", 560)),+ "speech_lengths": ("INT32", ("batch_size",)),+ "language": ("INT32", ("batch_size",)),+ "textnorm": ("INT32", ("batch_size",)),+ }+ valid_outputs = {+ "ctc_logits": ("FLOAT", ("batch_size", "logits_length", 25055)),+ "encoder_out_lens": ("INT32", ("batch_size",)),+ }++ builder.validate_tensor_contract(valid_inputs, valid_outputs)++ invalid_inputs = dict(valid_inputs)+ invalid_inputs["speech"] = ("FLOAT", ("batch_size", "feats_length", 80))+ with pytest.raises(ValueError, match=r"speech.*560"):+ builder.validate_tensor_contract(invalid_inputs, valid_outputs)+++def test_cli_reports_expected_input_errors_without_a_traceback(tmp_path):+ missing = tmp_path / "missing.onnx"+ engine = tmp_path / "model.plan"++ result = subprocess.run(+ [sys.executable, str(SCRIPT), str(missing), str(engine)],+ capture_output=True,+ text=True,+ check=False,+ )++ assert result.returncode == 2+ assert f"error: ONNX model does not exist: {missing}" in result.stderr+ assert "Traceback" not in result.stderr