Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat(runtime): add SenseVoice TensorRT deployment (#3463)

zhifu gao6408aaa96d54
diff --git a/runtime/triton_gpu/README.md b/runtime/triton_gpu/README.md
index 36bb8f6d7..b257424d4 100644
--- a/runtime/triton_gpu/README.md
+++ b/runtime/triton_gpu/README.md
@@ -20,7 +20,72 @@ docker run -it --name "sensevoice-server" --gpus all --net host -v $your_mount_d
```
### Export SenseVoice Model to Onnx
-Please follow the official guide of FunASR to export the sensevoice onnx file. Also, you need to download the tokenizer file by yourself.
+Please follow the official FunASR guide to export the SenseVoice ONNX file. Also,
+download the tokenizer file used by the scoring model.
+
+The default deployment uses Triton's ONNX Runtime backend. Export an unquantized
+graph when you plan to build a native TensorRT engine:
+
+```python
+from funasr import AutoModel
+
+model = AutoModel(model="iic/SenseVoiceSmall", device="cuda:0")
+model.export(
+ type="onnx",
+ quantize=False,
+ device="cuda:0",
+ output_dir="./sensevoice_onnx",
+ max_seq_len=4096,
+)
+```
+
+Do not use `model_quant.onnx` for native TensorRT. Dynamic ONNX quantization adds
+`DynamicQuantizeLinear` and `MatMulInteger`, which are not supported by this
+TensorRT path. The builder detects these operators and exits with an actionable
+error before starting an expensive engine build.
+
+### Build a Native TensorRT Engine
+
+Build the plan on the same GPU architecture and TensorRT version used by the
+target Triton server. TensorRT plans are not portable across GPU compute
+capabilities or arbitrary TensorRT versions.
+
+Inside the target Triton environment, install ONNX if needed and run:
+
+```sh
+pip install "onnx>=1.16"
+
+python runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py \
+ ./sensevoice_onnx/model.onnx \
+ runtime/triton_gpu/model_repo_sense_voice_small/encoder/1/model.plan \
+ --precision fp16 \
+ --min-batch 1 --opt-batch 8 --max-batch 16 \
+ --min-frames 1 --opt-frames 512 --max-frames 4096 \
+ --workspace-gb 8
+
+cp runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt \
+ runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt
+```
+
+The frame bounds apply after the SenseVoice LFR frontend. With the default
+frontend, one feature frame represents approximately 60 ms of audio. Tune the
+optimization profile to production traffic; larger maximum batch and frame
+bounds increase build time and may require more GPU memory. The script validates
+the ONNX checker result, exact SenseVoice tensor contract, profile ordering, GPU
+FP16 capability, TensorRT parser result, and atomic plan output.
+
+The maintained baseline was verified with TensorRT 10.0.1 on an NVIDIA H100:
+
+| Check | Result |
+|---|---|
+| FP32 ONNX parser | 0 TensorRT errors |
+| FP16 plan, batch 1-16, frames 1-4096 | 527,504,916 bytes; 113.9 s build |
+| Random features, 30 and 64 frames | 100% CTC top-1 agreement with PyTorch |
+| Bundled Chinese example | Exact transcript: `开饭时间早上九点至下午五点` |
+
+Keep `config.pbtxt` unchanged to continue using ONNX Runtime, or replace it with
+the provided `config.pbtxt.tensorrt` after placing `model.plan` in `encoder/1`.
+
### Launch Server
Log of directory tree:
```sh
diff --git a/runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt b/runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt
new file mode 100644
index 000000000..2d12292ce
--- /dev/null
+++ b/runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt
@@ -0,0 +1,68 @@
+# Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: "encoder"
+platform: "tensorrt_plan"
+default_model_filename: "model.plan"
+
+max_batch_size: 16
+
+input [
+ {
+ name: "speech"
+ data_type: TYPE_FP32
+ dims: [-1, 560]
+ },
+ {
+ name: "speech_lengths"
+ data_type: TYPE_INT32
+ dims: [1]
+ reshape: { shape: [] }
+ },
+ {
+ name: "language"
+ data_type: TYPE_INT32
+ dims: [1]
+ reshape: { shape: [] }
+ },
+ {
+ name: "textnorm"
+ data_type: TYPE_INT32
+ dims: [1]
+ reshape: { shape: [] }
+ }
+]
+
+output [
+ {
+ name: "ctc_logits"
+ data_type: TYPE_FP32
+ dims: [-1, 25055]
+ },
+ {
+ name: "encoder_out_lens"
+ data_type: TYPE_INT32
+ dims: [1]
+ reshape: { shape: [] }
+ }
+]
+
+dynamic_batching {}
+
+instance_group [
+ {
+ count: 1
+ kind: KIND_GPU
+ }
+]
diff --git a/runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py b/runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py
new file mode 100755
index 000000000..b58471b22
--- /dev/null
+++ b/runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py
@@ -0,0 +1,290 @@
+#!/usr/bin/env python3
+# Copyright FunASR (https://github.com/modelscope/FunASR). All Rights Reserved.
+# MIT License (https://opensource.org/licenses/MIT)
+
+import argparse
+from collections import Counter
+import hashlib
+import json
+import os
+from pathlib import Path
+import sys
+import tempfile
+import time
+
+
+FEATURE_WIDTH = 560
+VOCAB_SIZE = 25055
+CONTROL_INPUTS = ("speech_lengths", "language", "textnorm")
+UNSUPPORTED_QUANTIZED_OPS = frozenset({"DynamicQuantizeLinear", "MatMulInteger"})
+
+
+class ShapeProfile:
+ def __init__(
+ self,
+ *,
+ min_batch,
+ opt_batch,
+ max_batch,
+ min_frames,
+ opt_frames,
+ max_frames,
+ ):
+ self.min_batch = min_batch
+ self.opt_batch = opt_batch
+ self.max_batch = max_batch
+ self.min_frames = min_frames
+ self.opt_frames = opt_frames
+ self.max_frames = max_frames
+ self._validate()
+
+ def _validate(self):
+ batch = (self.min_batch, self.opt_batch, self.max_batch)
+ frames = (self.min_frames, self.opt_frames, self.max_frames)
+ if not (0 < batch[0] <= batch[1] <= batch[2]):
+ raise ValueError("batch bounds must satisfy 0 < min <= opt <= max")
+ if not (0 < frames[0] <= frames[1] <= frames[2]):
+ raise ValueError("frame bounds must satisfy 0 < min <= opt <= max")
+
+ def tensor_shapes(self):
+ batch = ((self.min_batch,), (self.opt_batch,), (self.max_batch,))
+ shapes = {
+ "speech": (
+ (self.min_batch, self.min_frames, FEATURE_WIDTH),
+ (self.opt_batch, self.opt_frames, FEATURE_WIDTH),
+ (self.max_batch, self.max_frames, FEATURE_WIDTH),
+ )
+ }
+ shapes.update({name: batch for name in CONTROL_INPUTS})
+ return shapes
+
+ def as_dict(self):
+ return {
+ "batch": [self.min_batch, self.opt_batch, self.max_batch],
+ "frames": [self.min_frames, self.opt_frames, self.max_frames],
+ }
+
+
+def reject_unsupported_quantization(op_types):
+ counts = Counter(op_types)
+ blocked = {name: counts[name] for name in sorted(UNSUPPORTED_QUANTIZED_OPS) if counts[name]}
+ if not blocked:
+ return
+ details = ", ".join(f"{name}={count}" for name, count in blocked.items())
+ raise ValueError(
+ "The ONNX graph is dynamically quantized and cannot be parsed by the "
+ f"supported TensorRT path ({details}). Export the FP32 graph with "
+ "model.export(type='onnx', quantize=False), then build FP16 or FP32 "
+ "inside TensorRT."
+ )
+
+
+def validate_tensor_contract(inputs, outputs):
+ expected_inputs = {"speech", *CONTROL_INPUTS}
+ expected_outputs = {"ctc_logits", "encoder_out_lens"}
+ if set(inputs) != expected_inputs:
+ raise ValueError(
+ f"SenseVoice inputs must be {sorted(expected_inputs)}, got {sorted(inputs)}"
+ )
+ if set(outputs) != expected_outputs:
+ raise ValueError(
+ f"SenseVoice outputs must be {sorted(expected_outputs)}, got {sorted(outputs)}"
+ )
+
+ speech_dtype, speech_shape = inputs["speech"]
+ if speech_dtype != "FLOAT" or len(speech_shape) != 3 or speech_shape[-1] != FEATURE_WIDTH:
+ raise ValueError(
+ f"speech must be FLOAT [batch, frames, {FEATURE_WIDTH}], got "
+ f"{speech_dtype} {speech_shape}"
+ )
+ for name in CONTROL_INPUTS:
+ dtype, shape = inputs[name]
+ if dtype != "INT32" or len(shape) != 1:
+ raise ValueError(f"{name} must be INT32 [batch], got {dtype} {shape}")
+
+ logits_dtype, logits_shape = outputs["ctc_logits"]
+ if logits_dtype != "FLOAT" or len(logits_shape) != 3 or logits_shape[-1] != VOCAB_SIZE:
+ raise ValueError(
+ f"ctc_logits must be FLOAT [batch, frames, {VOCAB_SIZE}], got "
+ f"{logits_dtype} {logits_shape}"
+ )
+ lens_dtype, lens_shape = outputs["encoder_out_lens"]
+ if lens_dtype != "INT32" or len(lens_shape) != 1:
+ raise ValueError(
+ "encoder_out_lens must be INT32 [batch], got " f"{lens_dtype} {lens_shape}"
+ )
+
+
+def _onnx_tensor_specs(values, onnx):
+ specs = {}
+ for value in values:
+ tensor_type = value.type.tensor_type
+ dtype = onnx.TensorProto.DataType.Name(tensor_type.elem_type)
+ shape = tuple(dim.dim_param or dim.dim_value for dim in tensor_type.shape.dim)
+ specs[value.name] = (dtype, shape)
+ return specs
+
+
+def inspect_onnx(onnx_path):
+ try:
+ import onnx
+ except ImportError as exc:
+ raise RuntimeError("Install the ONNX dependency with `pip install onnx`.") from exc
+
+ onnx.checker.check_model(str(onnx_path))
+ model = onnx.load(str(onnx_path), load_external_data=False)
+ reject_unsupported_quantization(node.op_type for node in model.graph.node)
+ inputs = _onnx_tensor_specs(model.graph.input, onnx)
+ outputs = _onnx_tensor_specs(model.graph.output, onnx)
+ validate_tensor_contract(inputs, outputs)
+ return {
+ "ir_version": model.ir_version,
+ "opsets": {item.domain or "ai.onnx": item.version for item in model.opset_import},
+ "node_count": len(model.graph.node),
+ "inputs": inputs,
+ "outputs": outputs,
+ }
+
+
+def _sha256(path):
+ digest = hashlib.sha256()
+ with path.open("rb") as handle:
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest()
+
+
+def _atomic_write(path, data):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ fd, temporary = tempfile.mkstemp(prefix=f".{path.name}.", dir=path.parent)
+ try:
+ with os.fdopen(fd, "wb") as handle:
+ handle.write(data)
+ handle.flush()
+ os.fsync(handle.fileno())
+ os.replace(temporary, path)
+ except Exception:
+ try:
+ os.unlink(temporary)
+ except FileNotFoundError:
+ pass
+ raise
+
+
+def build_engine(
+ onnx_path,
+ engine_path,
+ profile,
+ *,
+ precision="fp16",
+ workspace_gb=8.0,
+ verbose=False,
+):
+ try:
+ import tensorrt as trt
+ except ImportError as exc:
+ raise RuntimeError(
+ "Install TensorRT 10 or newer with `pip install tensorrt-cu12`."
+ ) from exc
+
+ severity = trt.Logger.VERBOSE if verbose else trt.Logger.WARNING
+ logger = trt.Logger(severity)
+ builder = trt.Builder(logger)
+ network = builder.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH))
+ parser = trt.OnnxParser(network, logger)
+ if not parser.parse_from_file(str(onnx_path)):
+ errors = "\n".join(str(parser.get_error(i)) for i in range(parser.num_errors))
+ raise RuntimeError(f"TensorRT failed to parse {onnx_path}:\n{errors}")
+
+ config = builder.create_builder_config()
+ config.set_memory_pool_limit(trt.MemoryPoolType.WORKSPACE, int(workspace_gb * (1 << 30)))
+ if precision == "fp16":
+ if not builder.platform_has_fast_fp16:
+ raise RuntimeError("This GPU does not report fast FP16 support.")
+ config.set_flag(trt.BuilderFlag.FP16)
+
+ optimization_profile = builder.create_optimization_profile()
+ for name, shapes in profile.tensor_shapes().items():
+ optimization_profile.set_shape(name, *shapes)
+ if not optimization_profile:
+ raise RuntimeError("TensorRT rejected the optimization profile.")
+ config.add_optimization_profile(optimization_profile)
+
+ started = time.monotonic()
+ serialized = builder.build_serialized_network(network, config)
+ if serialized is None:
+ raise RuntimeError("TensorRT could not build a serialized engine.")
+ _atomic_write(engine_path, serialized)
+ return {
+ "engine": str(engine_path),
+ "engine_bytes": engine_path.stat().st_size,
+ "engine_sha256": _sha256(engine_path),
+ "onnx": str(onnx_path),
+ "onnx_sha256": _sha256(onnx_path),
+ "precision": precision,
+ "profile": profile.as_dict(),
+ "tensorrt_version": trt.__version__,
+ "build_seconds": round(time.monotonic() - started, 3),
+ }
+
+
+def parse_args(argv=None):
+ parser = argparse.ArgumentParser(
+ description="Build a TensorRT plan for the FunASR SenseVoice encoder."
+ )
+ parser.add_argument("onnx_model", type=Path)
+ parser.add_argument("engine", type=Path)
+ parser.add_argument("--precision", choices=("fp16", "fp32"), default="fp16")
+ parser.add_argument("--workspace-gb", type=float, default=8.0)
+ parser.add_argument("--min-batch", type=int, default=1)
+ parser.add_argument("--opt-batch", type=int, default=8)
+ parser.add_argument("--max-batch", type=int, default=16)
+ parser.add_argument("--min-frames", type=int, default=1)
+ parser.add_argument("--opt-frames", type=int, default=512)
+ parser.add_argument("--max-frames", type=int, default=4096)
+ parser.add_argument("--force", action="store_true")
+ parser.add_argument("--verbose", action="store_true")
+ return parser.parse_args(argv)
+
+
+def main(argv=None):
+ args = parse_args(argv)
+ if not args.onnx_model.is_file():
+ raise FileNotFoundError(f"ONNX model does not exist: {args.onnx_model}")
+ if args.engine.exists() and not args.force:
+ raise FileExistsError(f"Engine already exists (pass --force): {args.engine}")
+ if args.workspace_gb <= 0:
+ raise ValueError("--workspace-gb must be greater than zero")
+
+ profile = ShapeProfile(
+ min_batch=args.min_batch,
+ opt_batch=args.opt_batch,
+ max_batch=args.max_batch,
+ min_frames=args.min_frames,
+ opt_frames=args.opt_frames,
+ max_frames=args.max_frames,
+ )
+ graph = inspect_onnx(args.onnx_model)
+ print(json.dumps({"onnx_validation": graph}, default=str, sort_keys=True))
+ result = build_engine(
+ args.onnx_model,
+ args.engine,
+ profile,
+ precision=args.precision,
+ workspace_gb=args.workspace_gb,
+ verbose=args.verbose,
+ )
+ print(json.dumps(result, sort_keys=True))
+
+
+def cli(argv=None):
+ try:
+ main(argv)
+ except (FileNotFoundError, FileExistsError, RuntimeError, ValueError) as exc:
+ print(f"error: {exc}", file=sys.stderr)
+ return 2
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(cli())
diff --git a/tests/test_sensevoice_tensorrt_builder.py b/tests/test_sensevoice_tensorrt_builder.py
new file mode 100644
index 000000000..bca6eb12f
--- /dev/null
+++ b/tests/test_sensevoice_tensorrt_builder.py
@@ -0,0 +1,120 @@
+import importlib.util
+from pathlib import Path
+import subprocess
+import sys
+
+import pytest
+
+
+SCRIPT = (
+ Path(__file__).parents[1]
+ / "runtime"
+ / "triton_gpu"
+ / "scripts"
+ / "build_sensevoice_tensorrt.py"
+)
+
+
+def load_builder_module():
+ spec = importlib.util.spec_from_file_location("build_sensevoice_tensorrt", SCRIPT)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_profile_expands_batch_and_frame_bounds_for_every_input():
+ builder = load_builder_module()
+
+ profile = builder.ShapeProfile(
+ min_batch=1,
+ opt_batch=8,
+ max_batch=16,
+ min_frames=1,
+ opt_frames=512,
+ max_frames=4096,
+ )
+
+ assert profile.tensor_shapes() == {
+ "speech": ((1, 1, 560), (8, 512, 560), (16, 4096, 560)),
+ "speech_lengths": ((1,), (8,), (16,)),
+ "language": ((1,), (8,), (16,)),
+ "textnorm": ((1,), (8,), (16,)),
+ }
+
+
+ "overrides",
+ [
+ {"min_batch": 2, "opt_batch": 1},
+ {"opt_batch": 17, "max_batch": 16},
+ {"min_frames": 0},
+ {"min_frames": 32, "opt_frames": 16},
+ {"opt_frames": 4097, "max_frames": 4096},
+ ],
+)
+def test_profile_rejects_invalid_or_non_monotonic_bounds(overrides):
+ builder = load_builder_module()
+ values = {
+ "min_batch": 1,
+ "opt_batch": 8,
+ "max_batch": 16,
+ "min_frames": 1,
+ "opt_frames": 512,
+ "max_frames": 4096,
+ }
+ values.update(overrides)
+
+ with pytest.raises(ValueError, match="min <= opt <= max"):
+ builder.ShapeProfile(**values)
+
+
+def test_quantized_onnx_is_rejected_before_tensorrt_parsing():
+ builder = load_builder_module()
+
+ with pytest.raises(ValueError, match=r"quantized.*DynamicQuantizeLinear.*281.*quantize=False"):
+ builder.reject_unsupported_quantization(
+ ["MatMul", "DynamicQuantizeLinear"] * 281 + ["MatMulInteger"] * 281
+ )
+
+
+def test_fp32_onnx_operator_set_passes_quantization_gate():
+ builder = load_builder_module()
+
+ builder.reject_unsupported_quantization(["MatMul", "LayerNormalization", "Conv", "Softmax"])
+
+
+def test_tensor_contract_requires_sensevoice_names_dtypes_and_feature_width():
+ builder = load_builder_module()
+ valid_inputs = {
+ "speech": ("FLOAT", ("batch_size", "feats_length", 560)),
+ "speech_lengths": ("INT32", ("batch_size",)),
+ "language": ("INT32", ("batch_size",)),
+ "textnorm": ("INT32", ("batch_size",)),
+ }
+ valid_outputs = {
+ "ctc_logits": ("FLOAT", ("batch_size", "logits_length", 25055)),
+ "encoder_out_lens": ("INT32", ("batch_size",)),
+ }
+
+ builder.validate_tensor_contract(valid_inputs, valid_outputs)
+
+ invalid_inputs = dict(valid_inputs)
+ invalid_inputs["speech"] = ("FLOAT", ("batch_size", "feats_length", 80))
+ with pytest.raises(ValueError, match=r"speech.*560"):
+ builder.validate_tensor_contract(invalid_inputs, valid_outputs)
+
+
+def test_cli_reports_expected_input_errors_without_a_traceback(tmp_path):
+ missing = tmp_path / "missing.onnx"
+ engine = tmp_path / "model.plan"
+
+ result = subprocess.run(
+ [sys.executable, str(SCRIPT), str(missing), str(engine)],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+
+ assert result.returncode == 2
+ assert f"error: ONNX model does not exist: {missing}" in result.stderr
+ assert "Traceback" not in result.stderr