Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(runtime): repair evaluation CLIs and honor GPU device selection

zhifu gao51b1a149a82d
diff --git a/.github/workflows/test-runtime-eval.yml b/.github/workflows/test-runtime-eval.yml
new file mode 100644
index 000000000..2e3a1901e
--- /dev/null
+++ b/.github/workflows/test-runtime-eval.yml
@@ -0,0 +1,58 @@
+name: Validate runtime evaluation tools
+
+on:
+ pull_request:
+ paths:
+ - "runtime/python/utils/**"
+ - "runtime/python/libtorch/funasr_torch/**"
+ - "runtime/python/onnxruntime/funasr_onnx/**"
+ - "tests/test_runtime_eval_*.py"
+ - ".github/workflows/test-runtime-eval.yml"
+ push:
+ branches: [main]
+ paths:
+ - "runtime/python/utils/**"
+ - "runtime/python/libtorch/funasr_torch/**"
+ - "runtime/python/onnxruntime/funasr_onnx/**"
+ - "tests/test_runtime_eval_*.py"
+ - ".github/workflows/test-runtime-eval.yml"
+
+permissions:
+ contents: read
+
+jobs:
+ runtime-eval:
+ runs-on: ubuntu-latest
+ timeout-minutes: 15
+ strategy:
+ fail-fast: false
+ matrix:
+ python: ["3.10", "3.12"]
+ env:
+ HF_HUB_OFFLINE: "1"
+ MODELSCOPE_OFFLINE: "1"
+ OMP_NUM_THREADS: "1"
+ steps:
+ - uses: actions/checkout@v4
+ with:
+ ref: ${{ github.event.pull_request.head.sha || github.sha }}
+ - uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Install CPU runtime test dependencies
+ run: |
+ python -m pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cpu
+ python -m pip install pytest "numpy<2" onnx==1.19.0 onnxruntime==1.23.2 librosa kaldi-native-fbank PyYAML jieba sentencepiece
+ - name: Run CLI and native offline runtime regressions
+ run: |
+ git rev-parse HEAD > source-commit.txt
+ python -m pip freeze > resolved-dependencies.txt
+ python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py --junitxml=runtime-eval-results.xml
+ - uses: actions/upload-artifact@v4
+ if: always()
+ with:
+ name: runtime-eval-${{ matrix.python }}
+ path: |
+ source-commit.txt
+ resolved-dependencies.txt
+ runtime-eval-results.xml
diff --git a/runtime/python/libtorch/funasr_torch/paraformer_bin.py b/runtime/python/libtorch/funasr_torch/paraformer_bin.py
index 8e404126a..fe405753c 100644
--- a/runtime/python/libtorch/funasr_torch/paraformer_bin.py
+++ b/runtime/python/libtorch/funasr_torch/paraformer_bin.py
@@ -75,7 +75,8 @@ class Paraformer:
self.converter = TokenIDConverter(token_list)
self.tokenizer = CharTokenizer()
self.frontend = WavFrontend(cmvn_file=cmvn_file, **config["frontend_conf"])
- self.ort_infer = torch.jit.load(model_file)
+ self.device = torch.device("cpu" if int(device_id) == -1 else f"cuda:{int(device_id)}")
+ self.ort_infer = torch.jit.load(model_file, map_location=self.device).eval()
self.batch_size = batch_size
self.device_id = device_id
self.plot_timestamp_to = plot_timestamp_to
@@ -98,12 +99,11 @@ class Paraformer:
feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx])
try:
with torch.no_grad():
- if int(self.device_id) == -1:
- outputs = self.ort_infer(feats, feats_len)
- am_scores, valid_token_lens = outputs[0], outputs[1]
- else:
- outputs = self.ort_infer(feats.cuda(), feats_len.cuda())
- am_scores, valid_token_lens = outputs[0].cpu(), outputs[1].cpu()
+ outputs = self.ort_infer(
+ feats.to(self.device), feats_len.to(self.device)
+ )
+ outputs = tuple(output.cpu() for output in outputs)
+ am_scores, valid_token_lens = outputs[0], outputs[1]
if len(outputs) == 4:
# for BiCifParaformer Inference
us_alphas, us_peaks = outputs[2], outputs[3]
diff --git a/runtime/python/utils/README.md b/runtime/python/utils/README.md
new file mode 100644
index 000000000..e0beee747
--- /dev/null
+++ b/runtime/python/utils/README.md
@@ -0,0 +1,50 @@
+# Runtime Evaluation Tools
+
+Run these commands from a FunASR checkout with the chosen runtime and its
+dependencies available. Importing the scripts or using `--help` does not load a
+runtime, audio, or model. ONNX and Torch backends are selected independently.
+
+Use the checkout's runtime modules when evaluating a source change:
+
+```sh
+export PYTHONPATH="$PWD/runtime/python/onnxruntime:$PWD/runtime/python/libtorch${PYTHONPATH:+:$PYTHONPATH}"
+python runtime/python/utils/test_rtf.py --model_dir /path/to/exported-model --wav_file wav.scp
+python runtime/python/utils/test_rtf_gpu.py --model_dir /path/to/exported-model --wav_file wav.scp --device_id 0 --batch_size 8
+python runtime/python/utils/test_cer.py --backend torch --model_dir /path/to/exported-model --wav_file wav.scp --output_dir transcripts
+```
+
+`wav.scp` contains an utterance ID followed by whitespace and an audio path.
+Paths may contain spaces. Blank lines are ignored; empty or malformed lists
+are rejected. Supply existing local exported models to avoid runtime download
+or export behavior. Torch uses `model.torchscript`; ONNX uses `model.onnx`.
+The corresponding quantized names are selected with `--quantize true`.
+
+The serial RTF and transcript tools default to CPU (`--device_id -1`). The
+batched GPU tool defaults to CUDA device 0 and rejects negative device IDs.
+It requires CUDA-capable Torch or ONNX Runtime, with compatible CUDA/cuDNN
+libraries. Missing ONNX CUDA providers cause an error instead of a CPU timing
+being reported as a GPU benchmark. ONNX may still place supported graph
+operations on CPU; provider availability alone is not proof that every node
+runs on CUDA. Use runtime profiling when auditing actual kernel placement.
+
+RTF is elapsed wall time divided by actual input audio duration. There is an
+untimed audio-duration prepass and, by default, 30 warmups on the first audio
+file (`--warmup 0` disables warmups). Timed model calls include file loading,
+frontend, inference and decoding; Torch CUDA is synchronized around timing.
+The serial tool invokes each file separately. The GPU tool passes the list to
+the backend, whose `--batch_size` controls batching. These numbers are not
+model-only kernel timings and depend on cache state and hardware.
+
+The transcript tool writes UTF-8 `text` and space-delimited `token` files. It
+does not compute CER/WER or alter references. A missing inference result is an
+error, not a successful empty transcript or benchmark.
+
+Offline regression tests use small synthetic graphs and generated audio:
+
+```sh
+python -m pytest tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py
+FUNASR_TEST_DEVICE=0 python -m pytest tests/test_runtime_eval_native.py
+```
+
+Native tests require Torch, ONNX, ONNX Runtime and the runtimes' audio/tokenizer
+dependencies. They test device and CLI mechanics, not pretrained ASR accuracy.
diff --git a/runtime/python/utils/runtime_eval.py b/runtime/python/utils/runtime_eval.py
new file mode 100644
index 000000000..a689cb468
--- /dev/null
+++ b/runtime/python/utils/runtime_eval.py
@@ -0,0 +1,160 @@
+"""Shared command-line boundaries for the standalone runtime evaluation tools."""
+
+import argparse
+import importlib
+import math
+from pathlib import Path
+import time
+
+
+def _boolean(value):
+ lowered = value.lower()
+ if lowered in {"true", "yes", "t", "y", "1"}:
+ return True
+ if lowered in {"false", "no", "f", "n", "0"}:
+ return False
+ raise argparse.ArgumentTypeError("expected a boolean")
+
+
+def _positive(value):
+ number = int(value)
+ if number < 1:
+ raise argparse.ArgumentTypeError("must be positive")
+ return number
+
+
+def _nonnegative(value):
+ number = int(value)
+ if number < 0:
+ raise argparse.ArgumentTypeError("must be nonnegative")
+ return number
+
+
+def parse_args(argv=None, *, gpu=False, cer=False):
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model_dir", required=True)
+ parser.add_argument("--backend", choices=["onnx", "torch"], default="onnx")
+ parser.add_argument("--wav_file", required=True, help="WAV SCP: utterance ID and audio path")
+ parser.add_argument("--quantize", type=_boolean, default=False)
+ parser.add_argument("--intra_op_num_threads", type=_positive, default=1)
+ parser.add_argument(
+ "--device_id",
+ type=_nonnegative if gpu else int,
+ default=0 if gpu else -1,
+ help="CUDA device index; -1 for CPU",
+ )
+ parser.add_argument("--batch_size", type=_positive, default=1)
+ if cer:
+ parser.add_argument("--output_dir", required=True)
+ else:
+ parser.add_argument("--warmup", type=_nonnegative, default=30)
+ args = parser.parse_args(argv)
+ if args.device_id < -1:
+ parser.error("--device_id must be -1 or a CUDA device index")
+ return args
+
+
+def read_wav_scp(path):
+ entries = []
+ with open(path, encoding="utf-8") as source:
+ for number, line in enumerate(source, 1):
+ if not line.strip():
+ continue
+ fields = line.strip().split(maxsplit=1)
+ if len(fields) != 2:
+ raise ValueError(f"{path}:{number}: expected utterance ID and audio path")
+ entries.append(tuple(fields))
+ if not entries:
+ raise ValueError(f"{path}: empty WAV list")
+ return entries
+
+
+def load_model(args):
+ backend = importlib.import_module("funasr_" + args.backend)
+ model = backend.Paraformer(
+ args.model_dir,
+ batch_size=args.batch_size,
+ device_id=str(args.device_id),
+ quantize=args.quantize,
+ intra_op_num_threads=args.intra_op_num_threads,
+ )
+ if args.backend == "onnx" and args.device_id >= 0:
+ providers = model.ort_infer.session.get_providers()
+ if "CUDAExecutionProvider" not in providers:
+ raise RuntimeError("CUDAExecutionProvider is unavailable; refusing a CPU GPU benchmark")
+ return model
+
+
+def _infer(model, paths):
+ result = model(paths)
+ expected = 1 if isinstance(paths, str) else len(paths)
+ if len(result) != expected:
+ raise RuntimeError(f"inference returned {len(result)} results for {expected} audio files")
+ return result
+
+
+def transcribe(args):
+ entries = read_wav_scp(args.wav_file)
+ model = load_model(args)
+ output = Path(args.output_dir)
+ output.mkdir(parents=True, exist_ok=True)
+ with (output / "text").open("w", encoding="utf-8") as text_file, (output / "token").open(
+ "w", encoding="utf-8"
+ ) as token_file:
+ for utterance, path in entries:
+ row = _infer(model, path)[0]
+ if isinstance(row["preds"], str):
+ text, tokens = row["preds"], row["raw_tokens"]
+ else:
+ text, tokens = row["preds"]
+ if not isinstance(tokens, str):
+ tokens = " ".join(tokens)
+ text_file.write(f"{utterance} {text}\n")
+ token_file.write(f"{utterance} {tokens}\n")
+
+
+def benchmark(args, *, batched=False):
+ entries = read_wav_scp(args.wav_file)
+ import librosa
+
+ durations = []
+ for _, path in entries:
+ audio, sample_rate = librosa.load(path, sr=None)
+ duration = len(audio) / sample_rate
+ if not math.isfinite(duration) or duration <= 0:
+ raise ValueError(f"{path}: audio duration must be positive")
+ durations.append(duration)
+ model = load_model(args)
+ print(f"backend: {args.backend}, device_id: {args.device_id}")
+
+ def synchronize():
+ if args.backend == "torch" and args.device_id >= 0:
+ import torch
+
+ torch.cuda.synchronize(args.device_id)
+
+ warmup_elapsed = 0.0
+ for index in range(args.warmup):
+ synchronize()
+ start = time.perf_counter()
+ result = _infer(model, entries[0][1])
+ synchronize()
+ elapsed = time.perf_counter() - start
+ warmup_elapsed += elapsed
+ average = warmup_elapsed / (index + 1)
+ print(result)
+ print(f"num: 1, time, {elapsed}, avg: {average}, rtf: {average / durations[0]}")
+
+ synchronize()
+ start = time.perf_counter()
+ if batched:
+ _infer(model, [path for _, path in entries])
+ else:
+ for _, path in entries:
+ _infer(model, path)
+ synchronize()
+ elapsed = time.perf_counter() - start
+ duration = sum(durations)
+ print(f"total_time_comput_ms: {elapsed * 1000:.6f}")
+ print(f"total_time_wav_ms: {duration * 1000:.6f}")
+ print(f"total_rtf: {elapsed / duration}")
diff --git a/runtime/python/utils/test_cer.py b/runtime/python/utils/test_cer.py
index 58285c248..06a57a9a6 100644
--- a/runtime/python/utils/test_cer.py
+++ b/runtime/python/utils/test_cer.py
@@ -1,56 +1,14 @@
-import os
-import time
-import sys
-import librosa
-from funasr.utils.types import str2bool
+"""Write runtime transcripts and tokens for external CER/WER scoring."""
-import argparse
-parser = argparse.ArgumentParser()
-parser.add_argument("--model_dir", type=str, required=True)
-parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]')
-parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number")
-parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model")
-parser.add_argument(
- "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx"
-)
-parser.add_argument("--output_dir", type=str, default=None, help="amp fallback number")
-args = parser.parse_args()
+def main(argv=None):
+ if __package__:
+ from .runtime_eval import parse_args, transcribe
+ else:
+ from runtime_eval import parse_args, transcribe
+ transcribe(parse_args(argv, cer=True))
-from funasr.runtime.python.libtorch.funasr_torch import Paraformer
-if args.backend == "onnx":
- from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer
-
-model = Paraformer(
- args.model_dir,
- batch_size=1,
- quantize=args.quantize,
- intra_op_num_threads=args.intra_op_num_threads,
-)
-
-wav_file_f = open(args.wav_file, "r")
-wav_files = wav_file_f.readlines()
-
-output_dir = args.output_dir
-if not os.path.exists(output_dir):
- os.makedirs(output_dir)
-if os.name == "nt": # Windows
- newline = "\r\n"
-else: # Linux Mac
- newline = "\n"
-text_f = open(os.path.join(output_dir, "text"), "w", newline=newline)
-token_f = open(os.path.join(output_dir, "token"), "w", newline=newline)
-
-for i, wav_path_i in enumerate(wav_files):
- wav_name, wav_path = wav_path_i.strip().split()
- result = model(wav_path)
- text_i = "{} {}\n".format(wav_name, result[0]["preds"][0])
- token_i = "{} {}\n".format(wav_name, result[0]["preds"][1])
- text_f.write(text_i)
- text_f.flush()
- token_f.write(token_i)
- token_f.flush()
-text_f.close()
-token_f.close()
+if __name__ == "__main__":
+ main()
diff --git a/runtime/python/utils/test_rtf.py b/runtime/python/utils/test_rtf.py
index 8e50c34f8..16c899c22 100644
--- a/runtime/python/utils/test_rtf.py
+++ b/runtime/python/utils/test_rtf.py
@@ -1,79 +1,14 @@
-import time
-import sys
-import librosa
-from funasr.utils.type_utils import str2bool
+"""Measure serial end-to-end runtime RTF, including frontend and audio loading."""
-import argparse
-parser = argparse.ArgumentParser()
-parser.add_argument("--model_dir", type=str, required=True)
-parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]')
-parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number")
-parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model")
-parser.add_argument(
- "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx"
-)
-args = parser.parse_args()
+def main(argv=None):
+ if __package__:
+ from .runtime_eval import benchmark, parse_args
+ else:
+ from runtime_eval import benchmark, parse_args
+ benchmark(parse_args(argv))
-from funasr.runtime.python.libtorch.funasr_torch import Paraformer
-if args.backend == "onnx":
- from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer
-
-model = Paraformer(
- args.model_dir,
- batch_size=1,
- quantize=args.quantize,
- intra_op_num_threads=args.intra_op_num_threads,
-)
-
-wav_file_f = open(args.wav_file, "r")
-wav_files = wav_file_f.readlines()
-
-# warm-up
-total = 0.0
-num = 30
-wav_path = (
- wav_files[0].split("\t")[1].strip()
- if "\t" in wav_files[0]
- else wav_files[0].split(" ")[1].strip()
-)
-for i in range(num):
- beg_time = time.time()
- result = model(wav_path)
- end_time = time.time()
- duration = end_time - beg_time
- total += duration
- print(result)
- print(
- "num: {}, time, {}, avg: {}, rtf: {}".format(
- len(wav_path), duration, total / (i + 1), (total / (i + 1)) / 5.53
- )
- )
-
-# infer time
-beg_time = time.time()
-for i, wav_path_i in enumerate(wav_files):
- wav_path = (
- wav_path_i.split("\t")[1].strip()
- if "\t" in wav_path_i
- else wav_path_i.split(" ")[1].strip()
- )
- result = model(wav_path)
-end_time = time.time()
-duration = (end_time - beg_time) * 1000
-print("total_time_comput_ms: {}".format(int(duration)))
-
-duration_time = 0.0
-for i, wav_path_i in enumerate(wav_files):
- wav_path = (
- wav_path_i.split("\t")[1].strip()
- if "\t" in wav_path_i
- else wav_path_i.split(" ")[1].strip()
- )
- waveform, _ = librosa.load(wav_path, sr=16000)
- duration_time += len(waveform) / 16.0
-print("total_time_wav_ms: {}".format(int(duration_time)))
-
-print("total_rtf: {:.5}".format(duration / duration_time))
+if __name__ == "__main__":
+ main()
diff --git a/runtime/python/utils/test_rtf_gpu.py b/runtime/python/utils/test_rtf_gpu.py
index 38d429401..172400d03 100644
--- a/runtime/python/utils/test_rtf_gpu.py
+++ b/runtime/python/utils/test_rtf_gpu.py
@@ -1,82 +1,14 @@
-import time
-import sys
-import librosa
-from funasr.utils.types import str2bool
+"""Measure batched GPU runtime RTF without silently benchmarking CPU fallback."""
-import argparse
-parser = argparse.ArgumentParser()
-parser.add_argument("--model_dir", type=str, required=True)
-parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]')
-parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number")
-parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model")
-parser.add_argument(
- "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx"
-)
-parser.add_argument("--batch_size", type=int, default=1, help="batch_size for onnx")
-args = parser.parse_args()
+def main(argv=None):
+ if __package__:
+ from .runtime_eval import benchmark, parse_args
+ else:
+ from runtime_eval import benchmark, parse_args
+ benchmark(parse_args(argv, gpu=True), batched=True)
-from funasr.runtime.python.libtorch.funasr_torch import Paraformer
-if args.backend == "onnx":
- from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer
-
-model = Paraformer(
- args.model_dir,
- batch_size=args.batch_size,
- quantize=args.quantize,
- intra_op_num_threads=args.intra_op_num_threads,
-)
-
-wav_file_f = open(args.wav_file, "r")
-wav_files = wav_file_f.readlines()
-
-# warm-up
-total = 0.0
-num = 30
-wav_path = (
- wav_files[0].split("\t")[1].strip()
- if "\t" in wav_files[0]
- else wav_files[0].split(" ")[1].strip()
-)
-for i in range(num):
- beg_time = time.time()
- result = model(wav_path)
- end_time = time.time()
- duration = end_time - beg_time
- total += duration
- print(result)
- print(
- "num: {}, time, {}, avg: {}, rtf: {}".format(
- len(wav_path), duration, total / (i + 1), (total / (i + 1)) / 5.53
- )
- )
-
-# infer time
-wav_path = []
-beg_time = time.time()
-for i, wav_path_i in enumerate(wav_files):
- wav_path_i = (
- wav_path_i.split("\t")[1].strip()
- if "\t" in wav_path_i
- else wav_path_i.split(" ")[1].strip()
- )
- wav_path += [wav_path_i]
-result = model(wav_path)
-end_time = time.time()
-duration = (end_time - beg_time) * 1000
-print("total_time_comput_ms: {}".format(int(duration)))
-
-duration_time = 0.0
-for i, wav_path_i in enumerate(wav_files):
- wav_path = (
- wav_path_i.split("\t")[1].strip()
- if "\t" in wav_path_i
- else wav_path_i.split(" ")[1].strip()
- )
- waveform, _ = librosa.load(wav_path, sr=16000)
- duration_time += len(waveform) / 16.0
-print("total_time_wav_ms: {}".format(int(duration_time)))
-
-print("total_rtf: {:.5}".format(duration / duration_time))
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_runtime_eval_cli.py b/tests/test_runtime_eval_cli.py
new file mode 100644
index 000000000..9eca635a2
--- /dev/null
+++ b/tests/test_runtime_eval_cli.py
@@ -0,0 +1,246 @@
+"""CLI boundary tests; backend stand-ins do not claim native ASR/GPU coverage."""
+
+import importlib.util
+import os
+from pathlib import Path
+import subprocess
+import sys
+from types import SimpleNamespace
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+SCRIPTS = ROOT / "runtime/python/utils"
+NAMES = ["test_cer.py", "test_rtf.py", "test_rtf_gpu.py"]
+
+
+def invoke(name, *args, env=None):
+ return subprocess.run(
+ [sys.executable, "-S", str(SCRIPTS / name), *map(str, args)],
+ text=True,
+ capture_output=True,
+ timeout=30,
+ env=env,
+ )
+
+
[email protected]("name", NAMES)
+def test_import_without_runtime_dependencies_or_cli_arguments(name):
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-S",
+ "-c",
+ "import runpy; runpy.run_path(" + repr(str(SCRIPTS / name)) + ")",
+ ],
+ text=True,
+ capture_output=True,
+ timeout=30,
+ )
+ assert result.returncode == 0, result.stderr
+ assert not result.stdout
+
+
[email protected]("name", NAMES)
+def test_help_without_runtime_dependencies(name):
+ result = invoke(name, "--help")
+ assert result.returncode == 0, result.stderr
+ assert "--model_dir" in result.stdout and "--wav_file" in result.stdout
+
+
[email protected]("name", NAMES)
+def test_missing_required_arguments_are_parser_errors(name):
+ result = invoke(name)
+ assert result.returncode == 2
+ assert "--model_dir" in result.stderr and "--wav_file" in result.stderr
+
+
+def helper():
+ path = SCRIPTS / "runtime_eval.py"
+ assert path.exists(), "Shared runtime CLI boundary is missing"
+ spec = importlib.util.spec_from_file_location("runtime_eval_under_test", path)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def test_wav_list_preserves_spaces_and_tabs(tmp_path):
+ scp = tmp_path / "wav.scp"
+ scp.write_text("first\t/some path/a.wav\n\nsecond /b path/b.wav\n")
+ assert helper().read_wav_scp(scp) == [
+ ("first", "/some path/a.wav"),
+ ("second", "/b path/b.wav"),
+ ]
+
+
[email protected]("content", ["", "\n \n", "missing-path\n"])
+def test_invalid_wav_list_is_rejected(tmp_path, content):
+ scp = tmp_path / "wav.scp"
+ scp.write_text(content)
+ with pytest.raises(ValueError):
+ helper().read_wav_scp(scp)
+
+
+def backends(tmp_path):
+ # Only model execution and media decoding are external here. Native device
+ # placement and inference have separate real-backend validation.
+ code = """
+import os
+class Session:
+ def get_providers(self):
+ return ["CPUExecutionProvider"] if os.environ.get("CPU_FALLBACK") else ["CUDAExecutionProvider", "CPUExecutionProvider"]
+class Engine:
+ session = Session()
+class Paraformer:
+ def __init__(self, model_dir, batch_size=1, device_id="-1", **kwargs):
+ assert str(device_id) == os.environ["EXPECTED_DEVICE"], device_id
+ assert batch_size == int(os.environ.get("EXPECTED_BATCH", "1"))
+ self.ort_infer = Engine()
+ def __call__(self, paths):
+ if os.environ.get("EMPTY_RESULT"):
+ return []
+ if isinstance(paths, str):
+ paths = [paths]
+ assert all(path.endswith("audio file.wav") for path in paths), paths
+ return [{"preds": ("fixture text", ["fixture", "text"])} for _ in paths]
+"""
+ packages = tmp_path / "packages"
+ packages.mkdir()
+ for name in ["funasr_onnx", "funasr_torch"]:
+ target = packages / name
+ target.mkdir()
+ (target / "__init__.py").write_text(code)
+ (packages / "librosa.py").write_text("def load(path, sr=None):\n return [0] * 16000, 8000\n")
+ (packages / "torch.py").write_text(
+ "class CUDA:\n def synchronize(self, device=None): pass\n" "cuda = CUDA()\n"
+ )
+ scp = tmp_path / "wav.scp"
+ scp.write_text("a /some path/audio file.wav\nb\t/other/audio file.wav\n")
+ env = dict(os.environ, PYTHONPATH=str(packages), EXPECTED_DEVICE="-1")
+ return packages, scp, env
+
+
[email protected]("name", NAMES)
[email protected]("backend", ["onnx", "torch"])
+def test_cli_uses_only_selected_backend_and_passes_device(tmp_path, backends, name, backend):
+ packages, scp, env = backends
+ other = "funasr_torch" if backend == "onnx" else "funasr_onnx"
+ (packages / other / "__init__.py").write_text('raise RuntimeError("wrong backend imported")\n')
+ args = ["--model_dir", "local-model", "--wav_file", scp, "--backend", backend]
+ if name == "test_cer.py":
+ args += ["--output_dir", tmp_path / "output"]
+ else:
+ args += ["--warmup", "1"]
+ if name == "test_rtf_gpu.py":
+ env.update(EXPECTED_DEVICE="2", EXPECTED_BATCH="2")
+ args += ["--device_id", "2", "--batch_size", "2"]
+ result = invoke(name, *args, env=env)
+ assert result.returncode == 0, result.stderr
+ if name == "test_cer.py":
+ assert (tmp_path / "output/text").read_text() == "a fixture text\nb fixture text\n"
+ assert (tmp_path / "output/token").read_text() == "a fixture text\nb fixture text\n"
+ else:
+ assert "total_time_wav_ms: 4000" in result.stdout
+ assert "num: 1," in result.stdout
+ assert "total_rtf:" in result.stdout
+
+
+def test_gpu_cli_defaults_to_device_zero(backends):
+ _, scp, env = backends
+ env["EXPECTED_DEVICE"] = "0"
+ result = invoke(
+ "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env
+ )
+ assert result.returncode == 0, result.stderr
+
+
+def test_gpu_cli_refuses_silent_onnx_cpu_fallback(backends):
+ _, scp, env = backends
+ env.update(EXPECTED_DEVICE="0", CPU_FALLBACK="1")
+ result = invoke(
+ "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env
+ )
+ assert result.returncode != 0
+ assert "CUDAExecutionProvider" in result.stderr
+ assert "total_rtf:" not in result.stdout
+
+
+ "option,value",
+ [("--batch_size", "0"), ("--warmup", "-1"), ("--device_id", "-1"), ("--backend", "bogus")],
+)
+def test_gpu_cli_rejects_invalid_arguments(option, value, backends):
+ _, scp, env = backends
+ result = invoke(
+ "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, option, value, env=env
+ )
+ assert result.returncode == 2, result.stderr
+
+
[email protected]("name", ["test_rtf.py", "test_rtf_gpu.py"])
+def test_incomplete_inference_is_not_a_successful_benchmark(name, backends):
+ _, scp, env = backends
+ env.update(EMPTY_RESULT="1", EXPECTED_DEVICE="0" if "gpu" in name else "-1")
+ result = invoke(name, "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env)
+ assert result.returncode != 0
+ assert "inference returned" in result.stderr
+ assert "total_rtf:" not in result.stdout
+
+
+def test_zero_duration_is_rejected(backends):
+ packages, scp, env = backends
+ (packages / "librosa.py").write_text("def load(path, sr=None): return [], 16000\n")
+ result = invoke(
+ "test_rtf.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env
+ )
+ assert result.returncode != 0
+ assert "duration" in result.stderr
+ assert "total_rtf:" not in result.stdout
+
+
+ "value,wanted",
+ [("t", True), ("f", False), ("True", True), ("False", False), ("1", True), ("0", False)],
+)
+def test_quantize_retains_legacy_boolean_spellings(value, wanted):
+ args = helper().parse_args(
+ ["--model_dir", "local-model", "--wav_file", "wav.scp", "--quantize", value]
+ )
+ assert args.quantize is wanted
+
+
+def test_rtf_uses_measured_duration_for_warmup_and_total(backends, monkeypatch, capsys):
+ _, scp, _ = backends
+ module = helper()
+ ticks = iter([0.0, 1.0, 2.0, 3.0])
+ monkeypatch.setattr(module.time, "perf_counter", lambda: next(ticks))
+ monkeypatch.setitem(
+ sys.modules, "librosa", SimpleNamespace(load=lambda *a, **k: ([0] * 16000, 8000))
+ )
+ monkeypatch.setattr(
+ module,
+ "load_model",
+ lambda args: lambda paths: [{}] * (1 if isinstance(paths, str) else len(paths)),
+ )
+ args = module.parse_args(
+ ["--model_dir", "local-model", "--wav_file", str(scp), "--warmup", "1"]
+ )
+ module.benchmark(args, batched=True)
+ output = capsys.readouterr().out
+ assert "avg: 1.0, rtf: 0.5" in output
+ assert "total_rtf: 0.25" in output
+
+
[email protected]("name", NAMES)
+def test_module_mode_help(name):
+ result = subprocess.run(
+ [sys.executable, "-S", "-m", "runtime.python.utils." + Path(name).stem, "--help"],
+ cwd=ROOT,
+ text=True,
+ capture_output=True,
+ timeout=30,
+ )
+ assert result.returncode == 0, result.stderr
+ assert "--model_dir" in result.stdout
diff --git a/tests/test_runtime_eval_native.py b/tests/test_runtime_eval_native.py
new file mode 100644
index 000000000..c3b60a8cc
--- /dev/null
+++ b/tests/test_runtime_eval_native.py
@@ -0,0 +1,163 @@
+"""Offline synthetic graphs exercise real frontends, devices and CLI inference.
+
+These fixed weights are test fixtures, not pretrained ASR or quality benchmarks.
+Set FUNASR_TEST_DEVICE to a CUDA index for the same tests on real GPU hardware.
+"""
+
+import importlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import wave
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[1]
+DEVICE = os.environ.get("FUNASR_TEST_DEVICE", "-1")
+
+
[email protected](scope="module")
+def native_fixture(tmp_path_factory):
+ torch = pytest.importorskip("torch")
+ np = pytest.importorskip("numpy")
+ onnx = pytest.importorskip("onnx")
+ pytest.importorskip("onnxruntime")
+ pytest.importorskip("kaldi_native_fbank")
+ directory = tmp_path_factory.mktemp("runtime-native")
+
+ class AcousticFixture(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.register_buffer("weight", torch.arange(20).reshape(4, 5).float() * 0.00001)
+ self.register_buffer("bias", torch.tensor([0.0, 0.0, 0.0, 5.0, 0.0]))
+
+ def forward(self, speech, lengths):
+ return speech[:, :2] @ self.weight + self.bias, torch.full_like(lengths, 2)
+
+ model = AcousticFixture().eval()
+ speech = torch.zeros(2, 10, 4)
+ lengths = torch.tensor([10, 8], dtype=torch.int32)
+ torch.jit.trace(model, (speech, lengths)).save(str(directory / "model.torchscript"))
+ torch.onnx.export(
+ model,
+ (speech, lengths),
+ str(directory / "model.onnx"),
+ dynamo=False,
+ input_names=["speech", "speech_lengths"],
+ output_names=["logits", "lengths"],
+ dynamic_axes={
+ "speech": {0: "B", 1: "T"},
+ "speech_lengths": {0: "B"},
+ "logits": {0: "B"},
+ "lengths": {0: "B"},
+ },
+ opset_version=17,
+ )
+ onnx.checker.check_model(str(directory / "model.onnx"))
+ (directory / "tokens.json").write_text(
+ json.dumps(["<blank>", "<s>", "</s>", "fixture", "unused"])
+ )
+ (directory / "config.yaml").write_text(
+ json.dumps(
+ {
+ "frontend_conf": {"n_mels": 4, "lfr_m": 1, "lfr_n": 1, "dither": 0.0},
+ "model_conf": {"predictor_bias": 0},
+ }
+ )
+ )
+ (directory / "am.mvn").write_text(
+ "<AddShift> 4 4\n<LearnRateCoef> 1 [ 0 0 0 0 ]\n"
+ "<Rescale> 4 4\n<LearnRateCoef> 1 [ 1 1 1 1 ]\n"
+ )
+ paths = []
+ for index, count in enumerate([16000, 8000]):
+ path = directory / f"audio {index}.wav"
+ signal = (
+ np.zeros(count, dtype="<i2")
+ if index == 0
+ else (np.sin(np.arange(count) * 2 * np.pi * 440 / 16000) * 3000).astype("<i2")
+ )
+ with wave.open(str(path), "wb") as output:
+ output.setnchannels(1)
+ output.setsampwidth(2)
+ output.setframerate(16000)
+ output.writeframes(signal.tobytes())
+ paths.append(path)
+ scp = directory / "wav.scp"
+ scp.write_text("".join(f"utt{i}\t{p}\n" for i, p in enumerate(paths)))
+ return directory, paths, scp
+
+
[email protected]("backend", ["torch", "onnx"])
+def test_real_backend_uses_requested_device_and_decodes(native_fixture, monkeypatch, backend):
+ directory, paths, _ = native_fixture
+ location = "libtorch" if backend == "torch" else "onnxruntime"
+ monkeypatch.syspath_prepend(str(ROOT / "runtime/python" / location))
+ module = importlib.import_module("funasr_" + backend)
+ model = module.Paraformer(
+ str(directory), batch_size=2, device_id=DEVICE, intra_op_num_threads=1
+ )
+ if backend == "torch":
+ expected = "cpu" if DEVICE == "-1" else "cuda:" + DEVICE
+ assert {str(item.device) for item in model.ort_infer.buffers()} == {expected}
+ else:
+ provider = "CPUExecutionProvider" if DEVICE == "-1" else "CUDAExecutionProvider"
+ assert model.ort_infer.session.get_providers()[0] == provider
+ result = model([str(p) for p in paths])
+ assert len(result) == 2
+ assert [row["preds"][0] for row in result] == ["fixture fixture", "fixture fixture"]
+ print(json.dumps({"backend": backend, "device": DEVICE, "raw": result}))
+
+
[email protected]("backend", ["torch", "onnx"])
[email protected]("script", ["test_cer.py", "test_rtf.py"])
+def test_real_cli_outputs_and_audio_duration(native_fixture, tmp_path, backend, script):
+ directory, _, scp = native_fixture
+ env = dict(os.environ)
+ env["PYTHONPATH"] = os.pathsep.join(
+ [
+ str(ROOT / "runtime/python/libtorch"),
+ str(ROOT / "runtime/python/onnxruntime"),
+ env.get("PYTHONPATH", ""),
+ ]
+ )
+ command = [
+ sys.executable,
+ str(ROOT / "runtime/python/utils" / script),
+ "--model_dir",
+ str(directory),
+ "--wav_file",
+ str(scp),
+ "--backend",
+ backend,
+ "--device_id",
+ DEVICE,
+ ]
+ if script == "test_cer.py":
+ command += ["--output_dir", str(tmp_path / "out")]
+ else:
+ command += ["--warmup", "1"]
+ result = subprocess.run(command, env=env, text=True, capture_output=True, timeout=120)
+ assert result.returncode == 0, result.stdout + result.stderr
+ if script == "test_cer.py":
+ assert (tmp_path / "out/text").read_text() == "utt0 fixture fixture\nutt1 fixture fixture\n"
+ assert (
+ tmp_path / "out/token"
+ ).read_text() == "utt0 fixture fixture\nutt1 fixture fixture\n"
+ else:
+ assert "total_time_wav_ms: 1500" in result.stdout
+ assert "total_rtf:" in result.stdout
+ print(
+ json.dumps(
+ {
+ "backend": backend,
+ "script": script,
+ "stdout": result.stdout,
+ "stderr": result.stderr,
+ "device": DEVICE,
+ }
+ )
+ )