Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(runtime): repair evaluation CLIs and honor GPU device selection
zhifu gao51b1a149a82d
diff --git a/.github/workflows/test-runtime-eval.yml b/.github/workflows/test-runtime-eval.ymlnew file mode 100644index 000000000..2e3a1901e--- /dev/null+++ b/.github/workflows/test-runtime-eval.yml@@ -0,0 +1,58 @@+name: Validate runtime evaluation tools++on:+ pull_request:+ paths:+ - "runtime/python/utils/**"+ - "runtime/python/libtorch/funasr_torch/**"+ - "runtime/python/onnxruntime/funasr_onnx/**"+ - "tests/test_runtime_eval_*.py"+ - ".github/workflows/test-runtime-eval.yml"+ push:+ branches: [main]+ paths:+ - "runtime/python/utils/**"+ - "runtime/python/libtorch/funasr_torch/**"+ - "runtime/python/onnxruntime/funasr_onnx/**"+ - "tests/test_runtime_eval_*.py"+ - ".github/workflows/test-runtime-eval.yml"++permissions:+ contents: read++jobs:+ runtime-eval:+ runs-on: ubuntu-latest+ timeout-minutes: 15+ strategy:+ fail-fast: false+ matrix:+ python: ["3.10", "3.12"]+ env:+ HF_HUB_OFFLINE: "1"+ MODELSCOPE_OFFLINE: "1"+ OMP_NUM_THREADS: "1"+ steps:+ - uses: actions/checkout@v4+ with:+ ref: ${{ github.event.pull_request.head.sha || github.sha }}+ - uses: actions/setup-python@v5+ with:+ python-version: ${{ matrix.python }}+ - name: Install CPU runtime test dependencies+ run: |+ python -m pip install torch==2.8.0 --index-url https://download.pytorch.org/whl/cpu+ python -m pip install pytest "numpy<2" onnx==1.19.0 onnxruntime==1.23.2 librosa kaldi-native-fbank PyYAML jieba sentencepiece+ - name: Run CLI and native offline runtime regressions+ run: |+ git rev-parse HEAD > source-commit.txt+ python -m pip freeze > resolved-dependencies.txt+ python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py --junitxml=runtime-eval-results.xml+ - uses: actions/upload-artifact@v4+ if: always()+ with:+ name: runtime-eval-${{ matrix.python }}+ path: |+ source-commit.txt+ resolved-dependencies.txt+ runtime-eval-results.xmldiff --git a/runtime/python/libtorch/funasr_torch/paraformer_bin.py b/runtime/python/libtorch/funasr_torch/paraformer_bin.pyindex 8e404126a..fe405753c 100644--- a/runtime/python/libtorch/funasr_torch/paraformer_bin.py+++ b/runtime/python/libtorch/funasr_torch/paraformer_bin.py@@ -75,7 +75,8 @@ class Paraformer:self.converter = TokenIDConverter(token_list)self.tokenizer = CharTokenizer()self.frontend = WavFrontend(cmvn_file=cmvn_file, **config["frontend_conf"])- self.ort_infer = torch.jit.load(model_file)+ self.device = torch.device("cpu" if int(device_id) == -1 else f"cuda:{int(device_id)}")+ self.ort_infer = torch.jit.load(model_file, map_location=self.device).eval()self.batch_size = batch_sizeself.device_id = device_idself.plot_timestamp_to = plot_timestamp_to@@ -98,12 +99,11 @@ class Paraformer:feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx])try:with torch.no_grad():- if int(self.device_id) == -1:- outputs = self.ort_infer(feats, feats_len)- am_scores, valid_token_lens = outputs[0], outputs[1]- else:- outputs = self.ort_infer(feats.cuda(), feats_len.cuda())- am_scores, valid_token_lens = outputs[0].cpu(), outputs[1].cpu()+ outputs = self.ort_infer(+ feats.to(self.device), feats_len.to(self.device)+ )+ outputs = tuple(output.cpu() for output in outputs)+ am_scores, valid_token_lens = outputs[0], outputs[1]if len(outputs) == 4:# for BiCifParaformer Inferenceus_alphas, us_peaks = outputs[2], outputs[3]diff --git a/runtime/python/utils/README.md b/runtime/python/utils/README.mdnew file mode 100644index 000000000..e0beee747--- /dev/null+++ b/runtime/python/utils/README.md@@ -0,0 +1,50 @@+# Runtime Evaluation Tools++Run these commands from a FunASR checkout with the chosen runtime and its+dependencies available. Importing the scripts or using `--help` does not load a+runtime, audio, or model. ONNX and Torch backends are selected independently.++Use the checkout's runtime modules when evaluating a source change:++```sh+export PYTHONPATH="$PWD/runtime/python/onnxruntime:$PWD/runtime/python/libtorch${PYTHONPATH:+:$PYTHONPATH}"+python runtime/python/utils/test_rtf.py --model_dir /path/to/exported-model --wav_file wav.scp+python runtime/python/utils/test_rtf_gpu.py --model_dir /path/to/exported-model --wav_file wav.scp --device_id 0 --batch_size 8+python runtime/python/utils/test_cer.py --backend torch --model_dir /path/to/exported-model --wav_file wav.scp --output_dir transcripts+```++`wav.scp` contains an utterance ID followed by whitespace and an audio path.+Paths may contain spaces. Blank lines are ignored; empty or malformed lists+are rejected. Supply existing local exported models to avoid runtime download+or export behavior. Torch uses `model.torchscript`; ONNX uses `model.onnx`.+The corresponding quantized names are selected with `--quantize true`.++The serial RTF and transcript tools default to CPU (`--device_id -1`). The+batched GPU tool defaults to CUDA device 0 and rejects negative device IDs.+It requires CUDA-capable Torch or ONNX Runtime, with compatible CUDA/cuDNN+libraries. Missing ONNX CUDA providers cause an error instead of a CPU timing+being reported as a GPU benchmark. ONNX may still place supported graph+operations on CPU; provider availability alone is not proof that every node+runs on CUDA. Use runtime profiling when auditing actual kernel placement.++RTF is elapsed wall time divided by actual input audio duration. There is an+untimed audio-duration prepass and, by default, 30 warmups on the first audio+file (`--warmup 0` disables warmups). Timed model calls include file loading,+frontend, inference and decoding; Torch CUDA is synchronized around timing.+The serial tool invokes each file separately. The GPU tool passes the list to+the backend, whose `--batch_size` controls batching. These numbers are not+model-only kernel timings and depend on cache state and hardware.++The transcript tool writes UTF-8 `text` and space-delimited `token` files. It+does not compute CER/WER or alter references. A missing inference result is an+error, not a successful empty transcript or benchmark.++Offline regression tests use small synthetic graphs and generated audio:++```sh+python -m pytest tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py+FUNASR_TEST_DEVICE=0 python -m pytest tests/test_runtime_eval_native.py+```++Native tests require Torch, ONNX, ONNX Runtime and the runtimes' audio/tokenizer+dependencies. They test device and CLI mechanics, not pretrained ASR accuracy.diff --git a/runtime/python/utils/runtime_eval.py b/runtime/python/utils/runtime_eval.pynew file mode 100644index 000000000..a689cb468--- /dev/null+++ b/runtime/python/utils/runtime_eval.py@@ -0,0 +1,160 @@+"""Shared command-line boundaries for the standalone runtime evaluation tools."""++import argparse+import importlib+import math+from pathlib import Path+import time+++def _boolean(value):+ lowered = value.lower()+ if lowered in {"true", "yes", "t", "y", "1"}:+ return True+ if lowered in {"false", "no", "f", "n", "0"}:+ return False+ raise argparse.ArgumentTypeError("expected a boolean")+++def _positive(value):+ number = int(value)+ if number < 1:+ raise argparse.ArgumentTypeError("must be positive")+ return number+++def _nonnegative(value):+ number = int(value)+ if number < 0:+ raise argparse.ArgumentTypeError("must be nonnegative")+ return number+++def parse_args(argv=None, *, gpu=False, cer=False):+ parser = argparse.ArgumentParser()+ parser.add_argument("--model_dir", required=True)+ parser.add_argument("--backend", choices=["onnx", "torch"], default="onnx")+ parser.add_argument("--wav_file", required=True, help="WAV SCP: utterance ID and audio path")+ parser.add_argument("--quantize", type=_boolean, default=False)+ parser.add_argument("--intra_op_num_threads", type=_positive, default=1)+ parser.add_argument(+ "--device_id",+ type=_nonnegative if gpu else int,+ default=0 if gpu else -1,+ help="CUDA device index; -1 for CPU",+ )+ parser.add_argument("--batch_size", type=_positive, default=1)+ if cer:+ parser.add_argument("--output_dir", required=True)+ else:+ parser.add_argument("--warmup", type=_nonnegative, default=30)+ args = parser.parse_args(argv)+ if args.device_id < -1:+ parser.error("--device_id must be -1 or a CUDA device index")+ return args+++def read_wav_scp(path):+ entries = []+ with open(path, encoding="utf-8") as source:+ for number, line in enumerate(source, 1):+ if not line.strip():+ continue+ fields = line.strip().split(maxsplit=1)+ if len(fields) != 2:+ raise ValueError(f"{path}:{number}: expected utterance ID and audio path")+ entries.append(tuple(fields))+ if not entries:+ raise ValueError(f"{path}: empty WAV list")+ return entries+++def load_model(args):+ backend = importlib.import_module("funasr_" + args.backend)+ model = backend.Paraformer(+ args.model_dir,+ batch_size=args.batch_size,+ device_id=str(args.device_id),+ quantize=args.quantize,+ intra_op_num_threads=args.intra_op_num_threads,+ )+ if args.backend == "onnx" and args.device_id >= 0:+ providers = model.ort_infer.session.get_providers()+ if "CUDAExecutionProvider" not in providers:+ raise RuntimeError("CUDAExecutionProvider is unavailable; refusing a CPU GPU benchmark")+ return model+++def _infer(model, paths):+ result = model(paths)+ expected = 1 if isinstance(paths, str) else len(paths)+ if len(result) != expected:+ raise RuntimeError(f"inference returned {len(result)} results for {expected} audio files")+ return result+++def transcribe(args):+ entries = read_wav_scp(args.wav_file)+ model = load_model(args)+ output = Path(args.output_dir)+ output.mkdir(parents=True, exist_ok=True)+ with (output / "text").open("w", encoding="utf-8") as text_file, (output / "token").open(+ "w", encoding="utf-8"+ ) as token_file:+ for utterance, path in entries:+ row = _infer(model, path)[0]+ if isinstance(row["preds"], str):+ text, tokens = row["preds"], row["raw_tokens"]+ else:+ text, tokens = row["preds"]+ if not isinstance(tokens, str):+ tokens = " ".join(tokens)+ text_file.write(f"{utterance} {text}\n")+ token_file.write(f"{utterance} {tokens}\n")+++def benchmark(args, *, batched=False):+ entries = read_wav_scp(args.wav_file)+ import librosa++ durations = []+ for _, path in entries:+ audio, sample_rate = librosa.load(path, sr=None)+ duration = len(audio) / sample_rate+ if not math.isfinite(duration) or duration <= 0:+ raise ValueError(f"{path}: audio duration must be positive")+ durations.append(duration)+ model = load_model(args)+ print(f"backend: {args.backend}, device_id: {args.device_id}")++ def synchronize():+ if args.backend == "torch" and args.device_id >= 0:+ import torch++ torch.cuda.synchronize(args.device_id)++ warmup_elapsed = 0.0+ for index in range(args.warmup):+ synchronize()+ start = time.perf_counter()+ result = _infer(model, entries[0][1])+ synchronize()+ elapsed = time.perf_counter() - start+ warmup_elapsed += elapsed+ average = warmup_elapsed / (index + 1)+ print(result)+ print(f"num: 1, time, {elapsed}, avg: {average}, rtf: {average / durations[0]}")++ synchronize()+ start = time.perf_counter()+ if batched:+ _infer(model, [path for _, path in entries])+ else:+ for _, path in entries:+ _infer(model, path)+ synchronize()+ elapsed = time.perf_counter() - start+ duration = sum(durations)+ print(f"total_time_comput_ms: {elapsed * 1000:.6f}")+ print(f"total_time_wav_ms: {duration * 1000:.6f}")+ print(f"total_rtf: {elapsed / duration}")diff --git a/runtime/python/utils/test_cer.py b/runtime/python/utils/test_cer.pyindex 58285c248..06a57a9a6 100644--- a/runtime/python/utils/test_cer.py+++ b/runtime/python/utils/test_cer.py@@ -1,56 +1,14 @@-import os-import time-import sys-import librosa-from funasr.utils.types import str2bool+"""Write runtime transcripts and tokens for external CER/WER scoring."""-import argparse-parser = argparse.ArgumentParser()-parser.add_argument("--model_dir", type=str, required=True)-parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]')-parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number")-parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model")-parser.add_argument(- "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx"-)-parser.add_argument("--output_dir", type=str, default=None, help="amp fallback number")-args = parser.parse_args()+def main(argv=None):+ if __package__:+ from .runtime_eval import parse_args, transcribe+ else:+ from runtime_eval import parse_args, transcribe+ transcribe(parse_args(argv, cer=True))-from funasr.runtime.python.libtorch.funasr_torch import Paraformer-if args.backend == "onnx":- from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer--model = Paraformer(- args.model_dir,- batch_size=1,- quantize=args.quantize,- intra_op_num_threads=args.intra_op_num_threads,-)--wav_file_f = open(args.wav_file, "r")-wav_files = wav_file_f.readlines()--output_dir = args.output_dir-if not os.path.exists(output_dir):- os.makedirs(output_dir)-if os.name == "nt": # Windows- newline = "\r\n"-else: # Linux Mac- newline = "\n"-text_f = open(os.path.join(output_dir, "text"), "w", newline=newline)-token_f = open(os.path.join(output_dir, "token"), "w", newline=newline)--for i, wav_path_i in enumerate(wav_files):- wav_name, wav_path = wav_path_i.strip().split()- result = model(wav_path)- text_i = "{} {}\n".format(wav_name, result[0]["preds"][0])- token_i = "{} {}\n".format(wav_name, result[0]["preds"][1])- text_f.write(text_i)- text_f.flush()- token_f.write(token_i)- token_f.flush()-text_f.close()-token_f.close()+if __name__ == "__main__":+ main()diff --git a/runtime/python/utils/test_rtf.py b/runtime/python/utils/test_rtf.pyindex 8e50c34f8..16c899c22 100644--- a/runtime/python/utils/test_rtf.py+++ b/runtime/python/utils/test_rtf.py@@ -1,79 +1,14 @@-import time-import sys-import librosa-from funasr.utils.type_utils import str2bool+"""Measure serial end-to-end runtime RTF, including frontend and audio loading."""-import argparse-parser = argparse.ArgumentParser()-parser.add_argument("--model_dir", type=str, required=True)-parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]')-parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number")-parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model")-parser.add_argument(- "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx"-)-args = parser.parse_args()+def main(argv=None):+ if __package__:+ from .runtime_eval import benchmark, parse_args+ else:+ from runtime_eval import benchmark, parse_args+ benchmark(parse_args(argv))-from funasr.runtime.python.libtorch.funasr_torch import Paraformer-if args.backend == "onnx":- from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer--model = Paraformer(- args.model_dir,- batch_size=1,- quantize=args.quantize,- intra_op_num_threads=args.intra_op_num_threads,-)--wav_file_f = open(args.wav_file, "r")-wav_files = wav_file_f.readlines()--# warm-up-total = 0.0-num = 30-wav_path = (- wav_files[0].split("\t")[1].strip()- if "\t" in wav_files[0]- else wav_files[0].split(" ")[1].strip()-)-for i in range(num):- beg_time = time.time()- result = model(wav_path)- end_time = time.time()- duration = end_time - beg_time- total += duration- print(result)- print(- "num: {}, time, {}, avg: {}, rtf: {}".format(- len(wav_path), duration, total / (i + 1), (total / (i + 1)) / 5.53- )- )--# infer time-beg_time = time.time()-for i, wav_path_i in enumerate(wav_files):- wav_path = (- wav_path_i.split("\t")[1].strip()- if "\t" in wav_path_i- else wav_path_i.split(" ")[1].strip()- )- result = model(wav_path)-end_time = time.time()-duration = (end_time - beg_time) * 1000-print("total_time_comput_ms: {}".format(int(duration)))--duration_time = 0.0-for i, wav_path_i in enumerate(wav_files):- wav_path = (- wav_path_i.split("\t")[1].strip()- if "\t" in wav_path_i- else wav_path_i.split(" ")[1].strip()- )- waveform, _ = librosa.load(wav_path, sr=16000)- duration_time += len(waveform) / 16.0-print("total_time_wav_ms: {}".format(int(duration_time)))--print("total_rtf: {:.5}".format(duration / duration_time))+if __name__ == "__main__":+ main()diff --git a/runtime/python/utils/test_rtf_gpu.py b/runtime/python/utils/test_rtf_gpu.pyindex 38d429401..172400d03 100644--- a/runtime/python/utils/test_rtf_gpu.py+++ b/runtime/python/utils/test_rtf_gpu.py@@ -1,82 +1,14 @@-import time-import sys-import librosa-from funasr.utils.types import str2bool+"""Measure batched GPU runtime RTF without silently benchmarking CPU fallback."""-import argparse-parser = argparse.ArgumentParser()-parser.add_argument("--model_dir", type=str, required=True)-parser.add_argument("--backend", type=str, default="onnx", help='["onnx", "torch"]')-parser.add_argument("--wav_file", type=str, default=None, help="amp fallback number")-parser.add_argument("--quantize", type=str2bool, default=False, help="quantized model")-parser.add_argument(- "--intra_op_num_threads", type=int, default=1, help="intra_op_num_threads for onnx"-)-parser.add_argument("--batch_size", type=int, default=1, help="batch_size for onnx")-args = parser.parse_args()+def main(argv=None):+ if __package__:+ from .runtime_eval import benchmark, parse_args+ else:+ from runtime_eval import benchmark, parse_args+ benchmark(parse_args(argv, gpu=True), batched=True)-from funasr.runtime.python.libtorch.funasr_torch import Paraformer-if args.backend == "onnx":- from funasr.runtime.python.onnxruntime.funasr_onnx import Paraformer--model = Paraformer(- args.model_dir,- batch_size=args.batch_size,- quantize=args.quantize,- intra_op_num_threads=args.intra_op_num_threads,-)--wav_file_f = open(args.wav_file, "r")-wav_files = wav_file_f.readlines()--# warm-up-total = 0.0-num = 30-wav_path = (- wav_files[0].split("\t")[1].strip()- if "\t" in wav_files[0]- else wav_files[0].split(" ")[1].strip()-)-for i in range(num):- beg_time = time.time()- result = model(wav_path)- end_time = time.time()- duration = end_time - beg_time- total += duration- print(result)- print(- "num: {}, time, {}, avg: {}, rtf: {}".format(- len(wav_path), duration, total / (i + 1), (total / (i + 1)) / 5.53- )- )--# infer time-wav_path = []-beg_time = time.time()-for i, wav_path_i in enumerate(wav_files):- wav_path_i = (- wav_path_i.split("\t")[1].strip()- if "\t" in wav_path_i- else wav_path_i.split(" ")[1].strip()- )- wav_path += [wav_path_i]-result = model(wav_path)-end_time = time.time()-duration = (end_time - beg_time) * 1000-print("total_time_comput_ms: {}".format(int(duration)))--duration_time = 0.0-for i, wav_path_i in enumerate(wav_files):- wav_path = (- wav_path_i.split("\t")[1].strip()- if "\t" in wav_path_i- else wav_path_i.split(" ")[1].strip()- )- waveform, _ = librosa.load(wav_path, sr=16000)- duration_time += len(waveform) / 16.0-print("total_time_wav_ms: {}".format(int(duration_time)))--print("total_rtf: {:.5}".format(duration / duration_time))+if __name__ == "__main__":+ main()diff --git a/tests/test_runtime_eval_cli.py b/tests/test_runtime_eval_cli.pynew file mode 100644index 000000000..9eca635a2--- /dev/null+++ b/tests/test_runtime_eval_cli.py@@ -0,0 +1,246 @@+"""CLI boundary tests; backend stand-ins do not claim native ASR/GPU coverage."""++import importlib.util+import os+from pathlib import Path+import subprocess+import sys+from types import SimpleNamespace++import pytest+++ROOT = Path(__file__).resolve().parents[1]+SCRIPTS = ROOT / "runtime/python/utils"+NAMES = ["test_cer.py", "test_rtf.py", "test_rtf_gpu.py"]+++def invoke(name, *args, env=None):+ return subprocess.run(+ [sys.executable, "-S", str(SCRIPTS / name), *map(str, args)],+ text=True,+ capture_output=True,+ timeout=30,+ env=env,+ )++[email protected]("name", NAMES)+def test_import_without_runtime_dependencies_or_cli_arguments(name):+ result = subprocess.run(+ [+ sys.executable,+ "-S",+ "-c",+ "import runpy; runpy.run_path(" + repr(str(SCRIPTS / name)) + ")",+ ],+ text=True,+ capture_output=True,+ timeout=30,+ )+ assert result.returncode == 0, result.stderr+ assert not result.stdout++[email protected]("name", NAMES)+def test_help_without_runtime_dependencies(name):+ result = invoke(name, "--help")+ assert result.returncode == 0, result.stderr+ assert "--model_dir" in result.stdout and "--wav_file" in result.stdout++[email protected]("name", NAMES)+def test_missing_required_arguments_are_parser_errors(name):+ result = invoke(name)+ assert result.returncode == 2+ assert "--model_dir" in result.stderr and "--wav_file" in result.stderr+++def helper():+ path = SCRIPTS / "runtime_eval.py"+ assert path.exists(), "Shared runtime CLI boundary is missing"+ spec = importlib.util.spec_from_file_location("runtime_eval_under_test", path)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ return module+++def test_wav_list_preserves_spaces_and_tabs(tmp_path):+ scp = tmp_path / "wav.scp"+ scp.write_text("first\t/some path/a.wav\n\nsecond /b path/b.wav\n")+ assert helper().read_wav_scp(scp) == [+ ("first", "/some path/a.wav"),+ ("second", "/b path/b.wav"),+ ]++[email protected]("content", ["", "\n \n", "missing-path\n"])+def test_invalid_wav_list_is_rejected(tmp_path, content):+ scp = tmp_path / "wav.scp"+ scp.write_text(content)+ with pytest.raises(ValueError):+ helper().read_wav_scp(scp)+++def backends(tmp_path):+ # Only model execution and media decoding are external here. Native device+ # placement and inference have separate real-backend validation.+ code = """+import os+class Session:+ def get_providers(self):+ return ["CPUExecutionProvider"] if os.environ.get("CPU_FALLBACK") else ["CUDAExecutionProvider", "CPUExecutionProvider"]+class Engine:+ session = Session()+class Paraformer:+ def __init__(self, model_dir, batch_size=1, device_id="-1", **kwargs):+ assert str(device_id) == os.environ["EXPECTED_DEVICE"], device_id+ assert batch_size == int(os.environ.get("EXPECTED_BATCH", "1"))+ self.ort_infer = Engine()+ def __call__(self, paths):+ if os.environ.get("EMPTY_RESULT"):+ return []+ if isinstance(paths, str):+ paths = [paths]+ assert all(path.endswith("audio file.wav") for path in paths), paths+ return [{"preds": ("fixture text", ["fixture", "text"])} for _ in paths]+"""+ packages = tmp_path / "packages"+ packages.mkdir()+ for name in ["funasr_onnx", "funasr_torch"]:+ target = packages / name+ target.mkdir()+ (target / "__init__.py").write_text(code)+ (packages / "librosa.py").write_text("def load(path, sr=None):\n return [0] * 16000, 8000\n")+ (packages / "torch.py").write_text(+ "class CUDA:\n def synchronize(self, device=None): pass\n" "cuda = CUDA()\n"+ )+ scp = tmp_path / "wav.scp"+ scp.write_text("a /some path/audio file.wav\nb\t/other/audio file.wav\n")+ env = dict(os.environ, PYTHONPATH=str(packages), EXPECTED_DEVICE="-1")+ return packages, scp, env++[email protected]("name", NAMES)[email protected]("backend", ["onnx", "torch"])+def test_cli_uses_only_selected_backend_and_passes_device(tmp_path, backends, name, backend):+ packages, scp, env = backends+ other = "funasr_torch" if backend == "onnx" else "funasr_onnx"+ (packages / other / "__init__.py").write_text('raise RuntimeError("wrong backend imported")\n')+ args = ["--model_dir", "local-model", "--wav_file", scp, "--backend", backend]+ if name == "test_cer.py":+ args += ["--output_dir", tmp_path / "output"]+ else:+ args += ["--warmup", "1"]+ if name == "test_rtf_gpu.py":+ env.update(EXPECTED_DEVICE="2", EXPECTED_BATCH="2")+ args += ["--device_id", "2", "--batch_size", "2"]+ result = invoke(name, *args, env=env)+ assert result.returncode == 0, result.stderr+ if name == "test_cer.py":+ assert (tmp_path / "output/text").read_text() == "a fixture text\nb fixture text\n"+ assert (tmp_path / "output/token").read_text() == "a fixture text\nb fixture text\n"+ else:+ assert "total_time_wav_ms: 4000" in result.stdout+ assert "num: 1," in result.stdout+ assert "total_rtf:" in result.stdout+++def test_gpu_cli_defaults_to_device_zero(backends):+ _, scp, env = backends+ env["EXPECTED_DEVICE"] = "0"+ result = invoke(+ "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env+ )+ assert result.returncode == 0, result.stderr+++def test_gpu_cli_refuses_silent_onnx_cpu_fallback(backends):+ _, scp, env = backends+ env.update(EXPECTED_DEVICE="0", CPU_FALLBACK="1")+ result = invoke(+ "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env+ )+ assert result.returncode != 0+ assert "CUDAExecutionProvider" in result.stderr+ assert "total_rtf:" not in result.stdout+++ "option,value",+ [("--batch_size", "0"), ("--warmup", "-1"), ("--device_id", "-1"), ("--backend", "bogus")],+)+def test_gpu_cli_rejects_invalid_arguments(option, value, backends):+ _, scp, env = backends+ result = invoke(+ "test_rtf_gpu.py", "--model_dir", "local-model", "--wav_file", scp, option, value, env=env+ )+ assert result.returncode == 2, result.stderr++[email protected]("name", ["test_rtf.py", "test_rtf_gpu.py"])+def test_incomplete_inference_is_not_a_successful_benchmark(name, backends):+ _, scp, env = backends+ env.update(EMPTY_RESULT="1", EXPECTED_DEVICE="0" if "gpu" in name else "-1")+ result = invoke(name, "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env)+ assert result.returncode != 0+ assert "inference returned" in result.stderr+ assert "total_rtf:" not in result.stdout+++def test_zero_duration_is_rejected(backends):+ packages, scp, env = backends+ (packages / "librosa.py").write_text("def load(path, sr=None): return [], 16000\n")+ result = invoke(+ "test_rtf.py", "--model_dir", "local-model", "--wav_file", scp, "--warmup", "0", env=env+ )+ assert result.returncode != 0+ assert "duration" in result.stderr+ assert "total_rtf:" not in result.stdout+++ "value,wanted",+ [("t", True), ("f", False), ("True", True), ("False", False), ("1", True), ("0", False)],+)+def test_quantize_retains_legacy_boolean_spellings(value, wanted):+ args = helper().parse_args(+ ["--model_dir", "local-model", "--wav_file", "wav.scp", "--quantize", value]+ )+ assert args.quantize is wanted+++def test_rtf_uses_measured_duration_for_warmup_and_total(backends, monkeypatch, capsys):+ _, scp, _ = backends+ module = helper()+ ticks = iter([0.0, 1.0, 2.0, 3.0])+ monkeypatch.setattr(module.time, "perf_counter", lambda: next(ticks))+ monkeypatch.setitem(+ sys.modules, "librosa", SimpleNamespace(load=lambda *a, **k: ([0] * 16000, 8000))+ )+ monkeypatch.setattr(+ module,+ "load_model",+ lambda args: lambda paths: [{}] * (1 if isinstance(paths, str) else len(paths)),+ )+ args = module.parse_args(+ ["--model_dir", "local-model", "--wav_file", str(scp), "--warmup", "1"]+ )+ module.benchmark(args, batched=True)+ output = capsys.readouterr().out+ assert "avg: 1.0, rtf: 0.5" in output+ assert "total_rtf: 0.25" in output++[email protected]("name", NAMES)+def test_module_mode_help(name):+ result = subprocess.run(+ [sys.executable, "-S", "-m", "runtime.python.utils." + Path(name).stem, "--help"],+ cwd=ROOT,+ text=True,+ capture_output=True,+ timeout=30,+ )+ assert result.returncode == 0, result.stderr+ assert "--model_dir" in result.stdoutdiff --git a/tests/test_runtime_eval_native.py b/tests/test_runtime_eval_native.pynew file mode 100644index 000000000..c3b60a8cc--- /dev/null+++ b/tests/test_runtime_eval_native.py@@ -0,0 +1,163 @@+"""Offline synthetic graphs exercise real frontends, devices and CLI inference.++These fixed weights are test fixtures, not pretrained ASR or quality benchmarks.+Set FUNASR_TEST_DEVICE to a CUDA index for the same tests on real GPU hardware.+"""++import importlib+import json+import os+from pathlib import Path+import subprocess+import sys+import wave++import pytest+++ROOT = Path(__file__).resolve().parents[1]+DEVICE = os.environ.get("FUNASR_TEST_DEVICE", "-1")++[email protected](scope="module")+def native_fixture(tmp_path_factory):+ torch = pytest.importorskip("torch")+ np = pytest.importorskip("numpy")+ onnx = pytest.importorskip("onnx")+ pytest.importorskip("onnxruntime")+ pytest.importorskip("kaldi_native_fbank")+ directory = tmp_path_factory.mktemp("runtime-native")++ class AcousticFixture(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.register_buffer("weight", torch.arange(20).reshape(4, 5).float() * 0.00001)+ self.register_buffer("bias", torch.tensor([0.0, 0.0, 0.0, 5.0, 0.0]))++ def forward(self, speech, lengths):+ return speech[:, :2] @ self.weight + self.bias, torch.full_like(lengths, 2)++ model = AcousticFixture().eval()+ speech = torch.zeros(2, 10, 4)+ lengths = torch.tensor([10, 8], dtype=torch.int32)+ torch.jit.trace(model, (speech, lengths)).save(str(directory / "model.torchscript"))+ torch.onnx.export(+ model,+ (speech, lengths),+ str(directory / "model.onnx"),+ dynamo=False,+ input_names=["speech", "speech_lengths"],+ output_names=["logits", "lengths"],+ dynamic_axes={+ "speech": {0: "B", 1: "T"},+ "speech_lengths": {0: "B"},+ "logits": {0: "B"},+ "lengths": {0: "B"},+ },+ opset_version=17,+ )+ onnx.checker.check_model(str(directory / "model.onnx"))+ (directory / "tokens.json").write_text(+ json.dumps(["<blank>", "<s>", "</s>", "fixture", "unused"])+ )+ (directory / "config.yaml").write_text(+ json.dumps(+ {+ "frontend_conf": {"n_mels": 4, "lfr_m": 1, "lfr_n": 1, "dither": 0.0},+ "model_conf": {"predictor_bias": 0},+ }+ )+ )+ (directory / "am.mvn").write_text(+ "<AddShift> 4 4\n<LearnRateCoef> 1 [ 0 0 0 0 ]\n"+ "<Rescale> 4 4\n<LearnRateCoef> 1 [ 1 1 1 1 ]\n"+ )+ paths = []+ for index, count in enumerate([16000, 8000]):+ path = directory / f"audio {index}.wav"+ signal = (+ np.zeros(count, dtype="<i2")+ if index == 0+ else (np.sin(np.arange(count) * 2 * np.pi * 440 / 16000) * 3000).astype("<i2")+ )+ with wave.open(str(path), "wb") as output:+ output.setnchannels(1)+ output.setsampwidth(2)+ output.setframerate(16000)+ output.writeframes(signal.tobytes())+ paths.append(path)+ scp = directory / "wav.scp"+ scp.write_text("".join(f"utt{i}\t{p}\n" for i, p in enumerate(paths)))+ return directory, paths, scp++[email protected]("backend", ["torch", "onnx"])+def test_real_backend_uses_requested_device_and_decodes(native_fixture, monkeypatch, backend):+ directory, paths, _ = native_fixture+ location = "libtorch" if backend == "torch" else "onnxruntime"+ monkeypatch.syspath_prepend(str(ROOT / "runtime/python" / location))+ module = importlib.import_module("funasr_" + backend)+ model = module.Paraformer(+ str(directory), batch_size=2, device_id=DEVICE, intra_op_num_threads=1+ )+ if backend == "torch":+ expected = "cpu" if DEVICE == "-1" else "cuda:" + DEVICE+ assert {str(item.device) for item in model.ort_infer.buffers()} == {expected}+ else:+ provider = "CPUExecutionProvider" if DEVICE == "-1" else "CUDAExecutionProvider"+ assert model.ort_infer.session.get_providers()[0] == provider+ result = model([str(p) for p in paths])+ assert len(result) == 2+ assert [row["preds"][0] for row in result] == ["fixture fixture", "fixture fixture"]+ print(json.dumps({"backend": backend, "device": DEVICE, "raw": result}))++[email protected]("backend", ["torch", "onnx"])[email protected]("script", ["test_cer.py", "test_rtf.py"])+def test_real_cli_outputs_and_audio_duration(native_fixture, tmp_path, backend, script):+ directory, _, scp = native_fixture+ env = dict(os.environ)+ env["PYTHONPATH"] = os.pathsep.join(+ [+ str(ROOT / "runtime/python/libtorch"),+ str(ROOT / "runtime/python/onnxruntime"),+ env.get("PYTHONPATH", ""),+ ]+ )+ command = [+ sys.executable,+ str(ROOT / "runtime/python/utils" / script),+ "--model_dir",+ str(directory),+ "--wav_file",+ str(scp),+ "--backend",+ backend,+ "--device_id",+ DEVICE,+ ]+ if script == "test_cer.py":+ command += ["--output_dir", str(tmp_path / "out")]+ else:+ command += ["--warmup", "1"]+ result = subprocess.run(command, env=env, text=True, capture_output=True, timeout=120)+ assert result.returncode == 0, result.stdout + result.stderr+ if script == "test_cer.py":+ assert (tmp_path / "out/text").read_text() == "utt0 fixture fixture\nutt1 fixture fixture\n"+ assert (+ tmp_path / "out/token"+ ).read_text() == "utt0 fixture fixture\nutt1 fixture fixture\n"+ else:+ assert "total_time_wav_ms: 1500" in result.stdout+ assert "total_rtf:" in result.stdout+ print(+ json.dumps(+ {+ "backend": backend,+ "script": script,+ "stdout": result.stdout,+ "stderr": result.stderr,+ "device": DEVICE,+ }+ )+ )