Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
test: numerical regression harness (frozen golden vs ggml/VAD/CIF/CTC output) (#3003)
zhifu gao3610b759ce9e
diff --git a/runtime/llama.cpp/tests/README.md b/runtime/llama.cpp/tests/README.mdnew file mode 100644index 000000000..0d7d9da34--- /dev/null+++ b/runtime/llama.cpp/tests/README.md@@ -0,0 +1,18 @@+# Regression tests++Run each runtime tool on a fixed clip (`sample.wav`, ~6 s) and diff the output against+the frozen golden in `golden/`. Catches regressions in the ggml graphs, the FSMN-VAD+state machine, the CIF predictor and CTC decode.++## Run+```bash+# from runtime/llama.cpp/, after building (cmake --build build):+./tests/run_regression.sh # VAD (tiny model auto-fetched) + any tool whose GGUF is already local+RUN_FULL=1 ./tests/run_regression.sh # also download the ASR GGUFs from Hugging Face and test every tool+```+`BIN_DIR` / `MODELS_DIR` override where binaries and GGUFs are found. Exit code is+non-zero if any test fails; tools with no binary or no model are skipped.++## Golden+Captured on Linux x86-64 (the reference platform) with the f16 GGUFs published at+`FunAudioLLM/*-GGUF`. Update a golden file only with a deliberate, reviewed output change.diff --git a/runtime/llama.cpp/tests/golden/nano.txt b/runtime/llama.cpp/tests/golden/nano.txtnew file mode 100644index 000000000..bcda42723--- /dev/null+++ b/runtime/llama.cpp/tests/golden/nano.txt@@ -0,0 +1 @@+我想问我在滨海新区有房diff --git a/runtime/llama.cpp/tests/golden/paraformer.txt b/runtime/llama.cpp/tests/golden/paraformer.txtnew file mode 100644index 000000000..bcda42723--- /dev/null+++ b/runtime/llama.cpp/tests/golden/paraformer.txt@@ -0,0 +1 @@+我想问我在滨海新区有房diff --git a/runtime/llama.cpp/tests/golden/sensevoice.txt b/runtime/llama.cpp/tests/golden/sensevoice.txtnew file mode 100644index 000000000..bcda42723--- /dev/null+++ b/runtime/llama.cpp/tests/golden/sensevoice.txt@@ -0,0 +1 @@+我想问我在滨海新区有房diff --git a/runtime/llama.cpp/tests/golden/vad.txt b/runtime/llama.cpp/tests/golden/vad.txtnew file mode 100644index 000000000..d9f88ff55--- /dev/null+++ b/runtime/llama.cpp/tests/golden/vad.txt@@ -0,0 +1 @@+770 5980diff --git a/runtime/llama.cpp/tests/run_regression.sh b/runtime/llama.cpp/tests/run_regression.shnew file mode 100755index 000000000..50eab5d5e--- /dev/null+++ b/runtime/llama.cpp/tests/run_regression.sh@@ -0,0 +1,52 @@+#!/usr/bin/env bash+# Numerical regression test for the FunASR llama.cpp runtime.+# Runs each available tool on a fixed clip (sample.wav) and diffs the output against+# the frozen golden in golden/. Catches regressions in the ggml graphs, the FSMN-VAD+# state machine, CIF predictor and CTC decode. Golden captured on Linux x86-64 (the+# reference platform) with the f16 GGUFs published on Hugging Face.+#+# ./run_regression.sh # test tools whose binary+model are present (VAD model auto-fetched)+# RUN_FULL=1 ./run_regression.sh # also download the ASR GGUFs and test every tool+# BIN_DIR=/path/to/bin MODELS_DIR=/path/to/gguf ./run_regression.sh+set -u+DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)+RT=$(cd "$DIR/.." && pwd)+BIN="${BIN_DIR:-$RT/build/bin}"+MODELS="${MODELS_DIR:-$DIR/models}"+SAMPLE="$DIR/sample.wav"+RUN_FULL="${RUN_FULL:-0}"+mkdir -p "$MODELS"+pass=0; fail=0; skip=0++bin(){ if [ -x "$BIN/$1" ]; then echo "$BIN/$1"; elif command -v "$1" >/dev/null 2>&1; then echo "$1"; fi; }+# ensure the listed model files exist in $MODELS (download via the repo script if allowed)+ensure_models(){ local key="$1"; shift+ local missing=0; for f in "$@"; do [ -f "$MODELS/$f" ] || missing=1; done+ [ "$missing" = 0 ] && return 0+ { [ "$key" = fsmn-vad ] || [ "$RUN_FULL" = 1 ]; } || return 1 # only auto-fetch the tiny VAD unless RUN_FULL+ "$RT/download-funasr-model.sh" "$key" "$MODELS" >/dev/null 2>&1 || return 1+ for f in "$@"; do [ -f "$MODELS/$f" ] || return 1; done; return 0+}+check(){ local name="$1" golden="$2" got="$3"+ if [ "$got" = "$(cat "$golden")" ]; then echo " PASS $name"; pass=$((pass+1))+ else echo " FAIL $name"; echo " expected: $(head -c 80 "$golden")"; echo " got: $(printf %s "$got" | head -c 80)"; fail=$((fail+1)); fi+}+skipper(){ echo " SKIP $1 ($2)"; skip=$((skip+1)); }++run_tool(){ # name binary golden key models... -- run...+ local name="$1" b key gold; b=$(bin "$2"); gold="$DIR/golden/$3"; key="$4"; shift 4+ local models=(); while [ "$1" != "--" ]; do models+=("$1"); shift; done; shift+ [ -n "$b" ] || { skipper "$name" "no binary"; return; }+ [ -f "$gold" ] || { skipper "$name" "no golden"; return; }+ ensure_models "$key" "${models[@]}" || { skipper "$name" "model missing (set RUN_FULL=1)"; return; }+ check "$name" "$gold" "$("$@" 2>/dev/null)"+}++echo "== FunASR llama.cpp regression (sample.wav) =="+B=$(bin llama-funasr-vad) && run_tool vad llama-funasr-vad vad.txt fsmn-vad fsmn-vad.gguf -- "$B" -m "$MODELS/fsmn-vad.gguf" -a "$SAMPLE"+B=$(bin llama-funasr-sensevoice) && run_tool sensevoice llama-funasr-sensevoice sensevoice.txt sensevoice sensevoice-small-f16.gguf -- "$B" -m "$MODELS/sensevoice-small-f16.gguf" -a "$SAMPLE"+B=$(bin llama-funasr-paraformer) && run_tool paraformer llama-funasr-paraformer paraformer.txt paraformer paraformer-f16.gguf -- "$B" -m "$MODELS/paraformer-f16.gguf" -a "$SAMPLE"+B=$(bin llama-funasr-cli) && run_tool nano llama-funasr-cli nano.txt nano funasr-encoder-f16.gguf qwen3-0.6b-q8_0.gguf -- "$B" --enc "$MODELS/funasr-encoder-f16.gguf" -m "$MODELS/qwen3-0.6b-q8_0.gguf" -a "$SAMPLE"++echo "== $pass passed, $fail failed, $skip skipped =="+[ "$fail" = 0 ]diff --git a/runtime/llama.cpp/tests/sample.wav b/runtime/llama.cpp/tests/sample.wavnew file mode 100644index 000000000..f5db87cf6Binary files /dev/null and b/runtime/llama.cpp/tests/sample.wav differ