Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
{
"schema_version": 1,
"verified": "2026-09-01",
"deployments": [
{
"id": "moss-transcribe-diarize",
"routes": {"zh": "/deploy/moss-transcribe-diarize.html", "en": "/en/deploy/moss-transcribe-diarize.html"},
"maturity": "community-verified",
"selector_rank": 12,
"workloads": ["batch", "private-api"],
"hardware": ["cpu", "nvidia-gpu", "desktop-edge-gpu", "kubernetes"],
"priorities": ["throughput"],
"models": ["OpenMOSS-Team/MOSS-Transcribe-Diarize (third-party Apache-2.0 model)"],
"operating_systems": ["Linux"],
"interfaces": ["OpenAI-compatible HTTP", "vLLM", "SGLang Omni", "Transformers", "LocalAI / moss-transcribe.cpp"],
"tested": {"funasr": "AutoModel HF + vLLM + SGLang adapters and OpenAI HTTP service; third-party model@e8681d68", "runtime": "Transformers 5.16.0.dev0 + Torch 2.11.0+cu130 and vLLM 0.27.1 + Torch 2.13.0+cu129 / H100 80GB", "verified": "2026-09-01"},
"commands": {
"install": [
"uv venv --python 3.12 .venv-moss && curl -fL https://github.com/vllm-project/vllm/releases/download/v0.27.1/vllm-0.27.1%2Bcu129-cp38-abi3-manylinux_2_28_x86_64.whl -o vllm-0.27.1+cu129-cp38-abi3-manylinux_2_28_x86_64.whl && echo 'bf0d52faa2a51e7a01c6856a7a8a2d1307fd0ff711415d34168a67ffac0fa47b vllm-0.27.1+cu129-cp38-abi3-manylinux_2_28_x86_64.whl' | sha256sum -c - && uv pip install --python .venv-moss/bin/python --torch-backend=auto \"vllm[audio] @ file://$PWD/vllm-0.27.1+cu129-cp38-abi3-manylinux_2_28_x86_64.whl\"",
"HF_HUB_ENABLE_HF_TRANSFER=1 hf download OpenMOSS-Team/MOSS-Transcribe-Diarize --revision e8681d68e7042738ffca8ac8212bc8fcb1131ab8 --local-dir .models/moss-transcribe-diarize"
],
"launch": [
"CUDA_VISIBLE_DEVICES=0 .venv-moss/bin/vllm serve OpenMOSS-Team/MOSS-Transcribe-Diarize --revision e8681d68e7042738ffca8ac8212bc8fcb1131ab8 --served-model-name moss-transcribe-diarize --trust-remote-code --host 127.0.0.1 --port 8898"
],
"health": [
"curl -fsS http://127.0.0.1:8898/health",
"curl -fsS http://127.0.0.1:8898/v1/models"
],
"smoke": [
"curl -fsS http://127.0.0.1:8898/v1/audio/transcriptions -F file=@runtime/llama.cpp/tests/sample.wav -F model=moss-transcribe-diarize -F response_format=diarized_json -F temperature=0 | tee /tmp/moss-transcription.json",
"python - <<'PY'\nimport json\n\nwith open('/tmp/moss-transcription.json', encoding='utf-8') as stream:\n payload = json.load(stream)\ntext = payload.get('text', '')\nsegments = payload.get('segments', [])\nassert text.strip() and segments, payload\nassert all(isinstance(item.get('speaker'), str) and item.get('text') and item.get('start') <= item.get('end') for item in segments), payload\nprint(text, sorted({item['speaker'] for item in segments}))\nPY",
"python - <<'PY'\nfrom funasr import AutoModel\n\nmodel = AutoModel(model='OpenMOSS-Team/MOSS-Transcribe-Diarize', backend='vllm', vllm_base_url='http://127.0.0.1:8898/v1', vllm_model='moss-transcribe-diarize', vllm_response_format='diarized_json', disable_update=True)\nresult = model.generate('runtime/llama.cpp/tests/sample.wav')[0]\nassert result['raw_text'] and result['sentence_info'], result\nprint(result['text'])\nprint(result['sentence_info'])\nPY"
]
},
"runtime_paths": [
{
"id": "funasr-server",
"tested": "Transformers 5.16.0.dev0 / Torch 2.11.0+cu130 / H100 80GB; real HTTP response verified 2026-09-01",
"translations": {
"zh": {"name": "FunASR OpenAI 兼容服务", "summary": "直接使用内置 funasr-server 加载固定 revision 的 HF 后端,返回带匿名说话人标签的 verbose_json;可使用专用 Docker Compose 与 Kubernetes GPU 配方。"},
"en": {"name": "FunASR OpenAI-compatible service", "summary": "Load the pinned HF backend directly with funasr-server and return verbose_json with anonymous speaker labels; dedicated Docker Compose and Kubernetes GPU recipes are included."}
},
"commands": {
"install": [
"python -m pip install 'transformers>=5.6,<6' fastapi uvicorn python-multipart"
],
"launch": [
"funasr-server --model moss-transcribe-diarize --device cuda:0 --port 8000",
"docker compose -f examples/openai_api/docker-compose.moss.yml up --build"
],
"health": [
"curl -fsS http://127.0.0.1:8000/health",
"curl -fsS http://127.0.0.1:8000/v1/models"
],
"smoke": [
"curl -fsS http://127.0.0.1:8000/v1/audio/transcriptions -F file=@runtime/llama.cpp/tests/sample.wav -F model=moss-transcribe-diarize -F response_format=verbose_json | tee /tmp/funasr-moss-transcription.json",
"python - <<'PY'\nimport json\n\nwith open('/tmp/funasr-moss-transcription.json', encoding='utf-8') as stream:\n payload = json.load(stream)\nsegments = payload.get('segments', [])\nassert payload.get('text', '').strip() and segments, payload\nassert all(item.get('speaker') and item.get('start') <= item.get('end') for item in segments), payload\nprint(payload['text'], sorted({item['speaker'] for item in segments}))\nPY"
]
}
},
{
"id": "vllm",
"tested": "vLLM 0.27.1 / Torch 2.13.0+cu129 / H100 80GB; FunASR adapter verified",
"translations": {
"zh": {"name": "vLLM + FunASR AutoModel", "summary": "使用已验证的 vLLM 0.27.1 OpenAI 音频接口,并通过 FunASR AutoModel 映射 diarized_json 为统一结构化结果。"},
"en": {"name": "vLLM + FunASR AutoModel", "summary": "Use the verified vLLM 0.27.1 OpenAI audio endpoint and map diarized_json into the common structured result through FunASR AutoModel."}
},
"commands": {
"install": [
"uv venv --python 3.12 .venv-moss && curl -fL https://github.com/vllm-project/vllm/releases/download/v0.27.1/vllm-0.27.1%2Bcu129-cp38-abi3-manylinux_2_28_x86_64.whl -o vllm-0.27.1+cu129-cp38-abi3-manylinux_2_28_x86_64.whl && echo 'bf0d52faa2a51e7a01c6856a7a8a2d1307fd0ff711415d34168a67ffac0fa47b vllm-0.27.1+cu129-cp38-abi3-manylinux_2_28_x86_64.whl' | sha256sum -c - && uv pip install --python .venv-moss/bin/python --torch-backend=auto \"vllm[audio] @ file://$PWD/vllm-0.27.1+cu129-cp38-abi3-manylinux_2_28_x86_64.whl\"",
"HF_HUB_ENABLE_HF_TRANSFER=1 hf download OpenMOSS-Team/MOSS-Transcribe-Diarize --revision e8681d68e7042738ffca8ac8212bc8fcb1131ab8 --local-dir .models/moss-transcribe-diarize"
],
"launch": [
"CUDA_VISIBLE_DEVICES=0 .venv-moss/bin/vllm serve OpenMOSS-Team/MOSS-Transcribe-Diarize --revision e8681d68e7042738ffca8ac8212bc8fcb1131ab8 --served-model-name moss-transcribe-diarize --trust-remote-code --host 127.0.0.1 --port 8898"
],
"health": [
"curl -fsS http://127.0.0.1:8898/health",
"curl -fsS http://127.0.0.1:8898/v1/models"
],
"smoke": [
"curl -fsS http://127.0.0.1:8898/v1/audio/transcriptions -F file=@runtime/llama.cpp/tests/sample.wav -F model=moss-transcribe-diarize -F response_format=diarized_json -F temperature=0 | tee /tmp/moss-transcription.json",
"python - <<'PY'\nimport json\n\nwith open('/tmp/moss-transcription.json', encoding='utf-8') as stream:\n payload = json.load(stream)\ntext = payload.get('text', '')\nsegments = payload.get('segments', [])\nassert text.strip() and segments, payload\nassert all(isinstance(item.get('speaker'), str) and item.get('text') and item.get('start') <= item.get('end') for item in segments), payload\nprint(text, sorted({item['speaker'] for item in segments}))\nPY",
"python - <<'PY'\nfrom funasr import AutoModel\n\nmodel = AutoModel(model='OpenMOSS-Team/MOSS-Transcribe-Diarize', backend='vllm', vllm_base_url='http://127.0.0.1:8898/v1', vllm_model='moss-transcribe-diarize', vllm_response_format='diarized_json', disable_update=True)\nresult = model.generate('runtime/llama.cpp/tests/sample.wav')[0]\nassert result['raw_text'] and result['sentence_info'], result\nprint(result['text'])\nprint(result['sentence_info'])\nPY"
]
}
},
{
"id": "sglang-omni",
"tested": "SGLang Omni 3f819f9c / FunASR adapter contract-tested / #914 H100 upstream benchmark",
"translations": {
"zh": {"name": "SGLang Omni + FunASR AutoModel", "summary": "使用已合并的原生 MOSS pipeline 和 OpenAI-compatible verbose_json 接口,并通过 FunASR AutoModel 校验 [Sxx] 前缀、统一为 sentence_info。"},
"en": {"name": "SGLang Omni + FunASR AutoModel", "summary": "Use the merged native MOSS pipeline and OpenAI-compatible verbose_json endpoint, then validate [Sxx] prefixes and normalize them into sentence_info through FunASR AutoModel."}
},
"commands": {
"install": [
"git clone https://github.com/sgl-project/sglang-omni.git && cd sglang-omni && git checkout 3f819f9cdae3d4eeec22f73306c9067a1ec2542e && uv venv .venv -p 3.12 && uv pip install --python .venv/bin/python -v -e .",
"HF_HUB_ENABLE_HF_TRANSFER=1 hf download OpenMOSS-Team/MOSS-Transcribe-Diarize --revision e8681d68e7042738ffca8ac8212bc8fcb1131ab8 --local-dir .models/moss-transcribe-diarize",
"curl -fsSL https://raw.githubusercontent.com/modelscope/FunASR/8d65a38a8f4f3b5301be72905096219dde443f73/runtime/llama.cpp/tests/sample.wav -o moss-sample.wav && echo 'ea03e1f473ad1618a03da3327a545369cb8f6f06cb0f4115535e5a866167d47e moss-sample.wav' | sha256sum -c -"
],
"launch": [
"CUDA_VISIBLE_DEVICES=0 .venv/bin/sgl-omni serve --model-path .models/moss-transcribe-diarize --host 127.0.0.1 --port 8898 --max-running-requests 16 --cuda-graph-max-bs 16 --mem-fraction-static 0.80"
],
"health": [
"curl -fsS http://127.0.0.1:8898/health",
"curl -fsS http://127.0.0.1:8898/v1/models"
],
"smoke": [
"curl -fsS http://127.0.0.1:8898/v1/audio/transcriptions -F [email protected] -F model=OpenMOSS-Team/MOSS-Transcribe-Diarize -F response_format=verbose_json | tee /tmp/moss-sglang-transcription.json",
"python - <<'PY'\nimport json\nimport re\n\nwith open('/tmp/moss-sglang-transcription.json', encoding='utf-8') as stream:\n payload = json.load(stream)\nsegments = payload.get('segments', [])\nassert payload.get('text', '').strip() and segments, payload\nassert all(item.get('start') <= item.get('end') and re.match(r'^\\[S\\d{2,}\\]', item.get('text', '')) for item in segments), payload\nprint(payload['text'])\nprint([(item['start'], item['end'], item['text'][:5]) for item in segments])\nPY",
"python - <<'PY'\nfrom funasr import AutoModel\n\nmodel = AutoModel(model='OpenMOSS-Team/MOSS-Transcribe-Diarize', backend='sglang', sglang_base_url='http://127.0.0.1:8898/v1', sglang_model='OpenMOSS-Team/MOSS-Transcribe-Diarize', max_new_tokens=65536, disable_update=True)\nresult = model.generate('moss-sample.wav', max_new_tokens=65536)[0]\nassert result['raw_text'] and result['sentence_info'], result\nprint(result['text'])\nprint(result['sentence_info'])\nPY"
]
}
}
],
"evidence": [
{"label": "OpenMOSS upstream repository and Apache-2.0 source license", "url": "https://github.com/OpenMOSS/MOSS-Transcribe-Diarize"},
{"label": "upstream source pinned at cb765f2", "url": "https://github.com/OpenMOSS/MOSS-Transcribe-Diarize/commit/cb765f2b0fe6f7a298aa2002e2281ae693d1f3c3"},
{"label": "official model snapshot pinned at e8681d68", "url": "https://huggingface.co/OpenMOSS-Team/MOSS-Transcribe-Diarize/tree/e8681d68e7042738ffca8ac8212bc8fcb1131ab8"},
{"label": "official vLLM and SGLang Omni serving guide", "url": "https://github.com/OpenMOSS/MOSS-Transcribe-Diarize/blob/cb765f2b0fe6f7a298aa2002e2281ae693d1f3c3/README.md"},
{"label": "vLLM official diarized_json response contract", "url": "https://github.com/vllm-project/vllm/pull/48543"},
{"label": "official vLLM MOSS deployment recipe", "url": "https://github.com/vllm-project/recipes/blob/d3f3136ad9d549ddf488f56c9b149ff4a92fc646/models/OpenMOSS-Team/MOSS-Transcribe-Diarize.yaml"},
{"label": "verified vLLM 0.27.1 release", "url": "https://github.com/vllm-project/vllm/releases/tag/v0.27.1"},
{"label": "merged native SGLang Omni MOSS runtime and H100 benchmark", "url": "https://github.com/sgl-project/sglang-omni/pull/914"},
{"label": "FunASR ecosystem integration guide", "url": "https://github.com/modelscope/FunASR/blob/main/docs/moss_transcribe_diarize.md"},
{"label": "FunASR AutoModel adapter merge and exact-head CI", "url": "https://github.com/modelscope/FunASR/pull/3558"},
{"label": "FunClip v2.2.0 MOSS speaker-clipping guide", "url": "https://www.funasr.com/en/blog/funclip-v2-2-0-moss-speaker-clipping.html"},
{"label": "FunClip v2.2.0 release and checksums", "url": "https://github.com/modelscope/FunClip/releases/tag/v2.2.0"},
{"label": "moss-transcribe.cpp third-party C++17 and GGUF runtime", "url": "https://github.com/localai-org/moss-transcribe.cpp"},
{"label": "LocalAI third-party moss-transcribe-cpp backend", "url": "https://github.com/mudler/LocalAI"}
],
"benchmarks": [
{
"model": "OpenMOSS-Team/MOSS-Transcribe-Diarize@e8681d68",
"runtime": "vLLM 0.27.1 / Torch 2.13.0+cu129",
"hardware": "NVIDIA H100 80GB HBM3",
"workload": "OpenAI-compatible diarized_json API smoke plus FunASR AutoModel structured-response mapping",
"audio": "15.1685 s two-speaker probe; SHA256 43dccc068506439cb633b382b6b98185baa837363d08cc5f7152ca89b0fdc3c8",
"settings": "Pinned model revision; vLLM 0.27.1; vllm[audio]; response_format=diarized_json; temperature=0; max_completion_tokens=768",
"timing_scope": "Functional response validation after /health became ready; no latency or throughput claim",
"result": "Direct vLLM HTTP and the FunASR adapter each returned two non-empty monotonic segments with speakers S01 and S02.",
"qualification": "A structured response contract smoke, not a diarization accuracy study, overlap test, throughput benchmark, or production capacity promise.",
"source": "https://github.com/modelscope/FunASR/blob/main/docs/moss_transcribe_diarize.md",
"verified": "2026-08-30"
},
{
"model": "OpenMOSS-Team/MOSS-Transcribe-Diarize@e8681d68",
"runtime": "SGLang Omni merge 8458f76a",
"hardware": "NVIDIA H100 80GB",
"workload": "Seed-TTS EN full set, non-streaming, max concurrency 16, one repeat",
"audio": "1088 single-speaker English clips; mean duration 4.736 s",
"settings": "Single H100; 1088 / 1088 requests completed; same upstream benchmark harness used for model comparison",
"timing_scope": "Upstream end-to-end benchmark report; throughput 55.487 req/s, mean latency 0.287 s, mean RTF 0.0625",
"result": "1088 / 1088 completed with 0 failures; raw WER 2.74% and WER excluding >50% outliers 2.45%.",
"qualification": "Single-speaker English ASR benchmark after stripping timestamp and speaker markup; diarization/timestamp correctness is not evaluated, and the result is not a production capacity promise.",
"source": "https://github.com/sgl-project/sglang-omni/pull/914",
"verified": "2026-07-04"
}
],
"translations": {
"zh": {
"name": "MOSS 统一转写与说话人分离",
"summary": "通过 FunASR AutoModel 或 OpenAI 兼容服务接入 OpenMOSS 发布的第三方 Apache-2.0 模型,一次生成长音频转写、时间戳和说话人标签。",
"fit": ["会议、访谈、播客和通话的多人长音频", "希望通过同一个模型输出文本、时间戳与说话人编号", "已有 NVIDIA GPU,并需要 vLLM 或 SGLang Omni 的 OpenAI 兼容音频转写接口", "通过 LocalAI / moss-transcribe.cpp 在 CPU 或桌面边缘 GPU 上运行第三方 GGUF 实现"],
"not_fit": ["实时流式字幕或低延迟端点检测", "要求 FunASR AutoModel 与 LocalAI C++ 路径输出完全相同结果的部署", "未经目标硬件复测的原生 Windows 部署", "要求 FunASR 自有模型权重的项目"],
"selection_reason": "模型端到端联合生成转写与说话人标签,用户不必再拼接外部 VAD、ASR 和 diarization 管线;FunASR AutoModel 为 Transformers、vLLM 与 SGLang Omni 提供统一的结构化结果,LocalAI 则提供独立的第三方 C++/GGUF 边缘路径。",
"primary_limitation": "这是 OpenMOSS 的第三方模型,不是 FunASR 模型;“无需外部 VAD”不表示模型内部没有分块或分段。vLLM 路径固定 0.27.1,SGLang Omni 路径固定 3f819f9c;升级前必须重跑真实多人长音频。SGLang Omni 的 verbose_json 把说话人编号保留在 segments[].text 的 [Sxx] 前缀,FunASR adapter 会校验该分段能够回溯到 raw_text 后再解析。LocalAI / moss-transcribe.cpp 也是独立第三方重写。",
"status_label": "社区验证",
"operations": ["固定模型 revision、FunASR adapter merge、vLLM 0.27.1、SGLang Omni 3f819f9c、CUDA/Torch 与 trust_remote_code 审计结果", "用真实会议验证说话人一致性、重叠语音、长静音、时间戳和最大生成长度", "SGLang Omni #914 的 1088 条单说话人英文 benchmark 不覆盖 diarization 或 timestamp 准确率,生产前必须单独验证", "LocalAI 路径固定第三方 C++ backend 与 GGUF revision,并分别验证 CPU/GPU backend、量化精度和 OpenAI 接口响应", "FunASR AutoModel 可把 vLLM diarized_json 与 SGLang Omni verbose_json 统一为 text、timestamp 和带 spk 的 sentence_info;SGLang 分段必须能回溯到 raw_text,避免接受上游合成的 S01 fallback"],
"security": ["把服务绑定内网,由网关完成认证、TLS、限流及音频大小/时长限制", "固定并审计 trust_remote_code 对应的模型 revision,不执行浮动 main 代码", "隔离模型缓存、上传临时目录和生成日志,按数据保留策略清理原始音频"],
"troubleshooting": ["CUDA 12 使用已校验 SHA256 的 vLLM 0.27.1 cu129 wheel;SGLang Omni 当前固定 CUDA 13 安装契约和 3f819f9c", "长音频被截断时,vLLM 提高 max_completion_tokens,SGLang Omni 提高 max_new_tokens,并同时监控显存、延迟和输出完整性", "vLLM diarized_json 提供独立 speaker 字段;SGLang Omni verbose_json 当前把说话人编号放在 segments[].text 的 [Sxx] 前缀"]
},
"en": {
"name": "MOSS unified transcription and diarization",
"summary": "Use FunASR AutoModel or an OpenAI-compatible service with the third-party Apache-2.0 model published by OpenMOSS to produce long-form transcripts, timestamps, and speaker labels in one pass.",
"fit": ["Multi-speaker meetings, interviews, podcasts, and calls", "One model output for text, timestamps, and speaker identifiers", "NVIDIA GPU deployments that need a vLLM or SGLang Omni OpenAI-compatible audio transcription endpoint", "The third-party LocalAI / moss-transcribe.cpp GGUF path on CPU or desktop-edge GPUs"],
"not_fit": ["Realtime streaming captions or low-latency endpoint detection", "Deployments that require identical output from FunASR AutoModel and the LocalAI C++ path", "Native Windows deployments without validation on the target hardware", "Projects that require FunASR-owned model weights"],
"selection_reason": "The model jointly emits transcription and speaker labels so users do not need to assemble an external VAD, ASR, and diarization pipeline; FunASR AutoModel provides one structured result across Transformers, vLLM, and SGLang Omni, while LocalAI exposes a separate third-party C++/GGUF edge path.",
"primary_limitation": "This is an OpenMOSS third-party model, not a FunASR model; no external VAD requirement does not imply an absence of internal segmentation. The vLLM path pins 0.27.1 and the SGLang Omni path pins 3f819f9c; revalidate real long multi-speaker audio before upgrades. SGLang Omni verbose_json keeps the speaker identifier in the [Sxx] prefix of segments[].text, which the FunASR adapter parses only after validating that the segment is backed by raw_text. LocalAI / moss-transcribe.cpp is another independent third-party reimplementation.",
"status_label": "Community verified",
"operations": ["Pin the model revision, FunASR adapter merge, vLLM 0.27.1, SGLang Omni 3f819f9c, CUDA/Torch stack, and trust_remote_code audit", "Validate speaker consistency, overlap, long silence, timestamps, and generation limits on real meetings", "The 1088-clip SGLang Omni #914 single-speaker English benchmark does not cover diarization or timestamp accuracy; validate both independently before production", "For LocalAI, pin the third-party C++ backend and GGUF revision, then validate CPU/GPU backends, quantization accuracy, and the OpenAI response independently", "FunASR AutoModel maps both vLLM diarized_json and SGLang Omni verbose_json into text, timestamp, and sentence_info with spk; SGLang segments must be backed by raw_text so an upstream synthesized S01 fallback is not accepted"],
"security": ["Bind workers to a private network and put authentication, TLS, rate limits, and audio size/duration limits at the gateway", "Pin and audit the trust_remote_code model revision instead of executing a floating main revision", "Isolate model caches, upload directories, and generation logs; remove source audio according to retention policy"],
"troubleshooting": ["For CUDA 12, use the SHA256-verified vLLM 0.27.1 cu129 wheel; the SGLang Omni path currently pins its CUDA 13 installation contract and 3f819f9c", "If long audio is truncated, raise max_completion_tokens for vLLM or max_new_tokens for SGLang Omni while monitoring memory, latency, and output completeness", "vLLM diarized_json returns a separate speaker field; SGLang Omni verbose_json currently keeps the speaker identifier in the [Sxx] prefix of segments[].text"]
}
}
},
{
"id": "vllm",
"routes": {"zh": "/deploy/vllm.html", "en": "/en/deploy/vllm.html"},
"maturity": "community-verified",
"selector_rank": 10,
"workloads": ["batch", "private-api"],
"hardware": ["nvidia-gpu", "kubernetes"],
"priorities": ["throughput", "latency"],
"models": [
"FunAudioLLM/Fun-ASR-Nano-2512-vllm (official checkpoint)"
],
"operating_systems": ["Linux"],
"interfaces": ["OpenAI-compatible HTTP"],
"tested": {
"funasr": "Official model revision a4362c943d48951f98ca2a62181cc028970270c5; not a FunASR package version",
"runtime": "vLLM 0.27.1+cu129 / Torch 2.13.0+cu129 / Transformers 5.15.0 / CUDA 12.9 / Python 3.12.3",
"verified": "2026-09-07"
},
"commands": {
"install": [
"export VLLM_PYTHON=\"$PWD/.venv/bin/python\"\nexport VALIDATION_DIR=\"$PWD/.official-native-validation\"\nexport MODEL_DIR=\"$VALIDATION_DIR/official-model/a4362c943d48951f98ca2a62181cc028970270c5\"",
"\"$VLLM_PYTHON\" - <<'PY'\nimport importlib.metadata as metadata\nimport torch\nimport vllm.model_executor.models.funasr\n\nfor name, expected in {\"vllm\": \"0.27.1+cu129\", \"torch\": \"2.13.0+cu129\",\n \"transformers\": \"5.15.0\"}.items():\n assert metadata.version(name) == expected, (name, metadata.version(name))\nassert torch.version.cuda == \"12.9\" and torch.cuda.is_available()\nPY",
"HF_HUB_DISABLE_TELEMETRY=1 HF_XET_CACHE=\"$VALIDATION_DIR/isolated-xet-cache\" \"$VLLM_PYTHON\" - <<'PY'\nimport hashlib\nimport json\nimport os\nfrom pathlib import Path\nfrom huggingface_hub import HfApi, snapshot_download\n\nmodel_id = \"FunAudioLLM/Fun-ASR-Nano-2512-vllm\"\nrevision = \"a4362c943d48951f98ca2a62181cc028970270c5\"\nroot = Path(os.environ[\"MODEL_DIR\"])\ninfo = HfApi().model_info(model_id, revision=revision, files_metadata=True)\nassert info.sha == revision and len(info.siblings) == 23\nsnapshot_download(model_id, revision=revision, local_dir=str(root),\n cache_dir=str(Path(os.environ[\"VALIDATION_DIR\"]) / \"isolated-hf-cache\"),\n max_workers=4)\nfiles = []\nfor item in info.siblings:\n content = (root / item.rfilename).read_bytes()\n digest = hashlib.sha256(content).hexdigest()\n assert len(content) == item.size, item.rfilename\n if item.lfs:\n assert digest == item.lfs.sha256, item.rfilename\n else:\n assert hashlib.sha1(f\"blob {len(content)}\\0\".encode() + content).hexdigest() == item.blob_id\n files.append({\"path\": item.rfilename, \"size\": len(content), \"sha256\": digest})\nassert json.loads((root / \"config.json\").read_text())[\"architectures\"] == [\"FunASRForConditionalGeneration\"]\n(root.parent / \"download-manifest.json\").write_text(json.dumps(files, indent=2) + \"\\n\")\nPY"
],
"launch": [
"CUDA_VISIBLE_DEVICES=0 PYTHONDONTWRITEBYTECODE=1 \\\nHF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \\\nHF_HUB_DISABLE_IMPLICIT_TOKEN=1 HF_HUB_DISABLE_TELEMETRY=1 VLLM_NO_USAGE_STATS=1 \\\nVLLM_CACHE_ROOT=\"$VALIDATION_DIR/runtime-cache\" TRITON_CACHE_DIR=\"$VALIDATION_DIR/triton-cache\" \\\n\"$VLLM_PYTHON\" -B -m vllm.entrypoints.openai.api_server \\\n --model \"$MODEL_DIR\" --served-model-name fun-asr-nano-official-a4362c94 \\\n --host 127.0.0.1 --port 57185 --dtype float32 \\\n --gpu-memory-utilization 0.40 --enforce-eager"
],
"health": [
"curl --max-time 15 -fsS http://127.0.0.1:57185/health",
"curl --max-time 15 -fsS http://127.0.0.1:57185/v1/models"
],
"smoke": [
"curl --max-time 45 -fsS http://127.0.0.1:57185/v1/audio/transcriptions -F \"file=@$MODEL_DIR/example/zh.mp3;type=audio/mpeg\" -F model=fun-asr-nano-official-a4362c94 -F language=zh -F response_format=json",
"curl --max-time 45 -fsS http://127.0.0.1:57185/v1/audio/transcriptions -F \"file=@$MODEL_DIR/example/en.mp3;type=audio/mpeg\" -F model=fun-asr-nano-official-a4362c94 -F language=en -F response_format=json",
"curl --max-time 45 -fsS http://127.0.0.1:57185/v1/audio/transcriptions -F \"file=@$MODEL_DIR/example/ja.mp3;type=audio/mpeg\" -F model=fun-asr-nano-official-a4362c94 -F language=ja -F response_format=json",
"curl --max-time 45 -fsS http://127.0.0.1:57185/v1/audio/transcriptions -F \"file=@$MODEL_DIR/example/zh.mp3;type=audio/mpeg\" -F model=fun-asr-nano-official-a4362c94 -F language=zh -F 'hotwords=开放时间,开放时间,开放时间' -F response_format=json"
]
},
"evidence": [
{
"label": "2026-09-07 official checkpoint native HTTP functional validation",
"url": "https://github.com/modelscope/FunASR/blob/main/docs/vllm_official_native_validation.md"
},
{
"label": "2026-09-07 官方 checkpoint 原生 HTTP 功能验证(中文)",
"url": "https://github.com/modelscope/FunASR/blob/main/docs/vllm_official_native_validation_zh.md"
},
{
"label": "23-file manifest and 8 HTTP response digests (private paths redacted)",
"url": "https://github.com/modelscope/FunASR/blob/main/docs/benchmark/vllm_official_native_20260907.json"
},
{
"label": "2026-08-13 historical community checkpoint validation (unchanged)",
"url": "https://github.com/modelscope/FunASR/blob/main/docs/vllm_native_funasr_validation.md"
},
{
"label": "Official reference in upstream docs/test registry; merged main, not a release claim (#54944)",
"url": "https://github.com/vllm-project/vllm/pull/54944"
},
{
"label": "Official checkpoint at immutable a4362c94 (validated separately on 2026-09-07)",
"url": "https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512-vllm/tree/a4362c943d48951f98ca2a62181cc028970270c5"
},
{
"label": "vLLM 0.27.1 release and CUDA 12.9 wheel",
"url": "https://github.com/vllm-project/vllm/releases/tag/v0.27.1"
},
{
"label": "native FunASR model integration",
"url": "https://github.com/vllm-project/vllm/pull/33247"
},
{
"label": "native FunASR hotword support",
"url": "https://github.com/vllm-project/vllm/pull/39674"
},
{
"label": "FunASR initialization fix",
"url": "https://github.com/vllm-project/vllm/pull/44215"
},
{
"label": "validated community conversion at e718b36e",
"url": "https://huggingface.co/allendou/Fun-ASR-Nano-2512-vllm/tree/e718b36e2578203ec893e9b488239225f8d668e2"
},
{
"label": "official FunASR split-engine guide",
"url": "https://github.com/modelscope/FunASR/blob/main/docs/vllm_guide.md"
},
{
"label": "FunASR deployment matrix",
"url": "https://github.com/modelscope/FunASR/blob/main/docs/deployment_matrix.md"
}
],
"benchmarks": [
{
"model": "allendou/Fun-ASR-Nano-2512-vllm@e718b36e",
"runtime": "vLLM 0.27.1+cu129 / Torch 2.13.0+cu129",
"hardware": "NVIDIA H100 80GB",
"workload": "OpenAI-compatible transcription; Chinese baseline and hotword probes plus two concurrent requests (English and Japanese)",
"audio": "Pinned model-repository examples: 6 s Chinese, 8 s English, and 8 s Japanese",
"settings": "FP32; eager mode; gpu-memory-utilization 0.40; max model length 40,960; explicit language; warmed server",
"timing_scope": "Client wall time after /health became ready; local HTTP and decoding included; cold model download and 20.3 s engine warmup excluded",
"result": "Chinese 200 in 0.968 s; repeated hotword corrected 开饭时间 to 开放时间 in 0.214 s; English and Japanese two-request batch completed in 1.123 s wall time",
"qualification": "Single-H100 correctness and concurrency probe with a community-converted checkpoint, not an accuracy study or production capacity promise. Revalidate the checkpoint, target GPU, languages, traffic, and hotword policy before rollout.",
"source": "https://github.com/modelscope/FunASR/blob/main/docs/vllm_native_funasr_validation.md",
"verified": "2026-08-13"
}
],
"translations": {
"zh": {
"name": "vLLM 原生 FunASR 服务",
"summary": "使用官方 FunAudioLLM/Fun-ASR-Nano-2512-vllm 固定快照,在 vLLM 0.27.1 上提供中英日转写与热词接口;已通过单张 H100 的独立功能验证。",
"fit": [
"需要标准 /v1/audio/transcriptions 接口",
"Fun-ASR-Nano 批量或并发转写",
"已有 NVIDIA GPU 和 Linux 服务环境"
],
"not_fit": [
"未经重新验证就直接替换下方固定版本 checkpoint",
"纯 CPU 或边缘设备",
"尚未完成显存、并发与业务音频压测的公网服务"
],
"selection_reason": "vLLM 已原生注册 FunASR 架构,并提供 OpenAI 兼容转写、显存调度、并发和热词参数。",
"primary_limitation": "当前命令对应官方 a4362c943d48951f98ca2a62181cc028970270c5,2026-09-07 在既有 vLLM 0.27.1+cu129 / Torch 2.13.0+cu129 / Transformers 5.15.0 / CUDA 12.9 环境验证,不是全新安装验证。准备步骤只检查版本和下载本地快照,不重新安装依赖。8 个 HTTP 请求均为功能验证;首次中文未预热,两请求 0.911 s 不代表容量。下方 2026-08-13 benchmark 仍是社区 checkpoint 的历史数据,没有重命名或与新结果比较。#54944 仅更新 main 的文档/测试 registry;审计时 v0.28.0 仍引用社区产物,不代表官方引用已发布。",
"status_label": "社区验证",
"operations": [
"固定官方模型 revision 与本地文件摘要,以及 vLLM 0.27.1+cu129、Torch 2.13.0+cu129、Transformers 5.15.0、CUDA 12.9;不要将模型 revision 当成 FunASR 包版本",
"等待 /health 就绪并完成预热后再采集容量数据",
"记录队列等待、最终结果延迟、显存和热词命中率"
],
"security": [
"在 API 网关实现认证、TLS、限流和音频大小/时长限制",
"worker 仅绑定 127.0.0.1;网关未在本次 smoke 中验证,不直接暴露公网",
"校验 wheel、模型 revision 和示例音频摘要,隔离模型缓存与上传临时目录"
],
"troubleshooting": [
"音频返回 400 时确认安装了 vllm[audio]",
"启动时 KV cache 不足则提高显存利用率或在业务允许时降低 max-model-len",
"非英文请求必须显式传 language;热词用逗号分隔并以真实业务样本验证偏置强度"
]
},
"en": {
"name": "Native FunASR on vLLM",
"summary": "Serve the pinned official FunAudioLLM/Fun-ASR-Nano-2512-vllm snapshot on vLLM 0.27.1; Chinese, English, Japanese and hotwords passed a separate single-H100 functional smoke.",
"fit": [
"A standard /v1/audio/transcriptions endpoint",
"Concurrent or batch Fun-ASR-Nano transcription",
"An NVIDIA GPU Linux serving environment"
],
"not_fit": [
"Replacing the pinned checkpoint below without revalidation",
"CPU-only or edge devices",
"Internet-facing service before memory, concurrency, and workload tests"
],
"selection_reason": "vLLM natively registers the FunASR architecture and provides OpenAI-compatible transcription, memory scheduling, concurrency, and hotword parameters.",
"primary_limitation": "Current commands use official FunAudioLLM/Fun-ASR-Nano-2512-vllm@a4362c943d48951f98ca2a62181cc028970270c5, tested 2026-09-07 in an existing environment: vLLM 0.27.1+cu129 / Torch 2.13.0+cu129 / Transformers 5.15.0 / CUDA 12.9. This is not a clean installation validation; preparation only checks versions and downloads a local snapshot. Eight HTTP requests establish functional compatibility only; first Chinese was not warmed, and the 0.911 s two-request probe is not capacity evidence. The 2026-08-13 benchmark below remains a historical community-converted checkpoint result, neither relabeled nor compared with the new run. #54944 updates main docs/test registry only; v0.28.0 still referenced the community artifact at audit time, not a claim of released official references.",
"status_label": "Community verified",
"operations": [
"Pin the official model revision and local file digests plus vLLM 0.27.1+cu129, Torch 2.13.0+cu129, Transformers 5.15.0 and CUDA 12.9; a model revision is not a FunASR package version",
"Wait for /health and warm the model before measuring capacity",
"Measure queue time, final-result latency, memory, and hotword hit rate"
],
"security": [
"Put authentication, TLS, rate limits, and audio size/duration limits at the gateway",
"Bind the worker to 127.0.0.1; the gateway was not validated by this smoke, and the worker must not be exposed directly",
"Verify wheel, model revision, and sample digests; isolate model cache and temporary uploads"
],
"troubleshooting": [
"For audio HTTP 400 responses, confirm vllm[audio] is installed",
"If startup reports insufficient KV cache, raise GPU utilization or lower max-model-len only when the workload permits",
"Pass language explicitly for non-English audio; comma-separate hotwords and validate bias strength on production samples"
]
}
}
},
{
"id": "sensevoice-tensorrt",
"routes": {"zh": "/deploy/sensevoice-tensorrt.html", "en": "/en/deploy/sensevoice-tensorrt.html"},
"maturity": "production-verified",
"selector_rank": 15,
"workloads": ["batch", "private-api"],
"hardware": ["nvidia-gpu", "kubernetes"],
"priorities": ["throughput", "latency"],
"models": ["SenseVoiceSmall"],
"operating_systems": ["Linux"],
"interfaces": ["Triton gRPC/HTTP", "TensorRT plan", "Docker"],
"tested": {"funasr": "main@6408aaa9", "runtime": "TensorRT 10.0.1 / Triton 24.05", "verified": "2026-08-04"},
"commands": {
"install": [
"git clone https://github.com/modelscope/FunASR.git && cd FunASR && git checkout 6408aaa96d54f89db467346010eff3c7d1fc485a",
"pip install -e . \"onnx>=1.16\" soundfile \"tritonclient[grpc]\"",
"python - <<'PY'\nfrom funasr import AutoModel\n\nmodel = AutoModel(model=\"iic/SenseVoiceSmall\", device=\"cuda:0\")\nmodel.export(\n type=\"onnx\",\n quantize=False,\n device=\"cuda:0\",\n output_dir=\"./sensevoice_onnx\",\n max_seq_len=4096,\n)\nPY",
"curl -fL https://huggingface.co/FunAudioLLM/SenseVoiceSmall/resolve/main/chn_jpn_yue_eng_ko_spectok.bpe.model \\\n -o runtime/triton_gpu/model_repo_sense_voice_small/scoring/chn_jpn_yue_eng_ko_spectok.bpe.model \\\n&& echo \"aa87f86064c3730d799ddf7af3c04659151102cba548bce325cf06ba4da4e6a8 runtime/triton_gpu/model_repo_sense_voice_small/scoring/chn_jpn_yue_eng_ko_spectok.bpe.model\" | sha256sum -c -"
],
"launch": [
"python runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py \\\n ./sensevoice_onnx/model.onnx \\\n runtime/triton_gpu/model_repo_sense_voice_small/encoder/1/model.plan \\\n --precision fp16 \\\n --min-batch 1 --opt-batch 8 --max-batch 16 \\\n --min-frames 1 --opt-frames 512 --max-frames 4096 \\\n --workspace-gb 8",
"cp runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt.tensorrt \\\n runtime/triton_gpu/model_repo_sense_voice_small/encoder/config.pbtxt",
"cd runtime/triton_gpu && tritonserver --model-repository ./model_repo_sense_voice_small \\\n --pinned-memory-pool-byte-size=512000000 \\\n --cuda-memory-pool-byte-size=0:1024000000"
],
"health": [
"curl -fsS http://localhost:8000/v2/health/ready",
"curl -fsS http://localhost:8000/v2/models/sensevoice/ready"
],
"smoke": [
"python - <<'PY'\nimport numpy as np\nimport soundfile as sf\nimport tritonclient.grpc as grpc\n\naudio, sample_rate = sf.read(\n \"runtime/triton_gpu/client/test_wavs/mid.wav\", dtype=\"float32\"\n)\nassert sample_rate == 16000\nvalues = {\n \"WAV\": audio[None, :],\n \"WAV_LENS\": np.array([[audio.size]], dtype=np.int32),\n \"LANGUAGE\": np.array([[0]], dtype=np.int32),\n \"TEXT_NORM\": np.array([[1]], dtype=np.int32),\n}\ninputs = []\nfor name, value in values.items():\n tensor = grpc.InferInput(name, value.shape, grpc.np_to_triton_dtype(value.dtype))\n tensor.set_data_from_numpy(value)\n inputs.append(tensor)\nresult = grpc.InferenceServerClient(\"localhost:8001\").infer(\"sensevoice\", inputs)\ntranscript = result.as_numpy(\"TRANSCRIPTS\")[0][0].decode()\nassert transcript.strip(), \"SenseVoice returned an empty transcript\"\nprint(transcript)\nPY"
]
},
"evidence": [
{"label": "TensorRT deployment guide", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/triton_gpu/README.md"},
{"label": "validated TensorRT builder", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/triton_gpu/scripts/build_sensevoice_tensorrt.py"},
{"label": "native TensorRT integration PR", "url": "https://github.com/modelscope/FunASR/pull/3463"},
{"label": "deployment request and validation record", "url": "https://github.com/modelscope/FunASR/issues/3453"}
],
"benchmarks": [
{
"model": "SenseVoiceSmall",
"runtime": "TensorRT 10.0.1 with Triton 24.05 model contract",
"hardware": "NVIDIA H100; device memory capacity and host CPU are not part of the recorded result",
"workload": "Native FP16 engine build, profile-bound execution, and PyTorch parity validation",
"audio": "Bundled Chinese example plus synthetic feature tensors at 30 and 64 frames",
"settings": "FP16; batch profile 1/8/16; post-LFR frame profile 1/512/4096; 8 GiB builder workspace",
"timing_scope": "Engine build wall time only; inference timings are correctness probes and not a throughput benchmark",
"result": "527,504,916 bytes; 113.9 s engine build; 100% CTC top-1 agreement; exact bundled-audio transcript",
"qualification": "Single-H100 compatibility and parity evidence. Build on the target GPU and TensorRT version, then load-test with production audio before capacity planning.",
"source": "https://github.com/modelscope/FunASR/blob/main/runtime/triton_gpu/README.md",
"verified": "2026-08-04"
}
],
"translations": {
"zh": {
"name": "SenseVoice TensorRT / Triton",
"summary": "把 SenseVoiceSmall 构建为原生 FP16 TensorRT engine,并通过 Triton 提供可批处理的 GPU 推理服务。",
"fit": ["固定 NVIDIA GPU 环境中的 SenseVoice 批量转写", "需要 Triton 动态批处理、健康检查和模型仓库", "愿意按生产音频分布定制 batch 与帧长 profile"],
"not_fit": ["CPU、macOS、Windows 或通用边缘设备", "需要跨 GPU 型号直接复制同一个 engine", "只有动态量化 model_quant.onnx 的部署"],
"selection_reason": "原生 TensorRT engine 和 Triton 调度适合固定 NVIDIA GPU 上以 SenseVoice 延迟与吞吐为核心的服务。",
"primary_limitation": "TensorRT plan 不保证跨 GPU 架构或 TensorRT 版本可移植,必须在目标环境重新构建并复测精度、显存和容量。",
"status_label": "生产验证",
"operations": ["在与生产相同的 GPU 架构和 TensorRT 版本上构建 engine", "按真实 batch、并发和音频时长重设优化 profile", "保存 ONNX、engine、构建日志、SHA-256 与 Triton 配置用于回滚"],
"security": ["Triton worker 仅绑定内网,由网关完成认证、TLS、限流和审计", "限制音频大小、时长、声道、采样率和并发", "只加载经过校验的 ONNX、TensorRT plan、分词器和模型仓库"],
"troubleshooting": ["构建前确认导出使用 quantize=False,不能使用 model_quant.onnx", "parser 报错时先运行 ONNX checker 并核对 SenseVoice tensor 名称和 dtype", "Triton 加载失败时确认 model.plan 与 config.pbtxt.tensorrt 位于同一 encoder 版本目录"]
},
"en": {
"name": "SenseVoice TensorRT / Triton",
"summary": "Build SenseVoiceSmall as a native FP16 TensorRT engine and serve batched GPU inference through Triton.",
"fit": ["SenseVoice batch transcription on a fixed NVIDIA GPU fleet", "Triton dynamic batching, health checks, and model repositories", "Optimization profiles tuned to production batch and audio distributions"],
"not_fit": ["CPU, macOS, Windows, or general edge targets", "Copying one engine across different GPU models", "Deployments that only have dynamically quantized model_quant.onnx"],
"selection_reason": "A native TensorRT engine and Triton scheduling fit SenseVoice services that prioritize latency and throughput on fixed NVIDIA GPUs.",
"primary_limitation": "A TensorRT plan is not portable across arbitrary GPU architectures or TensorRT versions; rebuild and revalidate accuracy, memory, and capacity on the target stack.",
"status_label": "Production verified",
"operations": ["Build on the same GPU architecture and TensorRT version used in production", "Retune optimization profiles for real batch, concurrency, and audio duration", "Retain ONNX, engine, build log, SHA-256, and Triton configuration for rollback"],
"security": ["Bind Triton workers to private networks and provide authentication, TLS, rate limits, and audit at the gateway", "Limit audio size, duration, channels, sample rate, and concurrency", "Load only verified ONNX, TensorRT plans, tokenizers, and model repositories"],
"troubleshooting": ["Confirm the export used quantize=False; model_quant.onnx is unsupported", "Run the ONNX checker and verify SenseVoice tensor names and dtypes before investigating parser errors", "When Triton cannot load the model, keep model.plan and config.pbtxt.tensorrt in the same encoder version directory"]
}
}
},
{
"id": "llama-cpp",
"routes": {"zh": "/deploy/llama-cpp.html", "en": "/en/deploy/llama-cpp.html"},
"maturity": "production-verified",
"selector_rank": 20,
"workloads": ["edge", "batch"],
"hardware": ["cpu", "desktop-edge-gpu"],
"priorities": ["portability", "compatibility"],
"models": ["SenseVoiceSmall-GGUF", "Paraformer-GGUF", "Fun-ASR-Nano-GGUF", "FSMN-VAD-GGUF"],
"operating_systems": ["Linux", "macOS", "Windows"],
"interfaces": ["CLI", "local HTTP server"],
"tested": {"funasr": "runtime-llamacpp-v0.2.6", "runtime": "llama.cpp@c8d43b10", "verified": "2026-08-30"},
"commands": {
"install": ["curl -fLO https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-linux-x64.tar.gz", "echo \"779967de1c528c2be966bcc47f246e7d3e6fcdb748d9491263062f4120f35e52 funasr-llamacpp-linux-x64.tar.gz\" | sha256sum -c -", "mkdir funasr-llamacpp && tar -xzf funasr-llamacpp-linux-x64.tar.gz -C funasr-llamacpp && cd funasr-llamacpp && bash download-funasr-model.sh sensevoice ./funasr-gguf f16"],
"launch": ["cd funasr-llamacpp && ./llama-funasr-sensevoice -m funasr-gguf/sensevoice-small-f16.gguf --vad funasr-gguf/fsmn-vad.gguf -a sample.wav"],
"health": ["cd funasr-llamacpp && ./llama-funasr-sensevoice --help"],
"smoke": ["cd funasr-llamacpp && ./llama-funasr-sensevoice -m funasr-gguf/sensevoice-small-f16.gguf --vad funasr-gguf/fsmn-vad.gguf -a sample.wav | tee transcript.txt && test -s transcript.txt"]
},
"downloads": [
{"operating_system": "Linux", "architecture": "arm64", "backend": "CPU", "archive": "funasr-llamacpp-linux-arm64.tar.gz", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-linux-arm64.tar.gz", "sha256": "7bca29cfa3c9a08e235a62212ca9e00f6656e59a8f07078966a2bfda1e5aa1f9"},
{"operating_system": "Linux", "architecture": "x64", "backend": "CPU", "archive": "funasr-llamacpp-linux-x64.tar.gz", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-linux-x64.tar.gz", "sha256": "779967de1c528c2be966bcc47f246e7d3e6fcdb748d9491263062f4120f35e52"},
{"operating_system": "Linux", "architecture": "x64 AVX2", "backend": "CPU", "archive": "funasr-llamacpp-linux-x64-avx2.tar.gz", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-linux-x64-avx2.tar.gz", "sha256": "aaebc5470f846ce915200b35d6e9f9bd0a0d3ed399d39e49bdeb7a1f1782bc70"},
{"operating_system": "Linux", "architecture": "x64", "backend": "Vulkan", "archive": "funasr-llamacpp-linux-x64-vulkan.tar.gz", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-linux-x64-vulkan.tar.gz", "sha256": "f02d41e98e9d4041f0896661007193810f025484d2175958f7c1313d5c90ec46"},
{"operating_system": "macOS", "architecture": "arm64", "backend": "CPU", "archive": "funasr-llamacpp-macos-arm64.tar.gz", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-macos-arm64.tar.gz", "sha256": "bda59474202b887190f59d25b7b42c714469efae71276072c12fa0a38de68792"},
{"operating_system": "Windows", "architecture": "x64", "backend": "CPU", "archive": "funasr-llamacpp-windows-x64.zip", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-windows-x64.zip", "sha256": "f6a73a548413ba9fbaf2145263ea66ec53cbdad1fb11790dbeeee493e339492e"},
{"operating_system": "Windows", "architecture": "x64 AVX2", "backend": "CPU", "archive": "funasr-llamacpp-windows-x64-avx2.zip", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-windows-x64-avx2.zip", "sha256": "062cda8fefadd31c3e811227116daccf448a8520f4b0bb168d225c896e65ebbd"},
{"operating_system": "Windows", "architecture": "x64", "backend": "Vulkan", "archive": "funasr-llamacpp-windows-x64-vulkan.zip", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-windows-x64-vulkan.zip", "sha256": "debf8007e55011cad06081e7b8a78972f1b8fe672bc324d41e650d68821f6a6a"},
{"operating_system": "Windows", "architecture": "x64", "backend": "CUDA", "archive": "funasr-llamacpp-windows-x64-cuda.zip", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-windows-x64-cuda.zip", "sha256": "148657911fb666b7af6ec43af2e23a0984e3259012b4c39f95631b717feb6840"},
{"operating_system": "Windows", "architecture": "x64 Blackwell (sm_120)", "backend": "CUDA", "archive": "funasr-llamacpp-windows-x64-cuda-blackwell.zip", "url": "https://github.com/modelscope/FunASR/releases/download/runtime-llamacpp-v0.2.6/funasr-llamacpp-windows-x64-cuda-blackwell.zip", "sha256": "e32961a753f40888182f352fa551159c5165a6a77718ae4ade316aedfea4b1c2"}
],
"evidence": [
{"label": "llama.cpp runtime", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/llama.cpp/README.md"},
{"label": "runtime v0.2.6 release", "url": "https://github.com/modelscope/FunASR/releases/tag/runtime-llamacpp-v0.2.6"},
{"label": "ten-platform release workflow", "url": "https://github.com/modelscope/FunASR/actions/runs/33290555297"},
{"label": "Blackwell sm_120 build and bundled CUDA runtime", "url": "https://github.com/modelscope/FunASR/pull/3570"},
{"label": "Vulkan weight upload fix", "url": "https://github.com/modelscope/FunASR/pull/3555"},
{"label": "F16 empty-transcript fix", "url": "https://github.com/modelscope/FunASR/pull/3550"},
{"label": "regression tests", "url": "https://github.com/modelscope/FunASR/tree/main/runtime/llama.cpp/tests"}
],
"benchmarks": [
{
"model": "SenseVoiceSmall F16 GGUF",
"runtime": "runtime-llamacpp-v0.2.4 Linux x64 AVX2 release asset",
"hardware": "Intel Xeon Platinum 8480+, 8 runtime threads",
"workload": "100 sequential cold-process SenseVoice transcriptions",
"audio": "One 6.00 s Mandarin PCM sample; mono, 16 kHz, 16-bit",
"settings": "Published AVX2 archive; 470,197,600-byte F16 GGUF; direct SenseVoice CLI without VAD",
"timing_scope": "Functional stability validation only; every process included model loading and decoding, and timings were not retained",
"result": "100/100 processes exited successfully; 0 empty transcripts; one output SHA-256; transcript 我想问我在滨海新区有房。",
"qualification": "Single-host, single-sample regression evidence for the former intermittent empty-output defect, not an accuracy study or production capacity promise.",
"source": "https://github.com/modelscope/FunASR/releases/tag/runtime-llamacpp-v0.2.4",
"verified": "2026-08-29"
},
{
"model": "SenseVoiceSmall Q8 GGUF",
"runtime": "llama.cpp/GGUF with built-in FSMN-VAD",
"hardware": "CPU, 8 threads; CPU model not recorded in the cited report",
"workload": "Offline Mandarin CPU",
"audio": "184 Mandarin clips, about 44-60 seconds each; total duration not recorded in the cited report",
"settings": "Q8 runtime; built-in FSMN-VAD; micro-CER normalize_zh; model load excluded",
"timing_scope": "Sum of compute time divided by sum of audio duration; model load excluded",
"result": "Approximately 20x realtime; Q8 CER 8.17%",
"qualification": "CPU model and full software stack are not recorded. Reproduce on target hardware before comparison.",
"source": "https://github.com/modelscope/FunASR/blob/main/runtime/llama.cpp/BENCHMARKS.md",
"verified": "2026-07-26"
},
{
"model": "Paraformer Q8 GGUF",
"runtime": "llama.cpp/GGUF with built-in FSMN-VAD",
"hardware": "CPU, 8 threads; CPU model not recorded in the cited report",
"workload": "Offline Mandarin CPU",
"audio": "184 Mandarin clips, about 44-60 seconds each; total duration not recorded in the cited report",
"settings": "Q8 runtime; built-in FSMN-VAD; micro-CER normalize_zh; model load excluded",
"timing_scope": "Sum of compute time divided by sum of audio duration; model load excluded",
"result": "Approximately 21x realtime; Q8 CER 9.89%",
"qualification": "CPU model and full software stack are not recorded. Reproduce on target hardware before comparison.",
"source": "https://github.com/modelscope/FunASR/blob/main/runtime/llama.cpp/BENCHMARKS.md",
"verified": "2026-07-26"
}
],
"translations": {
"zh": {
"name": "llama.cpp / GGUF 独立运行",
"summary": "使用 v0.2.6 的十个预编译包或源码构建,在 CPU、Vulkan、CUDA、Blackwell 和边缘设备上运行 FunASR GGUF 模型。",
"fit": ["不依赖 Python ML 环境", "桌面应用和离线边缘部署", "需要 Linux、macOS、Windows 的 CPU、Vulkan、CUDA 或 Blackwell 发布包"],
"not_fit": ["需要 vLLM 式大批量 GPU 调度", "没有目标硬件复测的 Blackwell 生产部署", "必须使用完整 Python 模型生态的流程"],
"selection_reason": "GGUF 运行时和独立二进制优先满足可移植、离线和低依赖部署。",
"primary_limitation": "v0.2.6 提供专用 sm_120 Blackwell 包,并捆绑官方 cuBLAS 与静态 MSVC 运行时;十平台构建、压缩包和依赖审计不等于真实 Blackwell 硬件验证。RX 9070 XT 的 v0.2.5 正向复测也不代表所有 AMD Windows 0xC0000005 报告均已解决;Android/Mali 不是本版预编译或验证目标。",
"status_label": "生产验证",
"operations": ["按页面列出的 SHA-256 校验十个发布资产", "模型和二进制使用同一发布清单", "保留旧二进制和模型目录用于回滚"],
"security": ["默认只读取本地音频和模型", "HTTP 服务绑定内网地址并限制上传大小", "不要从不可信地址加载 GGUF"],
"troubleshooting": ["CPU 路径先用 q8 模型验证", "CUDA 和 Vulkan 必须匹配本机驱动", "按顺序记录 initializing、resolving buffer type、backend ready、model ready、graph allocated 和 compute starting 的 stderr 边界", "Windows AMD Vulkan 异常时附上完整压缩包名、GPU、驱动、命令、退出码和最后到达的诊断边界"]
},
"en": {
"name": "llama.cpp / GGUF standalone",
"summary": "Use the ten v0.2.6 release packages or source builds to run FunASR GGUF models on CPU, Vulkan, CUDA, Blackwell, and edge devices.",
"fit": ["No Python ML environment", "Desktop applications and offline edge deployment", "CPU, Vulkan, CUDA, or Blackwell packages for Linux, macOS, and Windows"],
"not_fit": ["vLLM-style large GPU batch scheduling", "Blackwell production deployment without target-hardware validation", "Workflows that require the full Python model ecosystem"],
"selection_reason": "GGUF runtimes and standalone binaries prioritize portability, offline use, and low dependency count.",
"primary_limitation": "v0.2.6 provides a dedicated sm_120 Blackwell package with official cuBLAS and a static MSVC runtime; ten-platform build, archive, and dependency audits do not prove operation on physical Blackwell hardware. A positive RX 9070 XT v0.2.5 retest also does not resolve every AMD Windows 0xC0000005 report; Android/Mali remains outside the prebuilt and validated targets.",
"status_label": "Production verified",
"operations": ["Verify all ten release assets against the listed SHA-256 values", "Keep models and binaries on the same release manifest", "Retain the previous binary and model directory for rollback"],
"security": ["Read local audio and models by default", "Bind the HTTP server to a private address and limit uploads", "Do not load GGUF files from untrusted sources"],
"troubleshooting": ["Validate the q8 CPU path first", "Match CUDA or Vulkan to the local driver", "Capture the ordered stderr boundaries: initializing, resolving buffer type, backend ready, model ready, graph allocated, and compute starting", "For Windows AMD Vulkan failures, include the exact archive, GPU, driver, command, exit code, and last diagnostic boundary reached"]
}
}
},
{
"id": "sensevoice-native-server",
"routes": {"zh": "/deploy/sensevoice-native-server.html", "en": "/en/deploy/sensevoice-native-server.html"},
"maturity": "production-verified",
"selector_rank": 20.5,
"workloads": ["private-api", "realtime", "edge"],
"hardware": ["cpu", "nvidia-gpu"],
"priorities": ["compatibility", "portability"],
"models": ["SenseVoiceSmall-GGUF", "FSMN-VAD-GGUF"],
"operating_systems": ["Linux"],
"interfaces": ["OpenAI-compatible HTTP", "OpenAI realtime WebSocket", "WS /v1/realtime?intent=transcription", "SSE", "SRT/VTT"],
"tested": {
"funasr": "SenseVoice main@b054623c",
"runtime": "sensevoice-server@558bd67c",
"verified": "2026-08-13"
},
"commands": {
"install": [
"git clone https://github.com/QwenAudio/SenseVoice.git && cd SenseVoice",
"git checkout b054623cca8f015b73ec471dce4f473ac47413da",
"python3 -m pip install -U huggingface_hub",
"bash runtime/llama.cpp/download-funasr-model.sh sensevoice model",
"cmake -S runtime/llama.cpp -B runtime/llama.cpp/build -DCMAKE_BUILD_TYPE=Release",
"cmake --build runtime/llama.cpp/build --target sensevoice-server -j"
],
"launch": [
"runtime/llama.cpp/build/bin/sensevoice-server -m model/sensevoice-small-q8.gguf -vad model/fsmn-vad.gguf --threads 8 --max-connections 4 --max-audio-seconds 300 127.0.0.1 8040"
],
"health": [
"curl -fsS http://127.0.0.1:8040/health",
"curl -fsS http://127.0.0.1:8040/v1/models"
],
"smoke": [
"curl -fsS http://127.0.0.1:8040/v1/audio/transcriptions -F [email protected]",
"curl -fsS http://127.0.0.1:8040/v1/audio/transcriptions -F [email protected] -F response_format=vtt | tee transcript.vtt && grep -q WEBVTT transcript.vtt",
"curl -fsSN http://127.0.0.1:8040/v1/audio/transcriptions -F [email protected] -F stream=true | grep -q transcript.text.done",
"python3 runtime/llama.cpp/tests/stream_client.py 127.0.0.1 8040 speech_16k.wav 200"
]
},
"evidence": [
{
"label": "merged SenseVoice server contribution",
"url": "https://github.com/QwenAudio/SenseVoice/pull/341"
},
{
"label": "source and operating guide at the verified merge",
"url": "https://github.com/QwenAudio/SenseVoice/blob/b054623cca8f015b73ec471dce4f473ac47413da/runtime/llama.cpp/sensevoice-server/README.md"
},
{
"label": "pull-request build, unit, and contract workflow",
"url": "https://github.com/QwenAudio/SenseVoice/actions/runs/31633373807"
},
{
"label": "main-branch build, unit, and contract workflow",
"url": "https://github.com/QwenAudio/SenseVoice/actions/runs/31633730096"
},
{
"label": "server regression and contract tests",
"url": "https://github.com/QwenAudio/SenseVoice/tree/b054623cca8f015b73ec471dce4f473ac47413da/runtime/llama.cpp/tests"
}
],
"benchmarks": [],
"translations": {
"zh": {
"name": "SenseVoice 原生 OpenAI 实时服务",
"summary": "用一个无 Python 运行时的 C++ 二进制提供 SenseVoice REST、SSE、SRT/VTT 与 OpenAI Realtime WebSocket 转写。",
"fit": ["本地或内网 OpenAI 兼容语音接口", "CPU 优先、可选 CUDA 的低依赖部署", "需要文件转写、流式 partial/final 与服务端 VAD"],
"not_fit": ["需要预编译跨平台发布包", "需要多租户认证、计费或持久任务队列", "没有在目标硬件完成容量测试就直接暴露公网"],
"selection_reason": "原生 GGUF 服务把 REST 文件转写和实时 WebSocket 放进同一个受资源上限保护的二进制,适合私有兼容接口和边缘流式场景。",
"primary_limitation": "当前官方验证覆盖 Linux 源码构建;服务不内置生产认证、TLS、多租户配额或持久队列,必须由网关和基础设施补齐。",
"status_label": "生产验证",
"operations": ["固定 merge commit、SenseVoice GGUF 与 FSMN-VAD GGUF", "显式设置连接上限、单请求音频上限和读取超时", "发布前分别压测 REST、SSE 和 WebSocket,并保留旧二进制回滚"],
"security": ["默认绑定 127.0.0.1 或可信内网", "在反向代理实施 TLS、认证、MIME、请求体大小、速率和并发限制", "只加载经过来源与摘要校验的 GGUF"],
"troubleshooting": ["先用 CPU、已知 16 kHz WAV 和 REST JSON 路径验证", "确认模型文件名为 sensevoice-small-q8.gguf 与 fsmn-vad.gguf", "WebSocket 客户端发送 16 kHz 单声道 PCM16 的 base64 chunk"]
},
"en": {
"name": "SenseVoice native OpenAI realtime server",
"summary": "Use one C++ binary with no Python runtime to serve SenseVoice over REST, SSE, SRT/VTT, and OpenAI realtime WebSocket transcription.",
"fit": ["A local or private OpenAI-compatible speech endpoint", "CPU-first deployment with optional CUDA and few runtime dependencies", "File transcription, streaming partial/final results, and server-side VAD"],
"not_fit": ["Prebuilt cross-platform release packages are required", "Built-in multi-tenant authentication, billing, or durable job queues are required", "Public exposure before capacity testing on the target hardware"],
"selection_reason": "The native GGUF server combines REST file transcription and realtime WebSocket in one resource-bounded binary for private compatible APIs and edge streaming.",
"primary_limitation": "Official verification currently covers Linux source builds; the server does not include production authentication, TLS, tenant quotas, or durable queues, so provide them at the gateway and infrastructure layers.",
"status_label": "Production verified",
"operations": ["Pin the merge commit plus the SenseVoice and FSMN-VAD GGUF files", "Set the connection limit, per-request audio limit, and read timeout explicitly", "Load-test REST, SSE, and WebSocket separately before rollout, and retain the previous binary for rollback"],
"security": ["Bind to 127.0.0.1 or a trusted private network by default", "Enforce TLS, authentication, MIME, request-body size, rate, and concurrency at the reverse proxy", "Load only GGUF files with verified provenance and digests"],
"troubleshooting": ["Validate CPU, a known 16 kHz WAV, and the REST JSON path first", "Confirm the model files are named sensevoice-small-q8.gguf and fsmn-vad.gguf", "Send base64 chunks of 16 kHz mono PCM16 from WebSocket clients"]
}
}
},
{
"id": "audio-cpp",
"routes": {"zh": "/deploy/audio-cpp.html", "en": "/en/deploy/audio-cpp.html"},
"maturity": "community-verified",
"selector_rank": 21,
"workloads": ["edge", "batch", "private-api"],
"hardware": ["cpu", "nvidia-gpu", "desktop-edge-gpu"],
"priorities": ["portability", "compatibility"],
"models": ["Fun-ASR-Nano-2512", "SenseVoice-Small"],
"operating_systems": ["Linux", "macOS", "Windows"],
"interfaces": ["CLI", "OpenAI-compatible HTTP", "Buffered streaming CLI/SSE"],
"tested": {"funasr": "Fun-ASR-Nano-2512 + SenseVoice-Small", "runtime": "audio.cpp main@979e070f", "verified": "2026-08-13"},
"commands": {
"install": [
"git clone https://github.com/0xShug0/audio.cpp.git && cd audio.cpp",
"git checkout 979e070fc130bd499ad3fabeefc42b3884fff23a",
"bash scripts/build_linux.sh --backend cpu --model-set custom --models fun_asr_nano,sense_asr --target audiocpp_cli --target audiocpp_server",
"python3 tools/model_manager_v2.py install fun_asr_nano",
"python3 tools/model_manager_v2.py install sensevoice_small_q8"
],
"launch": [
"build/linux-cpu-release/bin/audiocpp_cli --task asr --family fun_asr_nano --model models/Fun-ASR-Nano-2512-GGUF/fun-asr-nano-2512-q8_0.gguf --backend cpu --audio speech.wav --text-out transcript.txt",
"build/linux-cpu-release/bin/audiocpp_cli --task asr --family sense_asr --model models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf --backend cpu --audio speech.wav --text-out sensevoice.txt",
"build/linux-cpu-release/bin/audiocpp_cli --task asr --family sense_asr --model models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf --backend cpu --mode streaming --audio - --request-option audio_chunk_duration_sec=5 --request-option audio_chunk_mode=none < 16k_s16.pcm",
"printf '%s\\n' '{\"host\":\"127.0.0.1\",\"port\":8080,\"backend\":\"cpu\",\"threads\":4,\"lazy_load\":true,\"models\":[{\"id\":\"fun-asr-nano\",\"family\":\"fun_asr_nano\",\"path\":\"models/Fun-ASR-Nano-2512-GGUF/fun-asr-nano-2512-q8_0.gguf\",\"task\":\"asr\",\"mode\":\"offline\"},{\"id\":\"sense-asr\",\"family\":\"sense_asr\",\"path\":\"models/SenseVoice-Small-GGUF/sensevoice-small-q8-audiocpp-v1.gguf\",\"task\":\"asr\",\"mode\":\"streaming\"}]}' > server.json",
"build/linux-cpu-release/bin/audiocpp_server --config server.json"
],
"health": [
"curl -fsS http://127.0.0.1:8080/health",
"curl -fsS http://127.0.0.1:8080/v1/models"
],
"smoke": [
"curl -fsS http://127.0.0.1:8080/v1/audio/transcriptions -F model=fun-asr-nano -F language=auto -F [email protected]",
"curl -fsS http://127.0.0.1:8080/v1/audio/transcriptions -F model=sense-asr -F language=auto -F [email protected]"
]
},
"evidence": [
{"label": "audio.cpp Fun-ASR-Nano guide", "url": "https://github.com/0xShug0/audio.cpp/blob/1778b23a5f6a4951c788e4bb0e7baa04f20012a2/docs/models/fun_asr_nano.md"},
{"label": "merged implementation", "url": "https://github.com/0xShug0/audio.cpp/pull/155"},
{"label": "pinned Fun-ASR-Nano GGUF package", "url": "https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512-GGUF/tree/ce72677f84900f0dc57f498ace253bfb3c9155b6"},
{"label": "merged SenseVoice implementation (6/6 CI)", "url": "https://github.com/0xShug0/audio.cpp/pull/219"},
{"label": "SenseVoice promotion to main", "url": "https://github.com/0xShug0/audio.cpp/pull/221"},
{"label": "pinned mainline integration commit", "url": "https://github.com/0xShug0/audio.cpp/commit/979e070fc130bd499ad3fabeefc42b3884fff23a"},
{"label": "pinned SenseVoice GGUF package (SHA-256 4dedf169f625437fb336f2959674f399819729a765e184128c0e25a6e16ff0ec)", "url": "https://huggingface.co/FunAudioLLM/SenseVoiceSmall-GGUF-audiocpp/tree/5c3fcfe748a8714216bc135476d5863084fddb72"}
],
"benchmarks": [
{
"model": "Fun-ASR-Nano-2512 Q8_0 GGUF",
"runtime": "audio.cpp native Fun-ASR-Nano runtime",
"hardware": "NVIDIA H100 CUDA; exact driver and toolchain versions are not recorded in the cited guide",
"workload": "Single offline transcription through the OpenAI-compatible server",
"audio": "One 14.07-second reference WAV",
"settings": "Standalone Q8_0 GGUF; CUDA backend; concurrency one; other runtime settings are not recorded",
"timing_scope": "End-to-end server validation; the cited guide does not state warmup, file I/O, or process-start boundaries",
"result": "Approximately 0.0725 RTF; transcript matched the official safetensors path",
"qualification": "Single-sample parity smoke, not a capacity benchmark. Reproduce on target hardware with production audio and concurrency.",
"source": "https://github.com/0xShug0/audio.cpp/blob/1778b23a5f6a4951c788e4bb0e7baa04f20012a2/docs/models/fun_asr_nano.md",
"verified": "2026-07-29"
},
{
"model": "SenseVoice-Small Q8_0 GGUF",
"runtime": "audio.cpp native sense_asr main@979e070f",
"hardware": "CPU; validation host details are not a cross-hardware benchmark contract",
"workload": "Offline Chinese transcription plus buffered streaming partials",
"audio": "Known 16 kHz reference WAV and PCM stream",
"settings": "Standalone schema-v1 Q8_0 GGUF; CPU backend; two-second streaming windows during validation",
"timing_scope": "Functional parity and loader validation; no capacity claim",
"result": "919 tensors loaded without sidecar overrides; transcript 开饭时间早上9点至下午5点。; GGUF SHA-256 4dedf169f625437fb336f2959674f399819729a765e184128c0e25a6e16ff0ec",
"qualification": "Merged into audio.cpp main with all six contribution CI jobs passing, but not yet included in a tagged release; pin the exact main commit.",
"source": "https://github.com/0xShug0/audio.cpp/commit/979e070fc130bd499ad3fabeefc42b3884fff23a",
"verified": "2026-08-13"
}
],
"translations": {
"zh": {
"name": "audio.cpp 原生 Fun-ASR-Nano 与 SenseVoice",
"summary": "用原生 C++ / GGML 在 CPU 或 GPU 上运行 Fun-ASR-Nano 与 SenseVoice Q8,并提供离线 CLI、兼容 OpenAI 的本地接口和 SenseVoice 缓冲流式结果。",
"fit": ["不希望安装 Python 推理环境", "桌面、边缘、离线批量与低依赖私有部署", "需要兼容 OpenAI 的本地音频接口或 SenseVoice 多语言流式 partial"],
"not_fit": ["需要词级时间戳或严格低延迟在线流式", "只允许使用正式 tagged release 的生产环境", "尚未在目标硬件完成容量与准确率复测的公网服务"],
"selection_reason": "Fun-ASR-Nano 与 SenseVoice 已进入 audio.cpp 主线,共用原生 GGML、Q8 模型及 CLI/HTTP 接口,适合低依赖部署。",
"primary_limitation": "SenseVoice 已合并到 audio.cpp main@979e070f,但尚未进入 tagged release;正式包发布前应固定该主线提交,且当前两条路径都不提供词级 timestamps。",
"status_label": "社区验证",
"operations": ["固定 audio.cpp 主线提交与两份 GGUF revision", "先用已知 WAV 核对两种模型的 transcript 再接业务流量", "分别记录冷启动、预热 RTF、内存、流式窗口与并发队列"],
"security": ["服务默认绑定 127.0.0.1 或可信内网", "在反向代理限制上传大小、MIME、认证与并发", "只加载经过 SHA-256 校验的 GGUF 和配置"],
"troubleshooting": ["先用 CPU + Q8_0 路径排除 GPU 环境问题", "构建 backend 必须与运行参数一致", "SenseVoice 输入异常时先核对 schema-v1 GGUF revision、WAV 或 16 kHz 单声道 PCM"]
},
"en": {
"name": "audio.cpp native Fun-ASR-Nano and SenseVoice",
"summary": "Run Fun-ASR-Nano and SenseVoice Q8 with native C++ and GGML on CPU or GPU, using offline CLI, a local OpenAI-compatible API, and buffered SenseVoice streaming results.",
"fit": ["No Python inference environment", "Desktop, edge, offline batch, and low-dependency private deployment", "A local OpenAI-compatible endpoint or multilingual SenseVoice streaming partials are required"],
"not_fit": ["Word-level timestamps or strict low-latency online streaming", "Production environments restricted to tagged releases", "Internet-facing service before capacity and accuracy tests on target hardware"],
"selection_reason": "Fun-ASR-Nano and SenseVoice are both on the audio.cpp main branch and share native GGML, Q8 weights, and CLI/HTTP surfaces for low-dependency deployment.",
"primary_limitation": "SenseVoice is merged into audio.cpp main@979e070f but is not yet in a tagged release. Pin that main commit until a release is published. Neither path currently provides word-level timestamps.",
"status_label": "Community verified",
"operations": ["Pin the audio.cpp main commit and both GGUF revisions", "Verify known WAV transcripts with both models before business traffic", "Measure cold start, warm RTF, memory, streaming windows, and concurrent queueing separately"],
"security": ["Bind the service to 127.0.0.1 or a trusted private network", "Enforce upload size, MIME, authentication, and concurrency at the proxy", "Load only GGUF and configuration files verified with SHA-256"],
"troubleshooting": ["Start with CPU and Q8_0 to isolate GPU setup", "Match the compiled backend to the runtime argument", "For SenseVoice input failures, verify the schema-v1 GGUF revision plus WAV or 16 kHz mono PCM"]
}
}
},
{
"id": "openai-api",
"routes": {"zh": "/deploy/openai-api.html", "en": "/en/deploy/openai-api.html"},
"maturity": "production-verified",
"selector_rank": 30,
"workloads": ["private-api", "batch"],
"hardware": ["cpu", "nvidia-gpu", "kubernetes"],
"priorities": ["compatibility", "portability"],
"models": ["sensevoice", "paraformer", "paraformer-en", "fun-asr-nano"],
"operating_systems": ["Linux", "macOS", "Windows"],
"interfaces": ["OpenAI-compatible HTTP", "OpenAPI", "Python SDK", "JavaScript"],
"tested": {"funasr": "1.3.29", "runtime": "FastAPI and Uvicorn example", "verified": "2026-07-26"},
"commands": {
"install": ["pip install funasr fastapi uvicorn python-multipart"],
"launch": ["cd examples/openai_api && python server.py --model sensevoice --device cpu --port 8000"],
"health": ["curl -fsS http://localhost:8000/health", "curl -fsS http://localhost:8000/v1/models"],
"smoke": ["cd examples/openai_api && python smoke_test.py --base-url http://localhost:8000"]
},
"evidence": [
{"label": "OpenAI API server", "url": "https://github.com/modelscope/FunASR/blob/main/examples/openai_api/README.md"},
{"label": "OpenAPI contract", "url": "https://github.com/modelscope/FunASR/blob/main/examples/openai_api/OPENAPI.md"},
{"label": "security guide", "url": "https://github.com/modelscope/FunASR/blob/main/examples/openai_api/SECURITY.md"}
],
"benchmarks": [],
"translations": {
"zh": {
"name": "OpenAI 兼容私有 API",
"summary": "把 FunASR 模型作为 `/v1/audio/transcriptions` 私有服务接入现有 SDK、代理和工作流。",
"fit": ["替换云端转写 API", "复用 OpenAI SDK 或 HTTP 客户端", "需要健康检查和模型枚举"],
"not_fit": ["需要浏览器直接匿名访问生产服务", "没有网关但准备暴露公网", "严格实时逐块字幕"],
"selection_reason": "稳定的 OpenAI 音频接口能以最少客户端改动接入现有应用。",
"primary_limitation": "示例服务不内置生产认证、租户配额和全套可观测性,这些必须由网关和基础设施补齐。",
"status_label": "生产验证",
"operations": ["用 `/health` 做就绪探针", "用 `/v1/models` 验证加载模型", "模型缓存使用持久卷并在发布前预热"],
"security": ["认证、配额、审计和 TLS 在网关实现", "限制 MIME、文件大小和请求时长", "服务仅接受可信反向代理流量"],
"troubleshooting": ["先运行仓库 smoke_test.py", "核对客户端 base_url 必须包含 `/v1`", "将模型加载错误与请求格式错误分开排查"]
},
"en": {
"name": "OpenAI-compatible private API",
"summary": "Serve FunASR models through `/v1/audio/transcriptions` for existing SDKs, agents, and workflows.",
"fit": ["Replacing a cloud transcription API", "Reusing OpenAI SDK or HTTP clients", "Health checks and model discovery are required"],
"not_fit": ["Anonymous browser access to a production worker", "Public exposure without a gateway", "Strict chunk-by-chunk live captions"],
"selection_reason": "A stable OpenAI audio contract integrates with existing applications with minimal client changes.",
"primary_limitation": "The example server does not include production authentication, tenant quotas, or full observability; provide them at the gateway and infrastructure layers.",
"status_label": "Production verified",
"operations": ["Use `/health` for readiness", "Use `/v1/models` to verify loaded models", "Persist and warm the model cache before rollout"],
"security": ["Implement authentication, quotas, audit, and TLS at the gateway", "Limit MIME types, file sizes, and request duration", "Accept traffic only from trusted reverse proxies"],
"troubleshooting": ["Run the repository smoke_test.py first", "Ensure the client base_url includes `/v1`", "Separate model-loading failures from request-format failures"]
}
}
},
{
"id": "realtime",
"routes": {"zh": "/deploy/realtime.html", "en": "/en/deploy/realtime.html"},
"maturity": "community-verified",
"selector_rank": 40,
"workloads": ["realtime"],
"hardware": ["cpu", "nvidia-gpu", "kubernetes"],
"priorities": ["latency", "compatibility"],
"models": ["Paraformer-online", "FSMN-VAD", "CT-Transformer punctuation", "Fun-ASR-Nano realtime"],
"operating_systems": ["Linux"],
"interfaces": ["WebSocket", "Python client", "C++ client", "Go client"],
"tested": {"funasr": "1.4.2", "runtime": "FunASR v1.4.2 Python WebSocket runtime", "verified": "2026-08-17"},
"commands": {
"install": ["python -m pip install -U \"funasr==1.4.2\"", "git clone --branch v1.4.2 --depth 1 https://github.com/modelscope/FunASR.git && python -m pip install -r FunASR/runtime/python/websocket/requirements_server.txt"],
"launch": ["cd FunASR/runtime/python/websocket && python funasr_wss_server.py --port 10095"],
"health": ["ss -ltn | grep 10095"],
"smoke": ["cd FunASR/runtime/python/websocket && python funasr_wss_client.py --host 127.0.0.1 --port 10095 --mode 2pass --chunk_size \"8,8,4\" --audio_in ../../funasr_api/asr_example.wav"]
},
"source_url": "/go/github",
"evidence": [
{"label": "Python WebSocket runtime", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/python/websocket/README.md"},
{"label": "WebSocket protocol", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/docs/websocket_protocol.md"},
{"label": "realtime benchmark method", "url": "https://github.com/modelscope/FunASR/blob/main/docs/benchmark/realtime_ws_benchmark.md"}
],
"benchmarks": [],
"translations": {
"zh": {
"name": "实时流式与字幕",
"summary": "通过 WebSocket 在线或 2-pass 协议处理麦克风、会议字幕和实时语音流。",
"fit": ["实时字幕", "客户端持续推送 PCM 音频", "需要在线初稿与离线修订"],
"not_fit": ["只转写完整文件", "客户端不能处理断线重连和背压", "未定义端点与最终结果语义"],
"selection_reason": "WebSocket 和 2-pass 协议为持续音频流提供低延迟初稿与最终结果。",
"primary_limitation": "真实容量由音频实时比、分块、VAD、连接数和客户端背压共同决定,文件 RTFx 不能替代并发测试。",
"status_label": "社区验证",
"operations": ["监控活动连接、积压音频秒数和最终结果延迟", "验证优雅断开与重连", "固定 chunk_size、VAD 和标点配置"],
"security": ["在反向代理限制来源、连接数和消息大小", "使用 TLS WebSocket", "不在 URL 查询参数传递长期凭证"],
"troubleshooting": ["先用仓库客户端和固定 WAV 验证", "确认 PCM16、16 kHz、单声道", "分别记录首稿和最终结果时刻"]
},
"en": {
"name": "Realtime streaming and captions",
"summary": "Use online or two-pass WebSocket protocols for microphones, meeting captions, and continuous speech streams.",
"fit": ["Live captions", "Clients continuously send PCM audio", "Online drafts plus offline correction"],
"not_fit": ["Complete-file transcription only", "Clients cannot handle reconnects and backpressure", "Endpoint and final-result semantics are undefined"],
"selection_reason": "WebSocket and two-pass protocols provide low-latency drafts and final results for continuous audio.",
"primary_limitation": "Capacity depends on realtime ratio, chunks, VAD, connection count, and client backpressure; file RTFx does not replace concurrency tests.",
"status_label": "Community verified",
"operations": ["Monitor active connections, queued audio seconds, and final latency", "Validate graceful disconnect and reconnect", "Pin chunk_size, VAD, and punctuation configuration"],
"security": ["Limit origins, connections, and message size at the proxy", "Use TLS WebSocket", "Do not put long-lived credentials in URL query strings"],
"troubleshooting": ["Start with the repository client and a fixed WAV", "Confirm PCM16, 16 kHz, mono audio", "Record draft and final-result timestamps separately"]
}
}
},
{
"id": "containers",
"routes": {"zh": "/deploy/containers.html", "en": "/en/deploy/containers.html"},
"maturity": "community-verified",
"selector_rank": 25,
"workloads": ["private-api", "batch", "realtime"],
"hardware": ["kubernetes", "cpu", "nvidia-gpu"],
"priorities": ["portability", "compatibility"],
"models": ["sensevoice", "paraformer", "fun-asr-nano"],
"operating_systems": ["Linux containers", "Kubernetes"],
"interfaces": ["Docker Compose", "Kubernetes ClusterIP", "OpenAI-compatible HTTP"],
"tested": {"funasr": "1.3.29", "runtime": "Docker Compose and Kubernetes templates", "verified": "2026-07-26"},
"commands": {
"install": ["cd examples/openai_api && cp .env.example .env"],
"launch": ["docker compose up --build"],
"health": ["curl -fsS http://localhost:8000/health"],
"smoke": ["python examples/openai_api/smoke_test.py --base-url http://localhost:8000"]
},
"evidence": [
{"label": "deployment matrix", "url": "https://github.com/modelscope/FunASR/blob/main/docs/deployment_matrix.md"},
{"label": "Kubernetes template", "url": "https://github.com/modelscope/FunASR/tree/main/examples/openai_api/kubernetes"},
{"label": "Docker installation", "url": "https://github.com/modelscope/FunASR/blob/main/docs/installation/docker.md"}
],
"benchmarks": [],
"translations": {
"zh": {
"name": "容器与 Kubernetes",
"summary": "用 Docker Compose 验证,再用带持久模型缓存、探针和 ClusterIP 的 Kubernetes 模板部署。",
"fit": ["需要可重复环境", "已有容器平台", "需要滚动发布和持久模型缓存"],
"not_fit": ["尚未验证本地模型和接口", "GPU 镜像与集群调度未准备", "把示例 Compose 直接当作公网生产架构"],
"selection_reason": "容器模板适合把已验证的 FunASR API 纳入现有发布和编排体系。",
"primary_limitation": "CPU 示例不是 CUDA 生产镜像;GPU 驱动、镜像、调度和容量仍需平台团队验证。",
"status_label": "社区验证",
"operations": ["模型缓存使用持久卷", "就绪探针检查 `/health`", "部署前完成镜像和模型预热"],
"security": ["默认使用私有 ClusterIP", "Secret 不写入镜像和仓库", "入口网关承担 TLS、认证和限流"],
"troubleshooting": ["先让 CPU Compose smoke test 通过", "区分镜像拉取、模型下载和推理错误", "GPU Pod 核对设备插件、驱动和运行时"]
},
"en": {
"name": "Containers and Kubernetes",
"summary": "Validate with Docker Compose, then deploy with the Kubernetes template, persistent model cache, probes, and private ClusterIP.",
"fit": ["Reproducible environments", "An existing container platform", "Rolling releases and persistent model cache"],
"not_fit": ["The local model and API are not yet validated", "GPU images and cluster scheduling are not ready", "Using the example Compose file as a public production architecture"],
"selection_reason": "Container templates bring an already validated FunASR API into an existing release and orchestration system.",
"primary_limitation": "The CPU example is not a CUDA production image; GPU drivers, image, scheduling, and capacity still require platform validation.",
"status_label": "Community verified",
"operations": ["Persist the model cache", "Use `/health` for readiness", "Warm images and models before rollout"],
"security": ["Use a private ClusterIP by default", "Keep secrets out of images and source control", "Put TLS, authentication, and rate limits at ingress"],
"troubleshooting": ["Pass the CPU Compose smoke test first", "Separate image pull, model download, and inference failures", "Check device plugin, driver, and runtime for GPU pods"]
}
}
},
{
"id": "cpu-runtime",
"routes": {"zh": "/deploy/cpu-runtime.html", "en": "/en/deploy/cpu-runtime.html"},
"maturity": "community-verified",
"selector_rank": 60,
"workloads": ["batch", "realtime", "edge"],
"hardware": ["cpu"],
"priorities": ["latency", "portability", "compatibility"],
"models": ["Paraformer", "Paraformer-online", "SenseVoiceSmall", "FSMN-VAD", "CT-Transformer punctuation"],
"operating_systems": ["Linux", "macOS", "Windows"],
"interfaces": ["C++", "ONNX Runtime", "Python ONNX", "HTTP", "WebSocket"],
"tested": {"funasr": "1.3.29", "runtime": "ONNX Runtime C++ and Python", "verified": "2026-07-26"},
"commands": {
"install": ["cd runtime/onnxruntime && cmake -B build -DCMAKE_BUILD_TYPE=Release", "cmake --build build -j"],
"launch": ["python runtime/python/onnxruntime/demo_sencevoice_small.py"],
"health": ["test -x runtime/onnxruntime/build/bin/funasr-onnx-offline"],
"smoke": ["python runtime/python/onnxruntime/demo_paraformer_offline.py"]
},
"evidence": [
{"label": "ONNX C++ runtime", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/onnxruntime/readme.md"},
{"label": "Python ONNX runtime", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/python/onnxruntime/README.md"},
{"label": "CPU benchmark guide", "url": "https://github.com/modelscope/FunASR/blob/main/runtime/docs/benchmark_onnx_cpp.md"}
],
"benchmarks": [],
"translations": {
"zh": {
"name": "ONNX / C++ CPU 运行时",
"summary": "在通用 CPU 上用 ONNX Runtime 和 C++ 路径部署离线、在线与 2-pass ASR。",
"fit": ["CPU 高并发服务", "嵌入式或本地应用", "现有 Paraformer 在线链路已经满足质量"],
"not_fit": ["需要 LLM 解码和开放式指令", "未经测试就迁移成熟 CPU 流量到 GPU", "依赖特定 Python 业务插件"],
"selection_reason": "ONNX/C++ 路径适合低依赖、可控延迟和已证明的 CPU 并发。",
"primary_limitation": "模型转换、算子支持和线程参数受平台影响;必须使用目标 CPU 与真实音频测容量。",
"status_label": "社区验证",
"operations": ["固定 ONNX Runtime 与模型版本", "分别设置模型、解码和 I/O 线程", "监控实时比、CPU 饱和度与队列"],
"security": ["限制本地文件和网络输入边界", "服务模式限制请求大小和并发", "模型目录只读并校验来源"],
"troubleshooting": ["先运行单文件离线示例", "线程增加不等于吞吐线性增加", "热词不足先评估确定性文本后处理"]
},
"en": {
"name": "ONNX / C++ CPU runtime",
"summary": "Deploy offline, online, and two-pass ASR on general CPUs with ONNX Runtime and C++ paths.",
"fit": ["High-concurrency CPU services", "Embedded or local applications", "An existing Paraformer online path already meets quality goals"],
"not_fit": ["LLM decoding and open-ended instructions", "Moving proven CPU traffic to GPU without tests", "Workflows that depend on Python-only business plugins"],
"selection_reason": "ONNX/C++ fits low-dependency, controlled-latency, proven CPU concurrency.",
"primary_limitation": "Model conversion, operator support, and thread settings vary by platform; measure capacity on the target CPU with real audio.",
"status_label": "Community verified",
"operations": ["Pin ONNX Runtime and model versions", "Tune model, decoder, and I/O threads separately", "Monitor realtime ratio, CPU saturation, and queue depth"],
"security": ["Bound local file and network inputs", "Limit request size and concurrency in service mode", "Keep model directories read-only and verify their source"],
"troubleshooting": ["Start with a single-file offline example", "More threads do not guarantee linear throughput", "Evaluate deterministic text post-processing before changing models for fixed terms"]
}
}
},
{
"id": "production",
"routes": {"zh": "/deploy/production.html", "en": "/en/deploy/production.html"},
"maturity": "production-verified",
"selector_rank": 70,
"workloads": ["private-api", "batch", "realtime", "edge"],
"hardware": ["cpu", "nvidia-gpu", "desktop-edge-gpu", "kubernetes"],
"priorities": ["compatibility", "latency", "throughput", "portability"],
"models": ["All supported deployment models"],
"operating_systems": ["Linux", "macOS", "Windows", "Kubernetes"],
"interfaces": ["HTTP", "WebSocket", "CLI", "C++"],
"tested": {"funasr": "1.3.29", "runtime": "cross-runtime production checklist", "verified": "2026-07-26"},
"commands": {
"install": ["git clone https://github.com/modelscope/FunASR.git"],
"launch": ["Run the selected runtime with pinned versions and an immutable model cache"],
"health": ["Verify readiness, model identity, and one known-audio transcript"],
"smoke": ["Record version, model digest, hardware, warmup, audio duration, concurrency, latency, and output before promotion"]
},
"evidence": [
{"label": "deployment matrix", "url": "https://github.com/modelscope/FunASR/blob/main/docs/deployment_matrix.md"},
{"label": "reproducible benchmark method", "url": "https://github.com/modelscope/FunASR/blob/main/docs/benchmark/rtf_reproducibility.md"},
{"label": "API security guide", "url": "https://github.com/modelscope/FunASR/blob/main/examples/openai_api/SECURITY.md"}
],
"benchmarks": [],
"translations": {
"zh": {
"name": "生产上线检查",
"summary": "把模型能运行推进到可观测、可扩缩、可回滚且边界清晰的生产服务。",
"fit": ["准备上线或扩容", "需要容量、就绪和安全边界", "需要可审计的发布与回滚"],
"not_fit": ["用单次演示结果代替容量测试", "省略网关和访问控制", "没有版本、模型和硬件记录的性能比较"],
"selection_reason": "无论选择哪种运行时,生产检查都是发布前的共同必经路径。",
"primary_limitation": "清单不能替代目标业务的负载、准确率、故障和安全演练。",
"status_label": "生产验证",
"operations": ["区分存活、就绪和业务 smoke test", "容量测试包含预热、队列和音频分布", "发布保留模型、配置和上一版本回滚点"],
"security": ["网络入口默认拒绝并最小授权", "上传、日志、缓存和临时文件有保留策略", "凭证、音频和预签名 URL 不写入普通日志"],
"troubleshooting": ["先确认模型和运行时精确版本", "使用已知音频区分基础设施与质量回归", "故障时先回滚再做在线试错"]
},
"en": {
"name": "Production rollout checklist",
"summary": "Move from a runnable model to an observable, scalable, reversible production service with explicit boundaries.",
"fit": ["Preparing a launch or scale-up", "Capacity, readiness, and security boundaries are required", "Auditable release and rollback are required"],
"not_fit": ["Using one demo result as a capacity test", "Omitting gateway and access control", "Comparing performance without version, model, and hardware records"],
"selection_reason": "Production readiness is a required path before release regardless of the selected runtime.",
"primary_limitation": "A checklist does not replace workload, accuracy, failure, and security exercises for the target product.",
"status_label": "Production verified",
"operations": ["Separate liveness, readiness, and business smoke tests", "Include warmup, queueing, and audio distribution in capacity tests", "Keep model, configuration, and previous-version rollback points"],
"security": ["Default-deny network access and grant minimum privileges", "Define retention for uploads, logs, caches, and temporary files", "Keep credentials, audio, and presigned URLs out of normal logs"],
"troubleshooting": ["Confirm exact model and runtime versions first", "Use known audio to separate infrastructure and quality regressions", "Roll back before experimenting during an incident"]
}
}
}
]
}