Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat: update non-streaming dynamic VAD schedule + unified vLLM server
xiaoyunchong.xyc7e2c3f44f2fb
diff --git a/benchmark_vllm.py b/benchmark_vllm.pynew file mode 100644index 000000000..1d8fb04ff--- /dev/null+++ b/benchmark_vllm.py@@ -0,0 +1,224 @@+#!/usr/bin/env python3+"""FunASR vLLM Benchmark: unified speed + CER comparison for all supported models.++Supports: Fun-ASR-Nano, GLM-ASR-Nano (and any model via AutoModelVLLM).++Usage:+ # Fun-ASR-Nano+ CUDA_VISIBLE_DEVICES=0 python benchmark_vllm.py \+ --model FunAudioLLM/Fun-ASR-Nano-2512 \+ --audio-dir /path/to/benchmark_audio \+ --label-json /path/to/benchmark_testset.json++ # GLM-ASR-Nano+ CUDA_VISIBLE_DEVICES=0 python benchmark_vllm.py \+ --model zai-org/GLM-ASR-Nano-2512 \+ --audio-dir /path/to/benchmark_audio \+ --label-json /path/to/benchmark_testset.json++ # Quick test (first N files)+ CUDA_VISIBLE_DEVICES=0 python benchmark_vllm.py \+ --model FunAudioLLM/Fun-ASR-Nano-2512 \+ --audio-dir /path/to/benchmark_audio \+ --label-json /path/to/benchmark_testset.json \+ --max-files 20++ # Skip PyTorch (only test vLLM)+ CUDA_VISIBLE_DEVICES=0 python benchmark_vllm.py \+ --model FunAudioLLM/Fun-ASR-Nano-2512 \+ --skip-pytorch \+ --audio-dir /path/to/benchmark_audio \+ --label-json /path/to/benchmark_testset.json+"""++import argparse+import json+import os+import re+import time++import kaldialign+import numpy as np+import soundfile as sf+import torch+++def normalize_zh(text):+ text = re.sub(r'[^\w一-鿿]', '', text)+ return text.upper()+++def compute_cer(refs, hyps):+ total_ref = 0+ total_errs = 0+ for ref, hyp in zip(refs, hyps):+ r = list(normalize_zh(ref))+ h = list(normalize_zh(hyp))+ total_ref += len(r)+ ali = kaldialign.align(r, h, '*')+ total_errs += sum(1 for a, b in ali if a != b)+ return total_errs / total_ref * 100 if total_ref > 0 else 0+++def vad_segment(files, device="cuda:0"):+ from funasr import AutoModel+ vad_model = AutoModel(model="fsmn-vad", device=device, disable_update=True)+ all_segments = []+ for fi, wav_path in enumerate(files):+ audio, sr = sf.read(wav_path)+ if audio.ndim > 1:+ audio = audio[:, 0]+ audio = audio.astype(np.float32)+ res = vad_model.generate(input=wav_path, dynamic_silence=False)+ for seg in res[0]["value"]:+ s0 = int(seg[0] * sr / 1000)+ s1 = int(seg[1] * sr / 1000)+ seg_audio = audio[s0:s1]+ if len(seg_audio) > sr * 0.5:+ all_segments.append((fi, seg_audio))+ return all_segments+++def concat_results(all_segments, seg_texts, n_files):+ file_texts = {}+ for (fi, _), text in zip(all_segments, seg_texts):+ file_texts.setdefault(fi, []).append(text)+ return ["".join(file_texts.get(fi, [])) for fi in range(n_files)]+++def run_pytorch(model_name, seg_files, device="cuda:0"):+ from funasr import AutoModel++ kwargs = {"model": model_name, "device": device, "disable_update": True}+ if "Fun-ASR-Nano" in model_name:+ kwargs["trust_remote_code"] = True+ kwargs["remote_code"] = os.path.join(+ os.path.dirname(__file__),+ "examples/industrial_data_pretraining/fun_asr_nano/model.py"+ )++ model = AutoModel(**kwargs)+ model.generate(input=seg_files[0]) # warmup++ t0 = time.perf_counter()+ texts = []+ for f in seg_files:+ res = model.generate(input=f)+ texts.append(res[0]["text"])+ t1 = time.perf_counter()+ return t1 - t0, texts+++def run_vllm(model_name, seg_files, device="cuda:0", hub="ms"):+ if "Fun-ASR-Nano" in model_name:+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM+ engine = FunASRNanoVLLM.from_pretrained(+ model=model_name, hub=hub, device=device, dtype="bf16",+ max_model_len=4096, gpu_memory_utilization=0.5)+ engine.generate(inputs=[seg_files[0]], language="中文") # warmup+ t0 = time.perf_counter()+ results = engine.generate(inputs=seg_files, language="中文", max_new_tokens=500)+ t1 = time.perf_counter()+ texts = [r["text"] for r in results]++ elif "GLM-ASR" in model_name:+ from funasr.models.glm_asr.inference_vllm import GLMASRVLLMEngine+ engine = GLMASRVLLMEngine.from_pretrained(+ model=model_name, hub=hub, device=device, dtype="bf16",+ gpu_memory_utilization=0.4, max_model_len=4096)+ engine.generate(inputs=[seg_files[0]]) # warmup+ t0 = time.perf_counter()+ results = engine.generate(inputs=seg_files, max_new_tokens=500)+ t1 = time.perf_counter()+ texts = [r["text"] for r in results]++ else:+ from funasr.auto.auto_model_vllm import AutoModelVLLM+ engine = AutoModelVLLM(model=model_name, hub=hub, device=device)+ engine.generate(inputs=[seg_files[0]])+ t0 = time.perf_counter()+ results = engine.generate(inputs=seg_files, max_new_tokens=500)+ t1 = time.perf_counter()+ texts = [r["text"] for r in results]++ return t1 - t0, texts+++if __name__ == '__main__':+ parser = argparse.ArgumentParser(description="FunASR vLLM Benchmark")+ parser.add_argument("--model", type=str, required=True, help="Model name or path")+ parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])+ parser.add_argument("--audio-dir", type=str, required=True)+ parser.add_argument("--label-json", type=str, required=True)+ parser.add_argument("--device", type=str, default="cuda:0")+ parser.add_argument("--max-files", type=int, default=0)+ parser.add_argument("--skip-pytorch", action="store_true")+ args = parser.parse_args()++ with open(args.label_json) as f:+ dataset = json.load(f)++ files = []+ refs = []+ for item in dataset:+ wav_path = os.path.join(args.audio_dir, f"{item['id']:03d}.wav")+ if os.path.exists(wav_path):+ files.append(wav_path)+ refs.append(item["ref"])+ if args.max_files > 0:+ files = files[:args.max_files]+ refs = refs[:args.max_files]++ total_audio = sum(sf.info(f).duration for f in files)+ print(f"{'='*60}")+ print(f"FunASR vLLM Benchmark")+ print(f"{'='*60}")+ print(f"Model: {args.model}")+ print(f"Dataset: {len(files)} files, {total_audio:.0f}s audio")++ # VAD+ print(f"\n>>> VAD pre-segmenting...")+ all_segments = vad_segment(files, device=args.device)+ print(f" {len(all_segments)} segments")++ os.makedirs("/tmp/benchmark_vllm_segs", exist_ok=True)+ seg_files = []+ for i, (fi, audio) in enumerate(all_segments):+ path = f"/tmp/benchmark_vllm_segs/{i:04d}.wav"+ sf.write(path, audio, 16000)+ seg_files.append(path)++ # PyTorch+ cer_pt = None+ pt_time = None+ if not args.skip_pytorch:+ print(f"\n>>> PyTorch native...")+ pt_time, pt_seg_texts = run_pytorch(args.model, seg_files, device=args.device)+ pt_texts = concat_results(all_segments, pt_seg_texts, len(files))+ cer_pt = compute_cer(refs, pt_texts)+ print(f" Time: {pt_time:.1f}s | RTFx: {total_audio/pt_time:.1f} | CER: {cer_pt:.2f}%")++ del torch.cuda.memory_allocated+ torch.cuda.empty_cache()++ # vLLM+ print(f"\n>>> vLLM...")+ vllm_time, vllm_seg_texts = run_vllm(args.model, seg_files, device=args.device, hub=args.hub)+ vllm_texts = concat_results(all_segments, vllm_seg_texts, len(files))+ cer_vllm = compute_cer(refs, vllm_texts)+ print(f" Time: {vllm_time:.1f}s | RTFx: {total_audio/vllm_time:.1f} | CER: {cer_vllm:.2f}%")++ # Summary+ print(f"\n{'='*60}")+ print(f"RESULTS")+ print(f"{'-'*60}")+ print(f"{'Method':<20} {'Time':<10} {'RTFx':<10} {'CER'}")+ print(f"{'-'*60}")+ if not args.skip_pytorch:+ print(f"{'PyTorch':<20} {pt_time:<10.1f} {total_audio/pt_time:<10.1f} {cer_pt:.2f}%")+ print(f"{'vLLM':<20} {vllm_time:<10.1f} {total_audio/vllm_time:<10.1f} {cer_vllm:.2f}%")+ if not args.skip_pytorch:+ print(f"{'-'*60}")+ speedup = (total_audio/vllm_time) / (total_audio/pt_time)+ print(f"Speedup: {speedup:.1f}x | CER diff: {cer_vllm - cer_pt:+.2f}%")+ print(f"{'='*60}")diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.pynew file mode 100644index 000000000..ef54c69eb--- /dev/null+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py@@ -0,0 +1,362 @@+#!/usr/bin/env python3+"""Fun-ASR-Nano vLLM Inference Server.++Unified server with three interfaces:+- HTTP REST: POST /asr (file upload)+- WebSocket: ws://host:port/ws (streaming audio)+- OpenAI API: POST /v1/audio/transcriptions (Whisper-compatible)++All endpoints share the same vLLM engine + dynamic VAD + SPK + timestamps.++Usage:+ CUDA_VISIBLE_DEVICES=0 python serve_vllm.py --port 8000+ CUDA_VISIBLE_DEVICES=0 python serve_vllm.py --port 8000 --model FunAudioLLM/Fun-ASR-Nano-2512+"""++import asyncio+import argparse+import io+import json+import logging+import os+import re+import time+import tempfile++import numpy as np+import soundfile as sf+import torch+import warnings++warnings.filterwarnings('ignore')+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')+logger = logging.getLogger(__name__)++try:+ from fastapi import FastAPI, File, UploadFile, Form, WebSocket, WebSocketDisconnect+ from fastapi.responses import JSONResponse+ import uvicorn+except ImportError:+ raise ImportError("pip install fastapi uvicorn python-multipart")++from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM+from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVAD+from funasr import AutoModel+++# ============================================================+# Global state+# ============================================================+_engine = None+_vad_model = None+_spk_model = None+_args = None+++def load_engine(args):+ global _engine, _vad_model, _spk_model, _args+ _args = args+ if _engine is None:+ logger.info(f"Loading vLLM engine: {args.model}")+ _engine = FunASRNanoVLLM.from_pretrained(+ model=args.model, hub=args.hub, device=args.device, dtype=args.dtype,+ max_model_len=args.max_model_len,+ gpu_memory_utilization=args.gpu_memory_utilization,+ )+ logger.info("Loading VAD: fsmn-vad")+ _vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True)+ logger.info("Loading SPK: eres2netv2")+ _spk_model = AutoModel(model="iic/speech_eres2netv2_sv_zh-cn_16k-common", device=args.device, disable_update=True)+ logger.info("All models ready!")+++def process_audio(audio_data, sr=16000, language=None, hotwords=None,+ use_vad=True, use_spk=False, use_timestamp=True):+ """Core processing: VAD segment → vLLM ASR → timestamps → SPK."""+ if sr != 16000:+ import librosa+ audio_data = librosa.resample(audio_data, orig_sr=sr, target_sr=16000)+ sr = 16000++ if audio_data.ndim > 1:+ audio_data = audio_data[:, 0]+ audio_data = audio_data.astype(np.float32)++ # VAD segmentation+ if use_vad and len(audio_data) > sr * 1:+ vad_res = _vad_model.generate(input=audio_data, fs=sr)+ segments = vad_res[0]["value"]+ else:+ segments = [[0, int(len(audio_data) * 1000 / sr)]]++ if not segments:+ return {"text": "", "segments": [], "duration": len(audio_data) / sr}++ # Extract segment audio+ seg_audios = []+ seg_times = []+ for seg in segments:+ s0 = int(seg[0] * sr / 1000)+ s1 = int(seg[1] * sr / 1000)+ seg_audio = audio_data[s0:s1]+ if len(seg_audio) > sr * 0.3:+ seg_audios.append(seg_audio)+ seg_times.append((seg[0], seg[1]))++ if not seg_audios:+ return {"text": "", "segments": [], "duration": len(audio_data) / sr}++ # vLLM batch ASR+ gen_kwargs = {"max_new_tokens": 500}+ if language:+ gen_kwargs["language"] = language+ if hotwords:+ gen_kwargs["hotwords"] = hotwords++ results = _engine.generate(inputs=seg_audios, **gen_kwargs)++ # Build segments with timestamps+ output_segments = []+ full_text_parts = []++ for i, (r, (start_ms, end_ms)) in enumerate(zip(results, seg_times)):+ seg_info = {+ "text": r["text"],+ "start": start_ms / 1000,+ "end": end_ms / 1000,+ }+ if use_timestamp and "timestamps" in r:+ # Offset timestamps by segment start+ offset = start_ms / 1000+ seg_info["words"] = [+ {"word": ts["token"], "start": ts["start_time"] + offset, "end": ts["end_time"] + offset}+ for ts in r["timestamps"]+ ]+ output_segments.append(seg_info)+ full_text_parts.append(r["text"])++ # SPK diarization+ if use_spk and _spk_model is not None:+ from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk+ from funasr.models.campplus.cluster_backend import ClusterBackend++ vad_segs = [[st, et, audio_data[int(st*sr):int(et*sr)]]+ for st, et in [(s["start"], s["end"]) for s in output_segments]]+ chunks = sv_chunk(vad_segs)+ if chunks:+ speech_list = [ch[2] for ch in chunks]+ spk_res = _spk_model.generate(input=speech_list, cache={}, is_final=True)+ embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)+ cluster = ClusterBackend(merge_thr=0.78).to(_args.device)+ labels = cluster(embs.cpu(), oracle_num=None)+ if not isinstance(labels, np.ndarray):+ labels = np.array(labels)+ all_sorted = sorted(chunks, key=lambda x: x[0])+ sv_output = postprocess(all_sorted, None, labels, embs.cpu())+ sentences = [{"text": s["text"], "start": int(s["start"]*1000), "end": int(s["end"]*1000)}+ for s in output_segments]+ distribute_spk(sentences, sv_output)+ for i, s in enumerate(sentences):+ output_segments[i]["speaker"] = f"SPK{s.get('spk', 0)}"++ return {+ "text": " ".join(full_text_parts),+ "segments": output_segments,+ "duration": len(audio_data) / sr,+ }+++# ============================================================+# FastAPI App+# ============================================================+app = FastAPI(title="Fun-ASR-Nano vLLM Server", version="1.0")++[email protected]_event("startup")+async def startup():+ load_engine(_args)+++# --- HTTP REST: POST /asr ---[email protected]("/asr")+async def asr_endpoint(+ file: UploadFile = File(...),+ language: str = Form(default=None),+ hotwords: str = Form(default=""),+ spk: bool = Form(default=False),+ timestamp: bool = Form(default=True),+):+ """ASR with file upload. Returns text + segments + timestamps + speaker."""+ content = await file.read()+ audio_data, sr = sf.read(io.BytesIO(content))++ hw_list = [w.strip() for w in hotwords.split(",") if w.strip()] if hotwords else None++ t0 = time.perf_counter()+ result = process_audio(audio_data, sr=sr, language=language,+ hotwords=hw_list, use_spk=spk, use_timestamp=timestamp)+ t1 = time.perf_counter()++ result["processing_time"] = round(t1 - t0, 3)+ result["rtf"] = round((t1 - t0) / result["duration"], 4) if result["duration"] > 0 else 0+ return JSONResponse(content=result)+++# --- OpenAI API: POST /v1/audio/transcriptions ---[email protected]("/v1/audio/transcriptions")+async def openai_transcriptions(+ file: UploadFile = File(...),+ model: str = Form(default="fun-asr-nano"),+ language: str = Form(default=None),+ response_format: str = Form(default="json"),+ timestamp_granularities: str = Form(default="word"),+):+ """OpenAI Whisper-compatible transcription API."""+ content = await file.read()+ audio_data, sr = sf.read(io.BytesIO(content))++ use_ts = "word" in timestamp_granularities or "segment" in timestamp_granularities+ result = process_audio(audio_data, sr=sr, language=language, use_timestamp=use_ts)++ if response_format == "text":+ return JSONResponse(content=result["text"])+ elif response_format == "verbose_json":+ return JSONResponse(content={+ "task": "transcribe",+ "language": language or "zh",+ "duration": result["duration"],+ "text": result["text"],+ "segments": [+ {+ "id": i,+ "start": seg["start"],+ "end": seg["end"],+ "text": seg["text"],+ "words": seg.get("words", []),+ }+ for i, seg in enumerate(result["segments"])+ ],+ })+ else:+ return JSONResponse(content={"text": result["text"]})+++# --- WebSocket: ws://host:port/ws ---[email protected]("/ws")+async def websocket_endpoint(websocket: WebSocket):+ """Streaming WebSocket ASR with dynamic VAD + SPK."""+ await websocket.accept()+ logger.info(f"WebSocket connected: {websocket.client}")++ vad = DynamicStreamingVAD(_vad_model)+ audio_buffer = np.array([], dtype=np.float32)+ locked_sentences = []+ language = None+ hotwords = None+ is_active = False++ try:+ while True:+ message = await websocket.receive()++ if "text" in message:+ cmd = message["text"].strip()+ if cmd.upper() == "START":+ vad.reset()+ audio_buffer = np.array([], dtype=np.float32)+ locked_sentences = []+ is_active = True+ await websocket.send_json({"event": "started"})+ elif cmd.upper().startswith("LANGUAGE:"):+ language = cmd[9:].strip() or None+ await websocket.send_json({"event": "language_set", "language": language})+ elif cmd.upper().startswith("HOTWORDS:"):+ hotwords = [w.strip() for w in cmd[9:].split(",") if w.strip()]+ await websocket.send_json({"event": "hotwords_set", "hotwords": hotwords})+ elif cmd.upper() == "STOP":+ if is_active and len(audio_buffer) > 0:+ # Final: process remaining audio+ final_segs = vad.finalize()+ for seg in final_segs:+ seg_audio = audio_buffer[int(seg[0]*16):int(seg[1]*16)]+ if len(seg_audio) > 8000:+ gen_kw = {"max_new_tokens": 500}+ if language: gen_kw["language"] = language+ if hotwords: gen_kw["hotwords"] = hotwords+ res = _engine.generate(inputs=[seg_audio], **gen_kw)+ if res[0]["text"].strip():+ locked_sentences.append({+ "text": res[0]["text"], "start": seg[0], "end": seg[1]+ })++ # Handle ongoing speech+ if vad.is_speaking:+ end_ms = int(len(audio_buffer) * 1000 / 16000)+ start_ms = int(vad.current_speech_start) if hasattr(vad, 'current_speech_start') and vad.current_speech_start else 0+ seg_audio = audio_buffer[int(start_ms*16):]+ if len(seg_audio) > 8000:+ gen_kw = {"max_new_tokens": 500}+ if language: gen_kw["language"] = language+ if hotwords: gen_kw["hotwords"] = hotwords+ res = _engine.generate(inputs=[seg_audio], **gen_kw)+ if res[0]["text"].strip():+ locked_sentences.append({+ "text": res[0]["text"], "start": start_ms, "end": end_ms+ })++ await websocket.send_json({+ "sentences": locked_sentences,+ "is_final": True,+ "duration_ms": int(len(audio_buffer) * 1000 / 16000),+ })+ is_active = False+ await websocket.send_json({"event": "stopped"})++ elif "bytes" in message and is_active:+ pcm = np.frombuffer(message["bytes"], dtype=np.int16).astype(np.float32) / 32768.0+ audio_buffer = np.concatenate([audio_buffer, pcm])++ # Feed VAD+ new_confirmed = vad.feed(torch.from_numpy(pcm).float())+ for seg in new_confirmed:+ seg_audio = audio_buffer[int(seg[0]*16):int(seg[1]*16)]+ if len(seg_audio) > 8000:+ gen_kw = {"max_new_tokens": 500}+ if language: gen_kw["language"] = language+ if hotwords: gen_kw["hotwords"] = hotwords+ res = _engine.generate(inputs=[seg_audio], **gen_kw)+ if res[0]["text"].strip():+ locked_sentences.append({+ "text": res[0]["text"], "start": seg[0], "end": seg[1]+ })++ # Send partial update+ await websocket.send_json({+ "sentences": locked_sentences,+ "is_final": False,+ "duration_ms": int(len(audio_buffer) * 1000 / 16000),+ })++ except WebSocketDisconnect:+ logger.info("WebSocket disconnected")+ except Exception as e:+ logger.error(f"WebSocket error: {e}", exc_info=True)+++# ============================================================+# Main+# ============================================================+if __name__ == "__main__":+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano vLLM Server")+ parser.add_argument("--port", type=int, default=8000)+ parser.add_argument("--host", type=str, default="0.0.0.0")+ parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")+ parser.add_argument("--hub", type=str, default="ms")+ parser.add_argument("--device", type=str, default="cuda:0")+ parser.add_argument("--dtype", type=str, default="bf16")+ parser.add_argument("--max-model-len", type=int, default=4096)+ parser.add_argument("--gpu-memory-utilization", type=float, default=0.5)+ _args = parser.parse_args()++ load_engine(_args)+ uvicorn.run(app, host=_args.host, port=_args.port)diff --git a/funasr/models/fsmn_vad_streaming/model.py b/funasr/models/fsmn_vad_streaming/model.pyindex dc0c394c1..15e472502 100644--- a/funasr/models/fsmn_vad_streaming/model.py+++ b/funasr/models/fsmn_vad_streaming/model.py@@ -31,11 +31,13 @@ STREAMING_SILENCE_SCHEDULE = [# Non-streaming mode: shorter patience (audio already complete, cut by natural pauses)DEFAULT_SILENCE_SCHEDULE = [- (5000, 800),- (10000, 600),- (20000, 500),- (30000, 400),- (float("inf"), 300),+ (10000, 2000),+ (20000, 1000),+ (30000, 800),+ (40000, 600),+ (50000, 400),+ (60000, 200),+ (float("inf"), 100),]