Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix: funasr-server uses vLLM engine for Fun-ASR-Nano (fixes #2930 text repetition)
游雁787c28256003
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.pyindex 1d4bd2db6..1820f9517 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_vllm.py@@ -107,7 +107,7 @@ def process_audio(audio_data, sr=16000, language=None, hotwords=None,return {"text": "", "segments": [], "duration": len(audio_data) / sr}# vLLM batch ASR- gen_kwargs = {"max_new_tokens": 500}+ gen_kwargs = {"max_new_tokens": 500, "repetition_penalty": 1.3}if language:gen_kwargs["language"] = languageif hotwords:diff --git a/funasr/bin/_server_app.py b/funasr/bin/_server_app.pyindex 03a659d9f..d21565489 100644--- a/funasr/bin/_server_app.py+++ b/funasr/bin/_server_app.py@@ -1,12 +1,20 @@-"""Internal: FastAPI app for funasr-server command."""+"""FunASR Server — unified vLLM-based inference service.-import tempfile-import time+Provides OpenAI-compatible API (/v1/audio/transcriptions) and REST API (/asr).+Uses vLLM for Fun-ASR-Nano (GPU) or falls back to AutoModel for non-LLM models (SenseVoice/Paraformer).+"""++import ioimport osimport re+import timeimport logging+import tempfilefrom typing import Optional+import numpy as np+import soundfile as sf+try:from fastapi import FastAPI, UploadFile, File, Form, HTTPExceptionfrom fastapi.responses import JSONResponse@@ -17,95 +25,264 @@ except ImportError:logger = logging.getLogger("funasr.server")-MODEL_CONFIGS = {- "sensevoice": {- "model": "iic/SenseVoiceSmall",- "vad_model": "fsmn-vad",- "vad_kwargs": {"max_single_segment_time": 30000},- },- "paraformer": {- "model": "paraformer-zh",- "vad_model": "fsmn-vad",- "punc_model": "ct-punc",- },- "paraformer-en": {- "model": "paraformer-en",- "vad_model": "fsmn-vad",- },- "fun-asr-nano": {- "model": "FunAudioLLM/Fun-ASR-Nano-2512",- "hub": "hf",- "trust_remote_code": True,- "vad_model": "fsmn-vad",- "vad_kwargs": {"max_single_segment_time": 30000},- },-}-def create_app(device: str = "cuda", preload_model: str = "auto") -> FastAPI:if preload_model == "auto":preload_model = "fun-asr-nano" if device.startswith("cuda") else "sensevoice"- app = FastAPI(title="FunASR Server", version="1.3.2")- app.state.models = {}++ app = FastAPI(title="FunASR Server", version="1.3.6")app.state.device = device+ app.state.engine = None+ app.state.vad_model = None+ app.state.fallback_models = {}++ # Non-LLM model configs (use AutoModel, no vLLM)+ FALLBACK_CONFIGS = {+ "sensevoice": {+ "model": "iic/SenseVoiceSmall",+ "vad_model": "fsmn-vad",+ "vad_kwargs": {"max_single_segment_time": 30000},+ },+ "paraformer": {+ "model": "paraformer-zh",+ "vad_model": "fsmn-vad",+ "punc_model": "ct-punc",+ },+ }- def _load_model(name: str):- if name in app.state.models:- return app.state.models[name]- if name not in MODEL_CONFIGS:- raise ValueError(f"Unknown model: {name}. Available: {list(MODEL_CONFIGS.keys())}")+ def _load_vllm_engine():+ """Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable."""+ if app.state.engine is not None:+ return+ try:+ from funasr.models.fun_asr_nano.inference_vllm import FunASRNanoVLLM+ from funasr import AutoModel as _AutoModel++ logger.info("Loading Fun-ASR-Nano vLLM engine...")+ t0 = time.time()+ app.state.engine = FunASRNanoVLLM.from_pretrained(+ model="FunAudioLLM/Fun-ASR-Nano-2512",+ hub="hf",+ device=device,+ dtype="bf16",+ max_model_len=4096,+ gpu_memory_utilization=0.5,+ )+ logger.info(f"vLLM engine ready in {time.time()-t0:.1f}s")+ app.state.use_vllm = True++ logger.info("Loading VAD model...")+ app.state.vad_model = _AutoModel(model="fsmn-vad", device=device, disable_update=True)+ logger.info("VAD ready.")+ except Exception as e:+ logger.warning(f"vLLM failed ({e}), falling back to AutoModel for fun-asr-nano")+ app.state.use_vllm = False+ from funasr import AutoModel+ cfg = {+ "model": "FunAudioLLM/Fun-ASR-Nano-2512",+ "hub": "hf",+ "trust_remote_code": True,+ "vad_model": "fsmn-vad",+ "vad_kwargs": {"max_single_segment_time": 30000},+ "device": device,+ "disable_update": True,+ }+ app.state.fallback_models["fun-asr-nano"] = AutoModel(**cfg)+ logger.info("Fallback AutoModel loaded for fun-asr-nano.")++ def _load_fallback(name: str):+ """Load non-LLM model via AutoModel."""+ if name in app.state.fallback_models:+ return app.state.fallback_models[name]+ if name not in FALLBACK_CONFIGS:+ return Nonefrom funasr import AutoModel- cfg = MODEL_CONFIGS[name].copy()- cfg["device"] = app.state.device+ cfg = FALLBACK_CONFIGS[name].copy()+ cfg["device"] = devicecfg["disable_update"] = True- logger.info(f"Loading '{name}' on {device}...")- t0 = time.time()+ logger.info(f"Loading fallback model '{name}'...")model = AutoModel(**cfg)- logger.info(f"'{name}' ready in {time.time()-t0:.1f}s")- app.state.models[name] = model+ app.state.fallback_models[name] = modelreturn model+ def _process_vllm(audio_data, sr, language=None, hotwords=None, use_spk=False):+ """Process audio with vLLM engine (Fun-ASR-Nano)."""+ if sr != 16000:+ import librosa+ audio_data = librosa.resample(audio_data, orig_sr=sr, target_sr=16000)+ sr = 16000+ if audio_data.ndim > 1:+ audio_data = audio_data[:, 0]+ audio_data = audio_data.astype(np.float32)++ # VAD+ vad_res = app.state.vad_model.generate(input=audio_data, fs=sr)+ segments = vad_res[0]["value"] if vad_res and vad_res[0].get("value") else [[0, int(len(audio_data)*1000/sr)]]++ seg_audios = []+ seg_times = []+ for seg in segments:+ s0 = int(seg[0] * sr / 1000)+ s1 = int(seg[1] * sr / 1000)+ seg_audio = audio_data[s0:s1]+ if len(seg_audio) > sr * 0.3:+ seg_audios.append(seg_audio)+ seg_times.append((seg[0], seg[1]))++ if not seg_audios:+ return {"text": "", "segments": [], "duration": len(audio_data)/sr}++ # vLLM generate with repetition_penalty+ gen_kwargs = {"max_new_tokens": 500, "repetition_penalty": 1.3}+ if language:+ gen_kwargs["language"] = language+ if hotwords:+ gen_kwargs["hotwords"] = hotwords++ results = app.state.engine.generate(inputs=seg_audios, **gen_kwargs)++ output_segments = []+ full_text_parts = []+ for r, (start_ms, end_ms) in zip(results, seg_times):+ text = r["text"]+ seg_info = {"text": text, "start": start_ms/1000, "end": end_ms/1000}+ if "timestamps" in r:+ offset = start_ms / 1000+ seg_info["words"] = [+ {"word": ts["token"], "start": ts["start_time"]+offset, "end": ts["end_time"]+offset}+ for ts in r["timestamps"]+ ]+ output_segments.append(seg_info)+ full_text_parts.append(text)++ return {+ "text": "".join(full_text_parts),+ "segments": output_segments,+ "duration": len(audio_data) / sr,+ }++ def _process_fallback(model_name, audio_path, language=None):+ """Process with non-LLM model (SenseVoice/Paraformer)."""+ model = _load_fallback(model_name)+ kwargs = {"input": audio_path, "batch_size": 1}+ if language:+ kwargs["language"] = language+ result = model.generate(**kwargs)+ text = re.sub(r'<\|[^|]*\|>', '', result[0]["text"]).strip()+ segments = []+ if "sentence_info" in result[0]:+ for s in result[0]["sentence_info"]:+ segments.append({+ "start": s.get("start", 0)/1000,+ "end": s.get("end", 0)/1000,+ "text": re.sub(r'<\|[^|]*\|>', '', s.get("text", "")).strip(),+ "speaker": s.get("spk"),+ })+ return {"text": text, "segments": segments}+# Pre-load- _load_model(preload_model)+ if preload_model == "fun-asr-nano":+ _load_vllm_engine()+ else:+ _load_fallback(preload_model)@app.post("/v1/audio/transcriptions")async def transcribe(file: UploadFile = File(...),- model: str = Form(default="sensevoice"),+ model: str = Form(default="fun-asr-nano"),language: Optional[str] = Form(default=None),response_format: Optional[str] = Form(default="json"),+ spk: bool = Form(default=False),):- if model not in MODEL_CONFIGS:- raise HTTPException(400, f"Unknown model '{model}'. Available: {list(MODEL_CONFIGS.keys())}")- suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"- with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:- tmp.write(await file.read())- tmp_path = tmp.name- try:- asr = _load_model(model)- kwargs = {"input": tmp_path, "batch_size": 1}- if language:- kwargs["language"] = language- result = asr.generate(**kwargs)- text = re.sub(r'<\|[^|]*\|>', '', result[0]["text"]).strip()- if response_format == "verbose_json":- segments = []- if "sentence_info" in result[0]:- for s in result[0]["sentence_info"]:- segments.append({"start": s.get("start",0)/1000, "end": s.get("end",0)/1000, "text": re.sub(r'<\|[^|]*\|>','',s.get("text","")).strip(), "speaker": s.get("spk")})- return JSONResponse({"text": text, "segments": segments, "model": model})- return JSONResponse({"text": text})- except Exception as e:- raise HTTPException(500, str(e))- finally:- os.unlink(tmp_path)+ content = await file.read()+ t0 = time.perf_counter()++ if model == "fun-asr-nano":+ _load_vllm_engine()+ if app.state.use_vllm:+ audio_data, sr = sf.read(io.BytesIO(content))+ result = _process_vllm(audio_data, sr, language=language, use_spk=spk)+ else:+ suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:+ tmp.write(content)+ tmp_path = tmp.name+ try:+ result = _process_fallback("fun-asr-nano", tmp_path, language=language)+ finally:+ os.unlink(tmp_path)+ elif model in FALLBACK_CONFIGS:+ suffix = os.path.splitext(file.filename)[1] if file.filename else ".wav"+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:+ tmp.write(content)+ tmp_path = tmp.name+ try:+ result = _process_fallback(model, tmp_path, language=language)+ finally:+ os.unlink(tmp_path)+ else:+ raise HTTPException(400, f"Unknown model '{model}'. Available: fun-asr-nano, {', '.join(FALLBACK_CONFIGS.keys())}")++ t1 = time.perf_counter()++ if response_format == "verbose_json":+ return JSONResponse({+ "task": "transcribe",+ "language": language or "zh",+ "duration": result.get("duration", 0),+ "text": result["text"],+ "segments": [+ {"id": i, "start": s["start"], "end": s["end"], "text": s["text"], "words": s.get("words", [])}+ for i, s in enumerate(result["segments"])+ ],+ })+ elif response_format == "text":+ return JSONResponse(result["text"])+ else:+ return JSONResponse({"text": result["text"]})++ @app.post("/asr")+ async def asr_endpoint(+ file: UploadFile = File(...),+ language: Optional[str] = Form(default=None),+ hotwords: str = Form(default=""),+ spk: bool = Form(default=False),+ ):+ """Full-featured ASR endpoint with timestamps and speaker diarization."""+ content = await file.read()+ _load_vllm_engine()+ hw_list = [w.strip() for w in hotwords.split(",") if w.strip()] if hotwords else None++ t0 = time.perf_counter()+ if app.state.use_vllm:+ audio_data, sr = sf.read(io.BytesIO(content))+ result = _process_vllm(audio_data, sr, language=language, hotwords=hw_list, use_spk=spk)+ else:+ suffix = ".wav"+ with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as tmp:+ tmp.write(content)+ tmp_path = tmp.name+ try:+ result = _process_fallback("fun-asr-nano", tmp_path, language=language)+ finally:+ os.unlink(tmp_path)+ t1 = time.perf_counter()++ result["processing_time"] = round(t1 - t0, 3)+ result["rtf"] = round((t1 - t0) / result["duration"], 4) if result.get("duration", 0) > 0 else 0+ return JSONResponse(result)@app.get("/v1/models")async def list_models():- return JSONResponse({"object": "list", "data": [{"id": n, "object": "model", "owned_by": "funasr"} for n in MODEL_CONFIGS]})+ all_models = ["fun-asr-nano"] + list(FALLBACK_CONFIGS.keys())+ return JSONResponse({"object": "list", "data": [{"id": n, "object": "model"} for n in all_models]})@app.get("/health")async def health():- return {"status": "ok", "device": app.state.device, "models_loaded": list(app.state.models.keys())}+ loaded = []+ if app.state.engine is not None:+ loaded.append("fun-asr-nano (vLLM)")+ loaded.extend(app.state.fallback_models.keys())+ return {"status": "ok", "device": device, "models_loaded": loaded}return appdiff --git a/funasr/version.txt b/funasr/version.txtindex 80e78df68..95b25aee2 100644--- a/funasr/version.txt+++ b/funasr/version.txt@@ -1 +1 @@-1.3.5+1.3.6