Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(vllm): honor fp32 decoder fallback

LauraGPT63d7098e7045
diff --git a/funasr/models/fun_asr_nano/inference_vllm.py b/funasr/models/fun_asr_nano/inference_vllm.py
index 3da3eecf1..331ba90cf 100644
--- a/funasr/models/fun_asr_nano/inference_vllm.py
+++ b/funasr/models/fun_asr_nano/inference_vllm.py
@@ -47,10 +47,11 @@ def _resolve_vllm_dtype(dtype: str) -> str:
"Fun-ASR-Nano's Qwen3 language model is numerically unstable in "
"float16; vLLM will use bfloat16 while audio components remain in "
"float16. This keeps the same memory footprint and avoids degraded "
- "or repetitive transcription."
+ "or repetitive transcription. On GPUs without bfloat16 support, "
+ "use dtype='fp32' so both audio and vLLM compute use float32."
)
return "bfloat16"
- return {"bf16": "bfloat16", "fp32": "auto"}.get(dtype, dtype)
+ return {"bf16": "bfloat16", "fp32": "float32"}.get(dtype, dtype)
def prepare_vllm_model_dir(model_dir: str, output_dir: str = None) -> str:
diff --git a/tests/test_fun_asr_nano_vllm_dtype.py b/tests/test_fun_asr_nano_vllm_dtype.py
index 48403ad85..0c83fd096 100644
--- a/tests/test_fun_asr_nano_vllm_dtype.py
+++ b/tests/test_fun_asr_nano_vllm_dtype.py
@@ -67,12 +67,13 @@ def test_fp16_keeps_audio_compute_but_promotes_vllm_to_bf16(monkeypatch, caplog)
assert streaming.torch_dtype is torch.float16
assert [call["dtype"] for call in _FakeLLM.calls] == ["bfloat16", "bfloat16"]
assert "audio components remain in float16" in caplog.text
+ assert "dtype='fp32'" in caplog.text
def test_vllm_dtype_mapping_preserves_supported_values(caplog):
with caplog.at_level(logging.WARNING):
assert inference_vllm._resolve_vllm_dtype("bf16") == "bfloat16"
- assert inference_vllm._resolve_vllm_dtype("fp32") == "auto"
+ assert inference_vllm._resolve_vllm_dtype("fp32") == "float32"
assert inference_vllm._resolve_vllm_dtype("custom") == "custom"
assert not caplog.text