Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(realtime): validate and forward vLLM HF overrides
zhifu gao73ae515788a4
diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.ymlindex 4a7bcec5c..6e708f0f4 100644--- a/.github/workflows/test-numpy-compatibility.yml+++ b/.github/workflows/test-numpy-compatibility.yml@@ -9,6 +9,9 @@ on:- "tests/test_cli.py"- "funasr/auto/auto_model.py"- "funasr/utils/load_utils.py"+ - "funasr/bin/realtime_ws.py"+ - "tests/test_realtime_ws_service.py"+ - "tests/test_realtime_ws_hf_overrides.py"- "funasr/utils/timestamp_tools.py"- "funasr/models/paraformer/model.py"- "funasr/models/contextual_paraformer/model.py"@@ -40,6 +43,9 @@ on:- "tests/test_cli.py"- "funasr/auto/auto_model.py"- "funasr/utils/load_utils.py"+ - "funasr/bin/realtime_ws.py"+ - "tests/test_realtime_ws_service.py"+ - "tests/test_realtime_ws_hf_overrides.py"- "funasr/utils/timestamp_tools.py"- "funasr/models/paraformer/model.py"- "funasr/models/contextual_paraformer/model.py"@@ -96,7 +102,7 @@ jobs:- name: Install CPU runtime and requested NumPy versionrun: |python -m pip install torch==2.10.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cpu- python -m pip install "numpy==$NUMPY_VERSION" -e . pytest einops pytorch-wpe --report install-report.json+ python -m pip install "numpy==$NUMPY_VERSION" -e . pytest websockets einops pytorch-wpe --report install-report.json- name: Record resolved environment and validate dependenciesrun: |git rev-parse HEAD | tee source-commit.txt@@ -113,6 +119,8 @@ jobs:tests/test_pypi_metadata.py \tests/test_sensevoice_word_timestamps.py \tests/test_pcm_input_format.py \+ tests/test_realtime_ws_service.py \+ tests/test_realtime_ws_hf_overrides.py \tests/test_load_audio_bytes.py \tests/test_timestamp_tools.py \tests/test_punc_model_none.py \diff --git a/funasr/bin/realtime_ws.py b/funasr/bin/realtime_ws.pyindex bce661b76..b6f96b54d 100644--- a/funasr/bin/realtime_ws.py+++ b/funasr/bin/realtime_ws.py@@ -1348,6 +1348,9 @@ def load_models(args):gpu_memory_utilization=getattr(args, 'gpu_memory_utilization', 0.8),max_model_len=getattr(args, 'max_model_len', 2048),enforce_eager=getattr(args, 'enforce_eager', False),+ vllm_kwargs={"hf_overrides": getattr(args, "hf_overrides", None)}+ if getattr(args, "hf_overrides", None)+ else {},)_vllm_engine = RealtimeBatchingEngine(engine,@@ -1614,12 +1617,25 @@ async def main(args):await asyncio.Future()+def _parse_hf_overrides(value):+ overrides = json.loads(value)+ if not isinstance(overrides, dict):+ raise argparse.ArgumentTypeError("--hf-overrides must be a JSON object")+ return overrides++def build_arg_parser():parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming WebSocket Server")parser.add_argument("--port", type=int, default=10095)parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])parser.add_argument("--device", type=str, default="cuda:0")+ parser.add_argument(+ "--hf-overrides",+ type=_parse_hf_overrides,+ default=None,+ help=("JSON object forwarded to the vLLM model configuration."),+ )parser.add_argument("--vad-device",type=str,diff --git a/tests/test_realtime_ws_hf_overrides.py b/tests/test_realtime_ws_hf_overrides.pynew file mode 100644index 000000000..f4ff6b7aa--- /dev/null+++ b/tests/test_realtime_ws_hf_overrides.py@@ -0,0 +1,41 @@+import pytest++from test_realtime_ws_service import load_service_module++[email protected]("value", ["[]", "[1]", '"model"', "1", "true", "false", "null"])+def test_cli_rejects_non_object_hf_overrides(value, capsys):+ parser = load_service_module().build_arg_parser()++ with pytest.raises(SystemExit) as exc:+ parser.parse_args(["--hf-overrides", value])++ assert exc.value.code == 2+ assert "JSON object" in capsys.readouterr().err++[email protected]("value", ["{", "{bad}", '{"architectures":}'])+def test_cli_rejects_malformed_hf_overrides(value):+ parser = load_service_module().build_arg_parser()++ with pytest.raises(SystemExit) as exc:+ parser.parse_args(["--hf-overrides", value])++ assert exc.value.code == 2+++ ("value", "expected"),+ [+ ("{}", {}),+ ('{"architectures":["Qwen2ForCausalLM"]}', {"architectures": ["Qwen2ForCausalLM"]}),+ ('{"rope_scaling":{"factor":2.0},"use_cache":false}',+ {"rope_scaling": {"factor": 2.0}, "use_cache": False}),+ ],+)+def test_cli_preserves_hf_override_objects(value, expected):+ parser = load_service_module().build_arg_parser()++ args = parser.parse_args(["--hf-overrides", value])++ assert args.hf_overrides == expecteddiff --git a/tests/test_realtime_ws_service.py b/tests/test_realtime_ws_service.pyindex 82292ecd5..21e5cd416 100644--- a/tests/test_realtime_ws_service.py+++ b/tests/test_realtime_ws_service.py@@ -119,6 +119,17 @@ def test_cli_defaults_disable_speaker_and_bound_partial_window():assert args.log_decode_profile is Falseassert args.vad_device == "cpu"assert args.vad_ncpu == 1+ assert args.hf_overrides is None+++def test_cli_accepts_hf_overrides_json_object():+ module = load_service_module()++ args = module.build_arg_parser().parse_args(+ ["--hf-overrides", '{"architectures":["FunASRForCausalLM"]}']+ )++ assert args.hf_overrides == {"architectures": ["FunASRForCausalLM"]}def test_cli_accepts_client_endpoint_mode():@@ -1806,6 +1817,39 @@ def test_load_models_reads_postprocess_hotword_file(monkeypatch, tmp_path):assert auto_model_calls == []+def test_load_models_forwards_hf_overrides_to_vllm(monkeypatch):+ module = load_service_module()+ module._vllm_engine = None+ module._asr_kwargs = None+ module._vad_model = None+ module._spk_model = None+ vllm_calls = []++ import funasr++ monkeypatch.setattr(funasr, "AutoModel", lambda *args, **kwargs: object())+ vllm_stub = types.ModuleType("funasr.auto.auto_model_vllm")+ vllm_stub.AutoModelVLLM = (+ lambda **kwargs: vllm_calls.append(kwargs) or object()+ )+ monkeypatch.setitem(sys.modules, "funasr.auto.auto_model_vllm", vllm_stub)++ args = module.build_arg_parser().parse_args(+ [+ "--endpoint-mode",+ "client",+ "--hf-overrides",+ '{"architectures":["FunASRForCausalLM"]}',+ ]+ )++ module.load_models(args)++ assert vllm_calls[0]["vllm_kwargs"] == {+ "hf_overrides": {"architectures": ["FunASRForCausalLM"]}+ }++def test_two_hour_session_keeps_audio_bounded_and_duration_absolute():module = load_service_module()sample_rate = 10