Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(realtime): validate and forward vLLM HF overrides

zhifu gao73ae515788a4
diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml
index 4a7bcec5c..6e708f0f4 100644
--- a/.github/workflows/test-numpy-compatibility.yml
+++ b/.github/workflows/test-numpy-compatibility.yml
@@ -9,6 +9,9 @@ on:
- "tests/test_cli.py"
- "funasr/auto/auto_model.py"
- "funasr/utils/load_utils.py"
+ - "funasr/bin/realtime_ws.py"
+ - "tests/test_realtime_ws_service.py"
+ - "tests/test_realtime_ws_hf_overrides.py"
- "funasr/utils/timestamp_tools.py"
- "funasr/models/paraformer/model.py"
- "funasr/models/contextual_paraformer/model.py"
@@ -40,6 +43,9 @@ on:
- "tests/test_cli.py"
- "funasr/auto/auto_model.py"
- "funasr/utils/load_utils.py"
+ - "funasr/bin/realtime_ws.py"
+ - "tests/test_realtime_ws_service.py"
+ - "tests/test_realtime_ws_hf_overrides.py"
- "funasr/utils/timestamp_tools.py"
- "funasr/models/paraformer/model.py"
- "funasr/models/contextual_paraformer/model.py"
@@ -96,7 +102,7 @@ jobs:
- name: Install CPU runtime and requested NumPy version
run: |
python -m pip install torch==2.10.0 torchaudio==2.10.0 --index-url https://download.pytorch.org/whl/cpu
- python -m pip install "numpy==$NUMPY_VERSION" -e . pytest einops pytorch-wpe --report install-report.json
+ python -m pip install "numpy==$NUMPY_VERSION" -e . pytest websockets einops pytorch-wpe --report install-report.json
- name: Record resolved environment and validate dependencies
run: |
git rev-parse HEAD | tee source-commit.txt
@@ -113,6 +119,8 @@ jobs:
tests/test_pypi_metadata.py \
tests/test_sensevoice_word_timestamps.py \
tests/test_pcm_input_format.py \
+ tests/test_realtime_ws_service.py \
+ tests/test_realtime_ws_hf_overrides.py \
tests/test_load_audio_bytes.py \
tests/test_timestamp_tools.py \
tests/test_punc_model_none.py \
diff --git a/funasr/bin/realtime_ws.py b/funasr/bin/realtime_ws.py
index bce661b76..b6f96b54d 100644
--- a/funasr/bin/realtime_ws.py
+++ b/funasr/bin/realtime_ws.py
@@ -1348,6 +1348,9 @@ def load_models(args):
gpu_memory_utilization=getattr(args, 'gpu_memory_utilization', 0.8),
max_model_len=getattr(args, 'max_model_len', 2048),
enforce_eager=getattr(args, 'enforce_eager', False),
+ vllm_kwargs={"hf_overrides": getattr(args, "hf_overrides", None)}
+ if getattr(args, "hf_overrides", None)
+ else {},
)
_vllm_engine = RealtimeBatchingEngine(
engine,
@@ -1614,12 +1617,25 @@ async def main(args):
await asyncio.Future()
+def _parse_hf_overrides(value):
+ overrides = json.loads(value)
+ if not isinstance(overrides, dict):
+ raise argparse.ArgumentTypeError("--hf-overrides must be a JSON object")
+ return overrides
+
+
def build_arg_parser():
parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming WebSocket Server")
parser.add_argument("--port", type=int, default=10095)
parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")
parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])
parser.add_argument("--device", type=str, default="cuda:0")
+ parser.add_argument(
+ "--hf-overrides",
+ type=_parse_hf_overrides,
+ default=None,
+ help=("JSON object forwarded to the vLLM model configuration."),
+ )
parser.add_argument(
"--vad-device",
type=str,
diff --git a/tests/test_realtime_ws_hf_overrides.py b/tests/test_realtime_ws_hf_overrides.py
new file mode 100644
index 000000000..f4ff6b7aa
--- /dev/null
+++ b/tests/test_realtime_ws_hf_overrides.py
@@ -0,0 +1,41 @@
+import pytest
+
+from test_realtime_ws_service import load_service_module
+
+
[email protected]("value", ["[]", "[1]", '"model"', "1", "true", "false", "null"])
+def test_cli_rejects_non_object_hf_overrides(value, capsys):
+ parser = load_service_module().build_arg_parser()
+
+ with pytest.raises(SystemExit) as exc:
+ parser.parse_args(["--hf-overrides", value])
+
+ assert exc.value.code == 2
+ assert "JSON object" in capsys.readouterr().err
+
+
[email protected]("value", ["{", "{bad}", '{"architectures":}'])
+def test_cli_rejects_malformed_hf_overrides(value):
+ parser = load_service_module().build_arg_parser()
+
+ with pytest.raises(SystemExit) as exc:
+ parser.parse_args(["--hf-overrides", value])
+
+ assert exc.value.code == 2
+
+
+ ("value", "expected"),
+ [
+ ("{}", {}),
+ ('{"architectures":["Qwen2ForCausalLM"]}', {"architectures": ["Qwen2ForCausalLM"]}),
+ ('{"rope_scaling":{"factor":2.0},"use_cache":false}',
+ {"rope_scaling": {"factor": 2.0}, "use_cache": False}),
+ ],
+)
+def test_cli_preserves_hf_override_objects(value, expected):
+ parser = load_service_module().build_arg_parser()
+
+ args = parser.parse_args(["--hf-overrides", value])
+
+ assert args.hf_overrides == expected
diff --git a/tests/test_realtime_ws_service.py b/tests/test_realtime_ws_service.py
index 82292ecd5..21e5cd416 100644
--- a/tests/test_realtime_ws_service.py
+++ b/tests/test_realtime_ws_service.py
@@ -119,6 +119,17 @@ def test_cli_defaults_disable_speaker_and_bound_partial_window():
assert args.log_decode_profile is False
assert args.vad_device == "cpu"
assert args.vad_ncpu == 1
+ assert args.hf_overrides is None
+
+
+def test_cli_accepts_hf_overrides_json_object():
+ module = load_service_module()
+
+ args = module.build_arg_parser().parse_args(
+ ["--hf-overrides", '{"architectures":["FunASRForCausalLM"]}']
+ )
+
+ assert args.hf_overrides == {"architectures": ["FunASRForCausalLM"]}
def test_cli_accepts_client_endpoint_mode():
@@ -1806,6 +1817,39 @@ def test_load_models_reads_postprocess_hotword_file(monkeypatch, tmp_path):
assert auto_model_calls == []
+def test_load_models_forwards_hf_overrides_to_vllm(monkeypatch):
+ module = load_service_module()
+ module._vllm_engine = None
+ module._asr_kwargs = None
+ module._vad_model = None
+ module._spk_model = None
+ vllm_calls = []
+
+ import funasr
+
+ monkeypatch.setattr(funasr, "AutoModel", lambda *args, **kwargs: object())
+ vllm_stub = types.ModuleType("funasr.auto.auto_model_vllm")
+ vllm_stub.AutoModelVLLM = (
+ lambda **kwargs: vllm_calls.append(kwargs) or object()
+ )
+ monkeypatch.setitem(sys.modules, "funasr.auto.auto_model_vllm", vllm_stub)
+
+ args = module.build_arg_parser().parse_args(
+ [
+ "--endpoint-mode",
+ "client",
+ "--hf-overrides",
+ '{"architectures":["FunASRForCausalLM"]}',
+ ]
+ )
+
+ module.load_models(args)
+
+ assert vllm_calls[0]["vllm_kwargs"] == {
+ "hf_overrides": {"architectures": ["FunASRForCausalLM"]}
+ }
+
+
def test_two_hour_session_keeps_audio_bounded_and_duration_absolute():
module = load_service_module()
sample_rate = 10