Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(auto-model): preserve resampled rate for speaker inference (#3763)
Hulkd98f6eee4d3a
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.pyindex e814cba84..4ff7f13a1 100644--- a/funasr/auto/auto_model.py+++ b/funasr/auto/auto_model.py@@ -1124,7 +1124,7 @@ class AutoModel:input_len=None,model=self.spk_model,kwargs=self.spk_kwargs,- **cfg,+ **{**cfg, "fs": fs},)spk_embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)results[_b]["spk_embedding"] = spk_embsdiff --git a/tests/test_pcm_input_format.py b/tests/test_pcm_input_format.pyindex 94e32a192..979b9bff2 100644--- a/tests/test_pcm_input_format.py+++ b/tests/test_pcm_input_format.py@@ -235,3 +235,137 @@ def test_vad_source_rates_are_not_reapplied_to_resampled_segments()assert len(loaded) == 3200np.testing.assert_allclose(loaded, expected, atol=1e-6, rtol=0)+++class _RecordingSpeaker(torch.nn.Module):+ """Use CAMPPlus preprocessing with a stand-in embedding network."""++ def __init__(self, **kwargs):+ super().__init__()+ self.anchor = torch.nn.Parameter(torch.zeros(1))+ self.calls = []++ def inference(self, data_in, key, **kwargs):+ from funasr.models.campplus.model import CAMPPlus++ self.calls.append(kwargs)+ return CAMPPlus.inference(self, data_in, key=key, **kwargs)++ def forward(self, features):+ return torch.ones(features.shape[0], 4)+++ "base_rate,call_rate,speaker_rate,source_rate",+ [+ (None, None, None, 16000),+ (8000, None, None, 8000),+ (None, 8000, None, 8000),+ (None, 48000, None, 48000),+ (8000, 48000, 8000, 48000),+ (48000, 8000, 48000, 8000),+ (16000, None, 48000, 16000),+ ],+)[email protected]("batch", [False, True])[email protected]("input_kind", ["array", "pcm"])+def test_vad_speaker_uses_resampled_rate(+ base_rate, call_rate, speaker_rate, source_rate, batch, input_kind, monkeypatch+):+ from funasr.models.campplus import model as campplus_model++ feature_inputs = []++ def capture_features(audio):+ feature_inputs.extend(sample.numpy().copy() for sample in audio)+ return torch.zeros(len(audio), 20, 80), [20] * len(audio), [len(a) for a in audio]++ monkeypatch.setattr(campplus_model, "extract_feature", capture_features)+ wrapper = _wrapper(vad=True)+ wrapper.vad_model.segments = [[0, 200]]+ wrapper.spk_model = _RecordingSpeaker()+ wrapper.spk_kwargs = {"device": "cpu"}+ wrapper.kwargs["return_spk_res"] = False+ if base_rate is not None:+ wrapper.kwargs["fs"] = base_rate+ if speaker_rate is not None:+ wrapper.spk_kwargs["fs"] = speaker_rate+ wrapper._store_base_configs()++ pcm = np.round(+ np.sin(2 * np.pi * 440 * np.arange(source_rate // 5) / source_rate) * 12000+ ).astype("<i2").tobytes()+ waveform = _expected(pcm)+ resampled = load_utils.load_audio_text_image_video(+ waveform, fs=16000, audio_fs=source_rate+ )+ # sv_chunk pads the already-resampled 200 ms segment to 1.5 seconds.+ expected = np.pad(resampled, (0, 24000 - len(resampled)))+ options = {} if call_rate is None else {"fs": call_rate}+ if input_kind == "pcm":+ options["input_format"] = "pcm_s16le"+ value = pcm if input_kind == "pcm" else waveform+ results = wrapper.generate(input=[value, value] if batch else value, **options)+ assert len(results) == (2 if batch else 1)+ assert len(feature_inputs) == len(results)+ for audio in feature_inputs:+ assert len(audio) == 24000+ np.testing.assert_allclose(audio, expected, atol=1e-6, rtol=0)+ for _, vad_options in wrapper.vad_model.calls:+ assert vad_options["fs"] == source_rate+ for _, asr_options in wrapper.model.calls:+ assert asr_options["fs"] == 16000+ for speaker_options in wrapper.spk_model.calls:+ assert speaker_options.get("fs", 16000) == 16000+ assert wrapper._base_kwargs_map["kwargs"].get("fs") == base_rate+ assert wrapper._base_kwargs_map["spk_kwargs"].get("fs") == speaker_rate++ # A subsequent call without an override must use the constructor source rate.+ feature_inputs.clear()+ next_rate = base_rate or 16000+ wrapper.generate(input=np.zeros(next_rate // 5, dtype=np.float32))+ assert wrapper.vad_model.calls[-1][1]["fs"] == next_rate+ assert wrapper.model.calls[-1][1]["fs"] == 16000+ assert wrapper.spk_model.calls[-1].get("fs", 16000) == 16000+ assert len(feature_inputs[0]) == 24000+++def test_speaker_rate_override_preserves_caller_config(monkeypatch):+ import copy+ import importlib++ from funasr.register import tables++ auto_module = importlib.import_module("funasr.auto.auto_model")++ class ConfiguredModel(_RecordingModel):+ def __init__(self, model, **kwargs):+ super().__init__(vad=model == "rate-test-vad")+ self.segments = [[0, 200]]++ monkeypatch.setitem(tables.model_classes, "rate-test-asr", ConfiguredModel)+ monkeypatch.setitem(tables.model_classes, "rate-test-vad", ConfiguredModel)+ monkeypatch.setitem(tables.model_classes, "rate-test-speaker", _RecordingSpeaker)+ monkeypatch.setitem(+ tables.frontend_classes, "rate-test-frontend", lambda **kw: SimpleNamespace(fs=16000)+ )+ monkeypatch.setattr(+ auto_module, "ClusterBackend", lambda **kw: torch.nn.Identity(), raising=False+ )+ monkeypatch.setattr(auto_module, "download_model", _forbid_decode)+ caller_config = {"model_conf": {}, "device": "cpu", "fs": 48000}+ original = copy.deepcopy(caller_config)+ wrapper = AutoModel(+ model="rate-test-asr", model_conf={}, frontend="rate-test-frontend",+ vad_model="rate-test-vad", vad_kwargs={"model_conf": {}},+ spk_model="rate-test-speaker", spk_kwargs=caller_config,+ fs=48000, device="cpu", ncpu=torch.get_num_threads(),+ disable_update=True, disable_pbar=True, return_spk_res=False,+ )+ for rate in (8000, 48000):+ options = {"fs": rate} if rate == 8000 else {}+ wrapper.generate(input=np.zeros(rate // 5, dtype=np.float32), **options)+ assert wrapper.vad_model.calls[-1][1]["fs"] == rate+ assert wrapper.spk_model.calls[-1]["fs"] == 16000+ assert caller_config == original+ assert wrapper._base_kwargs_map["spk_kwargs"]["fs"] == 48000