Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(server): return speaker labels for spk requests (#3433)
zhifu gaocbed336e56db
diff --git a/funasr/bin/_server_app.py b/funasr/bin/_server_app.pyindex 9c8a3b461..17e8ada6e 100644--- a/funasr/bin/_server_app.py+++ b/funasr/bin/_server_app.py@@ -127,11 +127,95 @@ def prepare_audio_for_inference(audio_data, sr, target_sr=16000):return audio_data.astype(np.float32), sr++def attach_speaker_labels(audio_data, sr, segments, speaker_model, device):+ """Run speaker diarization once and attach labels to timestamped segments."""+ if not segments:+ return segments++ import torch+ from funasr.models.campplus.cluster_backend import ClusterBackend+ from funasr.models.campplus.utils import distribute_spk, postprocess, sv_chunk++ audio_data, sr = prepare_audio_for_inference(audio_data, sr)+ duration = len(audio_data) / sr+ diarization_inputs = []+ segment_indexes = []+ for index, segment in enumerate(segments):+ start = max(float(segment.get("start", 0.0)), 0.0)+ end = min(max(float(segment.get("end", start)), start), duration)+ start_sample = int(start * sr)+ end_sample = int(end * sr)+ if end_sample <= start_sample:+ continue+ diarization_inputs.append([start, end, audio_data[start_sample:end_sample]])+ segment_indexes.append(index)++ chunks = sv_chunk(diarization_inputs, fs=sr)+ if not chunks:+ return segments++ speaker_results = speaker_model.generate(+ input=[chunk[2] for chunk in chunks], cache={}, is_final=True+ )+ embeddings = torch.cat(+ [speaker_result["spk_embedding"] for speaker_result in speaker_results], dim=0+ )+ labels = ClusterBackend(merge_thr=0.78).to(device)(embeddings.cpu(), oracle_num=None)+ if not isinstance(labels, np.ndarray):+ labels = np.asarray(labels)+ speaker_timeline = postprocess(+ sorted(chunks, key=lambda chunk: chunk[0]),+ None,+ labels,+ embeddings.detach().cpu().numpy(),+ )+ sentences = [+ {+ "text": segments[index]["text"],+ "start": int(float(segments[index]["start"]) * 1000),+ "end": int(float(segments[index]["end"]) * 1000),+ }+ for index in segment_indexes+ ]+ distribute_spk(sentences, speaker_timeline)+ for index, sentence in zip(segment_indexes, sentences):+ speaker = sentence.get("spk")+ if speaker is not None:+ segments[index]["speaker"] = f"SPK{speaker}"+ return segments+++def build_openai_verbose_json(result, requested_language=None):+ """Build OpenAI-compatible verbose JSON while preserving FunASR extensions."""+ segments = []+ for index, segment in enumerate(result.get("segments", [])):+ item = {+ "id": index,+ "start": segment["start"],+ "end": segment["end"],+ "text": segment["text"],+ "words": segment.get("words", []),+ }+ if segment.get("speaker") is not None:+ item["speaker"] = segment["speaker"]+ segments.append(item)++ return {+ "task": "transcribe",+ "language": resolve_transcription_language(requested_language, result),+ "duration": result.get("duration", 0),+ "text": result["text"],+ "segments": segments,+ }++def create_app(device: str = "cuda",preload_model: str = "auto",model_path: str = None,hub: str = "ms",+ spk_model: str = "cam++",cors_origins: Optional[Iterable[str]] = None,) -> FastAPI:if preload_model == "auto":@@ -141,6 +225,8 @@ def create_app(app.state.device = deviceapp.state.engine = Noneapp.state.vad_model = None+ app.state.spk_model = None+ app.state.spk_model_name = spk_modelapp.state.fallback_models = {}app.state.model_path = model_pathapp.state.hub = hub@@ -173,6 +259,24 @@ def create_app(},}+ def _load_spk_model():+ """Lazily load diarization only when a request opts in with spk=true."""+ if app.state.spk_model is not None:+ return app.state.spk_model+ if not app.state.spk_model_name:+ raise HTTPException(400, "Speaker diarization is disabled; configure --spk-model")++ from funasr import AutoModel++ logger.info(f"Loading speaker model: {app.state.spk_model_name}")+ app.state.spk_model = AutoModel(+ model=app.state.spk_model_name,+ device=device,+ disable_update=True,+ )+ logger.info("Speaker model ready.")+ return app.state.spk_model+def _load_vllm_engine():"""Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable."""if app.state.engine is not None or "fun-asr-nano" in app.state.fallback_models:@@ -286,13 +390,22 @@ def create_app(output_segments.append(seg_info)full_text_parts.append(text)+ if use_spk:+ attach_speaker_labels(+ audio_data,+ sr,+ output_segments,+ _load_spk_model(),+ device,+ )+return {"text": "".join(full_text_parts),"segments": output_segments,"duration": len(audio_data) / sr,}- def _process_fallback(model_name, audio_path, language=None):+ def _process_fallback(model_name, audio_path, language=None, use_spk=False):"""Process with non-LLM model (SenseVoice/Paraformer)."""model = _load_fallback(model_name)try:@@ -309,14 +422,27 @@ def create_app(segments = []if "sentence_info" in result[0]:for s in result[0]["sentence_info"]:- segments.append({+ segment = {"start": s.get("start", 0)/1000,"end": s.get("end", 0)/1000,- "text": re.sub(r'<\|[^|]*\|>', '', s.get("text", "")).strip(),- "speaker": s.get("spk"),- })+ "text": re.sub(+ r'<\|[^|]*\|>', '', s.get("text") or s.get("sentence", "")+ ).strip(),+ }+ if s.get("spk") is not None:+ segment["speaker"] = s["spk"]+ segments.append(segment)if not segments and text:segments = build_openai_fallback_segments(text, duration)+ if use_spk and segments:+ audio_data, sr = sf.read(audio_path)+ attach_speaker_labels(+ audio_data,+ sr,+ segments,+ _load_spk_model(),+ device,+ )return {"text": text,"segments": segments,@@ -356,7 +482,9 @@ def create_app(tmp.write(content)tmp_path = tmp.nametry:- result = _process_fallback("fun-asr-nano", tmp_path, language=language)+ result = _process_fallback(+ "fun-asr-nano", tmp_path, language=language, use_spk=spk+ )finally:os.unlink(tmp_path)elif model in FALLBACK_CONFIGS or model == "custom":@@ -365,7 +493,9 @@ def create_app(tmp.write(content)tmp_path = tmp.nametry:- result = _process_fallback(model, tmp_path, language=language)+ result = _process_fallback(+ model, tmp_path, language=language, use_spk=spk+ )finally:os.unlink(tmp_path)else:@@ -375,16 +505,7 @@ def create_app(t1 = time.perf_counter()if response_format == "verbose_json":- return JSONResponse({- "task": "transcribe",- "language": resolve_transcription_language(language, result),- "duration": result.get("duration", 0),- "text": result["text"],- "segments": [- {"id": i, "start": s["start"], "end": s["end"], "text": s["text"], "words": s.get("words", [])}- for i, s in enumerate(result["segments"])- ],- })+ return JSONResponse(build_openai_verbose_json(result, requested_language=language))elif response_format == "text":return JSONResponse(result["text"])else:@@ -412,7 +533,9 @@ def create_app(tmp.write(content)tmp_path = tmp.nametry:- result = _process_fallback("fun-asr-nano", tmp_path, language=language)+ result = _process_fallback(+ "fun-asr-nano", tmp_path, language=language, use_spk=spk+ )finally:os.unlink(tmp_path)t1 = time.perf_counter()diff --git a/funasr/bin/server.py b/funasr/bin/server.pyindex 2aa86f8ae..5aefda2e1 100644--- a/funasr/bin/server.py+++ b/funasr/bin/server.py@@ -48,6 +48,11 @@ Then use with OpenAI SDK:parser.add_argument("--model", default="auto", help="Pre-load model: auto (GPU=fun-asr-nano, CPU=sensevoice), sensevoice, paraformer, fun-asr-nano")parser.add_argument("--model-path", default=None, help="Local model path or model ID (overrides --model)")parser.add_argument("--hub", default="ms", help="Model hub: ms (ModelScope), hf (HuggingFace) (default: ms)")+ parser.add_argument(+ "--spk-model",+ default="cam++",+ help="Speaker model loaded on the first spk=true request (default: cam++)",+ )parser.add_argument("--cors-origin",action="append",@@ -81,6 +86,7 @@ def main():preload_model=args.model,model_path=args.model_path,hub=args.hub,+ spk_model=args.spk_model,cors_origins=args.cors_origin,)diff --git a/tests/test_server_app_openai_segments.py b/tests/test_server_app_openai_segments.pyindex 99e90f3fb..972b27dac 100644--- a/tests/test_server_app_openai_segments.py+++ b/tests/test_server_app_openai_segments.py@@ -82,7 +82,12 @@ def load_server_cli():return module-def install_dummy_funasr(monkeypatch, fail_once_for_models=(), generated_text="transcript"):+def install_dummy_funasr(+ monkeypatch,+ fail_once_for_models=(),+ generated_text="transcript",+ generated_result=None,+):remaining_failures = {model: 1 for model in fail_once_for_models}class DummyAutoModel:@@ -101,6 +106,8 @@ def install_dummy_funasr(monkeypatch, fail_once_for_models=(), generated_text="tdef generate(self, **kwargs):if self.kwargs.get("model") == "fsmn-vad":return [{"value": [[0, 1000]]}]+ if generated_result is not None:+ return [generated_result.copy()]return [{"text": generated_text}]funasr_stub = types.ModuleType("funasr")@@ -230,6 +237,211 @@ def test_verbose_json_reports_sensevoice_detected_language(monkeypatch):]+def test_openai_verbose_json_preserves_speaker_labels(monkeypatch):+ module = load_server_app(monkeypatch)++ response = module.build_openai_verbose_json(+ {+ "language": "zh",+ "duration": 1.25,+ "text": "hello",+ "segments": [+ {+ "start": 0.0,+ "end": 1.25,+ "text": "hello",+ "speaker": "SPK0",+ }+ ],+ },+ requested_language=None,+ )++ assert response["segments"] == [+ {+ "id": 0,+ "start": 0.0,+ "end": 1.25,+ "text": "hello",+ "words": [],+ "speaker": "SPK0",+ }+ ]+++def test_attach_speaker_labels_runs_diarization_pipeline(monkeypatch):+ import torch++ module = load_server_app(monkeypatch)+ calls = []++ class DummyClusterBackend:+ def __init__(self, merge_thr):+ assert merge_thr == 0.78++ def to(self, device):+ assert device == "cpu"+ return self++ def __call__(self, embeddings, oracle_num=None):+ assert embeddings.shape[1] == 2+ assert oracle_num is None+ calls.append("cluster")+ return module.np.zeros(embeddings.shape[0], dtype=int)++ class DummySpeakerModel:+ def generate(self, input, cache, is_final):+ assert input+ assert cache == {}+ assert is_final is True+ return [+ {"spk_embedding": torch.tensor([[1.0, 0.0]])}+ for _ in input+ ]++ def fake_sv_chunk(diarization_inputs, fs):+ assert fs == 16000+ assert diarization_inputs[0][:2] == [0.0, 2.0]+ calls.append("chunk")+ return [[0.0, 2.0, diarization_inputs[0][2]]]++ def fake_postprocess(chunks, vad_segments, labels, embeddings):+ assert vad_segments is None+ assert labels.tolist() == [0]+ assert embeddings.shape == (1, 2)+ calls.append("postprocess")+ return [[chunks[0][0], chunks[0][1], 0]]++ def fake_distribute_spk(sentences, speaker_timeline):+ assert speaker_timeline == [[0.0, 2.0, 0]]+ sentences[0]["spk"] = 0+ calls.append("distribute")++ funasr_stub = types.ModuleType("funasr")+ funasr_stub.__path__ = []+ models_stub = types.ModuleType("funasr.models")+ models_stub.__path__ = []+ campplus_stub = types.ModuleType("funasr.models.campplus")+ campplus_stub.__path__ = []+ cluster_stub = types.ModuleType("funasr.models.campplus.cluster_backend")+ cluster_stub.ClusterBackend = DummyClusterBackend+ utils_stub = types.ModuleType("funasr.models.campplus.utils")+ utils_stub.sv_chunk = fake_sv_chunk+ utils_stub.postprocess = fake_postprocess+ utils_stub.distribute_spk = fake_distribute_spk+ monkeypatch.setitem(sys.modules, "funasr", funasr_stub)+ monkeypatch.setitem(sys.modules, "funasr.models", models_stub)+ monkeypatch.setitem(sys.modules, "funasr.models.campplus", campplus_stub)+ monkeypatch.setitem(+ sys.modules, "funasr.models.campplus.cluster_backend", cluster_stub+ )+ monkeypatch.setitem(sys.modules, "funasr.models.campplus.utils", utils_stub)+ segments = [{"start": 0.0, "end": 2.0, "text": "hello"}]++ result = module.attach_speaker_labels(+ module.np.ones(32000, dtype=module.np.float32),+ 16000,+ segments,+ DummySpeakerModel(),+ "cpu",+ )++ assert result == [+ {"start": 0.0, "end": 2.0, "text": "hello", "speaker": "SPK0"}+ ]+ assert calls == ["chunk", "cluster", "postprocess", "distribute"]+++def test_fallback_reads_sentence_info_sentence_field(monkeypatch):+ module = load_server_app(monkeypatch)+ install_dummy_funasr(+ monkeypatch,+ generated_result={+ "text": "hello",+ "sentence_info": [+ {"start": 0, "end": 1250, "sentence": "hello", "spk": 1}+ ],+ },+ )+ monkeypatch.setattr(module.sf, "info", lambda path: types.SimpleNamespace(duration=1.25))+ app = module.create_app(device="cpu", preload_model="sensevoice")+ transcribe = app.routes[("POST", "/v1/audio/transcriptions")]++ response = asyncio.run(+ transcribe(+ file=DummyUpload(),+ model="sensevoice",+ language=None,+ response_format="verbose_json",+ spk=False,+ )+ )++ assert response["segments"] == [+ {+ "id": 0,+ "start": 0.0,+ "end": 1.25,+ "text": "hello",+ "words": [],+ "speaker": 1,+ }+ ]+++def test_spk_request_lazily_loads_and_reuses_speaker_model(monkeypatch):+ module = load_server_app(monkeypatch)+ DummyAutoModel = install_dummy_funasr(monkeypatch)+ monkeypatch.setattr(module.sf, "info", lambda path: types.SimpleNamespace(duration=1.25))+ monkeypatch.setattr(+ module.sf,+ "read",+ lambda path: (module.np.ones(20000, dtype=module.np.float32), 16000),+ )+ attach_calls = []++ def fake_attach(audio_data, sample_rate, segments, speaker_model, device):+ attach_calls.append((sample_rate, speaker_model.kwargs["model"], device))+ segments[0]["speaker"] = "SPK0"+ return segments++ monkeypatch.setattr(module, "attach_speaker_labels", fake_attach, raising=False)+ app = module.create_app(+ device="cpu",+ preload_model="sensevoice",+ spk_model="iic/speech_eres2netv2_sv_zh-cn_16k-common",+ )+ transcribe = app.routes[("POST", "/v1/audio/transcriptions")]++ assert not any(+ instance.get("model") == "iic/speech_eres2netv2_sv_zh-cn_16k-common"+ for instance in DummyAutoModel.instances+ )++ for _ in range(2):+ response = asyncio.run(+ transcribe(+ file=DummyUpload(),+ model="sensevoice",+ language=None,+ response_format="verbose_json",+ spk=True,+ )+ )+ assert response["segments"][0]["speaker"] == "SPK0"++ speaker_models = [+ instance+ for instance in DummyAutoModel.instances+ if instance.get("model") == "iic/speech_eres2netv2_sv_zh-cn_16k-common"+ ]+ assert len(speaker_models) == 1+ assert attach_calls == [+ (16000, "iic/speech_eres2netv2_sv_zh-cn_16k-common", "cpu"),+ (16000, "iic/speech_eres2netv2_sv_zh-cn_16k-common", "cpu"),+ ]++def test_server_versions_follow_package_version(monkeypatch):expected = (REPO_ROOT / "funasr" / "version.txt").read_text().strip()module = load_server_app(monkeypatch)@@ -261,6 +473,16 @@ def test_server_cli_collects_repeated_cors_origins():]+def test_server_cli_accepts_speaker_model():+ module = load_server_cli()++ args = module.build_parser().parse_args(+ ["--spk-model", "iic/speech_eres2netv2_sv_zh-cn-16k-common"]+ )++ assert args.spk_model == "iic/speech_eres2netv2_sv_zh-cn-16k-common"++def test_server_cors_is_disabled_by_default(monkeypatch):module = load_server_app(monkeypatch)install_dummy_funasr(monkeypatch)