Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(server): return speaker labels for spk requests (#3433)

zhifu gaocbed336e56db
diff --git a/funasr/bin/_server_app.py b/funasr/bin/_server_app.py
index 9c8a3b461..17e8ada6e 100644
--- a/funasr/bin/_server_app.py
+++ b/funasr/bin/_server_app.py
@@ -127,11 +127,95 @@ def prepare_audio_for_inference(audio_data, sr, target_sr=16000):
return audio_data.astype(np.float32), sr
+
+def attach_speaker_labels(audio_data, sr, segments, speaker_model, device):
+ """Run speaker diarization once and attach labels to timestamped segments."""
+ if not segments:
+ return segments
+
+ import torch
+ from funasr.models.campplus.cluster_backend import ClusterBackend
+ from funasr.models.campplus.utils import distribute_spk, postprocess, sv_chunk
+
+ audio_data, sr = prepare_audio_for_inference(audio_data, sr)
+ duration = len(audio_data) / sr
+ diarization_inputs = []
+ segment_indexes = []
+ for index, segment in enumerate(segments):
+ start = max(float(segment.get("start", 0.0)), 0.0)
+ end = min(max(float(segment.get("end", start)), start), duration)
+ start_sample = int(start * sr)
+ end_sample = int(end * sr)
+ if end_sample <= start_sample:
+ continue
+ diarization_inputs.append([start, end, audio_data[start_sample:end_sample]])
+ segment_indexes.append(index)
+
+ chunks = sv_chunk(diarization_inputs, fs=sr)
+ if not chunks:
+ return segments
+
+ speaker_results = speaker_model.generate(
+ input=[chunk[2] for chunk in chunks], cache={}, is_final=True
+ )
+ embeddings = torch.cat(
+ [speaker_result["spk_embedding"] for speaker_result in speaker_results], dim=0
+ )
+ labels = ClusterBackend(merge_thr=0.78).to(device)(embeddings.cpu(), oracle_num=None)
+ if not isinstance(labels, np.ndarray):
+ labels = np.asarray(labels)
+ speaker_timeline = postprocess(
+ sorted(chunks, key=lambda chunk: chunk[0]),
+ None,
+ labels,
+ embeddings.detach().cpu().numpy(),
+ )
+ sentences = [
+ {
+ "text": segments[index]["text"],
+ "start": int(float(segments[index]["start"]) * 1000),
+ "end": int(float(segments[index]["end"]) * 1000),
+ }
+ for index in segment_indexes
+ ]
+ distribute_spk(sentences, speaker_timeline)
+ for index, sentence in zip(segment_indexes, sentences):
+ speaker = sentence.get("spk")
+ if speaker is not None:
+ segments[index]["speaker"] = f"SPK{speaker}"
+ return segments
+
+
+def build_openai_verbose_json(result, requested_language=None):
+ """Build OpenAI-compatible verbose JSON while preserving FunASR extensions."""
+ segments = []
+ for index, segment in enumerate(result.get("segments", [])):
+ item = {
+ "id": index,
+ "start": segment["start"],
+ "end": segment["end"],
+ "text": segment["text"],
+ "words": segment.get("words", []),
+ }
+ if segment.get("speaker") is not None:
+ item["speaker"] = segment["speaker"]
+ segments.append(item)
+
+ return {
+ "task": "transcribe",
+ "language": resolve_transcription_language(requested_language, result),
+ "duration": result.get("duration", 0),
+ "text": result["text"],
+ "segments": segments,
+ }
+
+
def create_app(
device: str = "cuda",
preload_model: str = "auto",
model_path: str = None,
hub: str = "ms",
+ spk_model: str = "cam++",
cors_origins: Optional[Iterable[str]] = None,
) -> FastAPI:
if preload_model == "auto":
@@ -141,6 +225,8 @@ def create_app(
app.state.device = device
app.state.engine = None
app.state.vad_model = None
+ app.state.spk_model = None
+ app.state.spk_model_name = spk_model
app.state.fallback_models = {}
app.state.model_path = model_path
app.state.hub = hub
@@ -173,6 +259,24 @@ def create_app(
},
}
+ def _load_spk_model():
+ """Lazily load diarization only when a request opts in with spk=true."""
+ if app.state.spk_model is not None:
+ return app.state.spk_model
+ if not app.state.spk_model_name:
+ raise HTTPException(400, "Speaker diarization is disabled; configure --spk-model")
+
+ from funasr import AutoModel
+
+ logger.info(f"Loading speaker model: {app.state.spk_model_name}")
+ app.state.spk_model = AutoModel(
+ model=app.state.spk_model_name,
+ device=device,
+ disable_update=True,
+ )
+ logger.info("Speaker model ready.")
+ return app.state.spk_model
+
def _load_vllm_engine():
"""Load Fun-ASR-Nano vLLM engine. Falls back to AutoModel if vLLM unavailable."""
if app.state.engine is not None or "fun-asr-nano" in app.state.fallback_models:
@@ -286,13 +390,22 @@ def create_app(
output_segments.append(seg_info)
full_text_parts.append(text)
+ if use_spk:
+ attach_speaker_labels(
+ audio_data,
+ sr,
+ output_segments,
+ _load_spk_model(),
+ device,
+ )
+
return {
"text": "".join(full_text_parts),
"segments": output_segments,
"duration": len(audio_data) / sr,
}
- def _process_fallback(model_name, audio_path, language=None):
+ def _process_fallback(model_name, audio_path, language=None, use_spk=False):
"""Process with non-LLM model (SenseVoice/Paraformer)."""
model = _load_fallback(model_name)
try:
@@ -309,14 +422,27 @@ def create_app(
segments = []
if "sentence_info" in result[0]:
for s in result[0]["sentence_info"]:
- segments.append({
+ segment = {
"start": s.get("start", 0)/1000,
"end": s.get("end", 0)/1000,
- "text": re.sub(r'<\|[^|]*\|>', '', s.get("text", "")).strip(),
- "speaker": s.get("spk"),
- })
+ "text": re.sub(
+ r'<\|[^|]*\|>', '', s.get("text") or s.get("sentence", "")
+ ).strip(),
+ }
+ if s.get("spk") is not None:
+ segment["speaker"] = s["spk"]
+ segments.append(segment)
if not segments and text:
segments = build_openai_fallback_segments(text, duration)
+ if use_spk and segments:
+ audio_data, sr = sf.read(audio_path)
+ attach_speaker_labels(
+ audio_data,
+ sr,
+ segments,
+ _load_spk_model(),
+ device,
+ )
return {
"text": text,
"segments": segments,
@@ -356,7 +482,9 @@ def create_app(
tmp.write(content)
tmp_path = tmp.name
try:
- result = _process_fallback("fun-asr-nano", tmp_path, language=language)
+ result = _process_fallback(
+ "fun-asr-nano", tmp_path, language=language, use_spk=spk
+ )
finally:
os.unlink(tmp_path)
elif model in FALLBACK_CONFIGS or model == "custom":
@@ -365,7 +493,9 @@ def create_app(
tmp.write(content)
tmp_path = tmp.name
try:
- result = _process_fallback(model, tmp_path, language=language)
+ result = _process_fallback(
+ model, tmp_path, language=language, use_spk=spk
+ )
finally:
os.unlink(tmp_path)
else:
@@ -375,16 +505,7 @@ def create_app(
t1 = time.perf_counter()
if response_format == "verbose_json":
- return JSONResponse({
- "task": "transcribe",
- "language": resolve_transcription_language(language, result),
- "duration": result.get("duration", 0),
- "text": result["text"],
- "segments": [
- {"id": i, "start": s["start"], "end": s["end"], "text": s["text"], "words": s.get("words", [])}
- for i, s in enumerate(result["segments"])
- ],
- })
+ return JSONResponse(build_openai_verbose_json(result, requested_language=language))
elif response_format == "text":
return JSONResponse(result["text"])
else:
@@ -412,7 +533,9 @@ def create_app(
tmp.write(content)
tmp_path = tmp.name
try:
- result = _process_fallback("fun-asr-nano", tmp_path, language=language)
+ result = _process_fallback(
+ "fun-asr-nano", tmp_path, language=language, use_spk=spk
+ )
finally:
os.unlink(tmp_path)
t1 = time.perf_counter()
diff --git a/funasr/bin/server.py b/funasr/bin/server.py
index 2aa86f8ae..5aefda2e1 100644
--- a/funasr/bin/server.py
+++ b/funasr/bin/server.py
@@ -48,6 +48,11 @@ Then use with OpenAI SDK:
parser.add_argument("--model", default="auto", help="Pre-load model: auto (GPU=fun-asr-nano, CPU=sensevoice), sensevoice, paraformer, fun-asr-nano")
parser.add_argument("--model-path", default=None, help="Local model path or model ID (overrides --model)")
parser.add_argument("--hub", default="ms", help="Model hub: ms (ModelScope), hf (HuggingFace) (default: ms)")
+ parser.add_argument(
+ "--spk-model",
+ default="cam++",
+ help="Speaker model loaded on the first spk=true request (default: cam++)",
+ )
parser.add_argument(
"--cors-origin",
action="append",
@@ -81,6 +86,7 @@ def main():
preload_model=args.model,
model_path=args.model_path,
hub=args.hub,
+ spk_model=args.spk_model,
cors_origins=args.cors_origin,
)
diff --git a/tests/test_server_app_openai_segments.py b/tests/test_server_app_openai_segments.py
index 99e90f3fb..972b27dac 100644
--- a/tests/test_server_app_openai_segments.py
+++ b/tests/test_server_app_openai_segments.py
@@ -82,7 +82,12 @@ def load_server_cli():
return module
-def install_dummy_funasr(monkeypatch, fail_once_for_models=(), generated_text="transcript"):
+def install_dummy_funasr(
+ monkeypatch,
+ fail_once_for_models=(),
+ generated_text="transcript",
+ generated_result=None,
+):
remaining_failures = {model: 1 for model in fail_once_for_models}
class DummyAutoModel:
@@ -101,6 +106,8 @@ def install_dummy_funasr(monkeypatch, fail_once_for_models=(), generated_text="t
def generate(self, **kwargs):
if self.kwargs.get("model") == "fsmn-vad":
return [{"value": [[0, 1000]]}]
+ if generated_result is not None:
+ return [generated_result.copy()]
return [{"text": generated_text}]
funasr_stub = types.ModuleType("funasr")
@@ -230,6 +237,211 @@ def test_verbose_json_reports_sensevoice_detected_language(monkeypatch):
]
+def test_openai_verbose_json_preserves_speaker_labels(monkeypatch):
+ module = load_server_app(monkeypatch)
+
+ response = module.build_openai_verbose_json(
+ {
+ "language": "zh",
+ "duration": 1.25,
+ "text": "hello",
+ "segments": [
+ {
+ "start": 0.0,
+ "end": 1.25,
+ "text": "hello",
+ "speaker": "SPK0",
+ }
+ ],
+ },
+ requested_language=None,
+ )
+
+ assert response["segments"] == [
+ {
+ "id": 0,
+ "start": 0.0,
+ "end": 1.25,
+ "text": "hello",
+ "words": [],
+ "speaker": "SPK0",
+ }
+ ]
+
+
+def test_attach_speaker_labels_runs_diarization_pipeline(monkeypatch):
+ import torch
+
+ module = load_server_app(monkeypatch)
+ calls = []
+
+ class DummyClusterBackend:
+ def __init__(self, merge_thr):
+ assert merge_thr == 0.78
+
+ def to(self, device):
+ assert device == "cpu"
+ return self
+
+ def __call__(self, embeddings, oracle_num=None):
+ assert embeddings.shape[1] == 2
+ assert oracle_num is None
+ calls.append("cluster")
+ return module.np.zeros(embeddings.shape[0], dtype=int)
+
+ class DummySpeakerModel:
+ def generate(self, input, cache, is_final):
+ assert input
+ assert cache == {}
+ assert is_final is True
+ return [
+ {"spk_embedding": torch.tensor([[1.0, 0.0]])}
+ for _ in input
+ ]
+
+ def fake_sv_chunk(diarization_inputs, fs):
+ assert fs == 16000
+ assert diarization_inputs[0][:2] == [0.0, 2.0]
+ calls.append("chunk")
+ return [[0.0, 2.0, diarization_inputs[0][2]]]
+
+ def fake_postprocess(chunks, vad_segments, labels, embeddings):
+ assert vad_segments is None
+ assert labels.tolist() == [0]
+ assert embeddings.shape == (1, 2)
+ calls.append("postprocess")
+ return [[chunks[0][0], chunks[0][1], 0]]
+
+ def fake_distribute_spk(sentences, speaker_timeline):
+ assert speaker_timeline == [[0.0, 2.0, 0]]
+ sentences[0]["spk"] = 0
+ calls.append("distribute")
+
+ funasr_stub = types.ModuleType("funasr")
+ funasr_stub.__path__ = []
+ models_stub = types.ModuleType("funasr.models")
+ models_stub.__path__ = []
+ campplus_stub = types.ModuleType("funasr.models.campplus")
+ campplus_stub.__path__ = []
+ cluster_stub = types.ModuleType("funasr.models.campplus.cluster_backend")
+ cluster_stub.ClusterBackend = DummyClusterBackend
+ utils_stub = types.ModuleType("funasr.models.campplus.utils")
+ utils_stub.sv_chunk = fake_sv_chunk
+ utils_stub.postprocess = fake_postprocess
+ utils_stub.distribute_spk = fake_distribute_spk
+ monkeypatch.setitem(sys.modules, "funasr", funasr_stub)
+ monkeypatch.setitem(sys.modules, "funasr.models", models_stub)
+ monkeypatch.setitem(sys.modules, "funasr.models.campplus", campplus_stub)
+ monkeypatch.setitem(
+ sys.modules, "funasr.models.campplus.cluster_backend", cluster_stub
+ )
+ monkeypatch.setitem(sys.modules, "funasr.models.campplus.utils", utils_stub)
+ segments = [{"start": 0.0, "end": 2.0, "text": "hello"}]
+
+ result = module.attach_speaker_labels(
+ module.np.ones(32000, dtype=module.np.float32),
+ 16000,
+ segments,
+ DummySpeakerModel(),
+ "cpu",
+ )
+
+ assert result == [
+ {"start": 0.0, "end": 2.0, "text": "hello", "speaker": "SPK0"}
+ ]
+ assert calls == ["chunk", "cluster", "postprocess", "distribute"]
+
+
+def test_fallback_reads_sentence_info_sentence_field(monkeypatch):
+ module = load_server_app(monkeypatch)
+ install_dummy_funasr(
+ monkeypatch,
+ generated_result={
+ "text": "hello",
+ "sentence_info": [
+ {"start": 0, "end": 1250, "sentence": "hello", "spk": 1}
+ ],
+ },
+ )
+ monkeypatch.setattr(module.sf, "info", lambda path: types.SimpleNamespace(duration=1.25))
+ app = module.create_app(device="cpu", preload_model="sensevoice")
+ transcribe = app.routes[("POST", "/v1/audio/transcriptions")]
+
+ response = asyncio.run(
+ transcribe(
+ file=DummyUpload(),
+ model="sensevoice",
+ language=None,
+ response_format="verbose_json",
+ spk=False,
+ )
+ )
+
+ assert response["segments"] == [
+ {
+ "id": 0,
+ "start": 0.0,
+ "end": 1.25,
+ "text": "hello",
+ "words": [],
+ "speaker": 1,
+ }
+ ]
+
+
+def test_spk_request_lazily_loads_and_reuses_speaker_model(monkeypatch):
+ module = load_server_app(monkeypatch)
+ DummyAutoModel = install_dummy_funasr(monkeypatch)
+ monkeypatch.setattr(module.sf, "info", lambda path: types.SimpleNamespace(duration=1.25))
+ monkeypatch.setattr(
+ module.sf,
+ "read",
+ lambda path: (module.np.ones(20000, dtype=module.np.float32), 16000),
+ )
+ attach_calls = []
+
+ def fake_attach(audio_data, sample_rate, segments, speaker_model, device):
+ attach_calls.append((sample_rate, speaker_model.kwargs["model"], device))
+ segments[0]["speaker"] = "SPK0"
+ return segments
+
+ monkeypatch.setattr(module, "attach_speaker_labels", fake_attach, raising=False)
+ app = module.create_app(
+ device="cpu",
+ preload_model="sensevoice",
+ spk_model="iic/speech_eres2netv2_sv_zh-cn_16k-common",
+ )
+ transcribe = app.routes[("POST", "/v1/audio/transcriptions")]
+
+ assert not any(
+ instance.get("model") == "iic/speech_eres2netv2_sv_zh-cn_16k-common"
+ for instance in DummyAutoModel.instances
+ )
+
+ for _ in range(2):
+ response = asyncio.run(
+ transcribe(
+ file=DummyUpload(),
+ model="sensevoice",
+ language=None,
+ response_format="verbose_json",
+ spk=True,
+ )
+ )
+ assert response["segments"][0]["speaker"] == "SPK0"
+
+ speaker_models = [
+ instance
+ for instance in DummyAutoModel.instances
+ if instance.get("model") == "iic/speech_eres2netv2_sv_zh-cn_16k-common"
+ ]
+ assert len(speaker_models) == 1
+ assert attach_calls == [
+ (16000, "iic/speech_eres2netv2_sv_zh-cn_16k-common", "cpu"),
+ (16000, "iic/speech_eres2netv2_sv_zh-cn_16k-common", "cpu"),
+ ]
+
+
def test_server_versions_follow_package_version(monkeypatch):
expected = (REPO_ROOT / "funasr" / "version.txt").read_text().strip()
module = load_server_app(monkeypatch)
@@ -261,6 +473,16 @@ def test_server_cli_collects_repeated_cors_origins():
]
+def test_server_cli_accepts_speaker_model():
+ module = load_server_cli()
+
+ args = module.build_parser().parse_args(
+ ["--spk-model", "iic/speech_eres2netv2_sv_zh-cn-16k-common"]
+ )
+
+ assert args.spk_model == "iic/speech_eres2netv2_sv_zh-cn-16k-common"
+
+
def test_server_cors_is_disabled_by_default(monkeypatch):
module = load_server_app(monkeypatch)
install_dummy_funasr(monkeypatch)