Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat: support speaker diarization for Fun-ASR-Nano (#2844)

游雁a424f4c6b6aa
diff --git a/README.md b/README.md
index b5b9c8e2e..f80a0fe6a 100644
--- a/README.md
+++ b/README.md
@@ -34,6 +34,7 @@
## What's new:
+- 2026/05/19: Fun-ASR-Nano now supports speaker diarization. Use with `vad_model` + `spk_model` + `punc_model` to get per-sentence speaker labels. See [demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py).
- 2025/12/15: [Fun-ASR-Nano-2512](https://github.com/FunAudioLLM/Fun-ASR) is an end-to-end speech recognition large model trained on tens of millions of hours real speech data. It supports low-latency real-time transcription and covers 31 languages.
- 2024/10/29: Real-time Transcription Service 1.12 released, The 2pass-offline mode supports the SensevoiceSmal model;([docs](runtime/readme.md));
- 2024/10/10:Added support for the Whisper-large-v3-turbo model, a multitasking model that can perform multilingual speech recognition, speech translation, and language identification. It can be downloaded from the [modelscope](examples/industrial_data_pretraining/whisper/demo.py), and [openai](examples/industrial_data_pretraining/whisper/demo_from_openai.py).
@@ -170,6 +171,27 @@ Parameter Description:
- `vad_kwargs`: Specifies the configurations for the VAD model. `max_single_segment_time`: denotes the maximum duration for audio segmentation by the `vad_model`, with the unit being milliseconds (ms).
- `batch_size_s`: Indicates the use of dynamic batching, where the total duration of audio in the batch is measured in seconds (s).
+#### Fun-ASR-Nano with Speaker Diarization
+
+```python
+from funasr import AutoModel
+
+model = AutoModel(
+ model="FunAudioLLM/Fun-ASR-Nano-2512",
+ trust_remote_code=True,
+ remote_code="./model.py",
+ vad_model="fsmn-vad",
+ vad_kwargs={"max_single_segment_time": 30000},
+ spk_model="cam++",
+ punc_model="ct-punc",
+ device="cuda:0",
+)
+res = model.generate(input="example.wav", cache={}, batch_size=1, language="中文")
+# Result contains sentence_info with speaker labels
+print(res[0]["sentence_info"])
+# [{'text': '...', 'start': 420, 'end': 5580, 'timestamp': [...], 'spk': 0}]
+```
+
#### SenseVoice
```python
diff --git a/README_zh.md b/README_zh.md
index aadf46487..fdc6a07bf 100644
--- a/README_zh.md
+++ b/README_zh.md
@@ -34,6 +34,7 @@ FunASR 希望在语音识别的学术研究和工业应用之间架起一座桥
## 最新动态
+- 2026/05/19: Fun-ASR-Nano 现已支持说话人分离。配合 `vad_model` + `spk_model` + `punc_model` 使用,可获得带说话人标签的逐句结果。参见 [demo](examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py)。
- 2025/12/15: [Fun-ASR-Nano-2512](https://github.com/FunAudioLLM/Fun-ASR) 是一款基于数千万小时真实语音数据训练的端到端语音识别大模型。它支持低延迟实时转写,并涵盖 31 种语言识别功能。
- 2024/10/29: 中文实时语音听写服务 1.12 发布,2pass-offline 模式支持 SensevoiceSmall 模型;详细信息参阅([部署文档](runtime/readme_cn.md))
- 2024/10/10:新增加 Whisper-large-v3-turbo 模型支持,多语言语音识别/翻译/语种识别,支持从 [modelscope](examples/industrial_data_pretraining/whisper/demo.py)仓库下载,也支持从 [openai](examples/industrial_data_pretraining/whisper/demo_from_openai.py)仓库下载模型。
@@ -167,6 +168,27 @@ print(text)
+#### Fun-ASR-Nano 说话人分离
+
+```python
+from funasr import AutoModel
+
+model = AutoModel(
+ model="FunAudioLLM/Fun-ASR-Nano-2512",
+ trust_remote_code=True,
+ remote_code="./model.py",
+ vad_model="fsmn-vad",
+ vad_kwargs={"max_single_segment_time": 30000},
+ spk_model="cam++",
+ punc_model="ct-punc",
+ device="cuda:0",
+)
+res = model.generate(input="example.wav", cache={}, batch_size=1, language="中文")
+# 结果包含 sentence_info,带有说话人标签
+print(res[0]["sentence_info"])
+# [{'text': '...', 'start': 420, 'end': 5580, 'timestamp': [...], 'spk': 0}]
+```
+
#### SenseVoice
```python
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py b/examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py
new file mode 100644
index 000000000..d80657d6f
--- /dev/null
+++ b/examples/industrial_data_pretraining/fun_asr_nano/demo_spk.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+# -*- encoding: utf-8 -*-
+# Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
+# MIT License (https://opensource.org/licenses/MIT)
+
+"""
+Fun-ASR-Nano with Speaker Diarization
+
+This demo shows how to use Fun-ASR-Nano with VAD + Speaker Model + Punctuation Model
+to get per-sentence speaker labels.
+"""
+
+import torch
+from funasr import AutoModel
+
+
+def main():
+ model_dir = "FunAudioLLM/Fun-ASR-Nano-2512"
+ device = (
+ "cuda:0"
+ if torch.cuda.is_available()
+ else "mps" if torch.backends.mps.is_available() else "cpu"
+ )
+
+ model = AutoModel(
+ model=model_dir,
+ trust_remote_code=True,
+ remote_code="./model.py",
+ vad_model="fsmn-vad",
+ vad_kwargs={"max_single_segment_time": 30000},
+ spk_model="cam++",
+ punc_model="ct-punc",
+ device=device,
+ hub="ms",
+ )
+
+ wav_path = f"{model.model_path}/example/zh.mp3"
+ res = model.generate(
+ input=[wav_path],
+ cache={},
+ batch_size=1,
+ language="中文",
+ )
+
+ # Print full text
+ print("Text:", res[0]["text"])
+ print()
+
+ # Print per-sentence results with speaker labels
+ print("Speaker Diarization Results:")
+ for sent in res[0]["sentence_info"]:
+ print(
+ f" Speaker {sent['spk']}: [{sent['start']}ms - {sent['end']}ms] {sent['text']}"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py
index 54ef2d756..ddf19cc8c 100644
--- a/funasr/auto/auto_model.py
+++ b/funasr/auto/auto_model.py
@@ -546,7 +546,8 @@ class AutoModel:
spk_res = self.inference(
speech_b, input_len=None, model=self.spk_model, kwargs=kwargs, **cfg
)
- results[_b]["spk_embedding"] = spk_res[0]["spk_embedding"]
+ spk_embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)
+ results[_b]["spk_embedding"] = spk_embs
beg_idx = end_idx
end_idx += 1
max_len_in_batch = sample_length
@@ -606,6 +607,13 @@ class AutoModel:
else:
result[k] += restored_data[j][k]
+ # Convert dict-format timestamps (Fun-ASR-Nano) to list-format for downstream compatibility
+ if "timestamps" in result and "timestamp" not in result:
+ result["timestamp"] = [
+ [int(t["start_time"] * 1000), int(t["end_time"] * 1000)]
+ for t in result["timestamps"]
+ ]
+
if not len(result["text"].strip()):
continue
return_raw_text = kwargs.get("return_raw_text", False)
@@ -624,7 +632,10 @@ class AutoModel:
# speaker embedding cluster after resorted
if self.spk_model is not None and kwargs.get("return_spk_res", True):
- if raw_text is None:
+ if raw_text is None and self.spk_mode == "punc_segment":
+ logging.warning("punc_model is missing, falling back to vad_segment mode for speaker diarization.")
+ self.spk_mode = "vad_segment"
+ elif raw_text is None:
logging.error("Missing punc_model, which is required by spk_model.")
all_segments = sorted(all_segments, key=lambda x: x[0])
spk_embedding = result["spk_embedding"]
@@ -636,18 +647,22 @@ class AutoModel:
if self.spk_mode == "vad_segment": # recover sentence_list
sentence_list = []
for rest, vadsegment in zip(restored_data, vadsegments):
- if "timestamp" not in rest:
- logging.error(
- "Only 'iic/speech_paraformer-large-vad-punc_asr_nat-zh-cn-16k-common-vocab8404-pytorch' \
- and 'iic/speech_seaco_paraformer_large_asr_nat-zh-cn-16k-common-vocab8404-pytorch'\
- can predict timestamp, and speaker diarization relies on timestamps."
- )
+ if "timestamp" in rest:
+ ts = rest["timestamp"]
+ elif "timestamps" in rest:
+ ts = [
+ [int(t["start_time"] * 1000), int(t["end_time"] * 1000)]
+ for t in rest["timestamps"]
+ ]
+ else:
+ logging.error("No timestamp found in ASR result. Speaker diarization relies on timestamps.")
+ ts = []
sentence_list.append(
{
"start": vadsegment[0],
"end": vadsegment[1],
"sentence": rest["text"],
- "timestamp": rest["timestamp"],
+ "timestamp": ts,
}
)
elif self.spk_mode == "punc_segment":
diff --git a/tests_models/test_fun_asr_nano_spk.py b/tests_models/test_fun_asr_nano_spk.py
new file mode 100644
index 000000000..3c00d138c
--- /dev/null
+++ b/tests_models/test_fun_asr_nano_spk.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Test Fun-ASR-Nano + VAD + SPK + PUNC: full pipeline with speaker diarization"""
+import sys
+import time
+
+def main():
+ from funasr import AutoModel
+
+ print("[Fun-ASR-Nano-SPK] Loading model...")
+ t0 = time.time()
+ model = AutoModel(
+ model="FunAudioLLM/Fun-ASR-Nano-2512",
+ trust_remote_code=True,
+ remote_code="./model.py",
+ vad_model="iic/speech_fsmn_vad_zh-cn-16k-common-pytorch",
+ vad_kwargs={"max_single_segment_time": 30000},
+ spk_model="iic/speech_campplus_sv_zh-cn_16k-common",
+ punc_model="iic/punc_ct-transformer_zh-cn-common-vocab272727-pytorch",
+ device="cpu",
+ disable_update=True,
+ hub="hf",
+ )
+ print("[Fun-ASR-Nano-SPK] Model loaded in %.1fs" % (time.time() - t0))
+
+ wav_path = model.model_path + "/example/zh.mp3"
+
+ print("[Fun-ASR-Nano-SPK] Running inference...")
+ t0 = time.time()
+ res = model.generate(input=[wav_path], cache={}, batch_size=1, language="中文")
+ print("[Fun-ASR-Nano-SPK] Inference done in %.1fs" % (time.time() - t0))
+
+ if not res or len(res) == 0:
+ print("[Fun-ASR-Nano-SPK] FAILED - empty result")
+ return 1
+
+ result = res[0]
+ keys = list(result.keys())
+ print("[Fun-ASR-Nano-SPK] Result keys: %s" % keys)
+ print("[Fun-ASR-Nano-SPK] Text: %s" % result.get("text", ""))
+
+ # Verify timestamp
+ ts = result.get("timestamp", None)
+ if ts is None or len(ts) == 0:
+ print("[Fun-ASR-Nano-SPK] FAILED - no timestamp")
+ return 1
+ print("[Fun-ASR-Nano-SPK] Timestamp count: %d, first: %s" % (len(ts), ts[0]))
+
+ # Verify sentence_info with speaker labels
+ si = result.get("sentence_info", None)
+ if si is None or len(si) == 0:
+ print("[Fun-ASR-Nano-SPK] FAILED - no sentence_info")
+ return 1
+
+ print("[Fun-ASR-Nano-SPK] sentence_info:")
+ for s in si:
+ print(" spk=%d | [%d-%d] %s" % (s["spk"], s["start"], s["end"], s["text"]))
+
+ has_spk = all("spk" in s for s in si)
+ if not has_spk:
+ print("[Fun-ASR-Nano-SPK] FAILED - missing spk label in sentence_info")
+ return 1
+
+ print("[Fun-ASR-Nano-SPK] PASSED")
+ return 0
+
+if __name__ == "__main__":
+ sys.exit(main())