Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
Add client-driven realtime endpoints (#3245)
zhifu gaobd6e72142f1c
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md b/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.mdindex 3f40f1741..6dba0226e 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md+++ b/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md@@ -15,6 +15,25 @@ pip install -r requirements.txtCUDA_VISIBLE_DEVICES=0 python serve_realtime_ws.py --port 10095 --language 中文```+### 客户端 VAD / 低延迟对话模式++客户端已经持续执行 VAD 时,可以关闭服务端 VAD 模型,并由客户端明确提交每句话:++```bash+CUDA_VISIBLE_DEVICES=0 python serve_realtime_ws.py \+ --port 10095 --language 中文 --endpoint-mode client+```++同一条 WebSocket 连接上的协议为:++1. 发送文本 `START`;+2. 持续发送 16 kHz、16-bit PCM 二进制帧,服务端仍会返回 partial;+3. 客户端 VAD 检测到句末后发送文本 `COMMIT`;+4. 服务端立即返回一次 `is_final=true`,清理当前句状态但保持连接;+5. 重复步骤 2-4,整个会话结束时发送 `STOP`。++`COMMIT` 不受默认 960 ms final 门槛限制,因此短句也会提交。该模式信任客户端 endpoint,不运行 FSMN-VAD;需要服务端自动去噪和分句时继续使用默认的 `--endpoint-mode server`。+## 客户端```bashdiff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.pyindex 58355b3c0..db6888d09 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py@@ -3,6 +3,7 @@Features:- Streaming VAD segmentation (fsmn-vad)+- Client-driven utterance endpoints (COMMIT, without server VAD)- Per-segment ASR decoding (Fun-ASR-Nano via vLLM)- Speaker diarization (eres2netv2 + ClusterBackend)- Hotword customization@@ -75,6 +76,20 @@ def _clean_asr_text(text):from funasr.models.fsmn_vad_streaming.dynamic_vad import DynamicStreamingVAD+class ClientEndpointVAD:+ """Track an utterance start without loading or running a VAD model."""++ def __init__(self):+ self.current_speech_start = None++ def start_utterance(self, start_ms):+ if self.current_speech_start is None:+ self.current_speech_start = start_ms++ def reset(self):+ self.current_speech_start = None++class HybridSpeakerTracker:"""Speaker diarization: streaming ClusterBackend + final re-clustering."""@@ -312,10 +327,14 @@ class RealtimeASRSession:chunk_ms=960,partial_window_sec=15.0,audio_lookback_sec=5.0,+ endpoint_mode="server",):+ if endpoint_mode not in {"server", "client"}:+ raise ValueError(f"Unsupported endpoint mode: {endpoint_mode}")self.vllm_engine = vllm_engineself.asr_kwargs = asr_kwargsself.vad = vad+ self.endpoint_mode = endpoint_modeself.sample_rate = sample_rateself.chunk_samples = int(sample_rate * chunk_ms / 1000)self.first_chunk_samples = int(sample_rate * 480 / 1000)@@ -347,11 +366,19 @@ class RealtimeASRSession:def add_audio(self, pcm_bytes):audio_int16 = np.frombuffer(pcm_bytes, dtype=np.int16)audio_float = audio_int16.astype(np.float32) / 32768.0+ chunk_start_sample = self.total_samplesself.audio_buffer = np.concatenate([self.audio_buffer, audio_float])self.total_samples += len(audio_float)if len(audio_float) > 0:- new_confirmed = self.vad.feed(torch.from_numpy(audio_float).float(), is_final=False)+ if self.endpoint_mode == "client":+ start_ms = int(chunk_start_sample * 1000 / self.sample_rate)+ self.vad.start_utterance(start_ms)+ new_confirmed = []+ else:+ new_confirmed = self.vad.feed(+ torch.from_numpy(audio_float).float(), is_final=False+ )for seg in new_confirmed:seg_text = self._decode_segment(seg)@@ -398,11 +425,23 @@ class RealtimeASRSession:@torch.no_grad()def decode(self, is_final=False):- if self.total_samples < self.chunk_samples:+ if is_final and self.endpoint_mode == "client":+ return self.commit()++ if self.endpoint_mode == "server" and self.total_samples < self.chunk_samples:if is_final:self._release_audio_buffer()return self._build_response(is_final)+ if self.endpoint_mode == "client":+ if self.vad.current_speech_start is None:+ return self._build_response(is_final=False)+ utterance_start_sample = int(+ self.vad.current_speech_start * self.sample_rate / 1000+ )+ if self.total_samples - utterance_start_sample < self.first_chunk_samples:+ return self._build_response(is_final=False)+if is_final:if self.vad.current_speech_start is not None:end_ms = int(self.total_samples * 1000 / self.sample_rate)@@ -460,6 +499,52 @@ class RealtimeASRSession:return self._build_response(is_final)+ @torch.no_grad()+ def commit(self):+ """Finalize one client-delimited utterance while keeping the session open."""+ if self.endpoint_mode != "client":+ raise RuntimeError("COMMIT requires endpoint_mode='client'")++ if self.vad.current_speech_start is not None:+ end_ms = int(self.total_samples * 1000 / self.sample_rate)+ seg = [self.vad.current_speech_start, end_ms]+ seg_text = self._decode_segment(seg)+ if seg_text.strip():+ sentence = {+ "text": seg_text,+ "start": int(seg[0]),+ "end": int(seg[1]),+ }+ if self.spk_tracker:+ s0 = int(seg[0] * self.sample_rate / 1000)+ segment_audio = self._slice_audio(s0, self.total_samples).copy()+ self.spk_tracker.assign_streaming(+ segment_audio,+ seg[0] / 1000,+ seg[1] / 1000,+ sentence,+ )+ self.locked_sentences.append(sentence)+ self.vad.current_speech_start = None++ if self.spk_tracker and self.locked_sentences:+ self.locked_sentences = self.spk_tracker.finalize(self.locked_sentences)++ response = self._build_response(is_final=True)+ self._reset_utterance_state()+ return response++ def _reset_utterance_state(self):+ self._release_audio_buffer()+ self.first_decode_done = False+ self.vad.reset()+ self.prev_text = ""+ self.last_partial_text = ""+ self.last_partial_start_ms = 0+ self.last_decode_samples = self.total_samples+ self.locked_sentences = []+ self.prev_seg_text = ""+def get_partial_decode_audio(self):"""Return the bounded audio window used for unstable partial decoding."""seg_start_sample = int(self.vad.current_speech_start * self.sample_rate / 1000)@@ -564,8 +649,14 @@ def load_models(args):_asr_kwargs["language"] = args.languagelogger.info(f"Language: {args.language}")- logger.info("Loading VAD: fsmn-vad (streaming)")- _vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True)+ if getattr(args, "endpoint_mode", "server") == "server":+ logger.info("Loading VAD: fsmn-vad (streaming)")+ _vad_model = AutoModel(+ model="fsmn-vad", device=args.device, disable_update=True+ )+ else:+ _vad_model = None+ logger.info("Server VAD disabled; client COMMIT controls utterance endpoints")if getattr(args, "enable_spk", False):logger.info(f"Loading SPK: {args.spk_model}")@@ -597,7 +688,11 @@ async def run_session_work(args, operation, *operation_args, **operation_kwargs)async def handle_client(websocket, args):vllm_engine, asr_kwargs, vad_model, spk_model = load_models(args)- vad = DynamicStreamingVAD(vad_model)+ endpoint_mode = getattr(args, "endpoint_mode", "server")+ if endpoint_mode == "client":+ vad = ClientEndpointVAD()+ else:+ vad = DynamicStreamingVAD(vad_model)spk_tracker = create_speaker_tracker(spk_model, args)session = RealtimeASRSession(vllm_engine,@@ -605,6 +700,7 @@ async def handle_client(websocket, args):vad,spk_tracker=spk_tracker,partial_window_sec=getattr(args, 'partial_window_sec', 15.0),+ endpoint_mode=endpoint_mode,)logger.info(f"Client connected: {websocket.remote_address}")@@ -633,12 +729,55 @@ async def handle_client(websocket, args):session.asr_kwargs["language"] = lang if lang else Noneawait websocket.send(json.dumps({"event": "language_set", "language": lang}))logger.info(f"Language set: {lang}")+ elif cmd.upper() == "COMMIT":+ if endpoint_mode != "client":+ await websocket.send(+ json.dumps(+ {+ "event": "error",+ "error": "COMMIT requires --endpoint-mode client",+ }+ )+ )+ elif not session.is_active:+ await websocket.send(+ json.dumps(+ {+ "event": "error",+ "error": "Session is not active; send START first",+ }+ )+ )+ else:+ commit_started = time.perf_counter()+ result = await run_session_work(args, session.commit)+ await websocket.send(json.dumps(result))+ last_decode_time = time.time()+ elapsed_ms = (time.perf_counter() - commit_started) * 1000+ logger.info(+ "Commit final: %d sentences in %.1fms",+ len(result.get("sentences", [])),+ elapsed_ms,+ )elif cmd.upper() == "STOP":- if session.is_active and session.total_samples > 0:- result = await run_session_work(args, session.decode, is_final=True)+ if endpoint_mode == "client":+ has_pending_audio = (+ session.total_samples > session.audio_buffer_start_sample+ )+ else:+ has_pending_audio = session.total_samples > 0+ if session.is_active and has_pending_audio:+ if endpoint_mode == "client":+ result = await run_session_work(args, session.commit)+ else:+ result = await run_session_work(+ args, session.decode, is_final=True+ )await websocket.send(json.dumps(result))- logger.info(f"Final: {len(result['sentences'])} sentences")- session.is_active = False+ logger.info(+ "Final: %d sentences", len(result.get("sentences", []))+ )+ session.is_active = Falseawait websocket.send(json.dumps({"event": "stopped"}))elif isinstance(message, bytes) and session.is_active:await run_session_work(args, session.add_audio, message)@@ -693,6 +832,15 @@ def build_arg_parser():parser.add_argument("--use-context", action="store_true", default=True)parser.add_argument("--no-context", dest="use_context", action="store_false")parser.add_argument("--decode-interval", type=float, default=0.48)+ parser.add_argument(+ "--endpoint-mode",+ choices=["server", "client"],+ default="server",+ help=(+ "Use server VAD endpoints (default), or accept client COMMIT messages "+ "without loading the VAD model."+ ),+ )parser.add_argument("--partial-window-sec", type=float, default=15.0,help="Cap the interim partial re-decode window to the most recent N seconds. ""A long ongoing speech segment is otherwise re-encoded from its start on "diff --git a/tests/test_realtime_ws_service.py b/tests/test_realtime_ws_service.pyindex 00db6bafd..c7da0b994 100644--- a/tests/test_realtime_ws_service.py+++ b/tests/test_realtime_ws_service.py@@ -1,5 +1,6 @@import asyncioimport importlib.util+import jsonimport sysimport threadingimport time@@ -38,6 +39,15 @@ def test_cli_defaults_disable_speaker_and_bound_partial_window():assert args.enable_spk is Falseassert args.partial_window_sec == 15.0+ assert args.endpoint_mode == "server"+++def test_cli_accepts_client_endpoint_mode():+ module = load_service_module()++ args = module.build_arg_parser().parse_args(["--endpoint-mode", "client"])++ assert args.endpoint_mode == "client"def test_cli_accepts_float32_dtype_alias():@@ -214,6 +224,197 @@ class DummyEngine:return [{"text": "hello"}]+def test_client_endpoint_mode_skips_fsmn_vad_model(monkeypatch):+ module = load_service_module()+ auto_model_calls = []++ import funasr++ monkeypatch.setattr(+ funasr,+ "AutoModel",+ lambda model, **kwargs: auto_model_calls.append(model) or object(),+ )+ vllm_stub = types.ModuleType("funasr.auto.auto_model_vllm")+ vllm_stub.AutoModelVLLM = lambda **kwargs: object()+ monkeypatch.setitem(sys.modules, "funasr.auto.auto_model_vllm", vllm_stub)++ args = types.SimpleNamespace(+ model="FunAudioLLM/Fun-ASR-Nano-2512",+ hub="ms",+ device="cpu",+ dtype="fp32",+ tensor_parallel_size=1,+ gpu_memory_utilization=0.8,+ max_model_len=2048,+ hotword_file="",+ language=None,+ enable_spk=False,+ endpoint_mode="client",+ )++ _, _, vad_model, _ = module.load_models(args)++ assert vad_model is None+ assert auto_model_calls == []+++def make_client_endpoint_session(module):+ assert hasattr(module, "ClientEndpointVAD")+ return module.RealtimeASRSession(+ vllm_engine=DummyEngine(),+ asr_kwargs={},+ vad=module.ClientEndpointVAD(),+ sample_rate=16000,+ chunk_ms=960,+ endpoint_mode="client",+ )+++def test_client_endpoint_mode_emits_partial_at_first_decode_threshold():+ module = load_service_module()+ session = make_client_endpoint_session(module)+ audio = np.zeros(int(0.48 * session.sample_rate), dtype=np.int16).tobytes()++ session.add_audio(audio)+ response = session.decode(is_final=False)++ assert session.should_decode() is False+ assert response["partial"] == "hello"+ assert response["partial_start_ms"] == 0+ assert response["is_final"] is False+ assert session.vllm_engine.input_lengths == [int(0.48 * session.sample_rate)]+++def test_client_commits_short_utterances_once_with_monotonic_timestamps():+ module = load_service_module()+ session = make_client_endpoint_session(module)++ session.add_audio(np.zeros(int(0.4 * session.sample_rate), dtype=np.int16).tobytes())+ first = session.commit()+ session.add_audio(np.zeros(int(0.3 * session.sample_rate), dtype=np.int16).tobytes())+ second = session.commit()++ assert first == {+ "sentences": [{"text": "hello", "start": 0, "end": 400}],+ "partial": "",+ "partial_start_ms": 0,+ "duration_ms": 400,+ "is_final": True,+ }+ assert second == {+ "sentences": [{"text": "hello", "start": 400, "end": 700}],+ "partial": "",+ "partial_start_ms": 0,+ "duration_ms": 700,+ "is_final": True,+ }+ assert session.vllm_engine.input_lengths == [6400, 4800]+ assert len(session.audio_buffer) == 0+ assert session.audio_buffer_start_sample == session.total_samples+++def test_handler_commit_returns_one_final_and_reuses_connection(monkeypatch):+ module = load_service_module()+ dynamic_vad_calls = []+ client_vad_calls = []+ received_audio = []++ class ProtocolSession:+ def __init__(self, *args, **kwargs):+ self.is_active = False+ self.total_samples = 0+ self.audio_buffer_start_sample = 0+ self.audio_buffer = np.array([], dtype=np.float32)+ self.commit_count = 0++ def reset(self):+ self.total_samples = 0+ self.audio_buffer_start_sample = 0+ self.audio_buffer = np.array([], dtype=np.float32)++ def add_audio(self, message):+ received_audio.append(message)+ self.total_samples += 1+ self.audio_buffer = np.ones(1, dtype=np.float32)++ def should_decode(self):+ return False++ def commit(self):+ self.commit_count += 1+ self.audio_buffer = np.array([], dtype=np.float32)+ self.audio_buffer_start_sample = self.total_samples+ return {"is_final": True, "sequence": self.commit_count}++ def decode(self, is_final=False):+ return self.commit()++ class FakeWebSocket:+ remote_address = ("127.0.0.1", 12345)++ def __init__(self):+ self.messages = iter(+ [+ "START",+ b"first",+ "COMMIT",+ b"second",+ "COMMIT",+ "STOP",+ b"after-stop",+ ]+ )+ self.sent = []++ def __aiter__(self):+ return self++ async def __anext__(self):+ try:+ return next(self.messages)+ except StopIteration as error:+ raise StopAsyncIteration from error++ async def send(self, message):+ self.sent.append(json.loads(message))++ monkeypatch.setattr(module, "load_models", lambda args: (object(), {}, None, None))+ monkeypatch.setattr(+ module,+ "DynamicStreamingVAD",+ lambda model: dynamic_vad_calls.append(model) or object(),+ )+ monkeypatch.setattr(+ module,+ "ClientEndpointVAD",+ lambda: client_vad_calls.append(True) or object(),+ raising=False,+ )+ monkeypatch.setattr(module, "create_speaker_tracker", lambda model, args: None)+ monkeypatch.setattr(module, "RealtimeASRSession", ProtocolSession)+ websocket = FakeWebSocket()+ args = types.SimpleNamespace(+ device="cpu",+ decode_interval=0.48,+ partial_window_sec=15.0,+ endpoint_mode="client",+ )++ asyncio.run(module.handle_client(websocket, args))++ finals = [message for message in websocket.sent if message.get("is_final")]+ events = [message.get("event") for message in websocket.sent if message.get("event")]+ assert finals == [+ {"is_final": True, "sequence": 1},+ {"is_final": True, "sequence": 2},+ ]+ assert events == ["started", "stopped"]+ assert dynamic_vad_calls == []+ assert client_vad_calls == [True]+ assert received_audio == [b"first", b"second"]++def test_two_hour_session_keeps_audio_bounded_and_duration_absolute():module = load_service_module()sample_rate = 10