Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat: add real-time WebSocket ASR demo with streaming VAD

游雁ed0ed9b7d4f7
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html
new file mode 100644
index 000000000..f06aedb68
--- /dev/null
+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html
@@ -0,0 +1,243 @@
+<!DOCTYPE html>
+<html lang="zh-CN">
+<head>
+ <meta charset="UTF-8">
+ <title>Fun-ASR-Nano vLLM - Real-time ASR</title>
+ <style>
+ * { margin: 0; padding: 0; box-sizing: border-box; }
+ body {
+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+ background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%);
+ min-height: 100vh; display: flex; align-items: center; justify-content: center;
+ color: #e0e0e0;
+ }
+ .container {
+ background: rgba(255,255,255,0.05); backdrop-filter: blur(10px);
+ border: 1px solid rgba(255,255,255,0.1); border-radius: 20px;
+ padding: 40px; width: 90%; max-width: 700px;
+ }
+ h1 { text-align: center; font-size: 24px; margin-bottom: 8px; color: #64ffda; }
+ .subtitle { text-align: center; color: #aaa; margin-bottom: 30px; font-size: 14px; }
+ .config { display: flex; gap: 12px; margin-bottom: 20px; align-items: center; flex-wrap: wrap; }
+ .config label { font-size: 13px; color: #aaa; }
+ .config input, .config select {
+ background: rgba(255,255,255,0.1); border: 1px solid rgba(255,255,255,0.2);
+ border-radius: 8px; padding: 8px 12px; color: #fff; font-size: 14px;
+ }
+ .config input { width: 280px; }
+ .btn {
+ display: block; width: 200px; margin: 0 auto 20px; padding: 14px 0;
+ border: none; border-radius: 50px; font-size: 16px; font-weight: 600;
+ cursor: pointer; transition: all 0.3s;
+ }
+ .btn-start { background: #64ffda; color: #1a1a2e; }
+ .btn-start:hover { background: #4de8c2; transform: scale(1.05); }
+ .btn-stop { background: #ff6b6b; color: #fff; }
+ .btn-stop:hover { background: #ee5a5a; transform: scale(1.05); }
+ .btn:disabled { opacity: 0.5; cursor: not-allowed; transform: none; }
+ .status {
+ text-align: center; margin-bottom: 20px; font-size: 14px;
+ padding: 8px; border-radius: 8px;
+ }
+ .status.connected { background: rgba(100,255,218,0.1); color: #64ffda; }
+ .status.recording { background: rgba(255,107,107,0.1); color: #ff6b6b; }
+ .status.error { background: rgba(255,107,107,0.1); color: #ff6b6b; }
+ .result-box {
+ background: rgba(0,0,0,0.3); border: 1px solid rgba(255,255,255,0.1);
+ border-radius: 12px; padding: 20px; min-height: 150px; max-height: 400px; overflow-y: auto;
+ font-size: 18px; line-height: 1.8; word-break: break-all;
+ }
+ .result-box .final { color: #64ffda; }
+ .result-box .partial { color: #aaa; font-style: italic; }
+ .result-box .placeholder { color: #555; }
+ .visualizer { height: 60px; margin: 15px 0; display: flex; align-items: center; justify-content: center; gap: 3px; }
+ .visualizer .bar {
+ width: 4px; background: #64ffda; border-radius: 2px;
+ transition: height 0.1s; min-height: 4px;
+ }
+ .footer { text-align: center; margin-top: 20px; font-size: 12px; color: #666; }
+ </style>
+</head>
+<body>
+<div class="container">
+ <h1>Fun-ASR-Nano vLLM</h1>
+ <p class="subtitle">Real-time Speech Recognition</p>
+
+ <div class="config">
+ <label>Server:</label>
+ <input type="text" id="serverUrl" value="ws://localhost:10095" placeholder="ws://host:port">
+ <label>Language:</label>
+ <select id="language">
+ <option value="中文" selected>中文</option>
+ <option value="英文">English</option>
+ <option value="日文">日本語</option>
+ </select>
+ </div>
+
+ <div id="status" class="status">Click "Start" to begin</div>
+
+ <div class="visualizer" id="visualizer"></div>
+
+ <button class="btn btn-start" id="btnStart" onclick="startRecording()">Start Recording</button>
+ <button class="btn btn-stop" id="btnStop" onclick="stopRecording()" style="display:none">Stop</button>
+
+ <div class="result-box" id="resultBox">
+ <span class="placeholder">Recognition results will appear here...</span>
+ </div>
+
+ <p class="footer">Fun-ASR-Nano powered by vLLM | Audio: 16kHz PCM16 mono</p>
+</div>
+
+<script>
+let ws = null;
+let mediaStream = null;
+let audioContext = null;
+let processor = null;
+let isRecording = false;
+let finalText = '';
+
+// Create visualizer bars
+const visualizer = document.getElementById('visualizer');
+for (let i = 0; i < 40; i++) {
+ const bar = document.createElement('div');
+ bar.className = 'bar';
+ bar.style.height = '4px';
+ visualizer.appendChild(bar);
+}
+const bars = visualizer.querySelectorAll('.bar');
+
+function setStatus(msg, cls) {
+ const el = document.getElementById('status');
+ el.textContent = msg;
+ el.className = 'status ' + (cls || '');
+}
+
+function renderSentences(sentences, partial, partialStartMs, durationMs, isFinal) {
+ const box = document.getElementById('resultBox');
+ let html = '';
+ sentences.forEach(s => {
+ let startS = (s.start_ms / 1000).toFixed(1);
+ let endS = (s.end_ms / 1000).toFixed(1);
+ html += '<div style="margin:8px 0;display:flex;align-items:baseline">' +
+ '<span style="color:#64ffda;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ' + endS + 's</span>' +
+ '<span style="color:#e0e0e0;font-size:16px;margin-left:10px">' + s.text + '</span></div>';
+ });
+ if (partial) {
+ let startS = (partialStartMs / 1000).toFixed(1);
+ html += '<div style="margin:8px 0;display:flex;align-items:baseline">' +
+ '<span style="color:#ffb74d;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ...</span>' +
+ '<span style="color:#aaa;font-size:16px;margin-left:10px;font-style:italic">' + partial + '</span></div>';
+ }
+ if (!html) { html = '<span class="placeholder">Listening...</span>'; }
+ box.innerHTML = html;
+ box.scrollTop = box.scrollHeight;
+}
+
+async function startRecording() {
+ const serverUrl = document.getElementById('serverUrl').value;
+ finalText = '';
+ document.getElementById('resultBox').innerHTML = '<span class="placeholder">Connecting...</span>';
+
+ // Connect WebSocket
+ try {
+ ws = new WebSocket(serverUrl);
+ } catch(e) {
+ setStatus('Connection failed: ' + e.message, 'error');
+ return;
+ }
+
+ ws.onopen = async () => {
+ setStatus('Connected. Recording...', 'recording');
+ ws.send('START');
+
+ // Get microphone
+ try {
+ mediaStream = await navigator.mediaDevices.getUserMedia({
+ audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true }
+ });
+ } catch(e) {
+ setStatus('Microphone access denied', 'error');
+ ws.close();
+ return;
+ }
+
+ audioContext = new AudioContext({ sampleRate: 16000 });
+ const source = audioContext.createMediaStreamSource(mediaStream);
+
+ // AudioWorklet or ScriptProcessor
+ processor = audioContext.createScriptProcessor(4096, 1, 1);
+ processor.onaudioprocess = (e) => {
+ if (!isRecording) return;
+ const float32 = e.inputBuffer.getChannelData(0);
+
+ // Update visualizer
+ const rms = Math.sqrt(float32.reduce((sum, x) => sum + x*x, 0) / float32.length);
+ bars.forEach((bar, i) => {
+ const h = Math.max(4, Math.min(50, rms * 500 * (0.5 + Math.random())));
+ bar.style.height = h + 'px';
+ });
+
+ // Convert to PCM16 and send
+ const int16 = new Int16Array(float32.length);
+ for (let i = 0; i < float32.length; i++) {
+ int16[i] = Math.max(-32768, Math.min(32767, Math.round(float32[i] * 32768)));
+ }
+ if (ws && ws.readyState === WebSocket.OPEN) {
+ ws.send(int16.buffer);
+ }
+ };
+ source.connect(processor);
+ processor.connect(audioContext.destination);
+
+ isRecording = true;
+ document.getElementById('btnStart').style.display = 'none';
+ document.getElementById('btnStop').style.display = 'block';
+ document.getElementById('resultBox').innerHTML = '<span class="placeholder">Listening...</span>';
+ };
+
+ ws.onmessage = (event) => {
+ const data = JSON.parse(event.data);
+ if (data.sentences !== undefined) {
+ renderSentences(data.sentences, data.partial, data.partial_start_ms, data.duration_ms, data.is_final);
+ }
+ if (data.event === 'stopped') {
+ setStatus('Stopped', 'connected');
+ }
+ };
+
+ ws.onerror = (e) => {
+ setStatus('WebSocket error', 'error');
+ };
+
+ ws.onclose = () => {
+ if (isRecording) {
+ stopRecording();
+ }
+ };
+}
+
+function stopRecording() {
+ isRecording = false;
+
+ if (ws && ws.readyState === WebSocket.OPEN) {
+ ws.send('STOP');
+ }
+
+ if (processor) { processor.disconnect(); processor = null; }
+ if (audioContext) { audioContext.close(); audioContext = null; }
+ if (mediaStream) { mediaStream.getTracks().forEach(t => t.stop()); mediaStream = null; }
+
+ // Reset visualizer
+ bars.forEach(bar => bar.style.height = '4px');
+
+ setTimeout(() => {
+ if (ws) { ws.close(); ws = null; }
+ }, 1000);
+
+ document.getElementById('btnStart').style.display = 'block';
+ document.getElementById('btnStop').style.display = 'none';
+ setStatus('Stopped. Click Start to record again.', '');
+}
+</script>
+</body>
+</html>
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
new file mode 100644
index 000000000..367816e8a
--- /dev/null
+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
@@ -0,0 +1,292 @@
+#!/usr/bin/env python3
+"""
+Fun-ASR-Nano Real-time WebSocket Server.
+
+Key fix: Only decode CURRENT VAD segment (not entire cumulative audio).
+Confirmed segments get their text locked and never re-decoded.
+This prevents model collapse on long recordings.
+"""
+
+import asyncio
+import json
+import logging
+import os
+import time
+import argparse
+import numpy as np
+import torch
+import warnings
+
+warnings.filterwarnings('ignore')
+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
+logger = logging.getLogger(__name__)
+
+try:
+ import websockets
+except ImportError:
+ os.system("pip install websockets --break-system-packages -q")
+ import websockets
+
+
+class StreamingVAD:
+ """Streaming VAD: [start,-1]=speech start, [-1,end]=speech end."""
+
+ def __init__(self, vad_model):
+ self.model = vad_model
+ self.cache = {}
+ self.confirmed_segments = []
+ self.current_speech_start = None
+
+ def feed(self, audio_chunk_tensor, is_final=False):
+ res = self.model.generate(
+ input=[audio_chunk_tensor], cache=self.cache,
+ is_final=is_final, chunk_size=200,
+ )
+ signals = res[0].get("value", [])
+ new_confirmed = []
+ for sig in signals:
+ if sig[0] >= 0 and sig[1] == -1:
+ self.current_speech_start = sig[0]
+ elif sig[0] == -1 and sig[1] >= 0:
+ start = self.current_speech_start if self.current_speech_start is not None else 0
+ self.confirmed_segments.append([start, sig[1]])
+ new_confirmed.append([start, sig[1]])
+ self.current_speech_start = None
+ elif sig[0] >= 0 and sig[1] >= 0:
+ self.confirmed_segments.append(sig)
+ new_confirmed.append(sig)
+ self.current_speech_start = None
+ return new_confirmed
+
+ def reset(self):
+ self.cache = {}
+ self.confirmed_segments = []
+ self.current_speech_start = None
+
+
+class RealtimeASRSession:
+ def __init__(self, asr_model, asr_kwargs, tokenizer, vad, sample_rate=16000, chunk_ms=720):
+ self.asr_model = asr_model
+ self.asr_kwargs = asr_kwargs
+ self.tokenizer = tokenizer
+ self.vad = vad
+ self.sample_rate = sample_rate
+ self.chunk_samples = int(sample_rate * chunk_ms / 1000)
+
+ self.audio_buffer = np.array([], dtype=np.float32)
+ self.vad_fed_samples = 0
+ self.prev_text = ""
+ self.last_partial_text = ""
+ self.last_decode_samples = 0
+ self.locked_sentences = [] # text locked for confirmed segments
+ self.is_active = False
+
+ def add_audio(self, pcm_bytes):
+ audio_int16 = np.frombuffer(pcm_bytes, dtype=np.int16)
+ audio_float = audio_int16.astype(np.float32) / 32768.0
+ self.audio_buffer = np.concatenate([self.audio_buffer, audio_float])
+
+ # Feed new audio to streaming VAD
+ new_audio = self.audio_buffer[self.vad_fed_samples:]
+ if len(new_audio) > 0:
+ new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)
+ self.vad_fed_samples = len(self.audio_buffer)
+
+ # Lock text for newly confirmed segments
+ for seg in new_confirmed:
+ seg_text = self._decode_segment(seg)
+ self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})
+ self.prev_text = "" # reset for new segment
+ logger.info(f" Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:30]}\"")
+
+ def should_decode(self):
+ return (len(self.audio_buffer) - self.last_decode_samples) >= self.chunk_samples
+
+ @torch.no_grad()
+ def decode(self, is_final=False):
+ if len(self.audio_buffer) < self.chunk_samples:
+ return self._build_response(is_final)
+
+ if is_final:
+ # Feed remaining to VAD
+ remaining = self.audio_buffer[self.vad_fed_samples:]
+ if len(remaining) > 0:
+ new_confirmed = self.vad.feed(torch.from_numpy(remaining).float(), is_final=True)
+ self.vad_fed_samples = len(self.audio_buffer)
+ for seg in new_confirmed:
+ seg_text = self._decode_segment(seg)
+ self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})
+
+ # If still in speech, close and decode it
+ if self.vad.current_speech_start is not None:
+ end_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)
+ seg = [self.vad.current_speech_start, end_ms]
+ seg_text = self._decode_segment(seg)
+ self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})
+ self.vad.current_speech_start = None
+
+ return self._build_response(is_final)
+
+ # Decode ONLY current ongoing segment (not full buffer!)
+ if self.vad.current_speech_start is not None:
+ seg_start_sample = int(self.vad.current_speech_start * self.sample_rate / 1000)
+ seg_audio = self.audio_buffer[seg_start_sample:]
+ else:
+ # Not in speech - nothing to decode
+ self.last_decode_samples = len(self.audio_buffer)
+ self.last_partial_text = ""
+ return self._build_response(is_final)
+
+ if len(seg_audio) < self.chunk_samples // 2:
+ return self._build_response(is_final)
+
+ audio_tensor = torch.from_numpy(seg_audio).float()
+ try:
+ res = self.asr_model.inference(
+ [audio_tensor], prev_text=self.prev_text, **self.asr_kwargs
+ )
+ text = res[0][0]["text"]
+ except Exception as e:
+ logger.error(f"ASR error: {e}")
+ return self._build_response(is_final)
+
+ self.last_decode_samples = len(self.audio_buffer)
+ self.last_partial_text = text
+
+ # Update prev_text (rollback 5 tokens)
+ encoded = self.tokenizer.encode(text)
+ if len(encoded) > 5:
+ self.prev_text = self.tokenizer.decode(encoded[:-5]).replace("�", "")
+ else:
+ self.prev_text = ""
+
+ return self._build_response(is_final)
+
+ @torch.no_grad()
+ def _decode_segment(self, seg):
+ """Decode a specific completed VAD segment."""
+ start_sample = int(seg[0] * self.sample_rate / 1000)
+ end_sample = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))
+ seg_audio = self.audio_buffer[start_sample:end_sample]
+ if len(seg_audio) < 1600:
+ return ""
+ audio_tensor = torch.from_numpy(seg_audio).float()
+ try:
+ res = self.asr_model.inference([audio_tensor], prev_text="", **self.asr_kwargs)
+ return res[0][0]["text"]
+ except Exception as e:
+ logger.error(f"Segment decode error: {e}")
+ return ""
+
+ def _build_response(self, is_final):
+ duration_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)
+ sentences = list(self.locked_sentences)
+ partial = self.last_partial_text
+ partial_start = self.vad.current_speech_start or duration_ms
+
+ if is_final:
+ return {"sentences": sentences, "partial": "", "partial_start_ms": 0,
+ "duration_ms": duration_ms, "is_final": True}
+
+ return {"sentences": sentences, "partial": partial,
+ "partial_start_ms": partial_start,
+ "duration_ms": duration_ms, "is_final": False}
+
+ def reset(self):
+ self.audio_buffer = np.array([], dtype=np.float32)
+ self.vad_fed_samples = 0
+ self.vad.reset()
+ self.prev_text = ""
+ self.last_partial_text = ""
+ self.last_decode_samples = 0
+ self.locked_sentences = []
+
+
+_asr_model = None
+_asr_kwargs = None
+_tokenizer = None
+_vad_model = None
+
+
+def load_models(args):
+ global _asr_model, _asr_kwargs, _tokenizer, _vad_model
+ if _asr_model is None:
+ from funasr import AutoModel
+ logger.info(f"Loading ASR: {args.model}")
+ model = AutoModel(
+ model=args.model, trust_remote_code=True,
+ remote_code=os.path.join(os.path.dirname(__file__), "model.py"),
+ device=args.device, hub=args.hub, disable_update=True,
+ )
+ _asr_model = model.model
+ _asr_kwargs = model.kwargs
+ _tokenizer = model.kwargs["tokenizer"]
+
+ logger.info("Loading VAD: fsmn-vad (streaming)")
+ _vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True)
+ logger.info("All models ready!")
+ return _asr_model, _asr_kwargs, _tokenizer, _vad_model
+
+
+async def handle_client(websocket, args):
+ asr_model, asr_kwargs, tokenizer, vad_automodel = load_models(args)
+ vad = StreamingVAD(vad_automodel)
+ session = RealtimeASRSession(asr_model, asr_kwargs, tokenizer, vad)
+ logger.info(f"Client connected: {websocket.remote_address}")
+
+ decode_interval = args.decode_interval
+ last_decode_time = 0
+
+ try:
+ async for message in websocket:
+ if isinstance(message, str):
+ cmd = message.strip().upper()
+ if cmd == "START":
+ session.reset()
+ session.is_active = True
+ await websocket.send(json.dumps({"event": "started"}))
+ logger.info("Session started")
+ elif cmd == "STOP":
+ if session.is_active and len(session.audio_buffer) > 0:
+ result = session.decode(is_final=True)
+ await websocket.send(json.dumps(result))
+ n = len(result["sentences"])
+ logger.info(f"Final: {n} sentences")
+ session.is_active = False
+ await websocket.send(json.dumps({"event": "stopped"}))
+ elif isinstance(message, bytes) and session.is_active:
+ session.add_audio(message)
+
+ now = time.time()
+ if now - last_decode_time >= decode_interval and session.should_decode():
+ result = session.decode(is_final=False)
+ await websocket.send(json.dumps(result))
+ last_decode_time = now
+
+ except websockets.exceptions.ConnectionClosed:
+ logger.info("Client disconnected")
+ except Exception as e:
+ logger.error(f"Error: {e}")
+ import traceback
+ traceback.print_exc()
+
+
+async def main(args):
+ load_models(args)
+ logger.info(f"Server on ws://0.0.0.0:{args.port}")
+ async with websockets.serve(
+ lambda ws: handle_client(ws, args), "0.0.0.0", args.port,
+ max_size=10*1024*1024,
+ ):
+ await asyncio.Future()
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--port", type=int, default=10095)
+ parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")
+ parser.add_argument("--hub", type=str, default="ms")
+ parser.add_argument("--device", type=str, default="cuda:0")
+ parser.add_argument("--decode-interval", type=float, default=0.7)
+ args = parser.parse_args()
+ asyncio.run(main(args))