Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat: add real-time WebSocket ASR demo with streaming VAD
游雁ed0ed9b7d4f7
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.htmlnew file mode 100644index 000000000..f06aedb68--- /dev/null+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html@@ -0,0 +1,243 @@+<!DOCTYPE html>+<html lang="zh-CN">+<head>+ <meta charset="UTF-8">+ <title>Fun-ASR-Nano vLLM - Real-time ASR</title>+ <style>+ * { margin: 0; padding: 0; box-sizing: border-box; }+ body {+ font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;+ background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%);+ min-height: 100vh; display: flex; align-items: center; justify-content: center;+ color: #e0e0e0;+ }+ .container {+ background: rgba(255,255,255,0.05); backdrop-filter: blur(10px);+ border: 1px solid rgba(255,255,255,0.1); border-radius: 20px;+ padding: 40px; width: 90%; max-width: 700px;+ }+ h1 { text-align: center; font-size: 24px; margin-bottom: 8px; color: #64ffda; }+ .subtitle { text-align: center; color: #aaa; margin-bottom: 30px; font-size: 14px; }+ .config { display: flex; gap: 12px; margin-bottom: 20px; align-items: center; flex-wrap: wrap; }+ .config label { font-size: 13px; color: #aaa; }+ .config input, .config select {+ background: rgba(255,255,255,0.1); border: 1px solid rgba(255,255,255,0.2);+ border-radius: 8px; padding: 8px 12px; color: #fff; font-size: 14px;+ }+ .config input { width: 280px; }+ .btn {+ display: block; width: 200px; margin: 0 auto 20px; padding: 14px 0;+ border: none; border-radius: 50px; font-size: 16px; font-weight: 600;+ cursor: pointer; transition: all 0.3s;+ }+ .btn-start { background: #64ffda; color: #1a1a2e; }+ .btn-start:hover { background: #4de8c2; transform: scale(1.05); }+ .btn-stop { background: #ff6b6b; color: #fff; }+ .btn-stop:hover { background: #ee5a5a; transform: scale(1.05); }+ .btn:disabled { opacity: 0.5; cursor: not-allowed; transform: none; }+ .status {+ text-align: center; margin-bottom: 20px; font-size: 14px;+ padding: 8px; border-radius: 8px;+ }+ .status.connected { background: rgba(100,255,218,0.1); color: #64ffda; }+ .status.recording { background: rgba(255,107,107,0.1); color: #ff6b6b; }+ .status.error { background: rgba(255,107,107,0.1); color: #ff6b6b; }+ .result-box {+ background: rgba(0,0,0,0.3); border: 1px solid rgba(255,255,255,0.1);+ border-radius: 12px; padding: 20px; min-height: 150px; max-height: 400px; overflow-y: auto;+ font-size: 18px; line-height: 1.8; word-break: break-all;+ }+ .result-box .final { color: #64ffda; }+ .result-box .partial { color: #aaa; font-style: italic; }+ .result-box .placeholder { color: #555; }+ .visualizer { height: 60px; margin: 15px 0; display: flex; align-items: center; justify-content: center; gap: 3px; }+ .visualizer .bar {+ width: 4px; background: #64ffda; border-radius: 2px;+ transition: height 0.1s; min-height: 4px;+ }+ .footer { text-align: center; margin-top: 20px; font-size: 12px; color: #666; }+ </style>+</head>+<body>+<div class="container">+ <h1>Fun-ASR-Nano vLLM</h1>+ <p class="subtitle">Real-time Speech Recognition</p>++ <div class="config">+ <label>Server:</label>+ <input type="text" id="serverUrl" value="ws://localhost:10095" placeholder="ws://host:port">+ <label>Language:</label>+ <select id="language">+ <option value="中文" selected>中文</option>+ <option value="英文">English</option>+ <option value="日文">日本語</option>+ </select>+ </div>++ <div id="status" class="status">Click "Start" to begin</div>++ <div class="visualizer" id="visualizer"></div>++ <button class="btn btn-start" id="btnStart" onclick="startRecording()">Start Recording</button>+ <button class="btn btn-stop" id="btnStop" onclick="stopRecording()" style="display:none">Stop</button>++ <div class="result-box" id="resultBox">+ <span class="placeholder">Recognition results will appear here...</span>+ </div>++ <p class="footer">Fun-ASR-Nano powered by vLLM | Audio: 16kHz PCM16 mono</p>+</div>++<script>+let ws = null;+let mediaStream = null;+let audioContext = null;+let processor = null;+let isRecording = false;+let finalText = '';++// Create visualizer bars+const visualizer = document.getElementById('visualizer');+for (let i = 0; i < 40; i++) {+ const bar = document.createElement('div');+ bar.className = 'bar';+ bar.style.height = '4px';+ visualizer.appendChild(bar);+}+const bars = visualizer.querySelectorAll('.bar');++function setStatus(msg, cls) {+ const el = document.getElementById('status');+ el.textContent = msg;+ el.className = 'status ' + (cls || '');+}++function renderSentences(sentences, partial, partialStartMs, durationMs, isFinal) {+ const box = document.getElementById('resultBox');+ let html = '';+ sentences.forEach(s => {+ let startS = (s.start_ms / 1000).toFixed(1);+ let endS = (s.end_ms / 1000).toFixed(1);+ html += '<div style="margin:8px 0;display:flex;align-items:baseline">' ++ '<span style="color:#64ffda;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ' + endS + 's</span>' ++ '<span style="color:#e0e0e0;font-size:16px;margin-left:10px">' + s.text + '</span></div>';+ });+ if (partial) {+ let startS = (partialStartMs / 1000).toFixed(1);+ html += '<div style="margin:8px 0;display:flex;align-items:baseline">' ++ '<span style="color:#ffb74d;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ...</span>' ++ '<span style="color:#aaa;font-size:16px;margin-left:10px;font-style:italic">' + partial + '</span></div>';+ }+ if (!html) { html = '<span class="placeholder">Listening...</span>'; }+ box.innerHTML = html;+ box.scrollTop = box.scrollHeight;+}++async function startRecording() {+ const serverUrl = document.getElementById('serverUrl').value;+ finalText = '';+ document.getElementById('resultBox').innerHTML = '<span class="placeholder">Connecting...</span>';++ // Connect WebSocket+ try {+ ws = new WebSocket(serverUrl);+ } catch(e) {+ setStatus('Connection failed: ' + e.message, 'error');+ return;+ }++ ws.onopen = async () => {+ setStatus('Connected. Recording...', 'recording');+ ws.send('START');++ // Get microphone+ try {+ mediaStream = await navigator.mediaDevices.getUserMedia({+ audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true }+ });+ } catch(e) {+ setStatus('Microphone access denied', 'error');+ ws.close();+ return;+ }++ audioContext = new AudioContext({ sampleRate: 16000 });+ const source = audioContext.createMediaStreamSource(mediaStream);++ // AudioWorklet or ScriptProcessor+ processor = audioContext.createScriptProcessor(4096, 1, 1);+ processor.onaudioprocess = (e) => {+ if (!isRecording) return;+ const float32 = e.inputBuffer.getChannelData(0);++ // Update visualizer+ const rms = Math.sqrt(float32.reduce((sum, x) => sum + x*x, 0) / float32.length);+ bars.forEach((bar, i) => {+ const h = Math.max(4, Math.min(50, rms * 500 * (0.5 + Math.random())));+ bar.style.height = h + 'px';+ });++ // Convert to PCM16 and send+ const int16 = new Int16Array(float32.length);+ for (let i = 0; i < float32.length; i++) {+ int16[i] = Math.max(-32768, Math.min(32767, Math.round(float32[i] * 32768)));+ }+ if (ws && ws.readyState === WebSocket.OPEN) {+ ws.send(int16.buffer);+ }+ };+ source.connect(processor);+ processor.connect(audioContext.destination);++ isRecording = true;+ document.getElementById('btnStart').style.display = 'none';+ document.getElementById('btnStop').style.display = 'block';+ document.getElementById('resultBox').innerHTML = '<span class="placeholder">Listening...</span>';+ };++ ws.onmessage = (event) => {+ const data = JSON.parse(event.data);+ if (data.sentences !== undefined) {+ renderSentences(data.sentences, data.partial, data.partial_start_ms, data.duration_ms, data.is_final);+ }+ if (data.event === 'stopped') {+ setStatus('Stopped', 'connected');+ }+ };++ ws.onerror = (e) => {+ setStatus('WebSocket error', 'error');+ };++ ws.onclose = () => {+ if (isRecording) {+ stopRecording();+ }+ };+}++function stopRecording() {+ isRecording = false;++ if (ws && ws.readyState === WebSocket.OPEN) {+ ws.send('STOP');+ }++ if (processor) { processor.disconnect(); processor = null; }+ if (audioContext) { audioContext.close(); audioContext = null; }+ if (mediaStream) { mediaStream.getTracks().forEach(t => t.stop()); mediaStream = null; }++ // Reset visualizer+ bars.forEach(bar => bar.style.height = '4px');++ setTimeout(() => {+ if (ws) { ws.close(); ws = null; }+ }, 1000);++ document.getElementById('btnStart').style.display = 'block';+ document.getElementById('btnStop').style.display = 'none';+ setStatus('Stopped. Click Start to record again.', '');+}+</script>+</body>+</html>diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.pynew file mode 100644index 000000000..367816e8a--- /dev/null+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py@@ -0,0 +1,292 @@+#!/usr/bin/env python3+"""+Fun-ASR-Nano Real-time WebSocket Server.++Key fix: Only decode CURRENT VAD segment (not entire cumulative audio).+Confirmed segments get their text locked and never re-decoded.+This prevents model collapse on long recordings.+"""++import asyncio+import json+import logging+import os+import time+import argparse+import numpy as np+import torch+import warnings++warnings.filterwarnings('ignore')+logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')+logger = logging.getLogger(__name__)++try:+ import websockets+except ImportError:+ os.system("pip install websockets --break-system-packages -q")+ import websockets+++class StreamingVAD:+ """Streaming VAD: [start,-1]=speech start, [-1,end]=speech end."""++ def __init__(self, vad_model):+ self.model = vad_model+ self.cache = {}+ self.confirmed_segments = []+ self.current_speech_start = None++ def feed(self, audio_chunk_tensor, is_final=False):+ res = self.model.generate(+ input=[audio_chunk_tensor], cache=self.cache,+ is_final=is_final, chunk_size=200,+ )+ signals = res[0].get("value", [])+ new_confirmed = []+ for sig in signals:+ if sig[0] >= 0 and sig[1] == -1:+ self.current_speech_start = sig[0]+ elif sig[0] == -1 and sig[1] >= 0:+ start = self.current_speech_start if self.current_speech_start is not None else 0+ self.confirmed_segments.append([start, sig[1]])+ new_confirmed.append([start, sig[1]])+ self.current_speech_start = None+ elif sig[0] >= 0 and sig[1] >= 0:+ self.confirmed_segments.append(sig)+ new_confirmed.append(sig)+ self.current_speech_start = None+ return new_confirmed++ def reset(self):+ self.cache = {}+ self.confirmed_segments = []+ self.current_speech_start = None+++class RealtimeASRSession:+ def __init__(self, asr_model, asr_kwargs, tokenizer, vad, sample_rate=16000, chunk_ms=720):+ self.asr_model = asr_model+ self.asr_kwargs = asr_kwargs+ self.tokenizer = tokenizer+ self.vad = vad+ self.sample_rate = sample_rate+ self.chunk_samples = int(sample_rate * chunk_ms / 1000)++ self.audio_buffer = np.array([], dtype=np.float32)+ self.vad_fed_samples = 0+ self.prev_text = ""+ self.last_partial_text = ""+ self.last_decode_samples = 0+ self.locked_sentences = [] # text locked for confirmed segments+ self.is_active = False++ def add_audio(self, pcm_bytes):+ audio_int16 = np.frombuffer(pcm_bytes, dtype=np.int16)+ audio_float = audio_int16.astype(np.float32) / 32768.0+ self.audio_buffer = np.concatenate([self.audio_buffer, audio_float])++ # Feed new audio to streaming VAD+ new_audio = self.audio_buffer[self.vad_fed_samples:]+ if len(new_audio) > 0:+ new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)+ self.vad_fed_samples = len(self.audio_buffer)++ # Lock text for newly confirmed segments+ for seg in new_confirmed:+ seg_text = self._decode_segment(seg)+ self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})+ self.prev_text = "" # reset for new segment+ logger.info(f" Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:30]}\"")++ def should_decode(self):+ return (len(self.audio_buffer) - self.last_decode_samples) >= self.chunk_samples++ @torch.no_grad()+ def decode(self, is_final=False):+ if len(self.audio_buffer) < self.chunk_samples:+ return self._build_response(is_final)++ if is_final:+ # Feed remaining to VAD+ remaining = self.audio_buffer[self.vad_fed_samples:]+ if len(remaining) > 0:+ new_confirmed = self.vad.feed(torch.from_numpy(remaining).float(), is_final=True)+ self.vad_fed_samples = len(self.audio_buffer)+ for seg in new_confirmed:+ seg_text = self._decode_segment(seg)+ self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})++ # If still in speech, close and decode it+ if self.vad.current_speech_start is not None:+ end_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)+ seg = [self.vad.current_speech_start, end_ms]+ seg_text = self._decode_segment(seg)+ self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})+ self.vad.current_speech_start = None++ return self._build_response(is_final)++ # Decode ONLY current ongoing segment (not full buffer!)+ if self.vad.current_speech_start is not None:+ seg_start_sample = int(self.vad.current_speech_start * self.sample_rate / 1000)+ seg_audio = self.audio_buffer[seg_start_sample:]+ else:+ # Not in speech - nothing to decode+ self.last_decode_samples = len(self.audio_buffer)+ self.last_partial_text = ""+ return self._build_response(is_final)++ if len(seg_audio) < self.chunk_samples // 2:+ return self._build_response(is_final)++ audio_tensor = torch.from_numpy(seg_audio).float()+ try:+ res = self.asr_model.inference(+ [audio_tensor], prev_text=self.prev_text, **self.asr_kwargs+ )+ text = res[0][0]["text"]+ except Exception as e:+ logger.error(f"ASR error: {e}")+ return self._build_response(is_final)++ self.last_decode_samples = len(self.audio_buffer)+ self.last_partial_text = text++ # Update prev_text (rollback 5 tokens)+ encoded = self.tokenizer.encode(text)+ if len(encoded) > 5:+ self.prev_text = self.tokenizer.decode(encoded[:-5]).replace("�", "")+ else:+ self.prev_text = ""++ return self._build_response(is_final)++ @torch.no_grad()+ def _decode_segment(self, seg):+ """Decode a specific completed VAD segment."""+ start_sample = int(seg[0] * self.sample_rate / 1000)+ end_sample = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))+ seg_audio = self.audio_buffer[start_sample:end_sample]+ if len(seg_audio) < 1600:+ return ""+ audio_tensor = torch.from_numpy(seg_audio).float()+ try:+ res = self.asr_model.inference([audio_tensor], prev_text="", **self.asr_kwargs)+ return res[0][0]["text"]+ except Exception as e:+ logger.error(f"Segment decode error: {e}")+ return ""++ def _build_response(self, is_final):+ duration_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)+ sentences = list(self.locked_sentences)+ partial = self.last_partial_text+ partial_start = self.vad.current_speech_start or duration_ms++ if is_final:+ return {"sentences": sentences, "partial": "", "partial_start_ms": 0,+ "duration_ms": duration_ms, "is_final": True}++ return {"sentences": sentences, "partial": partial,+ "partial_start_ms": partial_start,+ "duration_ms": duration_ms, "is_final": False}++ def reset(self):+ self.audio_buffer = np.array([], dtype=np.float32)+ self.vad_fed_samples = 0+ self.vad.reset()+ self.prev_text = ""+ self.last_partial_text = ""+ self.last_decode_samples = 0+ self.locked_sentences = []+++_asr_model = None+_asr_kwargs = None+_tokenizer = None+_vad_model = None+++def load_models(args):+ global _asr_model, _asr_kwargs, _tokenizer, _vad_model+ if _asr_model is None:+ from funasr import AutoModel+ logger.info(f"Loading ASR: {args.model}")+ model = AutoModel(+ model=args.model, trust_remote_code=True,+ remote_code=os.path.join(os.path.dirname(__file__), "model.py"),+ device=args.device, hub=args.hub, disable_update=True,+ )+ _asr_model = model.model+ _asr_kwargs = model.kwargs+ _tokenizer = model.kwargs["tokenizer"]++ logger.info("Loading VAD: fsmn-vad (streaming)")+ _vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True)+ logger.info("All models ready!")+ return _asr_model, _asr_kwargs, _tokenizer, _vad_model+++async def handle_client(websocket, args):+ asr_model, asr_kwargs, tokenizer, vad_automodel = load_models(args)+ vad = StreamingVAD(vad_automodel)+ session = RealtimeASRSession(asr_model, asr_kwargs, tokenizer, vad)+ logger.info(f"Client connected: {websocket.remote_address}")++ decode_interval = args.decode_interval+ last_decode_time = 0++ try:+ async for message in websocket:+ if isinstance(message, str):+ cmd = message.strip().upper()+ if cmd == "START":+ session.reset()+ session.is_active = True+ await websocket.send(json.dumps({"event": "started"}))+ logger.info("Session started")+ elif cmd == "STOP":+ if session.is_active and len(session.audio_buffer) > 0:+ result = session.decode(is_final=True)+ await websocket.send(json.dumps(result))+ n = len(result["sentences"])+ logger.info(f"Final: {n} sentences")+ session.is_active = False+ await websocket.send(json.dumps({"event": "stopped"}))+ elif isinstance(message, bytes) and session.is_active:+ session.add_audio(message)++ now = time.time()+ if now - last_decode_time >= decode_interval and session.should_decode():+ result = session.decode(is_final=False)+ await websocket.send(json.dumps(result))+ last_decode_time = now++ except websockets.exceptions.ConnectionClosed:+ logger.info("Client disconnected")+ except Exception as e:+ logger.error(f"Error: {e}")+ import traceback+ traceback.print_exc()+++async def main(args):+ load_models(args)+ logger.info(f"Server on ws://0.0.0.0:{args.port}")+ async with websockets.serve(+ lambda ws: handle_client(ws, args), "0.0.0.0", args.port,+ max_size=10*1024*1024,+ ):+ await asyncio.Future()+++if __name__ == "__main__":+ parser = argparse.ArgumentParser()+ parser.add_argument("--port", type=int, default=10095)+ parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")+ parser.add_argument("--hub", type=str, default="ms")+ parser.add_argument("--device", type=str, default="cuda:0")+ parser.add_argument("--decode-interval", type=float, default=0.7)+ args = parser.parse_args()+ asyncio.run(main(args))