Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat: polish streaming ASR service - cleanup, clients, docs
xiaoyunchong.xycf720a289124a
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.htmlindex f06aedb68..b9312a1af 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html@@ -1,243 +1,146 @@<!DOCTYPE html><html lang="zh-CN"><head>- <meta charset="UTF-8">- <title>Fun-ASR-Nano vLLM - Real-time ASR</title>- <style>- * { margin: 0; padding: 0; box-sizing: border-box; }- body {- font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;- background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%);- min-height: 100vh; display: flex; align-items: center; justify-content: center;- color: #e0e0e0;- }- .container {- background: rgba(255,255,255,0.05); backdrop-filter: blur(10px);- border: 1px solid rgba(255,255,255,0.1); border-radius: 20px;- padding: 40px; width: 90%; max-width: 700px;- }- h1 { text-align: center; font-size: 24px; margin-bottom: 8px; color: #64ffda; }- .subtitle { text-align: center; color: #aaa; margin-bottom: 30px; font-size: 14px; }- .config { display: flex; gap: 12px; margin-bottom: 20px; align-items: center; flex-wrap: wrap; }- .config label { font-size: 13px; color: #aaa; }- .config input, .config select {- background: rgba(255,255,255,0.1); border: 1px solid rgba(255,255,255,0.2);- border-radius: 8px; padding: 8px 12px; color: #fff; font-size: 14px;- }- .config input { width: 280px; }- .btn {- display: block; width: 200px; margin: 0 auto 20px; padding: 14px 0;- border: none; border-radius: 50px; font-size: 16px; font-weight: 600;- cursor: pointer; transition: all 0.3s;- }- .btn-start { background: #64ffda; color: #1a1a2e; }- .btn-start:hover { background: #4de8c2; transform: scale(1.05); }- .btn-stop { background: #ff6b6b; color: #fff; }- .btn-stop:hover { background: #ee5a5a; transform: scale(1.05); }- .btn:disabled { opacity: 0.5; cursor: not-allowed; transform: none; }- .status {- text-align: center; margin-bottom: 20px; font-size: 14px;- padding: 8px; border-radius: 8px;- }- .status.connected { background: rgba(100,255,218,0.1); color: #64ffda; }- .status.recording { background: rgba(255,107,107,0.1); color: #ff6b6b; }- .status.error { background: rgba(255,107,107,0.1); color: #ff6b6b; }- .result-box {- background: rgba(0,0,0,0.3); border: 1px solid rgba(255,255,255,0.1);- border-radius: 12px; padding: 20px; min-height: 150px; max-height: 400px; overflow-y: auto;- font-size: 18px; line-height: 1.8; word-break: break-all;- }- .result-box .final { color: #64ffda; }- .result-box .partial { color: #aaa; font-style: italic; }- .result-box .placeholder { color: #555; }- .visualizer { height: 60px; margin: 15px 0; display: flex; align-items: center; justify-content: center; gap: 3px; }- .visualizer .bar {- width: 4px; background: #64ffda; border-radius: 2px;- transition: height 0.1s; min-height: 4px;- }- .footer { text-align: center; margin-top: 20px; font-size: 12px; color: #666; }- </style>+<meta charset="UTF-8">+<title>Fun-ASR-Nano · Streaming ASR with Speaker Diarization</title>+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800;900&display=swap" rel="stylesheet">+<style>+* { margin:0; padding:0; box-sizing:border-box; }+html, body { height:100%; }+body { font-family:'Inter',system-ui,sans-serif; background:#080810; color:#e0e0e0; display:flex; justify-content:center; height:100vh; overflow:hidden; }++/* === Hero Section === */+.hero { padding:24px 28px 16px; flex-shrink:0; }+.hero-top { display:flex; align-items:center; gap:16px; margin-bottom:14px; }+.logo { font-size:30px; font-weight:900; letter-spacing:-1px; }+.logo .g { background:linear-gradient(135deg,#64ffda,#00bfa5); -webkit-background-clip:text; -webkit-text-fill-color:transparent; }+.logo .w { color:#fff; }+.logo .s { color:rgba(255,255,255,0.3); font-weight:400; font-size:15px; margin-left:8px; }+.hero-links { margin-left:auto; display:flex; gap:10px; }+.hero-links a { font-size:12px; color:#aaa; text-decoration:none; padding:6px 14px; border-radius:8px; background:rgba(255,255,255,0.04); border:1px solid rgba(255,255,255,0.08); transition:all 0.2s; font-weight:500; }+.hero-links a:hover { color:#64ffda; border-color:rgba(100,255,218,0.3); }++.features { display:flex; gap:10px; flex-wrap:wrap; margin-bottom:14px; }+.feat { font-size:11px; padding:5px 12px; border-radius:20px; font-weight:500; }+.feat-green { color:#64ffda; background:rgba(100,255,218,0.08); border:1px solid rgba(100,255,218,0.2); }+.feat-yellow { color:#fbbf24; background:rgba(251,191,36,0.08); border:1px solid rgba(251,191,36,0.2); }+.feat-purple { color:#a78bfa; background:rgba(167,139,250,0.08); border:1px solid rgba(167,139,250,0.2); }+.feat-blue { color:#38bdf8; background:rgba(56,189,248,0.08); border:1px solid rgba(56,189,248,0.2); }++.hero-desc { font-size:13px; color:rgba(255,255,255,0.45); line-height:1.8; }+.hero-desc b { color:rgba(255,255,255,0.75); font-weight:600; }+.hero-desc a { color:#64ffda; text-decoration:none; font-weight:600; }+.hero-desc a:hover { text-decoration:underline; }++/* === Main Panel === */+.main { flex:1; display:flex; flex-direction:column; padding:0 28px 20px; min-height:0; }+.panel { flex:1; display:flex; flex-direction:column; background:rgba(16,16,24,0.95); border:1px solid rgba(255,255,255,0.05); border-radius:16px; box-shadow:0 10px 40px rgba(0,0,0,0.4); min-height:0; }++.controls { padding:14px 20px; display:flex; align-items:center; gap:12px; border-bottom:1px solid rgba(255,255,255,0.04); flex-shrink:0; }+.controls input[type=text] { background:rgba(255,255,255,0.05); border:1px solid rgba(255,255,255,0.1); border-radius:8px; padding:8px 12px; color:#ccc; font-size:12px; width:180px; outline:none; }+.controls input[type=text]:focus { border-color:rgba(100,255,218,0.4); }+.controls input[type=file] { display:none; }+.controls label { font-size:11px; color:#888; display:flex; align-items:center; gap:5px; cursor:pointer; }+.controls input[type=checkbox] { accent-color:#64ffda; }+.btn { padding:8px 18px; border:none; border-radius:10px; font-size:12px; cursor:pointer; font-weight:600; transition:all 0.12s; }+.btn:hover { transform:scale(1.04); }+.btn:active { transform:scale(0.97); }+.btn-mic { background:linear-gradient(135deg,#64ffda,#00bfa5); color:#080810; }+.btn-file { background:linear-gradient(135deg,#fbbf24,#f59e0b); color:#1a1a2e; }+.btn-hw { background:rgba(167,139,250,0.12); color:#a78bfa; border:1px solid rgba(167,139,250,0.3); }+.btn-stop { background:linear-gradient(135deg,#ef4444,#dc2626); color:#fff; }+.sta { margin-left:auto; display:flex; align-items:center; gap:6px; }+.dot { width:7px; height:7px; border-radius:50%; background:#222; }+.dot.on { background:#64ffda; box-shadow:0 0 10px rgba(100,255,218,0.6); animation:g 1.5s infinite; }+@keyframes g { 0%,100%{opacity:1} 50%{opacity:0.3} }+.sta span { font-size:10px; color:#555; }+.hw-info { font-size:10px; color:#a78bfa; margin-left:2px; }++.result { flex:1; overflow-y:auto; padding:18px 22px; min-height:0; }+.result::-webkit-scrollbar { width:4px; }+.result::-webkit-scrollbar-thumb { background:rgba(255,255,255,0.06); border-radius:2px; }+.line { padding:6px 0; border-bottom:1px solid rgba(255,255,255,0.02); display:flex; align-items:baseline; gap:0; word-break:break-all; }+.time { color:rgba(100,255,218,0.5); font-size:11px; font-family:'SF Mono','Menlo',monospace; width:100px; min-width:100px; flex-shrink:0; padding-top:2px; }+.spk { font-size:10px; font-weight:600; padding:2px 8px; border-radius:5px; flex-shrink:0; margin-right:10px; }+.text { color:rgba(255,255,255,0.88); font-size:15px; line-height:1.7; flex:1; }+.partial .text { color:rgba(255,255,255,0.25); font-style:italic; }+.ph { color:rgba(255,255,255,0.1); text-align:center; padding:50px 20px; font-size:14px; line-height:2; }+</style></head>-<body>-<div class="container">- <h1>Fun-ASR-Nano vLLM</h1>- <p class="subtitle">Real-time Speech Recognition</p>-- <div class="config">- <label>Server:</label>- <input type="text" id="serverUrl" value="ws://localhost:10095" placeholder="ws://host:port">- <label>Language:</label>- <select id="language">- <option value="中文" selected>中文</option>- <option value="英文">English</option>- <option value="日文">日本語</option>- </select>+<body><div style="width:920px;height:100%;display:flex;flex-direction:column">++<div class="hero">+ <div class="hero-top">+ <div class="logo"><span class="g">Fun-ASR-Nano</span><span class="s">vLLM Engine</span></div>+ <div class="hero-links">+ <a href="https://github.com/modelscope/FunASR" target="_blank">GitHub</a>+ <a href="https://modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512" target="_blank">ModelScope</a>+ <a href="https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512" target="_blank">HuggingFace</a>+ </div></div>-- <div id="status" class="status">Click "Start" to begin</div>-- <div class="visualizer" id="visualizer"></div>-- <button class="btn btn-start" id="btnStart" onclick="startRecording()">Start Recording</button>- <button class="btn btn-stop" id="btnStop" onclick="stopRecording()" style="display:none">Stop</button>-- <div class="result-box" id="resultBox">- <span class="placeholder">Recognition results will appear here...</span>+ <div class="features">+ <span class="feat feat-green">Streaming ASR</span>+ <span class="feat feat-yellow">Speaker Diarization <span style="font-size:9px;opacity:0.7">(Beta)</span></span>+ <span class="feat feat-purple">Hotword Customization</span>+ <span class="feat feat-blue">31 Languages · 7 Dialects</span></div>+ <p class="hero-desc">+ 基于 <b>FunASR</b> 的 <b>vLLM 推理引擎</b>,实现流式语音识别服务。支持实时 VAD 分句、说话人分离 <span style="color:#f59e0b;font-size:11px">(Beta)</span>、<b>热词定制化</b>(加载人名、地名等实体词列表,提升专有名词识别准确率)、31种语言及中文方言。所有代码与模型已全部开源。<br>+ <span style="font-size:12px">Streaming ASR with vLLM engine, real-time VAD, speaker diarization <span style="color:#f59e0b">(Beta)</span>, <b>hotword customization</b> (names, places, entities), 31 languages & Chinese dialects. Fully open-sourced.</span>+ · <a href="https://www.funasr.com" target="_blank">www.funasr.com</a>+ </p>+</div>- <p class="footer">Fun-ASR-Nano powered by vLLM | Audio: 16kHz PCM16 mono</p>+<div class="main">+<div class="panel">+ <div class="controls">+ <input type="text" id="serverUrl" value="ws://localhost:10095">+ <label><input type="checkbox" id="showSpk" checked> Speaker <span style="font-size:9px;color:#f59e0b;background:rgba(245,158,11,0.08);padding:1px 5px;border-radius:4px">Beta</span></label>+ <button class="btn btn-mic" id="btnMic" onclick="startMic()">Mic</button>+ <input type="file" id="fileInput" accept="audio/*,.wav,.mp3,.flac,.mp4,.m4a">+ <button class="btn btn-file" id="btnFile" onclick="document.getElementById('fileInput').click()">Audio File</button>+ <input type="file" id="hotwordFile" accept=".txt" style="display:none">+ <button class="btn btn-hw" id="btnHw" onclick="document.getElementById('hotwordFile').click()">Hotwords</button>+ <span class="hw-info" id="hwInfo"></span>+ <button class="btn btn-stop" id="btnStop" onclick="stopAll()" style="display:none">Stop</button>+ <div class="sta"><div class="dot" id="dot"></div><span id="status">Ready</span></div>+ </div>+ <div class="result" id="resultBox">+ <div class="ph">Click <b>Mic</b> for real-time recognition or <b>Audio File</b> to transcribe a file<br>Load a <b>Hotwords</b> file (.txt, one word per line) to boost recognition of names, places & entities</div>+ </div>+</div></div>+<input type="text" id="hotwords" style="display:none"><script>-let ws = null;-let mediaStream = null;-let audioContext = null;-let processor = null;-let isRecording = false;-let finalText = '';--// Create visualizer bars-const visualizer = document.getElementById('visualizer');-for (let i = 0; i < 40; i++) {- const bar = document.createElement('div');- bar.className = 'bar';- bar.style.height = '4px';- visualizer.appendChild(bar);-}-const bars = visualizer.querySelectorAll('.bar');--function setStatus(msg, cls) {- const el = document.getElementById('status');- el.textContent = msg;- el.className = 'status ' + (cls || '');-}--function renderSentences(sentences, partial, partialStartMs, durationMs, isFinal) {- const box = document.getElementById('resultBox');- let html = '';- sentences.forEach(s => {- let startS = (s.start_ms / 1000).toFixed(1);- let endS = (s.end_ms / 1000).toFixed(1);- html += '<div style="margin:8px 0;display:flex;align-items:baseline">' +- '<span style="color:#64ffda;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ' + endS + 's</span>' +- '<span style="color:#e0e0e0;font-size:16px;margin-left:10px">' + s.text + '</span></div>';- });- if (partial) {- let startS = (partialStartMs / 1000).toFixed(1);- html += '<div style="margin:8px 0;display:flex;align-items:baseline">' +- '<span style="color:#ffb74d;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ...</span>' +- '<span style="color:#aaa;font-size:16px;margin-left:10px;font-style:italic">' + partial + '</span></div>';- }- if (!html) { html = '<span class="placeholder">Listening...</span>'; }- box.innerHTML = html;- box.scrollTop = box.scrollHeight;-}--async function startRecording() {- const serverUrl = document.getElementById('serverUrl').value;- finalText = '';- document.getElementById('resultBox').innerHTML = '<span class="placeholder">Connecting...</span>';-- // Connect WebSocket- try {- ws = new WebSocket(serverUrl);- } catch(e) {- setStatus('Connection failed: ' + e.message, 'error');- return;- }-- ws.onopen = async () => {- setStatus('Connected. Recording...', 'recording');- ws.send('START');-- // Get microphone- try {- mediaStream = await navigator.mediaDevices.getUserMedia({- audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true }- });- } catch(e) {- setStatus('Microphone access denied', 'error');- ws.close();- return;- }-- audioContext = new AudioContext({ sampleRate: 16000 });- const source = audioContext.createMediaStreamSource(mediaStream);-- // AudioWorklet or ScriptProcessor- processor = audioContext.createScriptProcessor(4096, 1, 1);- processor.onaudioprocess = (e) => {- if (!isRecording) return;- const float32 = e.inputBuffer.getChannelData(0);-- // Update visualizer- const rms = Math.sqrt(float32.reduce((sum, x) => sum + x*x, 0) / float32.length);- bars.forEach((bar, i) => {- const h = Math.max(4, Math.min(50, rms * 500 * (0.5 + Math.random())));- bar.style.height = h + 'px';- });-- // Convert to PCM16 and send- const int16 = new Int16Array(float32.length);- for (let i = 0; i < float32.length; i++) {- int16[i] = Math.max(-32768, Math.min(32767, Math.round(float32[i] * 32768)));- }- if (ws && ws.readyState === WebSocket.OPEN) {- ws.send(int16.buffer);- }- };- source.connect(processor);- processor.connect(audioContext.destination);-- isRecording = true;- document.getElementById('btnStart').style.display = 'none';- document.getElementById('btnStop').style.display = 'block';- document.getElementById('resultBox').innerHTML = '<span class="placeholder">Listening...</span>';- };-- ws.onmessage = (event) => {- const data = JSON.parse(event.data);- if (data.sentences !== undefined) {- renderSentences(data.sentences, data.partial, data.partial_start_ms, data.duration_ms, data.is_final);- }- if (data.event === 'stopped') {- setStatus('Stopped', 'connected');- }- };-- ws.onerror = (e) => {- setStatus('WebSocket error', 'error');- };-- ws.onclose = () => {- if (isRecording) {- stopRecording();- }- };-}--function stopRecording() {- isRecording = false;-- if (ws && ws.readyState === WebSocket.OPEN) {- ws.send('STOP');- }-- if (processor) { processor.disconnect(); processor = null; }- if (audioContext) { audioContext.close(); audioContext = null; }- if (mediaStream) { mediaStream.getTracks().forEach(t => t.stop()); mediaStream = null; }-- // Reset visualizer- bars.forEach(bar => bar.style.height = '4px');-- setTimeout(() => {- if (ws) { ws.close(); ws = null; }- }, 1000);-- document.getElementById('btnStart').style.display = 'block';- document.getElementById('btnStop').style.display = 'none';- setStatus('Stopped. Click Start to record again.', '');-}+let ws=null,mediaStream=null,audioContext=null,processor=null,isRecording=false;+const C=['#64ffda','#f472b6','#fbbf24','#34d399','#a78bfa','#fb923c','#67e8f9','#f87171','#38bdf8','#c084fc'];++document.getElementById('hotwordFile').addEventListener('change',function(e){+ const f=e.target.files[0];if(!f)return;+ const r=new FileReader();r.onload=ev=>{+ const words=ev.target.result.split('\n').map(l=>l.trim()).filter(l=>l);+ document.getElementById('hotwords').value=words.join(',');+ document.getElementById('hwInfo').textContent=words.length+' words loaded';+ setS('Hotwords: '+words.length+' loaded');+ };r.readAsText(f);+});+document.getElementById('fileInput').addEventListener('change',function(){if(this.files.length)startFile();});++function setS(m,on){document.getElementById('status').textContent=m;document.getElementById('dot').className=on?'dot on':'dot';}+function sStop(){document.getElementById('btnMic').style.display='none';document.getElementById('btnFile').style.display='none';document.getElementById('btnHw').style.display='none';document.getElementById('btnStop').style.display='inline';}+function sStart(){document.getElementById('btnMic').style.display='inline';document.getElementById('btnFile').style.display='inline';document.getElementById('btnHw').style.display='inline';document.getElementById('btnStop').style.display='none';}+function render(ss,p,ps,d,f){+ const b=document.getElementById('resultBox'),sk=document.getElementById('showSpk').checked;let h='';+ ss.forEach(s=>{let st=s.start!==undefined?s.start:(s.start_ms||0),en=s.end!==undefined?s.end:(s.end_ms||0),sp=s.spk!==undefined?s.spk:-1,sh='';+ if(sk&&sp>=0){let c=C[sp%C.length];sh='<span class="spk" style="color:'+c+';background:'+c+'12;border:1px solid '+c+'30">SPK'+sp+'</span>';}+ h+='<div class="line"><span class="time">'+(st/1000).toFixed(1)+' - '+(en/1000).toFixed(1)+'s</span>'+sh+'<span class="text">'+s.text+'</span></div>';});+ if(p)h+='<div class="line partial"><span class="time">'+(ps/1000).toFixed(1)+'s ...</span><span class="text">'+p+'</span></div>';+ if(!h)h='<div class="ph">Listening...</div>';b.innerHTML=h;b.scrollTop=b.scrollHeight;}+function con(cb){ws=new WebSocket(document.getElementById('serverUrl').value);ws.onopen=()=>{ws.send('START');var hw=document.getElementById('hotwords').value.trim();if(hw)ws.send('HOTWORDS:'+hw);cb();};ws.onmessage=e=>{const d=JSON.parse(e.data);if(d.sentences!==undefined)render(d.sentences,d.partial,d.partial_start_ms,d.duration_ms,d.is_final);};ws.onerror=()=>setS('Error');ws.onclose=()=>{if(isRecording)stopAll();};}+function startMic(){con(async()=>{setS('Recording',true);sStop();document.getElementById('resultBox').innerHTML='<div class="ph">Listening...</div>';try{mediaStream=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:16000,channelCount:1,echoCancellation:true}});}catch(e){setS('Mic denied');ws.close();sStart();return;}audioContext=new AudioContext({sampleRate:16000});const s=audioContext.createMediaStreamSource(mediaStream);processor=audioContext.createScriptProcessor(4096,1,1);processor.onaudioprocess=e=>{if(!isRecording)return;const f=e.inputBuffer.getChannelData(0),i=new Int16Array(f.length);for(let j=0;j<f.length;j++)i[j]=Math.max(-32768,Math.min(32767,Math.round(f[j]*32768)));if(ws&&ws.readyState===1)ws.send(i.buffer);};s.connect(processor);processor.connect(audioContext.destination);isRecording=true;});}+function startFile(){const fi=document.getElementById('fileInput');if(!fi.files.length){setS('Pick a file');return;}const file=fi.files[0];setS('Decoding...');document.getElementById('resultBox').innerHTML='<div class="ph">Decoding '+file.name+'...</div>';const r=new FileReader();r.onload=async e=>{const a=new AudioContext({sampleRate:16000});let buf;try{buf=await a.decodeAudioData(e.target.result);}catch(err){setS('Decode failed');return;}const p=buf.getChannelData(0),i=new Int16Array(p.length);for(let j=0;j<p.length;j++)i[j]=Math.max(-32768,Math.min(32767,Math.round(p[j]*32768)));a.close();con(async()=>{setS('Streaming '+(p.length/16000|0)+'s',true);sStop();isRecording=true;for(let j=0;j<i.length&&isRecording;j+=4096){if(ws&&ws.readyState===1)ws.send(i.slice(j,j+4096).buffer);await new Promise(r=>setTimeout(r,50));}if(isRecording)stopAll();});};r.readAsArrayBuffer(file);}+function stopAll(){isRecording=false;if(ws&&ws.readyState===1)ws.send('STOP');if(processor){processor.disconnect();processor=null;}if(audioContext){audioContext.close();audioContext=null;}if(mediaStream){mediaStream.getTracks().forEach(t=>t.stop());mediaStream=null;}setTimeout(()=>{if(ws){ws.close();ws=null;}},2000);sStart();setS('Done');}</script>-</body>+</div></body></html>diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_python.py b/examples/industrial_data_pretraining/fun_asr_nano/client_python.pynew file mode 100644index 000000000..570fd86cb--- /dev/null+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_python.py@@ -0,0 +1,233 @@+#!/usr/bin/env python3+"""Fun-ASR-Nano Python WebSocket Client.++Supports real-time microphone recording and audio file streaming.++Usage:+ # Microphone mode+ python client_python.py --server ws://localhost:10095 --mic++ # File mode+ python client_python.py --server ws://localhost:10095 --file audio.wav++ # With hotwords+ python client_python.py --server ws://localhost:10095 --file audio.wav --hotwords "张三,李四,北京"++ # Disable speaker diarization display+ python client_python.py --server ws://localhost:10095 --mic --no-spk+"""++import asyncio+import argparse+import json+import sys+import numpy as np++try:+ import websockets+except ImportError:+ print("Please install websockets: pip install websockets")+ sys.exit(1)+++SAMPLE_RATE = 16000+CHUNK_DURATION_MS = 100+CHUNK_SAMPLES = int(SAMPLE_RATE * CHUNK_DURATION_MS / 1000)++SPK_COLORS = [+ "\033[36m", "\033[35m", "\033[33m", "\033[32m",+ "\033[34m", "\033[91m", "\033[96m", "\033[95m",+]+RESET = "\033[0m"+GRAY = "\033[90m"+GREEN = "\033[92m"+++def format_time(ms):+ s = ms / 1000+ return f"{s:.1f}s"+++def print_result(data, show_spk=True):+ """Print ASR result to terminal."""+ sentences = data.get("sentences", [])+ partial = data.get("partial", "")+ partial_start = data.get("partial_start_ms", 0)+ is_final = data.get("is_final", False)++ sys.stdout.write("\033[2J\033[H")++ print(f"{GREEN}Fun-ASR-Nano Streaming ASR{RESET}")+ print(f"{GRAY}{'─' * 60}{RESET}")++ for s in sentences:+ start = s.get("start", s.get("start_ms", 0))+ end = s.get("end", s.get("end_ms", 0))+ spk = s.get("spk", -1)+ text = s["text"]++ time_str = f"{GRAY}[{format_time(start)}-{format_time(end)}]{RESET}"+ spk_str = ""+ if show_spk and spk >= 0:+ color = SPK_COLORS[spk % len(SPK_COLORS)]+ spk_str = f" {color}SPK{spk}{RESET}"++ print(f" {time_str}{spk_str} {text}")++ if partial:+ print(f" {GRAY}[{format_time(partial_start)}-...] {partial}{RESET}")++ if is_final:+ print(f"\n{GRAY}{'─' * 60}{RESET}")+ print(f"{GREEN}Done.{RESET} {len(sentences)} sentences")+ else:+ print(f"\n{GRAY}Recording... Press Ctrl+C to stop{RESET}")++ sys.stdout.flush()+++async def run_mic(args):+ """Stream from microphone."""+ try:+ import sounddevice as sd+ except ImportError:+ print("Please install sounddevice: pip install sounddevice")+ sys.exit(1)++ print(f"Connecting to {args.server}...")+ async with websockets.connect(args.server, ping_interval=None) as ws:+ await ws.send("START")+ resp = await ws.recv()+ event = json.loads(resp)+ if event.get("event") != "started":+ print(f"Unexpected response: {resp}")+ return++ if args.hotwords:+ await ws.send(f"HOTWORDS:{args.hotwords}")+ await ws.recv()++ print("Recording... Press Ctrl+C to stop\n")++ audio_queue = asyncio.Queue()++ def audio_callback(indata, frames, time_info, status):+ audio_queue.put_nowait(indata.copy())++ stream = sd.InputStream(+ samplerate=SAMPLE_RATE, channels=1, dtype='int16',+ blocksize=CHUNK_SAMPLES, callback=audio_callback,+ )++ async def send_audio():+ with stream:+ while True:+ chunk = await audio_queue.get()+ await ws.send(chunk.tobytes())++ async def recv_results():+ async for msg in ws:+ data = json.loads(msg)+ if "sentences" in data:+ print_result(data, show_spk=args.spk)+ if data.get("is_final") or data.get("event") == "stopped":+ break++ send_task = asyncio.create_task(send_audio())+ recv_task = asyncio.create_task(recv_results())++ try:+ await asyncio.gather(send_task, recv_task)+ except (KeyboardInterrupt, asyncio.CancelledError):+ pass+ finally:+ send_task.cancel()+ if ws.open:+ await ws.send("STOP")+ async for msg in ws:+ data = json.loads(msg)+ if "sentences" in data:+ print_result(data, show_spk=args.spk)+ if data.get("is_final") or data.get("event") == "stopped":+ break+++async def run_file(args):+ """Stream an audio file."""+ try:+ import soundfile as sf+ except ImportError:+ print("Please install soundfile: pip install soundfile")+ sys.exit(1)++ audio, sr = sf.read(args.file)+ if sr != SAMPLE_RATE:+ try:+ import librosa+ audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)+ except ImportError:+ print(f"Audio is {sr}Hz, need 16kHz. Install librosa: pip install librosa")+ sys.exit(1)+ if audio.ndim > 1:+ audio = audio[:, 0]+ audio = audio.astype(np.float32)++ duration = len(audio) / SAMPLE_RATE+ print(f"File: {args.file} ({duration:.1f}s)")+ print(f"Connecting to {args.server}...")++ async with websockets.connect(args.server, ping_interval=None) as ws:+ await ws.send("START")+ await ws.recv()++ if args.hotwords:+ await ws.send(f"HOTWORDS:{args.hotwords}")+ await ws.recv()++ int16 = (audio * 32768).clip(-32768, 32767).astype(np.int16)++ chunk_size = CHUNK_SAMPLES+ total_chunks = (len(int16) + chunk_size - 1) // chunk_size++ async def send_audio():+ for i in range(0, len(int16), chunk_size):+ chunk = int16[i:i+chunk_size]+ await ws.send(chunk.tobytes())+ await asyncio.sleep(CHUNK_DURATION_MS / 1000 * 0.5)+ await ws.send("STOP")++ async def recv_results():+ async for msg in ws:+ data = json.loads(msg)+ if "sentences" in data:+ print_result(data, show_spk=args.spk)+ if data.get("is_final") or data.get("event") == "stopped":+ break++ await asyncio.gather(send_audio(), recv_results())+++def main():+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano Python Client")+ parser.add_argument("--server", type=str, default="ws://localhost:10095")+ parser.add_argument("--mic", action="store_true", help="Use microphone input")+ parser.add_argument("--file", type=str, help="Audio file to transcribe")+ parser.add_argument("--hotwords", type=str, default="", help="Hotwords (comma-separated)")+ parser.add_argument("--spk", action="store_true", default=True, help="Show speaker IDs")+ parser.add_argument("--no-spk", dest="spk", action="store_false")+ args = parser.parse_args()++ if not args.mic and not args.file:+ parser.error("Specify --mic or --file")++ try:+ if args.mic:+ asyncio.run(run_mic(args))+ else:+ asyncio.run(run_file(args))+ except KeyboardInterrupt:+ print(f"\n{RESET}Interrupted.")+++if __name__ == "__main__":+ main()diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_test.py b/examples/industrial_data_pretraining/fun_asr_nano/client_test.pynew file mode 100644index 000000000..9efc9f260--- /dev/null+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_test.py@@ -0,0 +1,244 @@+#!/usr/bin/env python3+"""Fun-ASR-Nano Streaming ASR - Automated Test Script.++Tests the WebSocket server with audio files and validates the response format.++Usage:+ python client_test.py --server ws://localhost:10095 --file test_audio.wav+ python client_test.py --server ws://localhost:10095 --file test_audio.wav --hotwords "热词1,热词2"+"""++import asyncio+import argparse+import json+import time+import sys+import numpy as np++try:+ import websockets+except ImportError:+ print("ERROR: pip install websockets")+ sys.exit(1)++try:+ import soundfile as sf+except ImportError:+ print("ERROR: pip install soundfile")+ sys.exit(1)+++SAMPLE_RATE = 16000+++def load_audio(file_path):+ audio, sr = sf.read(file_path)+ if sr != SAMPLE_RATE:+ try:+ import librosa+ audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)+ except ImportError:+ print(f"ERROR: Audio is {sr}Hz. Install librosa for resampling.")+ sys.exit(1)+ if audio.ndim > 1:+ audio = audio[:, 0]+ return audio.astype(np.float32)+++async def test_basic(server, audio, hotwords=None):+ """Test basic streaming ASR flow."""+ print("=" * 60)+ print("TEST: Basic Streaming ASR")+ print("=" * 60)++ t_start = time.perf_counter()++ async with websockets.connect(server, ping_interval=None) as ws:+ await ws.send("START")+ resp = json.loads(await ws.recv())+ assert resp["event"] == "started", f"Expected 'started', got: {resp}"+ print(" [PASS] START -> event:started")++ if hotwords:+ await ws.send(f"HOTWORDS:{hotwords}")+ resp = json.loads(await ws.recv())+ assert resp["event"] == "hotwords_set", f"Expected 'hotwords_set', got: {resp}"+ print(f" [PASS] HOTWORDS -> {len(resp['hotwords'])} words set")++ int16 = (audio * 32768).clip(-32768, 32767).astype(np.int16)+ chunk_size = 4096+ partial_count = 0++ for i in range(0, len(int16), chunk_size):+ chunk = int16[i:i+chunk_size]+ await ws.send(chunk.tobytes())+ await asyncio.sleep(0.02)++ try:+ msg = await asyncio.wait_for(ws.recv(), timeout=0.01)+ data = json.loads(msg)+ if "sentences" in data:+ partial_count += 1+ assert "partial" in data+ assert "duration_ms" in data+ assert "is_final" in data+ assert isinstance(data["sentences"], list)+ except asyncio.TimeoutError:+ pass++ await ws.send("STOP")++ final_result = None+ while True:+ msg = await asyncio.wait_for(ws.recv(), timeout=30)+ data = json.loads(msg)+ if data.get("is_final"):+ final_result = data+ if data.get("event") == "stopped":+ break++ t_end = time.perf_counter()++ assert final_result is not None, "No final result received"+ assert final_result["is_final"] is True+ sentences = final_result["sentences"]+ assert len(sentences) > 0, "No sentences in final result"++ print(f" [PASS] Received final result: {len(sentences)} sentences")+ print(f" [PASS] Partial updates received: {partial_count}")++ for s in sentences:+ assert "text" in s, f"Missing 'text' in sentence: {s}"+ assert "start" in s, f"Missing 'start' in sentence: {s}"+ assert "end" in s, f"Missing 'end' in sentence: {s}"+ assert isinstance(s["text"], str) and len(s["text"]) > 0+ assert s["end"] > s["start"]++ print(" [PASS] All sentences have valid format (text, start, end)")++ has_spk = all("spk" in s for s in sentences)+ if has_spk:+ spk_ids = set(s["spk"] for s in sentences)+ print(f" [PASS] Speaker diarization: {len(spk_ids)} speakers detected")+ else:+ print(" [INFO] No speaker IDs in result")++ elapsed = t_end - t_start+ audio_duration = len(audio) / SAMPLE_RATE+ rtf = elapsed / audio_duration+ print(f"\n Audio: {audio_duration:.1f}s | Time: {elapsed:.2f}s | RTF: {rtf:.3f}")++ print("\n --- Results ---")+ for s in sentences:+ spk = f" SPK{s['spk']}" if "spk" in s else ""+ print(f" [{s['start']/1000:.1f}-{s['end']/1000:.1f}s]{spk}: {s['text']}")++ return True+++async def test_empty_audio(server):+ """Test with very short/empty audio."""+ print("\n" + "=" * 60)+ print("TEST: Empty/Short Audio Handling")+ print("=" * 60)++ async with websockets.connect(server, ping_interval=None) as ws:+ await ws.send("START")+ await ws.recv()++ short_audio = np.zeros(800, dtype=np.int16)+ await ws.send(short_audio.tobytes())+ await asyncio.sleep(0.1)++ await ws.send("STOP")+ final = None+ while True:+ msg = await asyncio.wait_for(ws.recv(), timeout=10)+ data = json.loads(msg)+ if data.get("is_final"):+ final = data+ if data.get("event") == "stopped":+ break++ assert final is not None+ assert final["sentences"] == [] or all(s["text"].strip() for s in final["sentences"])+ print(" [PASS] Short audio handled gracefully")++ return True+++async def test_multiple_sessions(server, audio):+ """Test multiple consecutive sessions on same connection."""+ print("\n" + "=" * 60)+ print("TEST: Multiple Sessions")+ print("=" * 60)++ audio_short = audio[:SAMPLE_RATE * 5]+ int16 = (audio_short * 32768).clip(-32768, 32767).astype(np.int16)++ async with websockets.connect(server, ping_interval=None) as ws:+ for session_num in range(2):+ await ws.send("START")+ resp = json.loads(await ws.recv())+ assert resp["event"] == "started"++ for i in range(0, len(int16), 4096):+ await ws.send(int16[i:i+4096].tobytes())+ await asyncio.sleep(0.01)++ await ws.send("STOP")+ got_final = False+ while True:+ msg = await asyncio.wait_for(ws.recv(), timeout=15)+ data = json.loads(msg)+ if data.get("is_final"):+ got_final = True+ if data.get("event") == "stopped":+ break+ assert got_final, f"Session {session_num+1}: no final result"+ print(f" [PASS] Session {session_num+1} completed")++ return True+++async def run_tests(args):+ audio = load_audio(args.file)+ print(f"Loaded: {args.file} ({len(audio)/SAMPLE_RATE:.1f}s)")+ print(f"Server: {args.server}\n")++ passed = 0+ failed = 0++ tests = [+ ("Basic Streaming", lambda: test_basic(args.server, audio, args.hotwords)),+ ("Empty Audio", lambda: test_empty_audio(args.server)),+ ("Multiple Sessions", lambda: test_multiple_sessions(args.server, audio)),+ ]++ for name, test_fn in tests:+ try:+ await test_fn()+ passed += 1+ except Exception as e:+ print(f" [FAIL] {name}: {e}")+ failed += 1++ print("\n" + "=" * 60)+ print(f"RESULTS: {passed} passed, {failed} failed")+ print("=" * 60)+ return failed == 0+++def main():+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming Test")+ parser.add_argument("--server", type=str, default="ws://localhost:10095")+ parser.add_argument("--file", type=str, required=True, help="Test audio file")+ parser.add_argument("--hotwords", type=str, default="", help="Hotwords (comma-separated)")+ args = parser.parse_args()++ success = asyncio.run(run_tests(args))+ sys.exit(0 if success else 1)+++if __name__ == "__main__":+ main()diff --git a/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md b/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.mdindex 85f536830..cb3f7bcbf 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md+++ b/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md@@ -1,114 +1,254 @@-# Fun-ASR-Nano 实时语音识别 Demo+# Fun-ASR-Nano 流式语音识别服务(Streaming ASR)-基于 WebSocket 的实时 ASR 演示,支持麦克风录音、流式 VAD 分句、逐句显示。+基于 WebSocket 的实时流式语音识别服务,支持 VAD 分句、说话人分离(Beta)、热词定制化。++## 功能特性++| 功能 | 说明 |+|------|------|+| 流式 ASR | 基于 Fun-ASR-Nano + vLLM 推理引擎,实时输出识别结果 |+| 流式 VAD | fsmn-vad 增量检测语音端点,动态调整静音阈值 |+| 说话人分离 (Beta) | eres2netv2 + ClusterBackend,流式分配+最终重聚类 |+| 热词定制化 | 加载人名、地名等实体词列表,提升专有名词识别准确率 |+| 幻觉检测 | 自动检测重复模式并截断,防止模型生成循环 |+| 多语言 | 31种语言、7大方言、26种地方口音 |## 架构```-浏览器(麦克风) 服务器(GPU)-┌──────────────┐ WebSocket PCM16 ┌────────────────────────┐-│ client_mic │ ──────────────────→ │ serve_realtime_ws.py │-│ .html │ │ │-│ │ ←────────────────── │ ┌─────────────────┐ │-│ 逐句显示结果 │ JSON sentences │ │ Streaming VAD │ │-└──────────────┘ │ │ (fsmn-vad) │ │- │ └────────┬────────┘ │- │ ↓ │- │ ┌─────────────────┐ │- │ │ ASR per-segment │ │- │ │ (Fun-ASR-Nano) │ │- │ └─────────────────┘ │- └────────────────────────┘+客户端(浏览器/Python) 服务端(GPU)+┌─────────────────┐ WebSocket ┌──────────────────────────┐+│ │ PCM16 16kHz │ serve_realtime_ws.py │+│ client_mic.html │ ────────────→ │ │+│ client_python.py│ │ ┌──────────┐ │+│ │ ←──────────── │ │ VAD │ 流式分句 │+│ 实时显示结果 │ JSON │ └────┬─────┘ │+└─────────────────┘ │ ↓ │+ │ ┌──────────┐ │+ │ │ ASR │ 逐段解码 │+ │ └────┬─────┘ │+ │ ↓ │+ │ ┌──────────┐ │+ │ │ SPK │ 说话人分离 │+ │ └──────────┘ │+ └──────────────────────────┘```-## 核心设计+## 快速部署-1. **流式 VAD 分句**:fsmn-vad 增量检测语音端点- - `[start, -1]` = 检测到语音开始- - `[-1, end]` = 检测到语音结束 → 确认分句-2. **逐段解码**:只对当前 VAD 段做 ASR(不累积全部音频)- - 已确认段锁定文字,永不重新解码- - 支持无限时长录音,不会因音频过长而崩溃-3. **prev_text 连续性**:当前段内使用 prev_text 保证文字稳定+### 环境要求-## 使用方法+- Python >= 3.8+- CUDA >= 11.8+- GPU 显存 >= 4GB-### 1. 启动服务+### 安装依赖```bashcd examples/industrial_data_pretraining/fun_asr_nano+pip install -r requirements.txt+```-# 单卡启动+### 启动服务++```bash+# 基本启动CUDA_VISIBLE_DEVICES=0 python serve_realtime_ws.py --port 10095-# 指定参数+# 完整参数CUDA_VISIBLE_DEVICES=0 python serve_realtime_ws.py \--port 10095 \--model FunAudioLLM/Fun-ASR-Nano-2512 \+ --hub ms \--device cuda:0 \- --decode-interval 0.7+ --decode-interval 0.48 \+ --hotword-file 热词列表```-### 2. 打开客户端+首次启动会自动下载模型(约 30 秒),之后即可接受连接。++### 服务端参数-直接用浏览器打开 `client_mic.html`,修改 Server 地址为:-- 本机:`ws://localhost:10095`-- 远程(需 SSH 端口转发):先 `ssh -L 10095:localhost:10095 <server>`+| 参数 | 默认值 | 说明 |+|------|--------|------|+| `--port` | 10095 | WebSocket 监听端口 |+| `--model` | `FunAudioLLM/Fun-ASR-Nano-2512` | ASR 模型名称 |+| `--hub` | `ms` | 模型来源(ms=ModelScope, hf=HuggingFace) |+| `--device` | `cuda:0` | GPU 设备 |+| `--decode-interval` | 0.48 | 实时解码间隔(秒) |+| `--hotword-file` | `热词列表` | 热词文件路径(一行一个词) |+| `--use-context` | True | 使用上下文辅助解码 |+| `--no-context` | - | 禁用上下文 |-### 3. 体验+## 客户端使用-1. 点击 **Start Recording** → 允许麦克风-2. 对着麦克风说话-3. 说话过程中,VAD 检测到停顿自动分句显示-4. 点击 **Stop** 结束+### 1. 浏览器客户端(client_mic.html)-## 显示效果+直接在浏览器中打开 `client_mic.html`:+```bash+# 本机访问+open client_mic.html # macOS+xdg-open client_mic.html # Linux++# 远程服务器(需 SSH 端口转发)+ssh -L 10095:localhost:10095 <server>+# 然后在本地浏览器打开 client_mic.html```-0.3s - 2.1s 今天天气不错。-2.5s - 5.0s 我想去公园散步。-5.3s - ... 现在正在说的话...(斜体,未确认)++功能:+- 麦克风实时录音+- 音频文件上传识别+- 热词列表文件加载(.txt,一行一个词)+- 说话人分离显示(可开关)++### 2. Python 客户端(client_python.py)++```bash+# 麦克风模式+python client_python.py --server ws://localhost:10095 --mic++# 文件模式+python client_python.py --server ws://localhost:10095 --file audio.wav++# 带热词+python client_python.py --server ws://localhost:10095 --file audio.wav \+ --hotwords "张三,李四,北京"++# 不显示说话人+python client_python.py --server ws://localhost:10095 --mic --no-spk```-- 白色 = 已确认句子(VAD 检测到结束,文字锁定)-- 灰色斜体 = 正在说的句子(实时更新)-- 绿色时间戳 = VAD 检测的起止时间+### 3. 测试脚本(client_test.py)-## 服务端参数+```bash+# 运行完整测试+python client_test.py --server ws://localhost:10095 --file test_audio.wav-| 参数 | 默认值 | 说明 |-|------|--------|------|-| `--port` | 10095 | WebSocket 端口 |-| `--model` | `FunAudioLLM/Fun-ASR-Nano-2512` | ASR 模型 |-| `--device` | `cuda:0` | GPU 设备 |-| `--decode-interval` | 0.7 | 解码间隔(秒),越小延迟越低但 GPU 占用越高 |+# 带热词测试+python client_test.py --server ws://localhost:10095 --file test_audio.wav \+ --hotwords "人名,地名"+```-## 协议+测试内容:+- 基本流式 ASR 流程验证+- 返回格式校验(sentences/partial/is_final)+- 空音频处理+- 多轮 session 复用+- 性能统计(RTF)+## API 协议文档++### 连接++```+WebSocket URL: ws://<host>:<port>+音频格式: PCM16, 16kHz, 单声道 (Little-Endian)+```++### 客户端 → 服务端++| 消息类型 | 格式 | 说明 |+|---------|------|------|+| 开始会话 | `"START"` (text) | 初始化/重置 session |+| 设置热词 | `"HOTWORDS:词1,词2,词3"` (text) | 可选,在 START 后发送 |+| 音频数据 | `bytes` (binary) | PCM16 音频帧,任意长度 |+| 结束会话 | `"STOP"` (text) | 触发最终解码,返回完整结果 |++### 服务端 → 客户端++#### 1. 事件消息++```json+{"event": "started"}+{"event": "hotwords_set", "hotwords": ["词1", "词2"]}+{"event": "stopped"}```-Client → Server:- "START" 开始录音- "STOP" 结束录音- bytes PCM16 音频数据(16kHz 单声道)-Server → Client:- {"event": "started"}- {"event": "stopped"}- {- "sentences": [- {"text": "已确认的句子", "start_ms": 300, "end_ms": 2100},- ...- ],- "partial": "正在说的...",- "partial_start_ms": 5300,- "duration_ms": 7200,- "is_final": false- }+#### 2. 实时结果(流式,在 START 和 STOP 之间持续推送)++```json+{+ "sentences": [+ {"text": "已确认句子", "start": 1700, "end": 5500, "spk": 0}+ ],+ "partial": "正在说的文字...",+ "partial_start_ms": 5800,+ "duration_ms": 7200,+ "is_final": false+}```+#### 3. 最终结果(STOP 后返回)++```json+{+ "sentences": [+ {"text": "第一句话。", "start": 1700, "end": 5500, "spk": 0},+ {"text": "第二句话。", "start": 6000, "end": 9200, "spk": 1}+ ],+ "partial": "",+ "partial_start_ms": 0,+ "duration_ms": 10000,+ "is_final": true+}+```++### 字段说明++| 字段 | 类型 | 说明 |+|------|------|------|+| `sentences` | array | 已确认的句子列表 |+| `sentences[].text` | string | 识别文本 |+| `sentences[].start` | int | 起始时间(毫秒) |+| `sentences[].end` | int | 结束时间(毫秒) |+| `sentences[].spk` | int | 说话人 ID(0-based),仅最终结果保证准确 |+| `partial` | string | 当前正在说的文本(未确认,会被更新) |+| `partial_start_ms` | int | partial 的起始时间(毫秒) |+| `duration_ms` | int | 已接收音频总时长(毫秒) |+| `is_final` | bool | 是否为最终结果 |++### 典型交互流程++```+Client Server+ │ │+ │──── "START" ─────────────────→│+ │←──── {"event":"started"} ─────│+ │ │+ │──── "HOTWORDS:张三,北京" ────→│ (可选)+ │←── {"event":"hotwords_set"} ──│+ │ │+ │──── [audio bytes] ───────────→│+ │──── [audio bytes] ───────────→│+ │←── {sentences,partial} ───────│ (实时推送)+ │──── [audio bytes] ───────────→│+ │←── {sentences,partial} ───────│+ │ ... │+ │──── "STOP" ──────────────────→│+ │←── {sentences,is_final:true} ─│ (最终结果)+ │←── {"event":"stopped"} ───────│+ │ │+```++## 热词文件格式++文件名默认为 `热词列表`(可通过 `--hotword-file` 指定),格式为纯文本,一行一个词:++```+张三+李四+北京大学+Fun-ASR-Nano+```++也可在运行时通过 WebSocket 动态设置:`HOTWORDS:词1,词2,词3`+## 注意事项-- 浏览器需要 HTTPS 或 localhost 才能使用麦克风(Chrome 安全限制)-- 首次使用需允许麦克风权限-- 模型加载约需 30 秒,之后即可接受连接-- 单个服务实例支持多个并发连接(每个连接独立 session)+1. **HTTPS 限制**:浏览器麦克风需要 HTTPS 或 localhost(Chrome 安全策略)+2. **远程访问**:使用 SSH 端口转发 `ssh -L 10095:localhost:10095 <server>`+3. **并发支持**:单个服务实例支持多个并发 WebSocket 连接(每个连接独立 session)+4. **显存占用**:ASR + VAD + SPK 模型共约 3-4GB 显存+5. **说话人分离**:Beta 功能,短音频或单人场景可能不准确+6. **模型来源**:默认从 ModelScope 下载,可用 `--hub hf` 切换 HuggingFacediff --git a/examples/industrial_data_pretraining/fun_asr_nano/requirements.txt b/examples/industrial_data_pretraining/fun_asr_nano/requirements.txtindex eac7d4a42..8bb51b590 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/requirements.txt+++ b/examples/industrial_data_pretraining/fun_asr_nano/requirements.txt@@ -1,9 +1,10 @@-torch>=2.9.0-torchaudio>=2.9.0-transformers>=4.51.3-funasr>=1.3.0-zhconv-whisper_normalizer-pyopenjtalk-plus-compute-wer-openai-whisper+torch>=2.0+funasr+websockets>=12.0+regex+numpy+soundfile++# Optional: for Python client microphone mode+# sounddevice+# librosadiff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.pyindex 367816e8a..ad6f98dc8 100644--- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py@@ -1,10 +1,12 @@#!/usr/bin/env python3-"""-Fun-ASR-Nano Real-time WebSocket Server.--Key fix: Only decode CURRENT VAD segment (not entire cumulative audio).-Confirmed segments get their text locked and never re-decoded.-This prevents model collapse on long recordings.+"""Fun-ASR-Nano Streaming WebSocket Server.++Features:+- Streaming VAD segmentation (fsmn-vad)+- Per-segment ASR decoding (Fun-ASR-Nano via vLLM)+- Speaker diarization (eres2netv2 + ClusterBackend)+- Hotword customization+- Hallucination detection & prevention"""import asyncio@@ -16,20 +18,50 @@ import argparseimport numpy as npimport torchimport warnings+import regex+import websocketswarnings.filterwarnings('ignore')logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')logger = logging.getLogger(__name__)-try:- import websockets-except ImportError:- os.system("pip install websockets --break-system-packages -q")- import websockets++def detect_and_fix_hallucination(text, max_ngram_length=12, max_occurrences=3):+ """Detect repeated patterns (hallucination) and truncate to keep one occurrence."""+ if not text or len(text) < max_ngram_length * 2:+ return text, False++ cleaned = regex.sub(r'\p{P}+', '', text)++ word_pattern = rf'(?<!\S)(?!\d+$)(\w+)(?:\s+\1){{{max_occurrences - 1},}}(?!\S)'+ if regex.search(word_pattern, cleaned, regex.IGNORECASE):+ match = regex.search(word_pattern, cleaned, regex.IGNORECASE)+ repeated = match.group(1)+ pos = text.find(repeated)+ if pos >= 0:+ end_pos = text.find(repeated, pos + len(repeated))+ if end_pos >= 0:+ return text[:end_pos + len(repeated)], True+ return text[:len(text)//2], True++ for length in range(1, max_ngram_length):+ pattern = rf'(?<!\d)(\S{{{length}}})\1{{{max_occurrences - 1},}}(?!\d)'+ combined = rf'(?=.*\D){pattern}'+ match = regex.search(combined, cleaned)+ if match:+ repeated = match.group(1)+ pos = text.find(repeated)+ if pos >= 0:+ end_pos = text.find(repeated, pos + len(repeated))+ if end_pos >= 0:+ return text[:end_pos + len(repeated)], True+ return text[:len(text)//2], True++ return text, Falseclass StreamingVAD:- """Streaming VAD: [start,-1]=speech start, [-1,end]=speech end."""+ """Streaming VAD with dynamic silence threshold."""def __init__(self, vad_model):self.model = vad_model@@ -40,7 +72,7 @@ class StreamingVAD:def feed(self, audio_chunk_tensor, is_final=False):res = self.model.generate(input=[audio_chunk_tensor], cache=self.cache,- is_final=is_final, chunk_size=200,+ is_final=is_final, chunk_size=60,)signals = res[0].get("value", [])new_confirmed = []@@ -64,21 +96,178 @@ class StreamingVAD:self.current_speech_start = None+class HybridSpeakerTracker:+ """Speaker diarization: streaming ClusterBackend + final re-clustering."""++ def __init__(self, spk_model, device, threshold=0.6):+ self.spk_model = spk_model+ self.device = device+ self.threshold = threshold+ self.speaker_centers = []+ from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk+ from funasr.models.campplus.cluster_backend import ClusterBackend+ self.sv_chunk = sv_chunk+ self.postprocess = postprocess+ self.distribute_spk = distribute_spk+ self.cluster_backend = ClusterBackend(merge_thr=0.78).to(device)+ self.all_chunks = []+ self.all_embeddings = []+ self.display_map = {}+ self.next_display_id = 0++ @torch.no_grad()+ def assign_streaming(self, audio_samples, seg_start_s, seg_end_s, sentence):+ """Assign speaker ID during streaming using ClusterBackend."""+ vad_seg = [[seg_start_s, seg_end_s, audio_samples]]+ chunks = self.sv_chunk(vad_seg)+ if not chunks:+ sentence["spk"] = self.next_display_id+ self.next_display_id += 1+ return++ self.all_chunks.extend(chunks)+ speech_list = [ch[2] for ch in chunks]+ spk_res = self.spk_model.generate(input=speech_list, cache={}, is_final=True)+ embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)+ self.all_embeddings.append(embs)++ all_embs = torch.cat(self.all_embeddings, dim=0)+ labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)+ if not isinstance(labels, np.ndarray):+ labels = np.array(labels)++ all_sorted = sorted(self.all_chunks, key=lambda x: x[0])+ sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())+ temp = [{"start": int(seg_start_s*1000), "end": int(seg_end_s*1000), "text": sentence["text"]}]+ self.distribute_spk(temp, sv_output)+ raw_spk = temp[0].get("spk", 0)++ if raw_spk not in self.display_map:+ self.display_map[raw_spk] = self.next_display_id+ self.next_display_id += 1+ sentence["spk"] = self.display_map[raw_spk]++ @torch.no_grad()+ def finalize(self, sentences, min_split_s=3.0):+ """Final re-clustering for accurate speaker assignment."""+ if not self.all_embeddings or not sentences:+ return sentences++ all_embs = torch.cat(self.all_embeddings, dim=0)+ labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)+ if not isinstance(labels, np.ndarray):+ labels = np.array(labels)++ all_sorted = sorted(self.all_chunks, key=lambda x: x[0])+ sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())++ for s in sentences:+ s.pop("spk", None)+ self.distribute_spk(sentences, sv_output)++ id_map = {}+ next_id = 0+ for s in sentences:+ raw = s.get("spk", 0)+ if raw not in id_map:+ id_map[raw] = next_id+ next_id += 1+ s["spk"] = id_map[raw]++ final_sentences = []+ for s in sentences:+ sub = self._try_split(s, sv_output, id_map, min_split_s)+ final_sentences.extend(sub)++ return final_sentences++ def _try_split(self, sentence, sv_output, id_map, min_split_s):+ """Split a sentence if multiple speakers detected within its time range."""+ sent_start = sentence["start"] / 1000.0+ sent_end = sentence["end"] / 1000.0+ text = sentence["text"]++ overlapping = []+ for sv_start, sv_end, sv_spk in sv_output:+ o_start = max(sent_start, sv_start)+ o_end = min(sent_end, sv_end)+ if o_end > o_start:+ mapped_spk = id_map.get(int(sv_spk), int(sv_spk))+ overlapping.append([o_start, o_end, mapped_spk])++ if len(overlapping) <= 1:+ return [sentence]++ filtered = [overlapping[0]]+ for i in range(1, len(overlapping)):+ cur = overlapping[i]+ prev = filtered[-1]+ if cur[2] == prev[2]:+ filtered[-1] = [prev[0], cur[1], prev[2]]+ elif (cur[1] - cur[0]) < min_split_s:+ filtered[-1] = [prev[0], cur[1], prev[2]]+ else:+ filtered.append(cur)++ merged = [filtered[0]]+ for i in range(1, len(filtered)):+ if (merged[-1][1] - merged[-1][0]) < min_split_s:+ merged[-1] = [merged[-1][0], filtered[i][1], filtered[i][2]]+ else:+ merged.append(filtered[i])+ if len(merged) > 1 and (merged[-1][1] - merged[-1][0]) < min_split_s:+ merged[-2] = [merged[-2][0], merged[-1][1], merged[-2][2]]+ merged.pop()++ if len(merged) <= 1:+ return [sentence]++ total_dur = sum(m[1] - m[0] for m in merged)+ sub_sentences = []+ char_pos = 0+ for i, (m_start, m_end, m_spk) in enumerate(merged):+ if i == len(merged) - 1:+ sub_text = text[char_pos:]+ else:+ n_chars = max(1, int(len(text) * (m_end - m_start) / total_dur))+ sub_text = text[char_pos:char_pos + n_chars]+ char_pos += n_chars+ if sub_text.strip():+ sub_sentences.append({"text": sub_text.strip(), "start": int(m_start*1000), "end": int(m_end*1000), "spk": m_spk})++ return sub_sentences if sub_sentences else [sentence]++ def reset(self):+ self.speaker_centers = []+ self.all_chunks = []+ self.all_embeddings = []+ self.display_map = {}+ self.next_display_id = 0++class RealtimeASRSession:- def __init__(self, asr_model, asr_kwargs, tokenizer, vad, sample_rate=16000, chunk_ms=720):+ """Manages a single streaming ASR session."""++ def __init__(self, asr_model, asr_kwargs, tokenizer, vad, spk_tracker=None, sample_rate=16000, chunk_ms=960):self.asr_model = asr_modelself.asr_kwargs = asr_kwargsself.tokenizer = tokenizerself.vad = vadself.sample_rate = sample_rateself.chunk_samples = int(sample_rate * chunk_ms / 1000)+ self.first_chunk_samples = int(sample_rate * 480 / 1000)+ self.first_decode_done = Falseself.audio_buffer = np.array([], dtype=np.float32)self.vad_fed_samples = 0self.prev_text = ""self.last_partial_text = ""self.last_decode_samples = 0- self.locked_sentences = [] # text locked for confirmed segments+ self.locked_sentences = []+ self.prev_seg_text = ""+ self.accumulated_since_cut_ms = 0+ self.spk_tracker = spk_tracker+ self.use_context = Trueself.is_active = Falsedef add_audio(self, pcm_bytes):@@ -86,21 +275,45 @@ class RealtimeASRSession:audio_float = audio_int16.astype(np.float32) / 32768.0self.audio_buffer = np.concatenate([self.audio_buffer, audio_float])- # Feed new audio to streaming VADnew_audio = self.audio_buffer[self.vad_fed_samples:]if len(new_audio) > 0:+ self.accumulated_since_cut_ms += len(new_audio) * 1000 // self.sample_rate+ if "stats" in self.vad.cache:+ self.vad.cache["stats"].speech_noise_thres = 0.5+ if self.accumulated_since_cut_ms <= 5000:+ desired_silence_ms = 2000+ elif self.accumulated_since_cut_ms <= 10000:+ desired_silence_ms = 1500+ elif self.accumulated_since_cut_ms <= 15000:+ desired_silence_ms = 1000+ elif self.accumulated_since_cut_ms <= 30000:+ desired_silence_ms = 800+ elif self.accumulated_since_cut_ms <= 45000:+ desired_silence_ms = 400+ else:+ desired_silence_ms = 100+ new_thresh = max(desired_silence_ms - 150, 0)+ self.vad.cache["stats"].max_end_sil_frame_cnt_thresh = new_thresh+new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)self.vad_fed_samples = len(self.audio_buffer)- # Lock text for newly confirmed segmentsfor seg in new_confirmed:seg_text = self._decode_segment(seg)- self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})- self.prev_text = "" # reset for new segment- logger.info(f" Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:30]}\"")+ self.prev_text = ""+ self.accumulated_since_cut_ms = 0+ if not seg_text.strip():+ continue+ self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})+ if self.spk_tracker:+ s0 = int(seg[0] * self.sample_rate / 1000)+ s1 = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))+ self.spk_tracker.assign_streaming(self.audio_buffer[s0:s1], seg[0]/1000, seg[1]/1000, self.locked_sentences[-1])+ logger.info(f"Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:40]}\"")def should_decode(self):- return (len(self.audio_buffer) - self.last_decode_samples) >= self.chunk_samples+ threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples+ return (len(self.audio_buffer) - self.last_decode_samples) >= threshold@torch.no_grad()def decode(self, is_final=False):@@ -108,31 +321,32 @@ class RealtimeASRSession:return self._build_response(is_final)if is_final:- # Feed remaining to VADremaining = self.audio_buffer[self.vad_fed_samples:]if len(remaining) > 0:new_confirmed = self.vad.feed(torch.from_numpy(remaining).float(), is_final=True)self.vad_fed_samples = len(self.audio_buffer)for seg in new_confirmed:seg_text = self._decode_segment(seg)- self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})+ if not seg_text.strip():+ continue+ self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})- # If still in speech, close and decode itif self.vad.current_speech_start is not None:end_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)seg = [self.vad.current_speech_start, end_ms]seg_text = self._decode_segment(seg)- self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})+ if seg_text.strip():+ self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})self.vad.current_speech_start = None+ if self.spk_tracker and self.locked_sentences:+ self.locked_sentences = self.spk_tracker.finalize(self.locked_sentences)return self._build_response(is_final)- # Decode ONLY current ongoing segment (not full buffer!)if self.vad.current_speech_start is not None:seg_start_sample = int(self.vad.current_speech_start * self.sample_rate / 1000)seg_audio = self.audio_buffer[seg_start_sample:]else:- # Not in speech - nothing to decodeself.last_decode_samples = len(self.audio_buffer)self.last_partial_text = ""return self._build_response(is_final)@@ -150,10 +364,15 @@ class RealtimeASRSession:logger.error(f"ASR error: {e}")return self._build_response(is_final)+ text, hallucinated = detect_and_fix_hallucination(text)+ if hallucinated:+ self.prev_text = ""+self.last_decode_samples = len(self.audio_buffer)self.last_partial_text = text+ if text.strip() and not self.first_decode_done:+ self.first_decode_done = True- # Update prev_text (rollback 5 tokens)encoded = self.tokenizer.encode(text)if len(encoded) > 5:self.prev_text = self.tokenizer.decode(encoded[:-5]).replace("�", "")@@ -164,15 +383,16 @@ class RealtimeASRSession:@torch.no_grad()def _decode_segment(self, seg):- """Decode a specific completed VAD segment."""+ """Decode a completed VAD segment with optional context."""start_sample = int(seg[0] * self.sample_rate / 1000)end_sample = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))seg_audio = self.audio_buffer[start_sample:end_sample]if len(seg_audio) < 1600:return ""audio_tensor = torch.from_numpy(seg_audio).float()+ ctx = self.prev_seg_text if self.use_context and self.prev_seg_text else ""try:- res = self.asr_model.inference([audio_tensor], prev_text="", **self.asr_kwargs)+ res = self.asr_model.inference([audio_tensor], prev_text=ctx, **self.asr_kwargs)return res[0][0]["text"]except Exception as e:logger.error(f"Segment decode error: {e}")@@ -187,7 +407,6 @@ class RealtimeASRSession:if is_final:return {"sentences": sentences, "partial": "", "partial_start_ms": 0,"duration_ms": duration_ms, "is_final": True}-return {"sentences": sentences, "partial": partial,"partial_start_ms": partial_start,"duration_ms": duration_ms, "is_final": False}@@ -195,21 +414,26 @@ class RealtimeASRSession:def reset(self):self.audio_buffer = np.array([], dtype=np.float32)self.vad_fed_samples = 0+ self.first_decode_done = Falseself.vad.reset()self.prev_text = ""self.last_partial_text = ""self.last_decode_samples = 0self.locked_sentences = []+ self.accumulated_since_cut_ms = 0+ if self.spk_tracker:+ self.spk_tracker.reset()_asr_model = None_asr_kwargs = None_tokenizer = None_vad_model = None+_spk_model = Nonedef load_models(args):- global _asr_model, _asr_kwargs, _tokenizer, _vad_model+ global _asr_model, _asr_kwargs, _tokenizer, _vad_model, _spk_modelif _asr_model is None:from funasr import AutoModellogger.info(f"Loading ASR: {args.model}")@@ -224,14 +448,26 @@ def load_models(args):logger.info("Loading VAD: fsmn-vad (streaming)")_vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True)++ logger.info("Loading SPK: eres2netv2")+ _spk_model = AutoModel(model="iic/speech_eres2netv2_sv_zh-cn_16k-common", device=args.device, disable_update=True)++ hw_file = getattr(args, 'hotword_file', '热词列表')+ if hw_file and os.path.isfile(hw_file):+ with open(hw_file, "r", encoding="utf-8") as hf:+ hotwords = [line.strip() for line in hf if line.strip()]+ _asr_kwargs["hotwords"] = hotwords+ logger.info(f"Loaded {len(hotwords)} hotwords from '{hw_file}'")+logger.info("All models ready!")- return _asr_model, _asr_kwargs, _tokenizer, _vad_model+ return _asr_model, _asr_kwargs, _tokenizer, _vad_model, _spk_modelasync def handle_client(websocket, args):- asr_model, asr_kwargs, tokenizer, vad_automodel = load_models(args)- vad = StreamingVAD(vad_automodel)- session = RealtimeASRSession(asr_model, asr_kwargs, tokenizer, vad)+ asr_model, asr_kwargs, tokenizer, vad_model, spk_model = load_models(args)+ vad = StreamingVAD(vad_model)+ spk_tracker = HybridSpeakerTracker(spk_model, args.device)+ session = RealtimeASRSession(asr_model, asr_kwargs, tokenizer, vad, spk_tracker=spk_tracker)logger.info(f"Client connected: {websocket.remote_address}")decode_interval = args.decode_interval@@ -240,23 +476,28 @@ async def handle_client(websocket, args):try:async for message in websocket:if isinstance(message, str):- cmd = message.strip().upper()- if cmd == "START":+ cmd = message.strip()+ if cmd.upper() == "START":session.reset()session.is_active = Trueawait websocket.send(json.dumps({"event": "started"}))logger.info("Session started")- elif cmd == "STOP":+ elif cmd.upper().startswith("HOTWORDS:"):+ hw_str = cmd[9:]+ hotwords = [w.strip() for w in hw_str.split(",") if w.strip()]+ session.asr_kwargs = dict(session.asr_kwargs)+ session.asr_kwargs["hotwords"] = hotwords+ await websocket.send(json.dumps({"event": "hotwords_set", "hotwords": hotwords}))+ logger.info(f"Hotwords set: {len(hotwords)} words")+ elif cmd.upper() == "STOP":if session.is_active and len(session.audio_buffer) > 0:result = session.decode(is_final=True)await websocket.send(json.dumps(result))- n = len(result["sentences"])- logger.info(f"Final: {n} sentences")+ logger.info(f"Final: {len(result['sentences'])} sentences")session.is_active = Falseawait websocket.send(json.dumps({"event": "stopped"}))elif isinstance(message, bytes) and session.is_active:session.add_audio(message)-now = time.time()if now - last_decode_time >= decode_interval and session.should_decode():result = session.decode(is_final=False)@@ -266,9 +507,7 @@ async def handle_client(websocket, args):except websockets.exceptions.ConnectionClosed:logger.info("Client disconnected")except Exception as e:- logger.error(f"Error: {e}")- import traceback- traceback.print_exc()+ logger.error(f"Error: {e}", exc_info=True)async def main(args):@@ -282,11 +521,14 @@ async def main(args):if __name__ == "__main__":- parser = argparse.ArgumentParser()+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming WebSocket Server")parser.add_argument("--port", type=int, default=10095)parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")- parser.add_argument("--hub", type=str, default="ms")+ parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])parser.add_argument("--device", type=str, default="cuda:0")- parser.add_argument("--decode-interval", type=float, default=0.7)+ parser.add_argument("--use-context", action="store_true", default=True)+ parser.add_argument("--no-context", dest="use_context", action="store_false")+ parser.add_argument("--decode-interval", type=float, default=0.48)+ parser.add_argument("--hotword-file", type=str, default="热词列表")args = parser.parse_args()asyncio.run(main(args))diff --git "a/examples/industrial_data_pretraining/fun_asr_nano/\347\203\255\350\257\215\345\210\227\350\241\250" "b/examples/industrial_data_pretraining/fun_asr_nano/\347\203\255\350\257\215\345\210\227\350\241\250"new file mode 100644index 000000000..d29477dde--- /dev/null+++ "b/examples/industrial_data_pretraining/fun_asr_nano/\347\203\255\350\257\215\345\210\227\350\241\250"@@ -0,0 +1,4 @@+兔子+窝边草+土匪+师爷