Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

feat: polish streaming ASR service - cleanup, clients, docs

xiaoyunchong.xycf720a289124a
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html
index f06aedb68..b9312a1af 100644
--- a/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html
+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_mic.html
@@ -1,243 +1,146 @@
<!DOCTYPE html>
<html lang="zh-CN">
<head>
- <meta charset="UTF-8">
- <title>Fun-ASR-Nano vLLM - Real-time ASR</title>
- <style>
- * { margin: 0; padding: 0; box-sizing: border-box; }
- body {
- font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
- background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%);
- min-height: 100vh; display: flex; align-items: center; justify-content: center;
- color: #e0e0e0;
- }
- .container {
- background: rgba(255,255,255,0.05); backdrop-filter: blur(10px);
- border: 1px solid rgba(255,255,255,0.1); border-radius: 20px;
- padding: 40px; width: 90%; max-width: 700px;
- }
- h1 { text-align: center; font-size: 24px; margin-bottom: 8px; color: #64ffda; }
- .subtitle { text-align: center; color: #aaa; margin-bottom: 30px; font-size: 14px; }
- .config { display: flex; gap: 12px; margin-bottom: 20px; align-items: center; flex-wrap: wrap; }
- .config label { font-size: 13px; color: #aaa; }
- .config input, .config select {
- background: rgba(255,255,255,0.1); border: 1px solid rgba(255,255,255,0.2);
- border-radius: 8px; padding: 8px 12px; color: #fff; font-size: 14px;
- }
- .config input { width: 280px; }
- .btn {
- display: block; width: 200px; margin: 0 auto 20px; padding: 14px 0;
- border: none; border-radius: 50px; font-size: 16px; font-weight: 600;
- cursor: pointer; transition: all 0.3s;
- }
- .btn-start { background: #64ffda; color: #1a1a2e; }
- .btn-start:hover { background: #4de8c2; transform: scale(1.05); }
- .btn-stop { background: #ff6b6b; color: #fff; }
- .btn-stop:hover { background: #ee5a5a; transform: scale(1.05); }
- .btn:disabled { opacity: 0.5; cursor: not-allowed; transform: none; }
- .status {
- text-align: center; margin-bottom: 20px; font-size: 14px;
- padding: 8px; border-radius: 8px;
- }
- .status.connected { background: rgba(100,255,218,0.1); color: #64ffda; }
- .status.recording { background: rgba(255,107,107,0.1); color: #ff6b6b; }
- .status.error { background: rgba(255,107,107,0.1); color: #ff6b6b; }
- .result-box {
- background: rgba(0,0,0,0.3); border: 1px solid rgba(255,255,255,0.1);
- border-radius: 12px; padding: 20px; min-height: 150px; max-height: 400px; overflow-y: auto;
- font-size: 18px; line-height: 1.8; word-break: break-all;
- }
- .result-box .final { color: #64ffda; }
- .result-box .partial { color: #aaa; font-style: italic; }
- .result-box .placeholder { color: #555; }
- .visualizer { height: 60px; margin: 15px 0; display: flex; align-items: center; justify-content: center; gap: 3px; }
- .visualizer .bar {
- width: 4px; background: #64ffda; border-radius: 2px;
- transition: height 0.1s; min-height: 4px;
- }
- .footer { text-align: center; margin-top: 20px; font-size: 12px; color: #666; }
- </style>
+<meta charset="UTF-8">
+<title>Fun-ASR-Nano · Streaming ASR with Speaker Diarization</title>
+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700;800;900&display=swap" rel="stylesheet">
+<style>
+* { margin:0; padding:0; box-sizing:border-box; }
+html, body { height:100%; }
+body { font-family:'Inter',system-ui,sans-serif; background:#080810; color:#e0e0e0; display:flex; justify-content:center; height:100vh; overflow:hidden; }
+
+/* === Hero Section === */
+.hero { padding:24px 28px 16px; flex-shrink:0; }
+.hero-top { display:flex; align-items:center; gap:16px; margin-bottom:14px; }
+.logo { font-size:30px; font-weight:900; letter-spacing:-1px; }
+.logo .g { background:linear-gradient(135deg,#64ffda,#00bfa5); -webkit-background-clip:text; -webkit-text-fill-color:transparent; }
+.logo .w { color:#fff; }
+.logo .s { color:rgba(255,255,255,0.3); font-weight:400; font-size:15px; margin-left:8px; }
+.hero-links { margin-left:auto; display:flex; gap:10px; }
+.hero-links a { font-size:12px; color:#aaa; text-decoration:none; padding:6px 14px; border-radius:8px; background:rgba(255,255,255,0.04); border:1px solid rgba(255,255,255,0.08); transition:all 0.2s; font-weight:500; }
+.hero-links a:hover { color:#64ffda; border-color:rgba(100,255,218,0.3); }
+
+.features { display:flex; gap:10px; flex-wrap:wrap; margin-bottom:14px; }
+.feat { font-size:11px; padding:5px 12px; border-radius:20px; font-weight:500; }
+.feat-green { color:#64ffda; background:rgba(100,255,218,0.08); border:1px solid rgba(100,255,218,0.2); }
+.feat-yellow { color:#fbbf24; background:rgba(251,191,36,0.08); border:1px solid rgba(251,191,36,0.2); }
+.feat-purple { color:#a78bfa; background:rgba(167,139,250,0.08); border:1px solid rgba(167,139,250,0.2); }
+.feat-blue { color:#38bdf8; background:rgba(56,189,248,0.08); border:1px solid rgba(56,189,248,0.2); }
+
+.hero-desc { font-size:13px; color:rgba(255,255,255,0.45); line-height:1.8; }
+.hero-desc b { color:rgba(255,255,255,0.75); font-weight:600; }
+.hero-desc a { color:#64ffda; text-decoration:none; font-weight:600; }
+.hero-desc a:hover { text-decoration:underline; }
+
+/* === Main Panel === */
+.main { flex:1; display:flex; flex-direction:column; padding:0 28px 20px; min-height:0; }
+.panel { flex:1; display:flex; flex-direction:column; background:rgba(16,16,24,0.95); border:1px solid rgba(255,255,255,0.05); border-radius:16px; box-shadow:0 10px 40px rgba(0,0,0,0.4); min-height:0; }
+
+.controls { padding:14px 20px; display:flex; align-items:center; gap:12px; border-bottom:1px solid rgba(255,255,255,0.04); flex-shrink:0; }
+.controls input[type=text] { background:rgba(255,255,255,0.05); border:1px solid rgba(255,255,255,0.1); border-radius:8px; padding:8px 12px; color:#ccc; font-size:12px; width:180px; outline:none; }
+.controls input[type=text]:focus { border-color:rgba(100,255,218,0.4); }
+.controls input[type=file] { display:none; }
+.controls label { font-size:11px; color:#888; display:flex; align-items:center; gap:5px; cursor:pointer; }
+.controls input[type=checkbox] { accent-color:#64ffda; }
+.btn { padding:8px 18px; border:none; border-radius:10px; font-size:12px; cursor:pointer; font-weight:600; transition:all 0.12s; }
+.btn:hover { transform:scale(1.04); }
+.btn:active { transform:scale(0.97); }
+.btn-mic { background:linear-gradient(135deg,#64ffda,#00bfa5); color:#080810; }
+.btn-file { background:linear-gradient(135deg,#fbbf24,#f59e0b); color:#1a1a2e; }
+.btn-hw { background:rgba(167,139,250,0.12); color:#a78bfa; border:1px solid rgba(167,139,250,0.3); }
+.btn-stop { background:linear-gradient(135deg,#ef4444,#dc2626); color:#fff; }
+.sta { margin-left:auto; display:flex; align-items:center; gap:6px; }
+.dot { width:7px; height:7px; border-radius:50%; background:#222; }
+.dot.on { background:#64ffda; box-shadow:0 0 10px rgba(100,255,218,0.6); animation:g 1.5s infinite; }
+@keyframes g { 0%,100%{opacity:1} 50%{opacity:0.3} }
+.sta span { font-size:10px; color:#555; }
+.hw-info { font-size:10px; color:#a78bfa; margin-left:2px; }
+
+.result { flex:1; overflow-y:auto; padding:18px 22px; min-height:0; }
+.result::-webkit-scrollbar { width:4px; }
+.result::-webkit-scrollbar-thumb { background:rgba(255,255,255,0.06); border-radius:2px; }
+.line { padding:6px 0; border-bottom:1px solid rgba(255,255,255,0.02); display:flex; align-items:baseline; gap:0; word-break:break-all; }
+.time { color:rgba(100,255,218,0.5); font-size:11px; font-family:'SF Mono','Menlo',monospace; width:100px; min-width:100px; flex-shrink:0; padding-top:2px; }
+.spk { font-size:10px; font-weight:600; padding:2px 8px; border-radius:5px; flex-shrink:0; margin-right:10px; }
+.text { color:rgba(255,255,255,0.88); font-size:15px; line-height:1.7; flex:1; }
+.partial .text { color:rgba(255,255,255,0.25); font-style:italic; }
+.ph { color:rgba(255,255,255,0.1); text-align:center; padding:50px 20px; font-size:14px; line-height:2; }
+</style>
</head>
-<body>
-<div class="container">
- <h1>Fun-ASR-Nano vLLM</h1>
- <p class="subtitle">Real-time Speech Recognition</p>
-
- <div class="config">
- <label>Server:</label>
- <input type="text" id="serverUrl" value="ws://localhost:10095" placeholder="ws://host:port">
- <label>Language:</label>
- <select id="language">
- <option value="中文" selected>中文</option>
- <option value="英文">English</option>
- <option value="日文">日本語</option>
- </select>
+<body><div style="width:920px;height:100%;display:flex;flex-direction:column">
+
+<div class="hero">
+ <div class="hero-top">
+ <div class="logo"><span class="g">Fun-ASR-Nano</span><span class="s">vLLM Engine</span></div>
+ <div class="hero-links">
+ <a href="https://github.com/modelscope/FunASR" target="_blank">GitHub</a>
+ <a href="https://modelscope.cn/models/FunAudioLLM/Fun-ASR-Nano-2512" target="_blank">ModelScope</a>
+ <a href="https://huggingface.co/FunAudioLLM/Fun-ASR-Nano-2512" target="_blank">HuggingFace</a>
+ </div>
</div>
-
- <div id="status" class="status">Click "Start" to begin</div>
-
- <div class="visualizer" id="visualizer"></div>
-
- <button class="btn btn-start" id="btnStart" onclick="startRecording()">Start Recording</button>
- <button class="btn btn-stop" id="btnStop" onclick="stopRecording()" style="display:none">Stop</button>
-
- <div class="result-box" id="resultBox">
- <span class="placeholder">Recognition results will appear here...</span>
+ <div class="features">
+ <span class="feat feat-green">Streaming ASR</span>
+ <span class="feat feat-yellow">Speaker Diarization <span style="font-size:9px;opacity:0.7">(Beta)</span></span>
+ <span class="feat feat-purple">Hotword Customization</span>
+ <span class="feat feat-blue">31 Languages · 7 Dialects</span>
</div>
+ <p class="hero-desc">
+ 基于 <b>FunASR</b> 的 <b>vLLM 推理引擎</b>,实现流式语音识别服务。支持实时 VAD 分句、说话人分离 <span style="color:#f59e0b;font-size:11px">(Beta)</span>、<b>热词定制化</b>(加载人名、地名等实体词列表,提升专有名词识别准确率)、31种语言及中文方言。所有代码与模型已全部开源。<br>
+ <span style="font-size:12px">Streaming ASR with vLLM engine, real-time VAD, speaker diarization <span style="color:#f59e0b">(Beta)</span>, <b>hotword customization</b> (names, places, entities), 31 languages &amp; Chinese dialects. Fully open-sourced.</span>
+ &nbsp;·&nbsp; <a href="https://www.funasr.com" target="_blank">www.funasr.com</a>
+ </p>
+</div>
- <p class="footer">Fun-ASR-Nano powered by vLLM | Audio: 16kHz PCM16 mono</p>
+<div class="main">
+<div class="panel">
+ <div class="controls">
+ <input type="text" id="serverUrl" value="ws://localhost:10095">
+ <label><input type="checkbox" id="showSpk" checked> Speaker <span style="font-size:9px;color:#f59e0b;background:rgba(245,158,11,0.08);padding:1px 5px;border-radius:4px">Beta</span></label>
+ <button class="btn btn-mic" id="btnMic" onclick="startMic()">Mic</button>
+ <input type="file" id="fileInput" accept="audio/*,.wav,.mp3,.flac,.mp4,.m4a">
+ <button class="btn btn-file" id="btnFile" onclick="document.getElementById('fileInput').click()">Audio File</button>
+ <input type="file" id="hotwordFile" accept=".txt" style="display:none">
+ <button class="btn btn-hw" id="btnHw" onclick="document.getElementById('hotwordFile').click()">Hotwords</button>
+ <span class="hw-info" id="hwInfo"></span>
+ <button class="btn btn-stop" id="btnStop" onclick="stopAll()" style="display:none">Stop</button>
+ <div class="sta"><div class="dot" id="dot"></div><span id="status">Ready</span></div>
+ </div>
+ <div class="result" id="resultBox">
+ <div class="ph">Click <b>Mic</b> for real-time recognition or <b>Audio File</b> to transcribe a file<br>Load a <b>Hotwords</b> file (.txt, one word per line) to boost recognition of names, places &amp; entities</div>
+ </div>
+</div>
</div>
+<input type="text" id="hotwords" style="display:none">
<script>
-let ws = null;
-let mediaStream = null;
-let audioContext = null;
-let processor = null;
-let isRecording = false;
-let finalText = '';
-
-// Create visualizer bars
-const visualizer = document.getElementById('visualizer');
-for (let i = 0; i < 40; i++) {
- const bar = document.createElement('div');
- bar.className = 'bar';
- bar.style.height = '4px';
- visualizer.appendChild(bar);
-}
-const bars = visualizer.querySelectorAll('.bar');
-
-function setStatus(msg, cls) {
- const el = document.getElementById('status');
- el.textContent = msg;
- el.className = 'status ' + (cls || '');
-}
-
-function renderSentences(sentences, partial, partialStartMs, durationMs, isFinal) {
- const box = document.getElementById('resultBox');
- let html = '';
- sentences.forEach(s => {
- let startS = (s.start_ms / 1000).toFixed(1);
- let endS = (s.end_ms / 1000).toFixed(1);
- html += '<div style="margin:8px 0;display:flex;align-items:baseline">' +
- '<span style="color:#64ffda;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ' + endS + 's</span>' +
- '<span style="color:#e0e0e0;font-size:16px;margin-left:10px">' + s.text + '</span></div>';
- });
- if (partial) {
- let startS = (partialStartMs / 1000).toFixed(1);
- html += '<div style="margin:8px 0;display:flex;align-items:baseline">' +
- '<span style="color:#ffb74d;font-size:12px;min-width:100px;font-family:monospace">' + startS + 's - ...</span>' +
- '<span style="color:#aaa;font-size:16px;margin-left:10px;font-style:italic">' + partial + '</span></div>';
- }
- if (!html) { html = '<span class="placeholder">Listening...</span>'; }
- box.innerHTML = html;
- box.scrollTop = box.scrollHeight;
-}
-
-async function startRecording() {
- const serverUrl = document.getElementById('serverUrl').value;
- finalText = '';
- document.getElementById('resultBox').innerHTML = '<span class="placeholder">Connecting...</span>';
-
- // Connect WebSocket
- try {
- ws = new WebSocket(serverUrl);
- } catch(e) {
- setStatus('Connection failed: ' + e.message, 'error');
- return;
- }
-
- ws.onopen = async () => {
- setStatus('Connected. Recording...', 'recording');
- ws.send('START');
-
- // Get microphone
- try {
- mediaStream = await navigator.mediaDevices.getUserMedia({
- audio: { sampleRate: 16000, channelCount: 1, echoCancellation: true }
- });
- } catch(e) {
- setStatus('Microphone access denied', 'error');
- ws.close();
- return;
- }
-
- audioContext = new AudioContext({ sampleRate: 16000 });
- const source = audioContext.createMediaStreamSource(mediaStream);
-
- // AudioWorklet or ScriptProcessor
- processor = audioContext.createScriptProcessor(4096, 1, 1);
- processor.onaudioprocess = (e) => {
- if (!isRecording) return;
- const float32 = e.inputBuffer.getChannelData(0);
-
- // Update visualizer
- const rms = Math.sqrt(float32.reduce((sum, x) => sum + x*x, 0) / float32.length);
- bars.forEach((bar, i) => {
- const h = Math.max(4, Math.min(50, rms * 500 * (0.5 + Math.random())));
- bar.style.height = h + 'px';
- });
-
- // Convert to PCM16 and send
- const int16 = new Int16Array(float32.length);
- for (let i = 0; i < float32.length; i++) {
- int16[i] = Math.max(-32768, Math.min(32767, Math.round(float32[i] * 32768)));
- }
- if (ws && ws.readyState === WebSocket.OPEN) {
- ws.send(int16.buffer);
- }
- };
- source.connect(processor);
- processor.connect(audioContext.destination);
-
- isRecording = true;
- document.getElementById('btnStart').style.display = 'none';
- document.getElementById('btnStop').style.display = 'block';
- document.getElementById('resultBox').innerHTML = '<span class="placeholder">Listening...</span>';
- };
-
- ws.onmessage = (event) => {
- const data = JSON.parse(event.data);
- if (data.sentences !== undefined) {
- renderSentences(data.sentences, data.partial, data.partial_start_ms, data.duration_ms, data.is_final);
- }
- if (data.event === 'stopped') {
- setStatus('Stopped', 'connected');
- }
- };
-
- ws.onerror = (e) => {
- setStatus('WebSocket error', 'error');
- };
-
- ws.onclose = () => {
- if (isRecording) {
- stopRecording();
- }
- };
-}
-
-function stopRecording() {
- isRecording = false;
-
- if (ws && ws.readyState === WebSocket.OPEN) {
- ws.send('STOP');
- }
-
- if (processor) { processor.disconnect(); processor = null; }
- if (audioContext) { audioContext.close(); audioContext = null; }
- if (mediaStream) { mediaStream.getTracks().forEach(t => t.stop()); mediaStream = null; }
-
- // Reset visualizer
- bars.forEach(bar => bar.style.height = '4px');
-
- setTimeout(() => {
- if (ws) { ws.close(); ws = null; }
- }, 1000);
-
- document.getElementById('btnStart').style.display = 'block';
- document.getElementById('btnStop').style.display = 'none';
- setStatus('Stopped. Click Start to record again.', '');
-}
+let ws=null,mediaStream=null,audioContext=null,processor=null,isRecording=false;
+const C=['#64ffda','#f472b6','#fbbf24','#34d399','#a78bfa','#fb923c','#67e8f9','#f87171','#38bdf8','#c084fc'];
+
+document.getElementById('hotwordFile').addEventListener('change',function(e){
+ const f=e.target.files[0];if(!f)return;
+ const r=new FileReader();r.onload=ev=>{
+ const words=ev.target.result.split('\n').map(l=>l.trim()).filter(l=>l);
+ document.getElementById('hotwords').value=words.join(',');
+ document.getElementById('hwInfo').textContent=words.length+' words loaded';
+ setS('Hotwords: '+words.length+' loaded');
+ };r.readAsText(f);
+});
+document.getElementById('fileInput').addEventListener('change',function(){if(this.files.length)startFile();});
+
+function setS(m,on){document.getElementById('status').textContent=m;document.getElementById('dot').className=on?'dot on':'dot';}
+function sStop(){document.getElementById('btnMic').style.display='none';document.getElementById('btnFile').style.display='none';document.getElementById('btnHw').style.display='none';document.getElementById('btnStop').style.display='inline';}
+function sStart(){document.getElementById('btnMic').style.display='inline';document.getElementById('btnFile').style.display='inline';document.getElementById('btnHw').style.display='inline';document.getElementById('btnStop').style.display='none';}
+function render(ss,p,ps,d,f){
+ const b=document.getElementById('resultBox'),sk=document.getElementById('showSpk').checked;let h='';
+ ss.forEach(s=>{let st=s.start!==undefined?s.start:(s.start_ms||0),en=s.end!==undefined?s.end:(s.end_ms||0),sp=s.spk!==undefined?s.spk:-1,sh='';
+ if(sk&&sp>=0){let c=C[sp%C.length];sh='<span class="spk" style="color:'+c+';background:'+c+'12;border:1px solid '+c+'30">SPK'+sp+'</span>';}
+ h+='<div class="line"><span class="time">'+(st/1000).toFixed(1)+' - '+(en/1000).toFixed(1)+'s</span>'+sh+'<span class="text">'+s.text+'</span></div>';});
+ if(p)h+='<div class="line partial"><span class="time">'+(ps/1000).toFixed(1)+'s ...</span><span class="text">'+p+'</span></div>';
+ if(!h)h='<div class="ph">Listening...</div>';b.innerHTML=h;b.scrollTop=b.scrollHeight;}
+function con(cb){ws=new WebSocket(document.getElementById('serverUrl').value);ws.onopen=()=>{ws.send('START');var hw=document.getElementById('hotwords').value.trim();if(hw)ws.send('HOTWORDS:'+hw);cb();};ws.onmessage=e=>{const d=JSON.parse(e.data);if(d.sentences!==undefined)render(d.sentences,d.partial,d.partial_start_ms,d.duration_ms,d.is_final);};ws.onerror=()=>setS('Error');ws.onclose=()=>{if(isRecording)stopAll();};}
+function startMic(){con(async()=>{setS('Recording',true);sStop();document.getElementById('resultBox').innerHTML='<div class="ph">Listening...</div>';try{mediaStream=await navigator.mediaDevices.getUserMedia({audio:{sampleRate:16000,channelCount:1,echoCancellation:true}});}catch(e){setS('Mic denied');ws.close();sStart();return;}audioContext=new AudioContext({sampleRate:16000});const s=audioContext.createMediaStreamSource(mediaStream);processor=audioContext.createScriptProcessor(4096,1,1);processor.onaudioprocess=e=>{if(!isRecording)return;const f=e.inputBuffer.getChannelData(0),i=new Int16Array(f.length);for(let j=0;j<f.length;j++)i[j]=Math.max(-32768,Math.min(32767,Math.round(f[j]*32768)));if(ws&&ws.readyState===1)ws.send(i.buffer);};s.connect(processor);processor.connect(audioContext.destination);isRecording=true;});}
+function startFile(){const fi=document.getElementById('fileInput');if(!fi.files.length){setS('Pick a file');return;}const file=fi.files[0];setS('Decoding...');document.getElementById('resultBox').innerHTML='<div class="ph">Decoding '+file.name+'...</div>';const r=new FileReader();r.onload=async e=>{const a=new AudioContext({sampleRate:16000});let buf;try{buf=await a.decodeAudioData(e.target.result);}catch(err){setS('Decode failed');return;}const p=buf.getChannelData(0),i=new Int16Array(p.length);for(let j=0;j<p.length;j++)i[j]=Math.max(-32768,Math.min(32767,Math.round(p[j]*32768)));a.close();con(async()=>{setS('Streaming '+(p.length/16000|0)+'s',true);sStop();isRecording=true;for(let j=0;j<i.length&&isRecording;j+=4096){if(ws&&ws.readyState===1)ws.send(i.slice(j,j+4096).buffer);await new Promise(r=>setTimeout(r,50));}if(isRecording)stopAll();});};r.readAsArrayBuffer(file);}
+function stopAll(){isRecording=false;if(ws&&ws.readyState===1)ws.send('STOP');if(processor){processor.disconnect();processor=null;}if(audioContext){audioContext.close();audioContext=null;}if(mediaStream){mediaStream.getTracks().forEach(t=>t.stop());mediaStream=null;}setTimeout(()=>{if(ws){ws.close();ws=null;}},2000);sStart();setS('Done');}
</script>
-</body>
+</div></body>
</html>
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_python.py b/examples/industrial_data_pretraining/fun_asr_nano/client_python.py
new file mode 100644
index 000000000..570fd86cb
--- /dev/null
+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_python.py
@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""Fun-ASR-Nano Python WebSocket Client.
+
+Supports real-time microphone recording and audio file streaming.
+
+Usage:
+ # Microphone mode
+ python client_python.py --server ws://localhost:10095 --mic
+
+ # File mode
+ python client_python.py --server ws://localhost:10095 --file audio.wav
+
+ # With hotwords
+ python client_python.py --server ws://localhost:10095 --file audio.wav --hotwords "张三,李四,北京"
+
+ # Disable speaker diarization display
+ python client_python.py --server ws://localhost:10095 --mic --no-spk
+"""
+
+import asyncio
+import argparse
+import json
+import sys
+import numpy as np
+
+try:
+ import websockets
+except ImportError:
+ print("Please install websockets: pip install websockets")
+ sys.exit(1)
+
+
+SAMPLE_RATE = 16000
+CHUNK_DURATION_MS = 100
+CHUNK_SAMPLES = int(SAMPLE_RATE * CHUNK_DURATION_MS / 1000)
+
+SPK_COLORS = [
+ "\033[36m", "\033[35m", "\033[33m", "\033[32m",
+ "\033[34m", "\033[91m", "\033[96m", "\033[95m",
+]
+RESET = "\033[0m"
+GRAY = "\033[90m"
+GREEN = "\033[92m"
+
+
+def format_time(ms):
+ s = ms / 1000
+ return f"{s:.1f}s"
+
+
+def print_result(data, show_spk=True):
+ """Print ASR result to terminal."""
+ sentences = data.get("sentences", [])
+ partial = data.get("partial", "")
+ partial_start = data.get("partial_start_ms", 0)
+ is_final = data.get("is_final", False)
+
+ sys.stdout.write("\033[2J\033[H")
+
+ print(f"{GREEN}Fun-ASR-Nano Streaming ASR{RESET}")
+ print(f"{GRAY}{'─' * 60}{RESET}")
+
+ for s in sentences:
+ start = s.get("start", s.get("start_ms", 0))
+ end = s.get("end", s.get("end_ms", 0))
+ spk = s.get("spk", -1)
+ text = s["text"]
+
+ time_str = f"{GRAY}[{format_time(start)}-{format_time(end)}]{RESET}"
+ spk_str = ""
+ if show_spk and spk >= 0:
+ color = SPK_COLORS[spk % len(SPK_COLORS)]
+ spk_str = f" {color}SPK{spk}{RESET}"
+
+ print(f" {time_str}{spk_str} {text}")
+
+ if partial:
+ print(f" {GRAY}[{format_time(partial_start)}-...] {partial}{RESET}")
+
+ if is_final:
+ print(f"\n{GRAY}{'─' * 60}{RESET}")
+ print(f"{GREEN}Done.{RESET} {len(sentences)} sentences")
+ else:
+ print(f"\n{GRAY}Recording... Press Ctrl+C to stop{RESET}")
+
+ sys.stdout.flush()
+
+
+async def run_mic(args):
+ """Stream from microphone."""
+ try:
+ import sounddevice as sd
+ except ImportError:
+ print("Please install sounddevice: pip install sounddevice")
+ sys.exit(1)
+
+ print(f"Connecting to {args.server}...")
+ async with websockets.connect(args.server, ping_interval=None) as ws:
+ await ws.send("START")
+ resp = await ws.recv()
+ event = json.loads(resp)
+ if event.get("event") != "started":
+ print(f"Unexpected response: {resp}")
+ return
+
+ if args.hotwords:
+ await ws.send(f"HOTWORDS:{args.hotwords}")
+ await ws.recv()
+
+ print("Recording... Press Ctrl+C to stop\n")
+
+ audio_queue = asyncio.Queue()
+
+ def audio_callback(indata, frames, time_info, status):
+ audio_queue.put_nowait(indata.copy())
+
+ stream = sd.InputStream(
+ samplerate=SAMPLE_RATE, channels=1, dtype='int16',
+ blocksize=CHUNK_SAMPLES, callback=audio_callback,
+ )
+
+ async def send_audio():
+ with stream:
+ while True:
+ chunk = await audio_queue.get()
+ await ws.send(chunk.tobytes())
+
+ async def recv_results():
+ async for msg in ws:
+ data = json.loads(msg)
+ if "sentences" in data:
+ print_result(data, show_spk=args.spk)
+ if data.get("is_final") or data.get("event") == "stopped":
+ break
+
+ send_task = asyncio.create_task(send_audio())
+ recv_task = asyncio.create_task(recv_results())
+
+ try:
+ await asyncio.gather(send_task, recv_task)
+ except (KeyboardInterrupt, asyncio.CancelledError):
+ pass
+ finally:
+ send_task.cancel()
+ if ws.open:
+ await ws.send("STOP")
+ async for msg in ws:
+ data = json.loads(msg)
+ if "sentences" in data:
+ print_result(data, show_spk=args.spk)
+ if data.get("is_final") or data.get("event") == "stopped":
+ break
+
+
+async def run_file(args):
+ """Stream an audio file."""
+ try:
+ import soundfile as sf
+ except ImportError:
+ print("Please install soundfile: pip install soundfile")
+ sys.exit(1)
+
+ audio, sr = sf.read(args.file)
+ if sr != SAMPLE_RATE:
+ try:
+ import librosa
+ audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
+ except ImportError:
+ print(f"Audio is {sr}Hz, need 16kHz. Install librosa: pip install librosa")
+ sys.exit(1)
+ if audio.ndim > 1:
+ audio = audio[:, 0]
+ audio = audio.astype(np.float32)
+
+ duration = len(audio) / SAMPLE_RATE
+ print(f"File: {args.file} ({duration:.1f}s)")
+ print(f"Connecting to {args.server}...")
+
+ async with websockets.connect(args.server, ping_interval=None) as ws:
+ await ws.send("START")
+ await ws.recv()
+
+ if args.hotwords:
+ await ws.send(f"HOTWORDS:{args.hotwords}")
+ await ws.recv()
+
+ int16 = (audio * 32768).clip(-32768, 32767).astype(np.int16)
+
+ chunk_size = CHUNK_SAMPLES
+ total_chunks = (len(int16) + chunk_size - 1) // chunk_size
+
+ async def send_audio():
+ for i in range(0, len(int16), chunk_size):
+ chunk = int16[i:i+chunk_size]
+ await ws.send(chunk.tobytes())
+ await asyncio.sleep(CHUNK_DURATION_MS / 1000 * 0.5)
+ await ws.send("STOP")
+
+ async def recv_results():
+ async for msg in ws:
+ data = json.loads(msg)
+ if "sentences" in data:
+ print_result(data, show_spk=args.spk)
+ if data.get("is_final") or data.get("event") == "stopped":
+ break
+
+ await asyncio.gather(send_audio(), recv_results())
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano Python Client")
+ parser.add_argument("--server", type=str, default="ws://localhost:10095")
+ parser.add_argument("--mic", action="store_true", help="Use microphone input")
+ parser.add_argument("--file", type=str, help="Audio file to transcribe")
+ parser.add_argument("--hotwords", type=str, default="", help="Hotwords (comma-separated)")
+ parser.add_argument("--spk", action="store_true", default=True, help="Show speaker IDs")
+ parser.add_argument("--no-spk", dest="spk", action="store_false")
+ args = parser.parse_args()
+
+ if not args.mic and not args.file:
+ parser.error("Specify --mic or --file")
+
+ try:
+ if args.mic:
+ asyncio.run(run_mic(args))
+ else:
+ asyncio.run(run_file(args))
+ except KeyboardInterrupt:
+ print(f"\n{RESET}Interrupted.")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/client_test.py b/examples/industrial_data_pretraining/fun_asr_nano/client_test.py
new file mode 100644
index 000000000..9efc9f260
--- /dev/null
+++ b/examples/industrial_data_pretraining/fun_asr_nano/client_test.py
@@ -0,0 +1,244 @@
+#!/usr/bin/env python3
+"""Fun-ASR-Nano Streaming ASR - Automated Test Script.
+
+Tests the WebSocket server with audio files and validates the response format.
+
+Usage:
+ python client_test.py --server ws://localhost:10095 --file test_audio.wav
+ python client_test.py --server ws://localhost:10095 --file test_audio.wav --hotwords "热词1,热词2"
+"""
+
+import asyncio
+import argparse
+import json
+import time
+import sys
+import numpy as np
+
+try:
+ import websockets
+except ImportError:
+ print("ERROR: pip install websockets")
+ sys.exit(1)
+
+try:
+ import soundfile as sf
+except ImportError:
+ print("ERROR: pip install soundfile")
+ sys.exit(1)
+
+
+SAMPLE_RATE = 16000
+
+
+def load_audio(file_path):
+ audio, sr = sf.read(file_path)
+ if sr != SAMPLE_RATE:
+ try:
+ import librosa
+ audio = librosa.resample(audio, orig_sr=sr, target_sr=SAMPLE_RATE)
+ except ImportError:
+ print(f"ERROR: Audio is {sr}Hz. Install librosa for resampling.")
+ sys.exit(1)
+ if audio.ndim > 1:
+ audio = audio[:, 0]
+ return audio.astype(np.float32)
+
+
+async def test_basic(server, audio, hotwords=None):
+ """Test basic streaming ASR flow."""
+ print("=" * 60)
+ print("TEST: Basic Streaming ASR")
+ print("=" * 60)
+
+ t_start = time.perf_counter()
+
+ async with websockets.connect(server, ping_interval=None) as ws:
+ await ws.send("START")
+ resp = json.loads(await ws.recv())
+ assert resp["event"] == "started", f"Expected 'started', got: {resp}"
+ print(" [PASS] START -> event:started")
+
+ if hotwords:
+ await ws.send(f"HOTWORDS:{hotwords}")
+ resp = json.loads(await ws.recv())
+ assert resp["event"] == "hotwords_set", f"Expected 'hotwords_set', got: {resp}"
+ print(f" [PASS] HOTWORDS -> {len(resp['hotwords'])} words set")
+
+ int16 = (audio * 32768).clip(-32768, 32767).astype(np.int16)
+ chunk_size = 4096
+ partial_count = 0
+
+ for i in range(0, len(int16), chunk_size):
+ chunk = int16[i:i+chunk_size]
+ await ws.send(chunk.tobytes())
+ await asyncio.sleep(0.02)
+
+ try:
+ msg = await asyncio.wait_for(ws.recv(), timeout=0.01)
+ data = json.loads(msg)
+ if "sentences" in data:
+ partial_count += 1
+ assert "partial" in data
+ assert "duration_ms" in data
+ assert "is_final" in data
+ assert isinstance(data["sentences"], list)
+ except asyncio.TimeoutError:
+ pass
+
+ await ws.send("STOP")
+
+ final_result = None
+ while True:
+ msg = await asyncio.wait_for(ws.recv(), timeout=30)
+ data = json.loads(msg)
+ if data.get("is_final"):
+ final_result = data
+ if data.get("event") == "stopped":
+ break
+
+ t_end = time.perf_counter()
+
+ assert final_result is not None, "No final result received"
+ assert final_result["is_final"] is True
+ sentences = final_result["sentences"]
+ assert len(sentences) > 0, "No sentences in final result"
+
+ print(f" [PASS] Received final result: {len(sentences)} sentences")
+ print(f" [PASS] Partial updates received: {partial_count}")
+
+ for s in sentences:
+ assert "text" in s, f"Missing 'text' in sentence: {s}"
+ assert "start" in s, f"Missing 'start' in sentence: {s}"
+ assert "end" in s, f"Missing 'end' in sentence: {s}"
+ assert isinstance(s["text"], str) and len(s["text"]) > 0
+ assert s["end"] > s["start"]
+
+ print(" [PASS] All sentences have valid format (text, start, end)")
+
+ has_spk = all("spk" in s for s in sentences)
+ if has_spk:
+ spk_ids = set(s["spk"] for s in sentences)
+ print(f" [PASS] Speaker diarization: {len(spk_ids)} speakers detected")
+ else:
+ print(" [INFO] No speaker IDs in result")
+
+ elapsed = t_end - t_start
+ audio_duration = len(audio) / SAMPLE_RATE
+ rtf = elapsed / audio_duration
+ print(f"\n Audio: {audio_duration:.1f}s | Time: {elapsed:.2f}s | RTF: {rtf:.3f}")
+
+ print("\n --- Results ---")
+ for s in sentences:
+ spk = f" SPK{s['spk']}" if "spk" in s else ""
+ print(f" [{s['start']/1000:.1f}-{s['end']/1000:.1f}s]{spk}: {s['text']}")
+
+ return True
+
+
+async def test_empty_audio(server):
+ """Test with very short/empty audio."""
+ print("\n" + "=" * 60)
+ print("TEST: Empty/Short Audio Handling")
+ print("=" * 60)
+
+ async with websockets.connect(server, ping_interval=None) as ws:
+ await ws.send("START")
+ await ws.recv()
+
+ short_audio = np.zeros(800, dtype=np.int16)
+ await ws.send(short_audio.tobytes())
+ await asyncio.sleep(0.1)
+
+ await ws.send("STOP")
+ final = None
+ while True:
+ msg = await asyncio.wait_for(ws.recv(), timeout=10)
+ data = json.loads(msg)
+ if data.get("is_final"):
+ final = data
+ if data.get("event") == "stopped":
+ break
+
+ assert final is not None
+ assert final["sentences"] == [] or all(s["text"].strip() for s in final["sentences"])
+ print(" [PASS] Short audio handled gracefully")
+
+ return True
+
+
+async def test_multiple_sessions(server, audio):
+ """Test multiple consecutive sessions on same connection."""
+ print("\n" + "=" * 60)
+ print("TEST: Multiple Sessions")
+ print("=" * 60)
+
+ audio_short = audio[:SAMPLE_RATE * 5]
+ int16 = (audio_short * 32768).clip(-32768, 32767).astype(np.int16)
+
+ async with websockets.connect(server, ping_interval=None) as ws:
+ for session_num in range(2):
+ await ws.send("START")
+ resp = json.loads(await ws.recv())
+ assert resp["event"] == "started"
+
+ for i in range(0, len(int16), 4096):
+ await ws.send(int16[i:i+4096].tobytes())
+ await asyncio.sleep(0.01)
+
+ await ws.send("STOP")
+ got_final = False
+ while True:
+ msg = await asyncio.wait_for(ws.recv(), timeout=15)
+ data = json.loads(msg)
+ if data.get("is_final"):
+ got_final = True
+ if data.get("event") == "stopped":
+ break
+ assert got_final, f"Session {session_num+1}: no final result"
+ print(f" [PASS] Session {session_num+1} completed")
+
+ return True
+
+
+async def run_tests(args):
+ audio = load_audio(args.file)
+ print(f"Loaded: {args.file} ({len(audio)/SAMPLE_RATE:.1f}s)")
+ print(f"Server: {args.server}\n")
+
+ passed = 0
+ failed = 0
+
+ tests = [
+ ("Basic Streaming", lambda: test_basic(args.server, audio, args.hotwords)),
+ ("Empty Audio", lambda: test_empty_audio(args.server)),
+ ("Multiple Sessions", lambda: test_multiple_sessions(args.server, audio)),
+ ]
+
+ for name, test_fn in tests:
+ try:
+ await test_fn()
+ passed += 1
+ except Exception as e:
+ print(f" [FAIL] {name}: {e}")
+ failed += 1
+
+ print("\n" + "=" * 60)
+ print(f"RESULTS: {passed} passed, {failed} failed")
+ print("=" * 60)
+ return failed == 0
+
+
+def main():
+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming Test")
+ parser.add_argument("--server", type=str, default="ws://localhost:10095")
+ parser.add_argument("--file", type=str, required=True, help="Test audio file")
+ parser.add_argument("--hotwords", type=str, default="", help="Hotwords (comma-separated)")
+ args = parser.parse_args()
+
+ success = asyncio.run(run_tests(args))
+ sys.exit(0 if success else 1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md b/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md
index 85f536830..cb3f7bcbf 100644
--- a/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md
+++ b/examples/industrial_data_pretraining/fun_asr_nano/docs/realtime_demo.md
@@ -1,114 +1,254 @@
-# Fun-ASR-Nano 实时语音识别 Demo
+# Fun-ASR-Nano 流式语音识别服务(Streaming ASR)
-基于 WebSocket 的实时 ASR 演示,支持麦克风录音、流式 VAD 分句、逐句显示。
+基于 WebSocket 的实时流式语音识别服务,支持 VAD 分句、说话人分离(Beta)、热词定制化。
+
+## 功能特性
+
+| 功能 | 说明 |
+|------|------|
+| 流式 ASR | 基于 Fun-ASR-Nano + vLLM 推理引擎,实时输出识别结果 |
+| 流式 VAD | fsmn-vad 增量检测语音端点,动态调整静音阈值 |
+| 说话人分离 (Beta) | eres2netv2 + ClusterBackend,流式分配+最终重聚类 |
+| 热词定制化 | 加载人名、地名等实体词列表,提升专有名词识别准确率 |
+| 幻觉检测 | 自动检测重复模式并截断,防止模型生成循环 |
+| 多语言 | 31种语言、7大方言、26种地方口音 |
## 架构
```
-浏览器(麦克风) 服务器(GPU)
-┌──────────────┐ WebSocket PCM16 ┌────────────────────────┐
-│ client_mic │ ──────────────────→ │ serve_realtime_ws.py │
-│ .html │ │ │
-│ │ ←────────────────── │ ┌─────────────────┐ │
-│ 逐句显示结果 │ JSON sentences │ │ Streaming VAD │ │
-└──────────────┘ │ │ (fsmn-vad) │ │
- │ └────────┬────────┘ │
- │ ↓ │
- │ ┌─────────────────┐ │
- │ │ ASR per-segment │ │
- │ │ (Fun-ASR-Nano) │ │
- │ └─────────────────┘ │
- └────────────────────────┘
+客户端(浏览器/Python) 服务端(GPU)
+┌─────────────────┐ WebSocket ┌──────────────────────────┐
+│ │ PCM16 16kHz │ serve_realtime_ws.py │
+│ client_mic.html │ ────────────→ │ │
+│ client_python.py│ │ ┌──────────┐ │
+│ │ ←──────────── │ │ VAD │ 流式分句 │
+│ 实时显示结果 │ JSON │ └────┬─────┘ │
+└─────────────────┘ │ ↓ │
+ │ ┌──────────┐ │
+ │ │ ASR │ 逐段解码 │
+ │ └────┬─────┘ │
+ │ ↓ │
+ │ ┌──────────┐ │
+ │ │ SPK │ 说话人分离 │
+ │ └──────────┘ │
+ └──────────────────────────┘
```
-## 核心设计
+## 快速部署
-1. **流式 VAD 分句**:fsmn-vad 增量检测语音端点
- - `[start, -1]` = 检测到语音开始
- - `[-1, end]` = 检测到语音结束 → 确认分句
-2. **逐段解码**:只对当前 VAD 段做 ASR(不累积全部音频)
- - 已确认段锁定文字,永不重新解码
- - 支持无限时长录音,不会因音频过长而崩溃
-3. **prev_text 连续性**:当前段内使用 prev_text 保证文字稳定
+### 环境要求
-## 使用方法
+- Python >= 3.8
+- CUDA >= 11.8
+- GPU 显存 >= 4GB
-### 1. 启动服务
+### 安装依赖
```bash
cd examples/industrial_data_pretraining/fun_asr_nano
+pip install -r requirements.txt
+```
-# 单卡启动
+### 启动服务
+
+```bash
+# 基本启动
CUDA_VISIBLE_DEVICES=0 python serve_realtime_ws.py --port 10095
-# 指定参数
+# 完整参数
CUDA_VISIBLE_DEVICES=0 python serve_realtime_ws.py \
--port 10095 \
--model FunAudioLLM/Fun-ASR-Nano-2512 \
+ --hub ms \
--device cuda:0 \
- --decode-interval 0.7
+ --decode-interval 0.48 \
+ --hotword-file 热词列表
```
-### 2. 打开客户端
+首次启动会自动下载模型(约 30 秒),之后即可接受连接。
+
+### 服务端参数
-直接用浏览器打开 `client_mic.html`,修改 Server 地址为:
-- 本机:`ws://localhost:10095`
-- 远程(需 SSH 端口转发):先 `ssh -L 10095:localhost:10095 <server>`
+| 参数 | 默认值 | 说明 |
+|------|--------|------|
+| `--port` | 10095 | WebSocket 监听端口 |
+| `--model` | `FunAudioLLM/Fun-ASR-Nano-2512` | ASR 模型名称 |
+| `--hub` | `ms` | 模型来源(ms=ModelScope, hf=HuggingFace) |
+| `--device` | `cuda:0` | GPU 设备 |
+| `--decode-interval` | 0.48 | 实时解码间隔(秒) |
+| `--hotword-file` | `热词列表` | 热词文件路径(一行一个词) |
+| `--use-context` | True | 使用上下文辅助解码 |
+| `--no-context` | - | 禁用上下文 |
-### 3. 体验
+## 客户端使用
-1. 点击 **Start Recording** → 允许麦克风
-2. 对着麦克风说话
-3. 说话过程中,VAD 检测到停顿自动分句显示
-4. 点击 **Stop** 结束
+### 1. 浏览器客户端(client_mic.html)
-## 显示效果
+直接在浏览器中打开 `client_mic.html`:
+```bash
+# 本机访问
+open client_mic.html # macOS
+xdg-open client_mic.html # Linux
+
+# 远程服务器(需 SSH 端口转发)
+ssh -L 10095:localhost:10095 <server>
+# 然后在本地浏览器打开 client_mic.html
```
-0.3s - 2.1s 今天天气不错。
-2.5s - 5.0s 我想去公园散步。
-5.3s - ... 现在正在说的话...(斜体,未确认)
+
+功能:
+- 麦克风实时录音
+- 音频文件上传识别
+- 热词列表文件加载(.txt,一行一个词)
+- 说话人分离显示(可开关)
+
+### 2. Python 客户端(client_python.py)
+
+```bash
+# 麦克风模式
+python client_python.py --server ws://localhost:10095 --mic
+
+# 文件模式
+python client_python.py --server ws://localhost:10095 --file audio.wav
+
+# 带热词
+python client_python.py --server ws://localhost:10095 --file audio.wav \
+ --hotwords "张三,李四,北京"
+
+# 不显示说话人
+python client_python.py --server ws://localhost:10095 --mic --no-spk
```
-- 白色 = 已确认句子(VAD 检测到结束,文字锁定)
-- 灰色斜体 = 正在说的句子(实时更新)
-- 绿色时间戳 = VAD 检测的起止时间
+### 3. 测试脚本(client_test.py)
-## 服务端参数
+```bash
+# 运行完整测试
+python client_test.py --server ws://localhost:10095 --file test_audio.wav
-| 参数 | 默认值 | 说明 |
-|------|--------|------|
-| `--port` | 10095 | WebSocket 端口 |
-| `--model` | `FunAudioLLM/Fun-ASR-Nano-2512` | ASR 模型 |
-| `--device` | `cuda:0` | GPU 设备 |
-| `--decode-interval` | 0.7 | 解码间隔(秒),越小延迟越低但 GPU 占用越高 |
+# 带热词测试
+python client_test.py --server ws://localhost:10095 --file test_audio.wav \
+ --hotwords "人名,地名"
+```
-## 协议
+测试内容:
+- 基本流式 ASR 流程验证
+- 返回格式校验(sentences/partial/is_final)
+- 空音频处理
+- 多轮 session 复用
+- 性能统计(RTF)
+## API 协议文档
+
+### 连接
+
+```
+WebSocket URL: ws://<host>:<port>
+音频格式: PCM16, 16kHz, 单声道 (Little-Endian)
+```
+
+### 客户端 → 服务端
+
+| 消息类型 | 格式 | 说明 |
+|---------|------|------|
+| 开始会话 | `"START"` (text) | 初始化/重置 session |
+| 设置热词 | `"HOTWORDS:词1,词2,词3"` (text) | 可选,在 START 后发送 |
+| 音频数据 | `bytes` (binary) | PCM16 音频帧,任意长度 |
+| 结束会话 | `"STOP"` (text) | 触发最终解码,返回完整结果 |
+
+### 服务端 → 客户端
+
+#### 1. 事件消息
+
+```json
+{"event": "started"}
+{"event": "hotwords_set", "hotwords": ["词1", "词2"]}
+{"event": "stopped"}
```
-Client → Server:
- "START" 开始录音
- "STOP" 结束录音
- bytes PCM16 音频数据(16kHz 单声道)
-Server → Client:
- {"event": "started"}
- {"event": "stopped"}
- {
- "sentences": [
- {"text": "已确认的句子", "start_ms": 300, "end_ms": 2100},
- ...
- ],
- "partial": "正在说的...",
- "partial_start_ms": 5300,
- "duration_ms": 7200,
- "is_final": false
- }
+#### 2. 实时结果(流式,在 START 和 STOP 之间持续推送)
+
+```json
+{
+ "sentences": [
+ {"text": "已确认句子", "start": 1700, "end": 5500, "spk": 0}
+ ],
+ "partial": "正在说的文字...",
+ "partial_start_ms": 5800,
+ "duration_ms": 7200,
+ "is_final": false
+}
```
+#### 3. 最终结果(STOP 后返回)
+
+```json
+{
+ "sentences": [
+ {"text": "第一句话。", "start": 1700, "end": 5500, "spk": 0},
+ {"text": "第二句话。", "start": 6000, "end": 9200, "spk": 1}
+ ],
+ "partial": "",
+ "partial_start_ms": 0,
+ "duration_ms": 10000,
+ "is_final": true
+}
+```
+
+### 字段说明
+
+| 字段 | 类型 | 说明 |
+|------|------|------|
+| `sentences` | array | 已确认的句子列表 |
+| `sentences[].text` | string | 识别文本 |
+| `sentences[].start` | int | 起始时间(毫秒) |
+| `sentences[].end` | int | 结束时间(毫秒) |
+| `sentences[].spk` | int | 说话人 ID(0-based),仅最终结果保证准确 |
+| `partial` | string | 当前正在说的文本(未确认,会被更新) |
+| `partial_start_ms` | int | partial 的起始时间(毫秒) |
+| `duration_ms` | int | 已接收音频总时长(毫秒) |
+| `is_final` | bool | 是否为最终结果 |
+
+### 典型交互流程
+
+```
+Client Server
+ │ │
+ │──── "START" ─────────────────→│
+ │←──── {"event":"started"} ─────│
+ │ │
+ │──── "HOTWORDS:张三,北京" ────→│ (可选)
+ │←── {"event":"hotwords_set"} ──│
+ │ │
+ │──── [audio bytes] ───────────→│
+ │──── [audio bytes] ───────────→│
+ │←── {sentences,partial} ───────│ (实时推送)
+ │──── [audio bytes] ───────────→│
+ │←── {sentences,partial} ───────│
+ │ ... │
+ │──── "STOP" ──────────────────→│
+ │←── {sentences,is_final:true} ─│ (最终结果)
+ │←── {"event":"stopped"} ───────│
+ │ │
+```
+
+## 热词文件格式
+
+文件名默认为 `热词列表`(可通过 `--hotword-file` 指定),格式为纯文本,一行一个词:
+
+```
+张三
+李四
+北京大学
+Fun-ASR-Nano
+```
+
+也可在运行时通过 WebSocket 动态设置:`HOTWORDS:词1,词2,词3`
+
## 注意事项
-- 浏览器需要 HTTPS 或 localhost 才能使用麦克风(Chrome 安全限制)
-- 首次使用需允许麦克风权限
-- 模型加载约需 30 秒,之后即可接受连接
-- 单个服务实例支持多个并发连接(每个连接独立 session)
+1. **HTTPS 限制**:浏览器麦克风需要 HTTPS 或 localhost(Chrome 安全策略)
+2. **远程访问**:使用 SSH 端口转发 `ssh -L 10095:localhost:10095 <server>`
+3. **并发支持**:单个服务实例支持多个并发 WebSocket 连接(每个连接独立 session)
+4. **显存占用**:ASR + VAD + SPK 模型共约 3-4GB 显存
+5. **说话人分离**:Beta 功能,短音频或单人场景可能不准确
+6. **模型来源**:默认从 ModelScope 下载,可用 `--hub hf` 切换 HuggingFace
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/requirements.txt b/examples/industrial_data_pretraining/fun_asr_nano/requirements.txt
index eac7d4a42..8bb51b590 100644
--- a/examples/industrial_data_pretraining/fun_asr_nano/requirements.txt
+++ b/examples/industrial_data_pretraining/fun_asr_nano/requirements.txt
@@ -1,9 +1,10 @@
-torch>=2.9.0
-torchaudio>=2.9.0
-transformers>=4.51.3
-funasr>=1.3.0
-zhconv
-whisper_normalizer
-pyopenjtalk-plus
-compute-wer
-openai-whisper
+torch>=2.0
+funasr
+websockets>=12.0
+regex
+numpy
+soundfile
+
+# Optional: for Python client microphone mode
+# sounddevice
+# librosa
diff --git a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
index 367816e8a..ad6f98dc8 100644
--- a/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
+++ b/examples/industrial_data_pretraining/fun_asr_nano/serve_realtime_ws.py
@@ -1,10 +1,12 @@
#!/usr/bin/env python3
-"""
-Fun-ASR-Nano Real-time WebSocket Server.
-
-Key fix: Only decode CURRENT VAD segment (not entire cumulative audio).
-Confirmed segments get their text locked and never re-decoded.
-This prevents model collapse on long recordings.
+"""Fun-ASR-Nano Streaming WebSocket Server.
+
+Features:
+- Streaming VAD segmentation (fsmn-vad)
+- Per-segment ASR decoding (Fun-ASR-Nano via vLLM)
+- Speaker diarization (eres2netv2 + ClusterBackend)
+- Hotword customization
+- Hallucination detection & prevention
"""
import asyncio
@@ -16,20 +18,50 @@ import argparse
import numpy as np
import torch
import warnings
+import regex
+import websockets
warnings.filterwarnings('ignore')
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger(__name__)
-try:
- import websockets
-except ImportError:
- os.system("pip install websockets --break-system-packages -q")
- import websockets
+
+def detect_and_fix_hallucination(text, max_ngram_length=12, max_occurrences=3):
+ """Detect repeated patterns (hallucination) and truncate to keep one occurrence."""
+ if not text or len(text) < max_ngram_length * 2:
+ return text, False
+
+ cleaned = regex.sub(r'\p{P}+', '', text)
+
+ word_pattern = rf'(?<!\S)(?!\d+$)(\w+)(?:\s+\1){{{max_occurrences - 1},}}(?!\S)'
+ if regex.search(word_pattern, cleaned, regex.IGNORECASE):
+ match = regex.search(word_pattern, cleaned, regex.IGNORECASE)
+ repeated = match.group(1)
+ pos = text.find(repeated)
+ if pos >= 0:
+ end_pos = text.find(repeated, pos + len(repeated))
+ if end_pos >= 0:
+ return text[:end_pos + len(repeated)], True
+ return text[:len(text)//2], True
+
+ for length in range(1, max_ngram_length):
+ pattern = rf'(?<!\d)(\S{{{length}}})\1{{{max_occurrences - 1},}}(?!\d)'
+ combined = rf'(?=.*\D){pattern}'
+ match = regex.search(combined, cleaned)
+ if match:
+ repeated = match.group(1)
+ pos = text.find(repeated)
+ if pos >= 0:
+ end_pos = text.find(repeated, pos + len(repeated))
+ if end_pos >= 0:
+ return text[:end_pos + len(repeated)], True
+ return text[:len(text)//2], True
+
+ return text, False
class StreamingVAD:
- """Streaming VAD: [start,-1]=speech start, [-1,end]=speech end."""
+ """Streaming VAD with dynamic silence threshold."""
def __init__(self, vad_model):
self.model = vad_model
@@ -40,7 +72,7 @@ class StreamingVAD:
def feed(self, audio_chunk_tensor, is_final=False):
res = self.model.generate(
input=[audio_chunk_tensor], cache=self.cache,
- is_final=is_final, chunk_size=200,
+ is_final=is_final, chunk_size=60,
)
signals = res[0].get("value", [])
new_confirmed = []
@@ -64,21 +96,178 @@ class StreamingVAD:
self.current_speech_start = None
+class HybridSpeakerTracker:
+ """Speaker diarization: streaming ClusterBackend + final re-clustering."""
+
+ def __init__(self, spk_model, device, threshold=0.6):
+ self.spk_model = spk_model
+ self.device = device
+ self.threshold = threshold
+ self.speaker_centers = []
+ from funasr.models.campplus.utils import sv_chunk, postprocess, distribute_spk
+ from funasr.models.campplus.cluster_backend import ClusterBackend
+ self.sv_chunk = sv_chunk
+ self.postprocess = postprocess
+ self.distribute_spk = distribute_spk
+ self.cluster_backend = ClusterBackend(merge_thr=0.78).to(device)
+ self.all_chunks = []
+ self.all_embeddings = []
+ self.display_map = {}
+ self.next_display_id = 0
+
+ @torch.no_grad()
+ def assign_streaming(self, audio_samples, seg_start_s, seg_end_s, sentence):
+ """Assign speaker ID during streaming using ClusterBackend."""
+ vad_seg = [[seg_start_s, seg_end_s, audio_samples]]
+ chunks = self.sv_chunk(vad_seg)
+ if not chunks:
+ sentence["spk"] = self.next_display_id
+ self.next_display_id += 1
+ return
+
+ self.all_chunks.extend(chunks)
+ speech_list = [ch[2] for ch in chunks]
+ spk_res = self.spk_model.generate(input=speech_list, cache={}, is_final=True)
+ embs = torch.cat([r["spk_embedding"] for r in spk_res], dim=0)
+ self.all_embeddings.append(embs)
+
+ all_embs = torch.cat(self.all_embeddings, dim=0)
+ labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)
+ if not isinstance(labels, np.ndarray):
+ labels = np.array(labels)
+
+ all_sorted = sorted(self.all_chunks, key=lambda x: x[0])
+ sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())
+ temp = [{"start": int(seg_start_s*1000), "end": int(seg_end_s*1000), "text": sentence["text"]}]
+ self.distribute_spk(temp, sv_output)
+ raw_spk = temp[0].get("spk", 0)
+
+ if raw_spk not in self.display_map:
+ self.display_map[raw_spk] = self.next_display_id
+ self.next_display_id += 1
+ sentence["spk"] = self.display_map[raw_spk]
+
+ @torch.no_grad()
+ def finalize(self, sentences, min_split_s=3.0):
+ """Final re-clustering for accurate speaker assignment."""
+ if not self.all_embeddings or not sentences:
+ return sentences
+
+ all_embs = torch.cat(self.all_embeddings, dim=0)
+ labels = self.cluster_backend(all_embs.cpu(), oracle_num=None)
+ if not isinstance(labels, np.ndarray):
+ labels = np.array(labels)
+
+ all_sorted = sorted(self.all_chunks, key=lambda x: x[0])
+ sv_output = self.postprocess(all_sorted, None, labels, all_embs.cpu())
+
+ for s in sentences:
+ s.pop("spk", None)
+ self.distribute_spk(sentences, sv_output)
+
+ id_map = {}
+ next_id = 0
+ for s in sentences:
+ raw = s.get("spk", 0)
+ if raw not in id_map:
+ id_map[raw] = next_id
+ next_id += 1
+ s["spk"] = id_map[raw]
+
+ final_sentences = []
+ for s in sentences:
+ sub = self._try_split(s, sv_output, id_map, min_split_s)
+ final_sentences.extend(sub)
+
+ return final_sentences
+
+ def _try_split(self, sentence, sv_output, id_map, min_split_s):
+ """Split a sentence if multiple speakers detected within its time range."""
+ sent_start = sentence["start"] / 1000.0
+ sent_end = sentence["end"] / 1000.0
+ text = sentence["text"]
+
+ overlapping = []
+ for sv_start, sv_end, sv_spk in sv_output:
+ o_start = max(sent_start, sv_start)
+ o_end = min(sent_end, sv_end)
+ if o_end > o_start:
+ mapped_spk = id_map.get(int(sv_spk), int(sv_spk))
+ overlapping.append([o_start, o_end, mapped_spk])
+
+ if len(overlapping) <= 1:
+ return [sentence]
+
+ filtered = [overlapping[0]]
+ for i in range(1, len(overlapping)):
+ cur = overlapping[i]
+ prev = filtered[-1]
+ if cur[2] == prev[2]:
+ filtered[-1] = [prev[0], cur[1], prev[2]]
+ elif (cur[1] - cur[0]) < min_split_s:
+ filtered[-1] = [prev[0], cur[1], prev[2]]
+ else:
+ filtered.append(cur)
+
+ merged = [filtered[0]]
+ for i in range(1, len(filtered)):
+ if (merged[-1][1] - merged[-1][0]) < min_split_s:
+ merged[-1] = [merged[-1][0], filtered[i][1], filtered[i][2]]
+ else:
+ merged.append(filtered[i])
+ if len(merged) > 1 and (merged[-1][1] - merged[-1][0]) < min_split_s:
+ merged[-2] = [merged[-2][0], merged[-1][1], merged[-2][2]]
+ merged.pop()
+
+ if len(merged) <= 1:
+ return [sentence]
+
+ total_dur = sum(m[1] - m[0] for m in merged)
+ sub_sentences = []
+ char_pos = 0
+ for i, (m_start, m_end, m_spk) in enumerate(merged):
+ if i == len(merged) - 1:
+ sub_text = text[char_pos:]
+ else:
+ n_chars = max(1, int(len(text) * (m_end - m_start) / total_dur))
+ sub_text = text[char_pos:char_pos + n_chars]
+ char_pos += n_chars
+ if sub_text.strip():
+ sub_sentences.append({"text": sub_text.strip(), "start": int(m_start*1000), "end": int(m_end*1000), "spk": m_spk})
+
+ return sub_sentences if sub_sentences else [sentence]
+
+ def reset(self):
+ self.speaker_centers = []
+ self.all_chunks = []
+ self.all_embeddings = []
+ self.display_map = {}
+ self.next_display_id = 0
+
+
class RealtimeASRSession:
- def __init__(self, asr_model, asr_kwargs, tokenizer, vad, sample_rate=16000, chunk_ms=720):
+ """Manages a single streaming ASR session."""
+
+ def __init__(self, asr_model, asr_kwargs, tokenizer, vad, spk_tracker=None, sample_rate=16000, chunk_ms=960):
self.asr_model = asr_model
self.asr_kwargs = asr_kwargs
self.tokenizer = tokenizer
self.vad = vad
self.sample_rate = sample_rate
self.chunk_samples = int(sample_rate * chunk_ms / 1000)
+ self.first_chunk_samples = int(sample_rate * 480 / 1000)
+ self.first_decode_done = False
self.audio_buffer = np.array([], dtype=np.float32)
self.vad_fed_samples = 0
self.prev_text = ""
self.last_partial_text = ""
self.last_decode_samples = 0
- self.locked_sentences = [] # text locked for confirmed segments
+ self.locked_sentences = []
+ self.prev_seg_text = ""
+ self.accumulated_since_cut_ms = 0
+ self.spk_tracker = spk_tracker
+ self.use_context = True
self.is_active = False
def add_audio(self, pcm_bytes):
@@ -86,21 +275,45 @@ class RealtimeASRSession:
audio_float = audio_int16.astype(np.float32) / 32768.0
self.audio_buffer = np.concatenate([self.audio_buffer, audio_float])
- # Feed new audio to streaming VAD
new_audio = self.audio_buffer[self.vad_fed_samples:]
if len(new_audio) > 0:
+ self.accumulated_since_cut_ms += len(new_audio) * 1000 // self.sample_rate
+ if "stats" in self.vad.cache:
+ self.vad.cache["stats"].speech_noise_thres = 0.5
+ if self.accumulated_since_cut_ms <= 5000:
+ desired_silence_ms = 2000
+ elif self.accumulated_since_cut_ms <= 10000:
+ desired_silence_ms = 1500
+ elif self.accumulated_since_cut_ms <= 15000:
+ desired_silence_ms = 1000
+ elif self.accumulated_since_cut_ms <= 30000:
+ desired_silence_ms = 800
+ elif self.accumulated_since_cut_ms <= 45000:
+ desired_silence_ms = 400
+ else:
+ desired_silence_ms = 100
+ new_thresh = max(desired_silence_ms - 150, 0)
+ self.vad.cache["stats"].max_end_sil_frame_cnt_thresh = new_thresh
+
new_confirmed = self.vad.feed(torch.from_numpy(new_audio).float(), is_final=False)
self.vad_fed_samples = len(self.audio_buffer)
- # Lock text for newly confirmed segments
for seg in new_confirmed:
seg_text = self._decode_segment(seg)
- self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})
- self.prev_text = "" # reset for new segment
- logger.info(f" Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:30]}\"")
+ self.prev_text = ""
+ self.accumulated_since_cut_ms = 0
+ if not seg_text.strip():
+ continue
+ self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})
+ if self.spk_tracker:
+ s0 = int(seg[0] * self.sample_rate / 1000)
+ s1 = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))
+ self.spk_tracker.assign_streaming(self.audio_buffer[s0:s1], seg[0]/1000, seg[1]/1000, self.locked_sentences[-1])
+ logger.info(f"Locked: [{seg[0]}-{seg[1]}ms] \"{seg_text[:40]}\"")
def should_decode(self):
- return (len(self.audio_buffer) - self.last_decode_samples) >= self.chunk_samples
+ threshold = self.first_chunk_samples if not self.first_decode_done else self.chunk_samples
+ return (len(self.audio_buffer) - self.last_decode_samples) >= threshold
@torch.no_grad()
def decode(self, is_final=False):
@@ -108,31 +321,32 @@ class RealtimeASRSession:
return self._build_response(is_final)
if is_final:
- # Feed remaining to VAD
remaining = self.audio_buffer[self.vad_fed_samples:]
if len(remaining) > 0:
new_confirmed = self.vad.feed(torch.from_numpy(remaining).float(), is_final=True)
self.vad_fed_samples = len(self.audio_buffer)
for seg in new_confirmed:
seg_text = self._decode_segment(seg)
- self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})
+ if not seg_text.strip():
+ continue
+ self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})
- # If still in speech, close and decode it
if self.vad.current_speech_start is not None:
end_ms = int(len(self.audio_buffer) * 1000 / self.sample_rate)
seg = [self.vad.current_speech_start, end_ms]
seg_text = self._decode_segment(seg)
- self.locked_sentences.append({"text": seg_text, "start_ms": seg[0], "end_ms": seg[1]})
+ if seg_text.strip():
+ self.locked_sentences.append({"text": seg_text, "start": int(seg[0]), "end": int(seg[1])})
self.vad.current_speech_start = None
+ if self.spk_tracker and self.locked_sentences:
+ self.locked_sentences = self.spk_tracker.finalize(self.locked_sentences)
return self._build_response(is_final)
- # Decode ONLY current ongoing segment (not full buffer!)
if self.vad.current_speech_start is not None:
seg_start_sample = int(self.vad.current_speech_start * self.sample_rate / 1000)
seg_audio = self.audio_buffer[seg_start_sample:]
else:
- # Not in speech - nothing to decode
self.last_decode_samples = len(self.audio_buffer)
self.last_partial_text = ""
return self._build_response(is_final)
@@ -150,10 +364,15 @@ class RealtimeASRSession:
logger.error(f"ASR error: {e}")
return self._build_response(is_final)
+ text, hallucinated = detect_and_fix_hallucination(text)
+ if hallucinated:
+ self.prev_text = ""
+
self.last_decode_samples = len(self.audio_buffer)
self.last_partial_text = text
+ if text.strip() and not self.first_decode_done:
+ self.first_decode_done = True
- # Update prev_text (rollback 5 tokens)
encoded = self.tokenizer.encode(text)
if len(encoded) > 5:
self.prev_text = self.tokenizer.decode(encoded[:-5]).replace("�", "")
@@ -164,15 +383,16 @@ class RealtimeASRSession:
@torch.no_grad()
def _decode_segment(self, seg):
- """Decode a specific completed VAD segment."""
+ """Decode a completed VAD segment with optional context."""
start_sample = int(seg[0] * self.sample_rate / 1000)
end_sample = min(int(seg[1] * self.sample_rate / 1000), len(self.audio_buffer))
seg_audio = self.audio_buffer[start_sample:end_sample]
if len(seg_audio) < 1600:
return ""
audio_tensor = torch.from_numpy(seg_audio).float()
+ ctx = self.prev_seg_text if self.use_context and self.prev_seg_text else ""
try:
- res = self.asr_model.inference([audio_tensor], prev_text="", **self.asr_kwargs)
+ res = self.asr_model.inference([audio_tensor], prev_text=ctx, **self.asr_kwargs)
return res[0][0]["text"]
except Exception as e:
logger.error(f"Segment decode error: {e}")
@@ -187,7 +407,6 @@ class RealtimeASRSession:
if is_final:
return {"sentences": sentences, "partial": "", "partial_start_ms": 0,
"duration_ms": duration_ms, "is_final": True}
-
return {"sentences": sentences, "partial": partial,
"partial_start_ms": partial_start,
"duration_ms": duration_ms, "is_final": False}
@@ -195,21 +414,26 @@ class RealtimeASRSession:
def reset(self):
self.audio_buffer = np.array([], dtype=np.float32)
self.vad_fed_samples = 0
+ self.first_decode_done = False
self.vad.reset()
self.prev_text = ""
self.last_partial_text = ""
self.last_decode_samples = 0
self.locked_sentences = []
+ self.accumulated_since_cut_ms = 0
+ if self.spk_tracker:
+ self.spk_tracker.reset()
_asr_model = None
_asr_kwargs = None
_tokenizer = None
_vad_model = None
+_spk_model = None
def load_models(args):
- global _asr_model, _asr_kwargs, _tokenizer, _vad_model
+ global _asr_model, _asr_kwargs, _tokenizer, _vad_model, _spk_model
if _asr_model is None:
from funasr import AutoModel
logger.info(f"Loading ASR: {args.model}")
@@ -224,14 +448,26 @@ def load_models(args):
logger.info("Loading VAD: fsmn-vad (streaming)")
_vad_model = AutoModel(model="fsmn-vad", device=args.device, disable_update=True)
+
+ logger.info("Loading SPK: eres2netv2")
+ _spk_model = AutoModel(model="iic/speech_eres2netv2_sv_zh-cn_16k-common", device=args.device, disable_update=True)
+
+ hw_file = getattr(args, 'hotword_file', '热词列表')
+ if hw_file and os.path.isfile(hw_file):
+ with open(hw_file, "r", encoding="utf-8") as hf:
+ hotwords = [line.strip() for line in hf if line.strip()]
+ _asr_kwargs["hotwords"] = hotwords
+ logger.info(f"Loaded {len(hotwords)} hotwords from '{hw_file}'")
+
logger.info("All models ready!")
- return _asr_model, _asr_kwargs, _tokenizer, _vad_model
+ return _asr_model, _asr_kwargs, _tokenizer, _vad_model, _spk_model
async def handle_client(websocket, args):
- asr_model, asr_kwargs, tokenizer, vad_automodel = load_models(args)
- vad = StreamingVAD(vad_automodel)
- session = RealtimeASRSession(asr_model, asr_kwargs, tokenizer, vad)
+ asr_model, asr_kwargs, tokenizer, vad_model, spk_model = load_models(args)
+ vad = StreamingVAD(vad_model)
+ spk_tracker = HybridSpeakerTracker(spk_model, args.device)
+ session = RealtimeASRSession(asr_model, asr_kwargs, tokenizer, vad, spk_tracker=spk_tracker)
logger.info(f"Client connected: {websocket.remote_address}")
decode_interval = args.decode_interval
@@ -240,23 +476,28 @@ async def handle_client(websocket, args):
try:
async for message in websocket:
if isinstance(message, str):
- cmd = message.strip().upper()
- if cmd == "START":
+ cmd = message.strip()
+ if cmd.upper() == "START":
session.reset()
session.is_active = True
await websocket.send(json.dumps({"event": "started"}))
logger.info("Session started")
- elif cmd == "STOP":
+ elif cmd.upper().startswith("HOTWORDS:"):
+ hw_str = cmd[9:]
+ hotwords = [w.strip() for w in hw_str.split(",") if w.strip()]
+ session.asr_kwargs = dict(session.asr_kwargs)
+ session.asr_kwargs["hotwords"] = hotwords
+ await websocket.send(json.dumps({"event": "hotwords_set", "hotwords": hotwords}))
+ logger.info(f"Hotwords set: {len(hotwords)} words")
+ elif cmd.upper() == "STOP":
if session.is_active and len(session.audio_buffer) > 0:
result = session.decode(is_final=True)
await websocket.send(json.dumps(result))
- n = len(result["sentences"])
- logger.info(f"Final: {n} sentences")
+ logger.info(f"Final: {len(result['sentences'])} sentences")
session.is_active = False
await websocket.send(json.dumps({"event": "stopped"}))
elif isinstance(message, bytes) and session.is_active:
session.add_audio(message)
-
now = time.time()
if now - last_decode_time >= decode_interval and session.should_decode():
result = session.decode(is_final=False)
@@ -266,9 +507,7 @@ async def handle_client(websocket, args):
except websockets.exceptions.ConnectionClosed:
logger.info("Client disconnected")
except Exception as e:
- logger.error(f"Error: {e}")
- import traceback
- traceback.print_exc()
+ logger.error(f"Error: {e}", exc_info=True)
async def main(args):
@@ -282,11 +521,14 @@ async def main(args):
if __name__ == "__main__":
- parser = argparse.ArgumentParser()
+ parser = argparse.ArgumentParser(description="Fun-ASR-Nano Streaming WebSocket Server")
parser.add_argument("--port", type=int, default=10095)
parser.add_argument("--model", type=str, default="FunAudioLLM/Fun-ASR-Nano-2512")
- parser.add_argument("--hub", type=str, default="ms")
+ parser.add_argument("--hub", type=str, default="ms", choices=["ms", "hf"])
parser.add_argument("--device", type=str, default="cuda:0")
- parser.add_argument("--decode-interval", type=float, default=0.7)
+ parser.add_argument("--use-context", action="store_true", default=True)
+ parser.add_argument("--no-context", dest="use_context", action="store_false")
+ parser.add_argument("--decode-interval", type=float, default=0.48)
+ parser.add_argument("--hotword-file", type=str, default="热词列表")
args = parser.parse_args()
asyncio.run(main(args))
diff --git "a/examples/industrial_data_pretraining/fun_asr_nano/\347\203\255\350\257\215\345\210\227\350\241\250" "b/examples/industrial_data_pretraining/fun_asr_nano/\347\203\255\350\257\215\345\210\227\350\241\250"
new file mode 100644
index 000000000..d29477dde
--- /dev/null
+++ "b/examples/industrial_data_pretraining/fun_asr_nano/\347\203\255\350\257\215\345\210\227\350\241\250"
@@ -0,0 +1,4 @@
+兔子
+窝边草
+土匪
+师爷