Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat: built-in FSMN-VAD (--vad) for all three runtimes — single-binary segmentation (#2998)
zhifu gaoe615ed0dc0dc
diff --git a/runtime/llama.cpp/BENCHMARKS.md b/runtime/llama.cpp/BENCHMARKS.mdindex 78c6b09bc..e8e4a214a 100644--- a/runtime/llama.cpp/BENCHMARKS.md+++ b/runtime/llama.cpp/BENCHMARKS.md@@ -38,12 +38,15 @@ separates model-load from compute.- **Segmentation differs by system, each using its natural strategy:** FunASR uses an`fsmn-vad` front end (segments → ASR → concatenate); whisper.cpp uses its built-in30 s windowing. This is a fair system-level comparison.-- **Engine-internal VAD is on the FunASR runtime roadmap.** Today the runtime reaches- the reference numbers via the `fsmn-vad` front end (or `--chunk`); a native ggml VAD- is planned so the bare binary hits the reference end-to-end.-- **Bare binary, no VAD (whole-clip)**, for full disclosure: SenseVoiceSmall **9.99 %**,- Paraformer **12.82 %** (micro, normalize_zh, full 184). Long clips decoded as one- segment are out-of-distribution; this is exactly what VAD fixes.+- **Engine-internal VAD is now implemented** — a native ggml FSMN-VAD built into the+ binaries (`--vad fsmn-vad.gguf`). The **bare binary, with no Python front end**, now+ reaches the reference end-to-end: SenseVoiceSmall **8.01 %**, Paraformer **9.85 %**,+ Fun-ASR-Nano **8.30 %** (micro, normalize_zh, full 184). The built-in C++ VAD matches+ the PyTorch `fsmn-vad` front end (segment boundaries within ~10 ms, slightly better+ CER), so the runtime is now fully self-contained.+- For full disclosure, **bare binary with no VAD at all (whole-clip)** is higher —+ SenseVoiceSmall 9.99 %, Paraformer 12.82 % — because long clips decoded as one segment+ are out-of-distribution; that is exactly what the built-in VAD fixes.## Why FunASR wins on Chinesediff --git a/runtime/llama.cpp/README.md b/runtime/llama.cpp/README.mdindex f3bf03d65..5d8a59f55 100644--- a/runtime/llama.cpp/README.md+++ b/runtime/llama.cpp/README.md@@ -36,6 +36,9 @@ echo 'add_subdirectory(<example-dir>)' >> examples/CMakeLists.txtcmake -B build -DGGML_NATIVE=ON -DLLAMA_CURL=OFFcmake --build build -j --target <target>```+The shared **FSMN-VAD** front end builds the same way (`funasr-vad/` + `funasr-common/`,+target `llama-funasr-vad`); export weights with `export_vad_gguf.py`. Pass+`--vad fsmn-vad.gguf` to any of the three tools for built-in long-audio segmentation.## Validation@@ -44,9 +47,11 @@ SenseVoice CTC token ids identical; Paraformer text identical; Fun-ASR-Nano aggrCER matches PyTorch within 0.02% under identical conditions). See per-model READMEs.## Status / notes-- WAV input currently assumes 16 kHz mono PCM16.-- For long audio, Fun-ASR-Nano supports `--chunk` windowing; a proper VAD front end- is on the roadmap.+- Any audio in (wav/mp3/flac, any rate/channels) via the bundled miniaudio loader.+- **Built-in FSMN-VAD (`--vad fsmn-vad.gguf`)** segments long audio inside the binary+ (native ggml, no Python front end); all three tools support it. Bare-binary full-184+ micro-CER: SenseVoice **8.01** / Paraformer **9.85** / Fun-ASR-Nano **8.30** (see+ [BENCHMARKS.md](BENCHMARKS.md)). `--chunk` fixed-window remains a simpler fallback.- This adds a new `runtime/llama.cpp/` directory only; no existing code is modified.## Further readingdiff --git a/runtime/llama.cpp/export_vad_gguf.py b/runtime/llama.cpp/export_vad_gguf.pynew file mode 100644index 000000000..aea122152--- /dev/null+++ b/runtime/llama.cpp/export_vad_gguf.py@@ -0,0 +1,33 @@+#!/usr/bin/env python3+"""Export FSMN-VAD (encoder + CMVN) to GGUF for the ggml C++ runtime."""+import argparse, os, re+import numpy as np, torch, gguf++def parse_mvn(path):+ b=[np.array([float(x) for x in m.split()],np.float32) for m in re.findall(r"\[([^\]]*)\]",open(path).read())]+ v=[x for x in b if x.size>1]; return v[0], v[1] # shift, scale (both 400-dim)++def main():+ ap=argparse.ArgumentParser()+ ap.add_argument("--model_pt",required=True); ap.add_argument("--mvn",required=True); ap.add_argument("--out",required=True)+ a=ap.parse_args()+ sd=torch.load(a.model_pt,map_location="cpu"); sd=sd.get("state_dict",sd)+ w=gguf.GGUFWriter(a.out,"fsmn-vad")+ w.add_uint32("vad.input_dim",400); w.add_uint32("vad.input_affine_dim",140)+ w.add_uint32("vad.linear_dim",250); w.add_uint32("vad.proj_dim",128)+ w.add_uint32("vad.fsmn_layers",4); w.add_uint32("vad.lorder",20)+ w.add_uint32("vad.output_affine_dim",140); w.add_uint32("vad.output_dim",248)+ w.add_uint32("vad.n_mels",80); w.add_uint32("vad.lfr_m",5); w.add_uint32("vad.lfr_n",1)+ shift,scale=parse_mvn(a.mvn); w.add_tensor("cmvn.shift",shift); w.add_tensor("cmvn.scale",scale)+ n=0+ for k,v in sd.items():+ if not k.startswith("encoder."): continue+ arr=v.detach().to(torch.float32).contiguous().numpy()+ if k.endswith("conv_left.weight"): # (C,1,lorder,1) -> (lorder,C) tap-major+ arr=np.ascontiguousarray(arr[:,0,:,0].T)+ w.add_tensor(k,arr); n+=1+ print(f"writing {n} tensors (+cmvn) to {a.out}")+ w.write_header_to_file(); w.write_kv_data_to_file(); w.write_tensors_to_file(); w.close()+ print(f"done: {a.out} ({os.path.getsize(a.out)/1e6:.1f} MB)")++if __name__=="__main__": main()diff --git a/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cppindex 9f4fba88e..f9266943b 100644--- a/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp+++ b/runtime/llama.cpp/fun-asr-nano/funasr-cli/funasr-cli.cpp@@ -26,6 +26,9 @@// any audio (wav/mp3/flac, any rate/channels) -> 16 kHz mono f32, via miniaudio#define FUNASR_AUDIO_IMPLEMENTATION#include "funasr_audio.h"+// built-in FSMN-VAD front end (single-binary --vad segmentation)+#include "funasr_vad.h"+#include <utility>// ======================= kaldi fbank + LFR =======================static const int FS=16000, WINLEN=400, SHIFT=160, NFFT=512, NMEL=80, LFR_M=7, LFR_N=6;@@ -164,15 +167,18 @@ static int decode_batch(llama_context*ctx,int n,llama_token*tok,float*embd,int n}int main(int argc,char**argv){- std::string enc_path,llm_path,wav_path; int npred=512; double chunk_sec=0; float rep=1.0f;+ std::string enc_path,llm_path,wav_path,vad_path; int npred=512; double chunk_sec=0; float rep=1.0f;+ int vad_maxseg=30000;for(int i=1;i<argc;i++){if(!strcmp(argv[i],"--enc")&&i+1<argc)enc_path=argv[++i];else if(!strcmp(argv[i],"-m")&&i+1<argc)llm_path=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];else if(!strcmp(argv[i],"-n")&&i+1<argc)npred=atoi(argv[++i]);else if(!strcmp(argv[i],"--chunk")&&i+1<argc)chunk_sec=atof(argv[++i]);+ else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];+ else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);else if(!strcmp(argv[i],"--rep")&&i+1<argc)rep=atof(argv[++i]);- else {fprintf(stderr,"usage: %s --enc enc.gguf -m llm.gguf -a audio.wav [-n npred] [--chunk sec]\n",argv[0]);return 1;}+ else {fprintf(stderr,"usage: %s --enc enc.gguf -m llm.gguf -a audio.wav [-n npred] [--chunk sec] [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;}}if(enc_path.empty()||llm_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}@@ -199,12 +205,24 @@ int main(int argc,char**argv){std::vector<llama_token> v(n); llama_tokenize(vocab,s,strlen(s),v.data(),n,false,true); return v;};auto pre=tokenize(prefix); auto suf=tokenize(suffix);- // split into chunks (chunk_sec<=0 -> whole file)- int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size();+ // Build the list of [offset,len] windows to transcribe (in samples).+ // --vad : FSMN-VAD speech segments (single-binary front end, replaces fixed chunking)+ // --chunk sec : fixed-size chunks ; otherwise the whole file in one window+ std::vector<std::pair<int,int>> wins; // {sample offset, sample len}+ if(!vad_path.empty()){+ std::vector<std::pair<int,int>> segs; // ms+ if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}+ for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);+ if(end>(int)wav.size())end=wav.size(); if(end-off>0) wins.push_back({off,end-off}); }+ fprintf(stderr,"[vad] %zu segments\n",wins.size());+ } else {+ int chunk_n = chunk_sec > 0 ? std::max(1, (int)(chunk_sec*16000)) : (int)wav.size();+ for(size_t off=0; off<wav.size(); off+=chunk_n) wins.push_back({(int)off,(int)std::min((size_t)chunk_n,wav.size()-off)});+ }std::string full;- for (size_t off = 0; off < wav.size(); off += chunk_n) {- int len = std::min((size_t)chunk_n, wav.size()-off);- if (len < WINLEN) break; // too short for one frame+ for (auto& w : wins) {+ int off = w.first, len = w.second;+ if (len < WINLEN) continue; // too short for one framestd::vector<float> seg(wav.begin()+off, wav.begin()+off+len);int T=0; auto fbank=compute_fbank(seg,T);int D=0; auto adp=run_encoder(em,fbank,T,560,D);diff --git a/runtime/llama.cpp/funasr-common/funasr_vad.h b/runtime/llama.cpp/funasr-common/funasr_vad.hnew file mode 100644index 000000000..6f55bf1fe--- /dev/null+++ b/runtime/llama.cpp/funasr-common/funasr_vad.h@@ -0,0 +1,145 @@+// funasr_vad.h — single-header FSMN-VAD for the funasr ggml runtime.+// Exposes funasr_vad_segments(): 16k mono wav -> speech segments [start_ms,end_ms].+// Front end (80-mel fbank + LFR m5n1 + CMVN) and FSMN encoder validated bit-exact vs+// PyTorch fsmn-vad; the host state machine reproduces E2EVadModel (DEFAULT_SILENCE_SCHEDULE,+// chunk-stepped) to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set.+#pragma once+#include "ggml.h"+#include "ggml-cpu.h"+#include "ggml-alloc.h"+#include "ggml-backend.h"+#include "gguf.h"+#include <cmath>+#include <cstdint>+#include <cstdio>+#include <cstring>+#include <map>+#include <string>+#include <utility>+#include <vector>++namespace funasr_vad_impl {+static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80;+static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;+static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);}+static void fftc(std::vector<float>&re,std::vector<float>&im,int n){for(int i=1,j=0;i<n;i++){int b=n>>1;for(;j&b;b>>=1)j^=b;j^=b;if(i<j){std::swap(re[i],re[j]);std::swap(im[i],im[j]);}}+ for(int len=2;len<=n;len<<=1){double a=-2.0*M_PI/len;float wr=cosf(a),wi=sinf(a);for(int i=0;i<n;i+=len){float cr=1,ci=0;for(int k=0;k<len/2;k++){float ur=re[i+k],ui=im[i+k];+ float vr=re[i+k+len/2]*cr-im[i+k+len/2]*ci,vi=re[i+k+len/2]*ci+im[i+k+len/2]*cr;re[i+k]=ur+vr;im[i+k]=ui+vi;re[i+k+len/2]=ur-vr;im[i+k+len/2]=ui-vi;float nc=cr*wr-ci*wi;ci=cr*wi+ci*wr;cr=nc;}}}}+static std::vector<std::vector<float>> fbank80(std::vector<float> wav){+ for(auto&v:wav)v*=32768.0f; std::vector<float>win(WINLEN);+ for(int i=0;i<WINLEN;i++)win[i]=0.54f-0.46f*cosf(2.0f*M_PI*i/(WINLEN-1));+ const int NB=NFFT/2+1; float bw=(float)FS/NFFT,ml=melf(LOWF),mh=melf(HIGHF),dm=(mh-ml)/(NMEL+1);+ std::vector<std::vector<float>>fb(NMEL,std::vector<float>(NB,0.0f));+ for(int m=0;m<NMEL;m++){float L=ml+m*dm,C=ml+(m+1)*dm,R=ml+(m+2)*dm;for(int k=0;k<NB;k++){float mf=melf(bw*k);if(mf>L&&mf<R)fb[m][k]=mf<=C?(mf-L)/(C-L):(R-mf)/(R-C);}}+ int N=wav.size(),T=(N-WINLEN)/SHIFT+1; if(T<1)T=0; std::vector<std::vector<float>>feat(T,std::vector<float>(NMEL));+ std::vector<float>re(NFFT),im(NFFT),fr(WINLEN);const float fl=1.1920929e-07f;+ for(int t=0;t<T;t++){const float*s=wav.data()+t*SHIFT;double mn=0;for(int i=0;i<WINLEN;i++)mn+=s[i];mn/=WINLEN;+ for(int i=0;i<WINLEN;i++)fr[i]=s[i]-(float)mn;for(int i=WINLEN-1;i>0;i--)fr[i]-=PREEMPH*fr[i-1];fr[0]-=PREEMPH*fr[0];+ for(int i=0;i<NFFT;i++){re[i]=i<WINLEN?fr[i]*win[i]:0.0f;im[i]=0.0f;}fftc(re,im,NFFT);+ for(int m=0;m<NMEL;m++){float e=0;for(int k=0;k<NB;k++)if(fb[m][k]>0)e+=fb[m][k]*(re[k]*re[k]+im[k]*im[k]);feat[t][m]=logf(e>fl?e:fl);}}+ return feat;+}+static std::vector<float> lfr(const std::vector<std::vector<float>>&feat,int m,int n,int&T_out){+ int T=feat.size(),D=NMEL,pad=(m-1)/2; int Tl=T>0?(T+n-1)/n:0;+ std::vector<std::vector<float>> pf; pf.reserve(T+pad+m);+ for(int i=0;i<pad;i++)pf.push_back(feat[0]);+ for(int t=0;t<T;t++)pf.push_back(feat[t]);+ while((int)pf.size()<(Tl-1)*n+m)pf.push_back(feat[T-1]);+ std::vector<float> out((size_t)Tl*m*D);+ for(int i=0;i<Tl;i++)for(int j=0;j<m;j++)memcpy(&out[((size_t)i*m+j)*D],pf[i*n+j].data(),D*sizeof(float));+ T_out=Tl; return out;+}+struct vad{ggml_context*ctx=nullptr;std::map<std::string,ggml_tensor*>t;+ ggml_tensor*g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"vad: missing %s\n",n.c_str());return nullptr;}return it->second;}};+static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;}+} // namespace funasr_vad_impl++// Run FSMN-VAD on a 16k mono float waveform; fills segs with [start_ms,end_ms] speech spans.+// max_seg_ms caps a single segment (e.g. 30000); pass <=0 to use 60000 (model default).+inline bool funasr_vad_segments(const std::string& gguf_path, const std::vector<float>& wav,+ int max_seg_ms, std::vector<std::pair<int,int>>& segs, int nthreads=8){+ using namespace funasr_vad_impl;+ segs.clear();+ vad m; gguf_init_params ip={false,&m.ctx}; gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),ip);+ if(!gg){fprintf(stderr,"vad: cannot load %s\n",gguf_path.c_str());return false;}+ auto rd=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};+ int idim=rd("vad.input_dim",400),pd=rd("vad.proj_dim",128),nl=rd("vad.fsmn_layers",4),lorder=rd("vad.lorder",20),+ od=rd("vad.output_dim",248),lm=rd("vad.lfr_m",5),ln=rd("vad.lfr_n",1);+ for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx,nm);}+ gguf_free(gg);++ auto feat=fbank80(wav); int T=0; auto feats=lfr(feat,lm,ln,T); // [T,400]+ if(T<1){if(m.ctx)ggml_free(m.ctx);return true;} // too short -> no speech+ float*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data;+ for(int t=0;t<T;t++)for(int d=0;d<idim;d++)feats[(size_t)t*idim+d]=(feats[(size_t)t*idim+d]+shift[d])*scale[d];++ ggml_backend_t be=ggml_backend_cpu_init();+ ggml_init_params cp={(size_t)512*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp);+ ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,idim,T); ggml_set_input(x);+ ggml_tensor*h=lin(c,m.g("encoder.in_linear1.linear.weight"),m.g("encoder.in_linear1.linear.bias"),x);+ h=lin(c,m.g("encoder.in_linear2.linear.weight"),m.g("encoder.in_linear2.linear.bias"),h); h=ggml_relu(c,h);+ for(int i=0;i<nl;i++){std::string p="encoder.fsmn."+std::to_string(i)+".";+ ggml_tensor*z=ggml_mul_mat(c,m.g(p+"linear.linear.weight"),h);+ ggml_tensor*fk=m.g(p+"fsmn_block.conv_left.weight"); ggml_tensor*zp=ggml_pad_ext(c,z,0,0,lorder-1,0,0,0,0,0); ggml_tensor*acc=z;+ for(int j=0;j<lorder;j++){auto sl=ggml_view_2d(c,zp,pd,T,zp->nb[1],(size_t)j*zp->nb[1]);auto wj=ggml_view_1d(c,fk,pd,(size_t)j*fk->nb[1]);acc=ggml_add(c,acc,ggml_mul(c,ggml_cont(c,sl),wj));}+ ggml_tensor*a=lin(c,m.g(p+"affine.linear.weight"),m.g(p+"affine.linear.bias"),acc); h=ggml_relu(c,a);}+ h=lin(c,m.g("encoder.out_linear1.linear.weight"),m.g("encoder.out_linear1.linear.bias"),h);+ h=lin(c,m.g("encoder.out_linear2.linear.weight"),m.g("encoder.out_linear2.linear.bias"),h);+ h=ggml_soft_max(c,h); ggml_set_output(h);+ ggml_cgraph*gf=ggml_new_graph(c); ggml_build_forward_expand(gf,h);+ ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf);+ ggml_backend_tensor_set(x,feats.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,nthreads);+ bool ok=ggml_backend_graph_compute(be,gf)==GGML_STATUS_SUCCESS;+ std::vector<float> sc((size_t)od*T); if(ok)ggml_backend_tensor_get(h,sc.data(),0,ggml_nbytes(h));+ ggml_gallocr_free(ga);ggml_free(c);ggml_backend_free(be);if(m.ctx)ggml_free(m.ctx);+ if(!ok)return false;++ // ===== E2EVadModel state machine (host) -> speech segments [start_ms,end_ms] =====+ const int FR=10; // ms per frame (frame_in_ms)+ const int win=20; // window_size_ms 200 / 10+ const int s2s=15, sp2s=15; // sil_to_speech / speech_to_sil thres (150/10)+ const int lookahead_end=100/FR; // lookahead_time_end_point 100/10 = 10 (do_extend)+ int max_seg = (max_seg_ms>0 ? max_seg_ms : 60000)/FR; // max_single_segment frames+ const int start_lookback = win + 200/FR; // LatencyFrmNumAtStartPoint = 40+ // End-silence threshold from DEFAULT_SILENCE_SCHEDULE, stepped at chunk boundaries (chunk_size+ // 60000ms = 6000 frames). At each boundary, if mid-segment (InSpeech) or in_speech latched,+ // accumulated_ms += 60000; threshold = schedule(accumulated). Reset to 0 on each segment emit.+ const int CHUNK=60000/FR;+ int acc=0, insp=0; int max_end_sil, end_lookback;+ auto recompute=[&](){+ int s; if(acc<=10000)s=2000; else if(acc<=20000)s=1000; else if(acc<=30000)s=800;+ else if(acc<=40000)s=600; else if(acc<=50000)s=400; else if(acc<=60000)s=200; else s=100;+ int ms=s-150; if(ms<0)ms=0; max_end_sil=ms/FR;+ end_lookback=max_end_sil-lookahead_end-1; if(end_lookback<0)end_lookback=0;+ };+ recompute();+ std::vector<int> wbuf(win,0); int wpos=0,wsum=0,pre=0;+ int st=0, cstart=-1, csil=0, prev_end=0;+ auto reset=[&](){ std::fill(wbuf.begin(),wbuf.end(),0); wpos=0; wsum=0; pre=0; csil=0; st=0; cstart=-1; acc=0; insp=0; };+ auto emit=[&](int s,int e){ if(s<prev_end)s=prev_end; if(s<0)s=0; if(e>T)e=T; if(e>s){segs.push_back({s,e}); prev_end=e;} };+ for(int t=0;t<T;t++){+ if(t>0 && t%CHUNK==0){ if(st==1||insp){acc+=60000; insp=1;} recompute(); }+ float sil=sc[(size_t)t*od+0];+ int fs = ((1.0f-sil) >= sil + 0.5f) ? 1 : 0; // speech_noise_thres=0.5+ wsum -= wbuf[wpos]; wsum += fs; wbuf[wpos]=fs; wpos=(wpos+1)%win;+ int ch;+ if(pre==0 && wsum>=s2s){pre=1; ch=3;}+ else if(pre==1 && wsum<=sp2s){pre=0; ch=1;}+ else ch = pre==0?0:2;+ if(ch==3){ csil=0;+ if(st==0){ cstart=t-start_lookback; if(cstart<prev_end)cstart=prev_end; if(cstart<0)cstart=0; st=1; }+ else if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); }+ } else if(ch==1||ch==2){ csil=0;+ if(st==1 && t-cstart+1>max_seg){ emit(cstart,t); reset(); }+ } else { csil++;+ if(st==1){+ if(csil>=max_end_sil){ emit(cstart, t-end_lookback); reset(); }+ else if(t-cstart+1>max_seg){ emit(cstart,t); reset(); }+ }+ }+ }+ if(st==1) emit(cstart,T);+ // convert frame indices -> ms+ for(auto&s:segs){ s.first*=FR; s.second*=FR; }+ return true;+}diff --git a/runtime/llama.cpp/funasr-vad/CMakeLists.txt b/runtime/llama.cpp/funasr-vad/CMakeLists.txtnew file mode 100644index 000000000..3bb1b7465--- /dev/null+++ b/runtime/llama.cpp/funasr-vad/CMakeLists.txt@@ -0,0 +1,6 @@+set(TARGET llama-funasr-vad)+add_executable(${TARGET} funasr-vad.cpp)+install(TARGETS ${TARGET} RUNTIME)+target_include_directories(${TARGET} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../funasr-common)+target_link_libraries(${TARGET} PRIVATE ggml ${CMAKE_THREAD_LIBS_INIT})+target_compile_features(${TARGET} PRIVATE cxx_std_17)diff --git a/runtime/llama.cpp/funasr-vad/funasr-vad.cpp b/runtime/llama.cpp/funasr-vad/funasr-vad.cppnew file mode 100644index 000000000..abf542730--- /dev/null+++ b/runtime/llama.cpp/funasr-vad/funasr-vad.cpp@@ -0,0 +1,25 @@+// funasr-vad: FSMN-VAD on ggml. WAV(any fmt/rate) -> speech segments [start_ms,end_ms].+// Front end + FSMN encoder validated bit-exact vs PyTorch; state machine reproduces+// E2EVadModel segmentation to within 1 frame (10ms) of fsmn-vad.generate on the 184-clip set.+#define FUNASR_AUDIO_IMPLEMENTATION+#include "funasr_audio.h"+#include "funasr_vad.h"+#include <cstdio>+#include <cstdlib>+#include <cstring>+#include <string>+#include <utility>+#include <vector>++int main(int argc,char**argv){+ std::string gp,wp;+ for(int i=1;i<argc;i++){if(!strcmp(argv[i],"-m")&&i+1<argc)gp=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wp=argv[++i];}+ if(gp.empty()||wp.empty()){fprintf(stderr,"usage: %s -m fsmn-vad.gguf -a audio.wav\n",argv[0]);return 1;}+ std::vector<float> wav; if(!funasr_load_audio_16k_mono(wp.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}+ int max_seg = getenv("VAD_MAXSEG") ? atoi(getenv("VAD_MAXSEG")) : 30000;+ std::vector<std::pair<int,int>> segs;+ if(!funasr_vad_segments(gp,wav,max_seg,segs)){fprintf(stderr,"vad failed\n");return 1;}+ for(auto&s:segs) printf("%d %d\n", s.first, s.second);+ fprintf(stderr,"[vad] %zu segments (max_seg=%dms)\n",segs.size(),max_seg);+ return 0;+}diff --git a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cppindex 18f90f221..9d0e17991 100644--- a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp+++ b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp@@ -22,6 +22,8 @@ static const float LN_EPS = 1e-5f;// ===== audio loader: any wav/mp3/flac, any rate/channels -> 16k mono (miniaudio) =====#define FUNASR_AUDIO_IMPLEMENTATION#include "funasr_audio.h"+#include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation)+#include <utility>static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6;static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);}@@ -118,9 +120,11 @@ static std::vector<float> run_graph(ggml_context*c,ggml_tensor*out,ggml_tensor*iggml_gallocr_free(ga);ggml_backend_free(be);return r;}int main(int argc,char**argv){- std::string gguf_path,wav_path;+ std::string gguf_path,wav_path,vad_path; int vad_maxseg=30000;for(int i=1;i<argc;i++){if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];- else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav\n",argv[0]);return 1;}}+ else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];+ else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);+ else{fprintf(stderr,"usage: %s -m paraformer.gguf -a audio.wav [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;}}if(gguf_path.empty()||wav_path.empty()){fprintf(stderr,"missing args\n");return 1;}model m;gguf_init_params gp={false,&m.ctx_w};gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),gp);if(!gg){fprintf(stderr,"gguf load failed\n");return 1;}auto rdi=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};@@ -130,80 +134,70 @@ int main(int argc,char**argv){for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}gguf_free(gg);const int D=m.c.d_model,F=560,V=m.c.vocab;- // fbank- std::vector<float>wav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}- int64_t t0=ggml_time_us();int T=0;auto fb=compute_fbank(wav,T);- // CMVN (Paraformer applies it): (x+shift)*scalefloat*shift=(float*)m.g("cmvn.shift")->data,*scale=(float*)m.g("cmvn.scale")->data;- for(int t=0;t<T;t++)for(int d=0;d<F;d++)fb[(size_t)t*F+d]=(fb[(size_t)t*F+d]+shift[d])*scale[d];- // encoder pre-scale + posenc- {float sc=sqrtf((float)D);for(auto&v:fb)v*=sc;}add_posenc(fb,T,F);-- // ===== encoder graph =====- std::vector<float>enc;- {ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);- ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T);ggml_set_input(x);- ggml_tensor*h=enc_layer(c,m,"encoder.encoders0.0.",x,T,false);- for(int i=0;i<m.c.enc_blocks-1;i++)h=enc_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,T,true);- h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));ggml_set_output(h);- enc=run_graph(c,h,x,fb.data(),nullptr,nullptr);ggml_free(c);} // enc: [D, T] -> stored row-major as T rows? ggml out ne0=D,ne1=T- // run_graph returns row-major [N=ne1 rows, D=ne0]; here rows=T, cols=D- int64_t t1=ggml_time_us();- // enc[t*D + d]-- // ===== CIF predictor (host) =====float*cw=(float*)m.g("predictor.cif_conv1d.weight")->data; // [512,512,3] = [o][i][j]float*cb=(float*)m.g("predictor.cif_conv1d.bias")->data; // [512]float*ow=(float*)m.g("predictor.cif_output.weight")->data; // [1,512]float ob=((float*)m.g("predictor.cif_output.bias")->data)[0];- // conv1d (k=3,pad1) + residual + relu, then cif_output -> sigmoid -> alpha- std::vector<float> alphas(T);- std::vector<float> outp((size_t)T*D);- for(int t=0;t<T;t++){- for(int o=0;o<D;o++){- float acc=cb[o];- for(int j=0;j<3;j++){int tt=t+j-1; if(tt<0||tt>=T)continue; const float*ev=&enc[(size_t)tt*D];- const float*wo=&cw[(size_t)o*D*3]; for(int i=0;i<D;i++) acc+=wo[i*3+j]*ev[i];}- outp[(size_t)t*D+o]=acc+enc[(size_t)t*D+o]; // + context residual- }- float a=ob; for(int o=0;o<D;o++){float r=outp[(size_t)t*D+o]; if(r<0)r=0; a+=ow[o]*r;} // relu then linear- float s=1.0f/(1.0f+expf(-a)); alphas[t]=s>0?s:0; // sigmoid, smooth=1 noise=0 -> relu(s)=s- }- // tail: append zero hidden frame + alpha=tail_threshold- std::vector<float> hid=enc; hid.resize((size_t)(T+1)*D,0.0f);- std::vector<float> al=alphas; al.push_back(m.c.tail);- int L=T+1;- // integrate-and-fire- std::vector<float> acoustic; acoustic.reserve(64*D);- float integrate=0; std::vector<float> frame(D,0.0f);- for(int t=0;t<L;t++){- float alpha=al[t]; float dc=1.0f-integrate; integrate+=alpha;- bool fire=integrate>=m.c.thresh; float cur=fire?dc:alpha; float rem=alpha-cur;- for(int d=0;d<D;d++) frame[d]+=cur*hid[(size_t)t*D+d];- if(fire){ acoustic.insert(acoustic.end(),frame.begin(),frame.end()); integrate-=1.0f;- for(int d=0;d<D;d++) frame[d]=rem*hid[(size_t)t*D+d]; }- }- int N=acoustic.size()/D;- if(N<1){printf("\n");fprintf(stderr,"[paraformer] no tokens\n");return 0;}- // ===== decoder graph =====- std::vector<float> logits;- {ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);- ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt);- ggml_tensor*mem=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,T);ggml_set_input(mem);- ggml_tensor*x=tgt;- for(int i=0;i<m.c.dec_att;i++)x=dec_layer(c,m,"decoder.decoders."+std::to_string(i)+".",x,mem,N,T);- for(int i=0;i<m.c.dec3;i++)x=dec3_layer(c,m,"decoder.decoders3."+std::to_string(i)+".",x);- x=lnorm(c,x,m.g("decoder.after_norm.weight"),m.g("decoder.after_norm.bias"));- x=lin(c,m.g("decoder.output_layer.weight"),m.g("decoder.output_layer.bias"),x); // [V,N]- ggml_set_output(x);- logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);}- int64_t t2=ggml_time_us();+ // Full pipeline (fbank -> CMVN -> encoder -> CIF -> decoder) on one wav window; prints token IDs.+ auto run_seg=[&](const std::vector<float>& wav){+ int T=0;auto fb=compute_fbank(wav,T); if(T<1)return;+ for(int t=0;t<T;t++)for(int d=0;d<F;d++)fb[(size_t)t*F+d]=(fb[(size_t)t*F+d]+shift[d])*scale[d]; // CMVN+ {float sc=sqrtf((float)D);for(auto&v:fb)v*=sc;}add_posenc(fb,T,F);+ std::vector<float>enc;+ {ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);+ ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,T);ggml_set_input(x);+ ggml_tensor*h=enc_layer(c,m,"encoder.encoders0.0.",x,T,false);+ for(int i=0;i<m.c.enc_blocks-1;i++)h=enc_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,T,true);+ h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));ggml_set_output(h);+ enc=run_graph(c,h,x,fb.data(),nullptr,nullptr);ggml_free(c);} // enc row-major [T,D]+ // ===== CIF predictor (host): conv1d(k3,pad1)+residual+relu -> cif_output -> sigmoid -> alpha =====+ std::vector<float> outp((size_t)T*D); std::vector<float> alphas(T);+ for(int t=0;t<T;t++){+ for(int o=0;o<D;o++){ float acc=cb[o];+ for(int j=0;j<3;j++){int tt=t+j-1; if(tt<0||tt>=T)continue; const float*ev=&enc[(size_t)tt*D];+ const float*wo=&cw[(size_t)o*D*3]; for(int i=0;i<D;i++) acc+=wo[i*3+j]*ev[i];}+ outp[(size_t)t*D+o]=acc+enc[(size_t)t*D+o]; }+ float a=ob; for(int o=0;o<D;o++){float r=outp[(size_t)t*D+o]; if(r<0)r=0; a+=ow[o]*r;}+ float s=1.0f/(1.0f+expf(-a)); alphas[t]=s>0?s:0; }+ std::vector<float> hid=enc; hid.resize((size_t)(T+1)*D,0.0f);+ std::vector<float> al=alphas; al.push_back(m.c.tail); int L=T+1;+ std::vector<float> acoustic; acoustic.reserve(64*D);+ float integrate=0; std::vector<float> frame(D,0.0f);+ for(int t=0;t<L;t++){+ float alpha=al[t]; float dc=1.0f-integrate; integrate+=alpha;+ bool fire=integrate>=m.c.thresh; float cur=fire?dc:alpha; float rem=alpha-cur;+ for(int d=0;d<D;d++) frame[d]+=cur*hid[(size_t)t*D+d];+ if(fire){ acoustic.insert(acoustic.end(),frame.begin(),frame.end()); integrate-=1.0f;+ for(int d=0;d<D;d++) frame[d]=rem*hid[(size_t)t*D+d]; } }+ int N=acoustic.size()/D; if(N<1)return;+ std::vector<float> logits;+ {ggml_init_params cp={(size_t)2048*1024*1024,nullptr,true};ggml_context*c=ggml_init(cp);+ ggml_tensor*tgt=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,N);ggml_set_input(tgt);+ ggml_tensor*mem=ggml_new_tensor_2d(c,GGML_TYPE_F32,D,T);ggml_set_input(mem);+ ggml_tensor*x=tgt;+ for(int i=0;i<m.c.dec_att;i++)x=dec_layer(c,m,"decoder.decoders."+std::to_string(i)+".",x,mem,N,T);+ for(int i=0;i<m.c.dec3;i++)x=dec3_layer(c,m,"decoder.decoders3."+std::to_string(i)+".",x);+ x=lnorm(c,x,m.g("decoder.after_norm.weight"),m.g("decoder.after_norm.bias"));+ x=lin(c,m.g("decoder.output_layer.weight"),m.g("decoder.output_layer.bias"),x); // [V,N]+ ggml_set_output(x);+ logits=run_graph(c,x,tgt,acoustic.data(),mem,enc.data());ggml_free(c);}+ for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}printf("%d ",am);}+ };- // argmax per slot -> ids- for(int n=0;n<N;n++){const float*col=&logits[(size_t)n*V];int am=0;float best=col[0];for(int v=1;v<V;v++)if(col[v]>best){best=col[v];am=v;}printf("%d ",am);}+ int64_t t0=ggml_time_us();+ std::vector<float>wav;if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}+ if(!vad_path.empty()){+ std::vector<std::pair<int,int>> segs;+ if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}+ for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);+ if(end>(int)wav.size())end=wav.size(); if(end-off<WINLEN)continue;+ std::vector<float> seg(wav.begin()+off,wav.begin()+end); run_seg(seg); }+ fprintf(stderr,"[paraformer] %zu vad segments\n",segs.size());+ } else { run_seg(wav); }printf("\n");- fprintf(stderr,"[paraformer] T=%d N_tok=%d enc %.2fs dec %.2fs\n",T,N,(t1-t0)/1e6,(t2-t1)/1e6);+ fprintf(stderr,"[paraformer] done %.2fs\n",(ggml_time_us()-t0)/1e6);if(m.ctx_w) ggml_free(m.ctx_w);return 0;}diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cppindex 47f663459..45eb8ac4b 100644--- a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp+++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp@@ -23,6 +23,8 @@ static const float LN_EPS = 1e-5f;// ---- audio loader: any wav/mp3/flac, any rate/channels -> 16k mono (miniaudio) ----#define FUNASR_AUDIO_IMPLEMENTATION#include "funasr_audio.h"+#include "funasr_vad.h" // built-in FSMN-VAD front end (--vad segmentation)+#include <utility>static const int FS=16000,WINLEN=400,SHIFT=160,NFFT=512,NMEL=80,LFR_M=7,LFR_N=6;static const float PREEMPH=0.97f,LOWF=20.0f,HIGHF=8000.0f;static inline float melf(float f){return 1127.0f*logf(1.0f+f/700.0f);}@@ -90,11 +92,13 @@ static void add_posenc(std::vector<float>&x,int T,int depth){}int main(int argc,char**argv){- std::string gguf_path,fbank_path,wav_path;+ std::string gguf_path,fbank_path,wav_path,vad_path; int vad_maxseg=30000;for(int i=1;i<argc;i++){ if(!strcmp(argv[i],"-m")&&i+1<argc)gguf_path=argv[++i];else if(!strcmp(argv[i],"-f")&&i+1<argc)fbank_path=argv[++i];else if(!strcmp(argv[i],"-a")&&i+1<argc)wav_path=argv[++i];- else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin)\n",argv[0]);return 1;} }+ else if(!strcmp(argv[i],"--vad")&&i+1<argc)vad_path=argv[++i];+ else if(!strcmp(argv[i],"--vad-maxseg")&&i+1<argc)vad_maxseg=atoi(argv[++i]);+ else {fprintf(stderr,"usage: %s -m sensevoice.gguf (-a audio.wav | -f fbank.bin) [--vad fsmn-vad.gguf [--vad-maxseg ms]]\n",argv[0]);return 1;} }if(gguf_path.empty()||(fbank_path.empty()&&wav_path.empty())){fprintf(stderr,"missing args\n");return 1;}// load model@@ -110,61 +114,61 @@ int main(int argc,char**argv){gguf_free(gg);const int F=560, D=m.c.d_model, V=m.c.vocab;- // fbank [T,F]: from wav (-a) or precomputed (-f)- int32_t T=0,Fc=F; std::vector<float> fb;- if(!wav_path.empty()){- std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}- int t=0; fb=compute_fbank(wav,t); T=t;- } else {- FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;}- if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;}- fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f);- }-// NOTE: SenseVoiceSmall inference() feeds the RAW log-mel fbank to the encoder;// it does NOT apply am.mvn CMVN (that path is unused at inference). Applying it// makes the encoder predict <|nospeech|>. So no CMVN here.-- // prepend nq query tokens (rows of embed.weight) -> input [nq+T, F]float*emb=(float*)m.g("embed.weight")->data; // [16, 560] row-major- int N=nq+T; std::vector<float> inp((size_t)N*F);- for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &emb[(size_t)qtok[i]*F], F*sizeof(float));- memcpy(&inp[(size_t)nq*F], fb.data(), (size_t)T*F*sizeof(float));+ // Run encoder+CTC on one fbank window [T,F]; prints greedy-CTC token IDs (no newline).+ auto run_seg=[&](const std::vector<float>& fb,int T){+ int N=nq+T; std::vector<float> inp((size_t)N*F);+ for(int i=0;i<nq;i++) memcpy(&inp[(size_t)i*F], &emb[(size_t)qtok[i]*F], F*sizeof(float));+ memcpy(&inp[(size_t)nq*F], fb.data(), (size_t)T*F*sizeof(float));+ float sc=sqrtf((float)D); for(auto&v:inp)v*=sc; add_posenc(inp,N,F);+ ggml_backend_t be=ggml_backend_cpu_init();+ ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp);+ ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,N); ggml_set_input(x);+ ggml_tensor*h=sanm_layer(c,m,"encoder.encoders0.0.",x,N,false);+ for(int i=0;i<m.c.num_blocks-1;i++) h=sanm_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,N,true);+ h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));+ for(int i=0;i<m.c.tp_blocks;i++) h=sanm_layer(c,m,"encoder.tp_encoders."+std::to_string(i)+".",h,N,true);+ h=lnorm(c,h,m.g("encoder.tp_norm.weight"),m.g("encoder.tp_norm.bias"));+ ggml_tensor*logits=lin(c,m.g("ctc.ctc_lo.weight"),m.g("ctc.ctc_lo.bias"),h); // [V, N]+ ggml_set_output(logits);+ ggml_cgraph*gf=ggml_new_graph_custom(c,32768,false); ggml_build_forward_expand(gf,logits);+ ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf);+ ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x)); ggml_backend_cpu_set_n_threads(be,8);+ if(ggml_backend_graph_compute(be,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");}+ std::vector<float> lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits));+ int prev=-1; // greedy CTC: argmax per frame -> collapse consecutive -> drop blank+ for(int n=0;n<N;n++){ const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];+ for(int v=1;v<V;v++) if(col[v]>best){best=col[v];am=v;}+ if(am!=prev && am!=m.c.blank) printf("%d ", am); prev=am; }+ ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be);+ };- // encoder pre-scale + posenc- float sc=sqrtf((float)D); for(auto&v:inp)v*=sc; add_posenc(inp,N,F);-- // build graph- ggml_backend_t be=ggml_backend_cpu_init();- ggml_init_params cp={(size_t)1024*1024*1024,nullptr,true}; ggml_context*c=ggml_init(cp);- ggml_tensor*x=ggml_new_tensor_2d(c,GGML_TYPE_F32,F,N); ggml_set_input(x);- ggml_tensor*h=sanm_layer(c,m,"encoder.encoders0.0.",x,N,false);- for(int i=0;i<m.c.num_blocks-1;i++) h=sanm_layer(c,m,"encoder.encoders."+std::to_string(i)+".",h,N,true);- h=lnorm(c,h,m.g("encoder.after_norm.weight"),m.g("encoder.after_norm.bias"));- for(int i=0;i<m.c.tp_blocks;i++) h=sanm_layer(c,m,"encoder.tp_encoders."+std::to_string(i)+".",h,N,true);- h=lnorm(c,h,m.g("encoder.tp_norm.weight"),m.g("encoder.tp_norm.bias"));- ggml_tensor*logits=lin(c,m.g("ctc.ctc_lo.weight"),m.g("ctc.ctc_lo.bias"),h); // [V, N]- ggml_set_output(logits);- ggml_cgraph*gf=ggml_new_graph_custom(c,32768,false); ggml_build_forward_expand(gf,logits);- ggml_gallocr_t ga=ggml_gallocr_new(ggml_backend_cpu_buffer_type()); ggml_gallocr_alloc_graph(ga,gf);- ggml_backend_tensor_set(x,inp.data(),0,ggml_nbytes(x));- ggml_backend_cpu_set_n_threads(be,8);int64_t t0=ggml_time_us();- if(ggml_backend_graph_compute(be,gf)!=GGML_STATUS_SUCCESS){fprintf(stderr,"compute failed\n");return 1;}- int64_t t1=ggml_time_us();-- std::vector<float> lg((size_t)V*N); ggml_backend_tensor_get(logits,lg.data(),0,ggml_nbytes(logits));- // greedy CTC: argmax per frame -> collapse consecutive -> drop blank- int prev=-1;- for(int n=0;n<N;n++){- const float*col=&lg[(size_t)n*V]; int am=0; float best=col[0];- for(int v=1;v<V;v++) if(col[v]>best){best=col[v];am=v;}- if(am!=prev && am!=m.c.blank) printf("%d ", am);- prev=am;+ if(!vad_path.empty()){+ std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}+ std::vector<std::pair<int,int>> segs;+ if(!funasr_vad_segments(vad_path,wav,vad_maxseg,segs)){fprintf(stderr,"vad failed\n");return 1;}+ for(auto&s:segs){ int off=(int)((int64_t)s.first*16000/1000), end=(int)((int64_t)s.second*16000/1000);+ if(end>(int)wav.size())end=wav.size(); if(end-off<WINLEN)continue;+ std::vector<float> seg(wav.begin()+off,wav.begin()+end); int t=0; auto fb=compute_fbank(seg,t); run_seg(fb,t); }+ fprintf(stderr,"[sensevoice] %zu vad segments\n",segs.size());+ } else {+ int32_t T=0,Fc=F; std::vector<float> fb;+ if(!wav_path.empty()){+ std::vector<float> wav; if(!funasr_load_audio_16k_mono(wav_path.c_str(),wav)){fprintf(stderr,"read audio failed\n");return 1;}+ int t=0; fb=compute_fbank(wav,t); T=t;+ } else {+ FILE*f=fopen(fbank_path.c_str(),"rb"); if(!f){fprintf(stderr,"open fbank\n");return 1;}+ if(fread(&T,4,1,f)!=1||fread(&Fc,4,1,f)!=1){fclose(f);return 1;}+ fb.resize((size_t)T*Fc); if((int)fread(fb.data(),4,fb.size(),f)!=(int)fb.size()){fclose(f);return 1;} fclose(f);+ }+ run_seg(fb,T);}printf("\n");- fprintf(stderr,"[sensevoice] N=%d (q%d+T%d) encode %.2fs\n",N,nq,T,(t1-t0)/1e6);- ggml_gallocr_free(ga); ggml_free(c); ggml_backend_free(be);+ fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6);if(m.ctx_w) ggml_free(m.ctx_w);return 0;}