Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(runtime): upload SenseVoice weights to backend buffer
LauraGPT949ccf46f32c
diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cppindex 11a808684..1e69bfdb0 100644--- a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp+++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp@@ -16,6 +16,7 @@#include <cstdint>#include <cstdio>#include <cstring>+#include <fstream>#include <map>#include <string>#include <vector>@@ -69,7 +70,8 @@ static std::vector<float> compute_fbank(std::vector<float> wav,int&T_out){}struct cfg { int d_model=512,n_head=4,num_blocks=50,tp_blocks=20,kernel=11,vocab=25055,blank=0; };-struct model { cfg c; ggml_context*ctx_w=nullptr; std::map<std::string,ggml_tensor*> t;+struct model { cfg c; gguf_context*gguf=nullptr; ggml_context*ctx_meta=nullptr; ggml_context*ctx_w=nullptr;+ ggml_backend_buffer_t weights_buffer=nullptr; std::map<std::string,ggml_tensor*> t;ggml_tensor* g(const std::string&n){auto it=t.find(n);if(it==t.end()){fprintf(stderr,"missing %s\n",n.c_str());exit(1);}return it->second;} };struct graph_backend {@@ -163,6 +165,52 @@ static graph_backend make_graph_backend(const std::string&name){return out;}+static void free_model(model&m){+ if(m.gguf){gguf_free(m.gguf);m.gguf=nullptr;}+ if(m.ctx_meta){ggml_free(m.ctx_meta);m.ctx_meta=nullptr;}+ if(m.weights_buffer){ggml_backend_buffer_free(m.weights_buffer);m.weights_buffer=nullptr;}+ if(m.ctx_w){ggml_free(m.ctx_w);m.ctx_w=nullptr;}+ m.t.clear();+}++static bool load_model_weights(const std::string&path,ggml_backend_buffer_type_t buffer_type,model&m){+ gguf_init_params gp={true,&m.ctx_meta};+ m.gguf=gguf_init_from_file(path.c_str(),gp);+ if(!m.gguf){fprintf(stderr,"load gguf failed\n");return false;}+ const int64_t n_tensors=gguf_get_n_tensors(m.gguf);+ ggml_init_params wp={(size_t)(n_tensors+1)*ggml_tensor_overhead(),nullptr,true};+ m.ctx_w=ggml_init(wp);+ if(!m.ctx_w){fprintf(stderr,"failed to initialize model tensor context\n");free_model(m);return false;}+ for(int64_t i=0;i<n_tensors;i++){+ const char*name=gguf_get_tensor_name(m.gguf,i);+ ggml_tensor*meta=ggml_get_tensor(m.ctx_meta,name);+ if(!meta){fprintf(stderr,"missing model tensor metadata: %s\n",name);free_model(m);return false;}+ ggml_tensor*weight=ggml_dup_tensor(m.ctx_w,meta);+ ggml_set_name(weight,name);+ m.t[name]=weight;+ }+ m.weights_buffer=ggml_backend_alloc_ctx_tensors_from_buft(m.ctx_w,buffer_type);+ if(!m.weights_buffer){fprintf(stderr,"failed to allocate model weights buffer\n");free_model(m);return false;}+ ggml_backend_buffer_set_usage(m.weights_buffer,GGML_BACKEND_BUFFER_USAGE_WEIGHTS);++ std::ifstream fin(path,std::ios::binary);+ if(!fin){fprintf(stderr,"failed to reopen model weights: %s\n",path.c_str());free_model(m);return false;}+ std::vector<uint8_t> read_buffer;+ const size_t data_offset=gguf_get_data_offset(m.gguf);+ for(int64_t i=0;i<n_tensors;i++){+ const char*name=gguf_get_tensor_name(m.gguf,i);+ ggml_tensor*weight=m.t[name];+ const size_t size=ggml_nbytes(weight);+ const size_t offset=data_offset+gguf_get_tensor_offset(m.gguf,i);+ read_buffer.resize(size);+ fin.seekg((std::streamoff)offset,std::ios::beg);+ fin.read((char*)read_buffer.data(),(std::streamsize)size);+ if(!fin){fprintf(stderr,"failed to read model tensor: %s\n",name);free_model(m);return false;}+ ggml_backend_tensor_set(weight,read_buffer.data(),0,size);+ }+ return true;+}+static ggml_tensor* lin(ggml_context*c,ggml_tensor*w,ggml_tensor*b,ggml_tensor*x){auto y=ggml_mul_mat(c,w,x);return b?ggml_add(c,y,b):y;}static ggml_tensor* lnorm(ggml_context*c,ggml_tensor*x,ggml_tensor*g,ggml_tensor*b){return ggml_add(c,ggml_mul(c,ggml_norm(c,x,LN_EPS),g),b);}static ggml_tensor* sanm_attn(ggml_context*c,model&m,const std::string&p,ggml_tensor*x,int T){@@ -223,8 +271,7 @@ int main(int argc,char**argv){// load modeltrace_stage("[sensevoice] loading model metadata");- model m; gguf_init_params gp={false,&m.ctx_w}; gguf_context*gg=gguf_init_from_file(gguf_path.c_str(),gp);- if(!gg){fprintf(stderr,"load gguf failed\n");return 1;}+ model m; if(!load_model_weights(gguf_path,graph_be.buffer_type,m))return 1; gguf_context*gg=m.gguf;auto rd=[&](const char*k,int d){int i=gguf_find_key(gg,k);return i<0?d:(int)gguf_get_val_u32(gg,i);};m.c.d_model=rd("sv.output_size",512); m.c.n_head=rd("sv.attention_heads",4);m.c.num_blocks=rd("sv.num_blocks",50); m.c.tp_blocks=rd("sv.tp_blocks",20);@@ -232,9 +279,8 @@ int main(int argc,char**argv){int qi=gguf_find_key(gg,"sv.query_tokens"); int nq=qi<0?0:(int)gguf_get_arr_n(gg,qi);std::vector<int> qtok(nq); for(int i=0;i<nq;i++) qtok[i]=((const int32_t*)gguf_get_arr_data(gg,qi))[i];std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++){const char*s=gguf_get_arr_str(gg,ki,i); vocab[i]=s?s:"";}}}- for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}trace_stage("[sensevoice] model ready: %d tensors",gguf_get_n_tensors(gg));- gguf_free(gg);+ gguf_free(m.gguf); m.gguf=nullptr; ggml_free(m.ctx_meta); m.ctx_meta=nullptr;const int F=560, D=m.c.d_model, V=m.c.vocab;bool emit_ids = ids_mode || vocab.empty(); // fall back to ids if the gguf has no vocab@@ -246,10 +292,11 @@ int main(int argc,char**argv){for(int id:qtok) if(id<0||id>=embed->ne[1]){fprintf(stderr,"query token out of embed range: %d\n",id);return 1;}std::vector<float> embed_f32((size_t)ggml_nelements(embed));if(embed->type==GGML_TYPE_F32){- memcpy(embed_f32.data(),embed->data,embed_f32.size()*sizeof(float));+ ggml_backend_tensor_get(embed,embed_f32.data(),0,ggml_nbytes(embed));} else if(embed->type==GGML_TYPE_F16){- const ggml_fp16_t*src=(const ggml_fp16_t*)embed->data;- for(size_t i=0;i<embed_f32.size();i++)embed_f32[i]=ggml_fp16_to_fp32(src[i]);+ std::vector<ggml_fp16_t> embed_f16(embed_f32.size());+ ggml_backend_tensor_get(embed,embed_f16.data(),0,ggml_nbytes(embed));+ for(size_t i=0;i<embed_f32.size();i++)embed_f32[i]=ggml_fp16_to_fp32(embed_f16[i]);} else {fprintf(stderr,"unsupported embed type: %s\n",ggml_type_name(embed->type));return 1;}@@ -332,7 +379,7 @@ int main(int argc,char**argv){}if(!srt_mode) printf("\n");fprintf(stderr,"[sensevoice] done %.2fs\n",(ggml_time_us()-t0)/1e6);+ free_model(m);ggml_backend_free(graph_be.backend);- if(m.ctx_w) ggml_free(m.ctx_w);return 0;}diff --git a/runtime/llama.cpp/tests/test_backend_flags.py b/runtime/llama.cpp/tests/test_backend_flags.pyindex 1cf383a4c..63dd4ec07 100644--- a/runtime/llama.cpp/tests/test_backend_flags.py+++ b/runtime/llama.cpp/tests/test_backend_flags.py@@ -19,6 +19,8 @@ def test_sensevoice_reads_query_embeddings_using_their_ggml_type():assert "embed->type==GGML_TYPE_F32" in sourceassert "embed->type==GGML_TYPE_F16" in sourceassert "ggml_fp16_to_fp32" in source+ assert source.count("ggml_backend_tensor_get(embed") == 2+ assert "embed->data" not in sourceassert 'float*emb=(float*)m.g("embed.weight")->data' not in source@@ -120,3 +122,25 @@ def test_sensevoice_traces_post_backend_pipeline_boundaries():("compute starting", "compute complete"),):assert source.index(before) < source.index(after)+++def test_sensevoice_uploads_model_weights_to_selected_backend_before_compute():+ source = SENSEVOICE.read_text(encoding="utf-8")++ assert "static bool load_model_weights(" in source+ loader = source.split("static bool load_model_weights(", maxsplit=1)[1].split(+ "static ggml_tensor*", maxsplit=1+ )[0]++ assert "gguf_init_from_file" in loader+ assert "ggml_dup_tensor" in loader+ assert "ggml_backend_alloc_ctx_tensors_from_buft" in loader+ assert "GGML_BACKEND_BUFFER_USAGE_WEIGHTS" in loader+ assert "gguf_get_data_offset" in loader+ assert "ggml_backend_tensor_set" in loader++ model_load = source.split("// load model", maxsplit=1)[1].split(+ "auto run_seg=", maxsplit=1+ )[0]+ assert "load_model_weights(gguf_path,graph_be.buffer_type,m)" in model_load+ assert "gguf_init_params gp={false,&m.ctx_w}" not in model_load