Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix: detok review findings (null gguf_get_arr_str guard + utf-8 tokens.json read) (#3005)

zhifu gao9cc0ebcc66be
diff --git a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py
index 8497d503d..8a61aace2 100644
--- a/runtime/llama.cpp/paraformer/export_paraformer_gguf.py
+++ b/runtime/llama.cpp/paraformer/export_paraformer_gguf.py
@@ -32,7 +32,8 @@ def main():
import json, glob
tp = a.tokens or (glob.glob(os.path.join(os.path.dirname(a.model_pt), "tokens.json")) + [None])[0]
if tp and os.path.exists(tp):
- toks = json.load(open(tp)); w.add_array("pf.vocab", toks)
+ with open(tp, encoding="utf-8") as f: toks = json.load(f)
+ w.add_array("pf.vocab", toks)
print(f"embedded pf.vocab ({len(toks)} tokens) from {tp}")
else:
print("WARNING: tokens.json not found - gguf will have no vocab (binary falls back to ids)")
diff --git a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp
index 810d5d078..10c0c4d03 100644
--- a/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp
+++ b/runtime/llama.cpp/paraformer/funasr-paraformer/funasr-paraformer.cpp
@@ -141,7 +141,7 @@ int main(int argc,char**argv){
auto rdf=[&](const char*k,float d){int i=gguf_find_key(gg,k);return i<0?d:gguf_get_val_f32(gg,i);};
m.c.enc_blocks=rdi("pf.enc.num_blocks",50);m.c.dec_blocks=rdi("pf.dec.num_blocks",16);m.c.dec_att=rdi("pf.dec.att_layer_num",16);
m.c.dec3=rdi("pf.dec.decoders3",1);m.c.vocab=rdi("pf.vocab_size",8404);m.c.tail=rdf("pf.predictor.tail_threshold",0.45f);m.c.thresh=rdf("pf.predictor.threshold",1.0f);
- std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
+ std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"pf.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++){const char*s=gguf_get_arr_str(gg,ki,i); vocab[i]=s?s:"";}}}
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}gguf_free(gg);
const int D=m.c.d_model,F=560,V=m.c.vocab;
bool emit_ids = ids_mode || vocab.empty(); // fall back to ids if the gguf has no vocab
diff --git a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp
index 88e008f33..778eaee56 100644
--- a/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp
+++ b/runtime/llama.cpp/sensevoice/funasr-sensevoice/funasr-sensevoice.cpp
@@ -123,7 +123,7 @@ int main(int argc,char**argv){
m.c.kernel=rd("sv.kernel_size",11); m.c.vocab=rd("sv.vocab_size",25055); m.c.blank=rd("sv.blank_id",0);
int qi=gguf_find_key(gg,"sv.query_tokens"); int nq=qi<0?0:(int)gguf_get_arr_n(gg,qi);
std::vector<int> qtok(nq); for(int i=0;i<nq;i++) qtok[i]=((const int32_t*)gguf_get_arr_data(gg,qi))[i];
- std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++)vocab[i]=gguf_get_arr_str(gg,ki,i);}}
+ std::vector<std::string> vocab; {int ki=gguf_find_key(gg,"sv.vocab"); if(ki>=0){int nv=gguf_get_arr_n(gg,ki); vocab.resize(nv); for(int i=0;i<nv;i++){const char*s=gguf_get_arr_str(gg,ki,i); vocab[i]=s?s:"";}}}
for(int i=0;i<gguf_get_n_tensors(gg);i++){const char*nm=gguf_get_tensor_name(gg,i);m.t[nm]=ggml_get_tensor(m.ctx_w,nm);}
gguf_free(gg);
const int F=560, D=m.c.d_model, V=m.c.vocab;