Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
add sensevoice-small
雾聪aa72e0ca5f75
diff --git a/runtime/onnxruntime/include/com-define.h b/runtime/onnxruntime/include/com-define.hindex 5f71f7b3c..6b4c08f0d 100644--- a/runtime/onnxruntime/include/com-define.h+++ b/runtime/onnxruntime/include/com-define.h@@ -36,6 +36,9 @@ namespace funasr {#define HOTWORD_SEP " "#define AUDIO_FS "audio-fs"+#define MODEL_PARA "Paraformer"+#define MODEL_SVS "SenseVoiceSmall"+// #define VAD_MODEL_PATH "vad-model"// #define VAD_CMVN_PATH "vad-cmvn"// #define VAD_CONFIG_PATH "vad-config"diff --git a/runtime/onnxruntime/include/funasrruntime.h b/runtime/onnxruntime/include/funasrruntime.hindex ac9a3bd3a..5dedaf7ec 100644--- a/runtime/onnxruntime/include/funasrruntime.h+++ b/runtime/onnxruntime/include/funasrruntime.h@@ -101,7 +101,8 @@ _FUNASRAPI void FunOfflineReset(FUNASR_HANDLE handle, FUNASR_DEC_HANDLE// buffer_FUNASRAPI FUNASR_RESULT FunOfflineInferBuffer(FUNASR_HANDLE handle, const char* sz_buf, int n_len,FUNASR_MODE mode, QM_CALLBACK fn_callback, const std::vector<std::vector<float>> &hw_emb,- int sampling_rate=16000, std::string wav_format="pcm", bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr);+ int sampling_rate=16000, std::string wav_format="pcm", bool itn=true, FUNASR_DEC_HANDLE dec_handle=nullptr,+ std::string svs_lang="auto", bool svs_itn=true);// file, support wav & pcm_FUNASRAPI FUNASR_RESULT FunOfflineInfer(FUNASR_HANDLE handle, const char* sz_filename, FUNASR_MODE mode,QM_CALLBACK fn_callback, const std::vector<std::vector<float>> &hw_emb,diff --git a/runtime/onnxruntime/include/model.h b/runtime/onnxruntime/include/model.hindex 1064c4c0f..a49baebd0 100644--- a/runtime/onnxruntime/include/model.h+++ b/runtime/onnxruntime/include/model.h@@ -24,6 +24,8 @@ class Model {virtual std::string Forward(float *din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr){return "";};virtual std::vector<std::string> Forward(float** din, int* len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr, int batch_in=1){return std::vector<string>();};+ virtual std::vector<std::string> Forward(float** din, int* len, bool input_finished, std::string svs_lang="auto", bool svs_itn=false, int batch_in=1)+ {return std::vector<string>();};virtual std::string Rescoring() = 0;virtual void InitHwCompiler(const std::string &hw_model, int thread_num){};virtual void InitSegDict(const std::string &seg_dict_model){};diff --git a/runtime/onnxruntime/include/offline-stream.h b/runtime/onnxruntime/include/offline-stream.hindex cc0f1c43b..10bd6df70 100644--- a/runtime/onnxruntime/include/offline-stream.h+++ b/runtime/onnxruntime/include/offline-stream.h@@ -9,6 +9,7 @@#include "vad-model.h"#if !defined(__APPLE__)#include "itn-model.h"+#include "com-define.h"#endifnamespace funasr {@@ -26,11 +27,13 @@ class OfflineStream {bool UseVad(){return use_vad;};bool UsePunc(){return use_punc;};bool UseITN(){return use_itn;};+ std::string GetModelType(){return model_type;};private:bool use_vad=false;bool use_punc=false;bool use_itn=false;+ std::string model_type = MODEL_PARA;};OfflineStream *CreateOfflineStream(std::map<std::string, std::string>& model_path, int thread_num=1, bool use_gpu=false, int batch_size=1);diff --git a/runtime/onnxruntime/src/sensevoice-small.cpp b/runtime/onnxruntime/src/sensevoice-small.cppnew file mode 100644index 000000000..10eb907eb--- /dev/null+++ b/runtime/onnxruntime/src/sensevoice-small.cpp@@ -0,0 +1,377 @@+/**+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+ * MIT License (https://opensource.org/licenses/MIT)+*/++#include "precomp.h"+#include "sensevoice-small.h"+#include <cstddef>++using namespace std;+namespace funasr {++SenseVoiceSmall::SenseVoiceSmall()+:use_hotword(false),+ env_(ORT_LOGGING_LEVEL_ERROR, "sensevoice"),session_options_{} {+}++// offline+void SenseVoiceSmall::InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num){+ LoadConfigFromYaml(am_config.c_str());+ // knf options+ fbank_opts_.frame_opts.dither = 0;+ fbank_opts_.mel_opts.num_bins = n_mels;+ fbank_opts_.frame_opts.samp_freq = asr_sample_rate;+ fbank_opts_.frame_opts.window_type = window_type;+ fbank_opts_.frame_opts.frame_shift_ms = frame_shift;+ fbank_opts_.frame_opts.frame_length_ms = frame_length;+ fbank_opts_.energy_floor = 0;+ fbank_opts_.mel_opts.debug_mel = false;++ // session_options_.SetInterOpNumThreads(1);+ session_options_.SetIntraOpNumThreads(thread_num);+ session_options_.SetGraphOptimizationLevel(ORT_ENABLE_ALL);+ // DisableCpuMemArena can improve performance+ session_options_.DisableCpuMemArena();++ try {+ m_session_ = std::make_unique<Ort::Session>(env_, ORTSTRING(am_model).c_str(), session_options_);+ LOG(INFO) << "Successfully load model from " << am_model;+ } catch (std::exception const &e) {+ LOG(ERROR) << "Error when load am onnx model: " << e.what();+ exit(-1);+ }++ string strName;+ GetInputName(m_session_.get(), strName);+ m_strInputNames.push_back(strName.c_str());+ GetInputName(m_session_.get(), strName,1);+ m_strInputNames.push_back(strName);+ GetInputName(m_session_.get(), strName,2);+ m_strInputNames.push_back(strName);+ GetInputName(m_session_.get(), strName,3);+ m_strInputNames.push_back(strName);++ size_t numOutputNodes = m_session_->GetOutputCount();+ for(int index=0; index<numOutputNodes; index++){+ GetOutputName(m_session_.get(), strName, index);+ m_strOutputNames.push_back(strName);+ }++ for (auto& item : m_strInputNames)+ m_szInputNames.push_back(item.c_str());+ for (auto& item : m_strOutputNames)+ m_szOutputNames.push_back(item.c_str());+ vocab = new Vocab(token_file.c_str());+ LoadCmvn(am_cmvn.c_str());+}++void SenseVoiceSmall::LoadConfigFromYaml(const char* filename){++ YAML::Node config;+ try{+ config = YAML::LoadFile(filename);+ }catch(exception const &e){+ LOG(ERROR) << "Error loading file, yaml file error or not exist.";+ exit(-1);+ }++ try{+ YAML::Node frontend_conf = config["frontend_conf"];+ YAML::Node encoder_conf = config["encoder_conf"];++ this->window_type = frontend_conf["window"].as<string>();+ this->n_mels = frontend_conf["n_mels"].as<int>();+ this->frame_length = frontend_conf["frame_length"].as<int>();+ this->frame_shift = frontend_conf["frame_shift"].as<int>();+ this->lfr_m = frontend_conf["lfr_m"].as<int>();+ this->lfr_n = frontend_conf["lfr_n"].as<int>();++ this->encoder_size = encoder_conf["output_size"].as<int>();+ this->fsmn_dims = encoder_conf["output_size"].as<int>();++ this->asr_sample_rate = frontend_conf["fs"].as<int>();+ }catch(exception const &e){+ LOG(ERROR) << "Error when load argument from vad config YAML.";+ exit(-1);+ }+}++SenseVoiceSmall::~SenseVoiceSmall()+{+ if(vocab){+ delete vocab;+ }+ if(lm_vocab){+ delete lm_vocab;+ }+ if(seg_dict){+ delete seg_dict;+ }+ if(phone_set_){+ delete phone_set_;+ }+}++void SenseVoiceSmall::StartUtterance()+{+}++void SenseVoiceSmall::EndUtterance()+{+}++void SenseVoiceSmall::Reset()+{+}++void SenseVoiceSmall::FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats) {+ knf::OnlineFbank fbank_(fbank_opts_);+ std::vector<float> buf(len);+ for (int32_t i = 0; i != len; ++i) {+ buf[i] = waves[i] * 32768;+ }+ fbank_.AcceptWaveform(sample_rate, buf.data(), buf.size());++ int32_t frames = fbank_.NumFramesReady();+ for (int32_t i = 0; i != frames; ++i) {+ const float *frame = fbank_.GetFrame(i);+ std::vector<float> frame_vector(frame, frame + fbank_opts_.mel_opts.num_bins);+ asr_feats.emplace_back(frame_vector);+ }+}++void SenseVoiceSmall::LoadCmvn(const char *filename)+{+ ifstream cmvn_stream(filename);+ if (!cmvn_stream.is_open()) {+ LOG(ERROR) << "Failed to open file: " << filename;+ exit(-1);+ }+ string line;++ while (getline(cmvn_stream, line)) {+ istringstream iss(line);+ vector<string> line_item{istream_iterator<string>{iss}, istream_iterator<string>{}};+ if (line_item[0] == "<AddShift>") {+ getline(cmvn_stream, line);+ istringstream means_lines_stream(line);+ vector<string> means_lines{istream_iterator<string>{means_lines_stream}, istream_iterator<string>{}};+ if (means_lines[0] == "<LearnRateCoef>") {+ for (int j = 3; j < means_lines.size() - 1; j++) {+ means_list_.push_back(stof(means_lines[j]));+ }+ continue;+ }+ }+ else if (line_item[0] == "<Rescale>") {+ getline(cmvn_stream, line);+ istringstream vars_lines_stream(line);+ vector<string> vars_lines{istream_iterator<string>{vars_lines_stream}, istream_iterator<string>{}};+ if (vars_lines[0] == "<LearnRateCoef>") {+ for (int j = 3; j < vars_lines.size() - 1; j++) {+ vars_list_.push_back(stof(vars_lines[j])*scale);+ }+ continue;+ }+ }+ }+}++string SenseVoiceSmall::CTCSearch(float * in, std::vector<int32_t> paraformer_length, std::vector<int64_t> outputShape)+{+ std::string unicodeChar = "▁";+ int32_t vocab_size = outputShape[2];++ std::vector<int64_t> tokens;+ std::string text="";+ int32_t prev_id = -1;+ for (int32_t t = 0; t != paraformer_length[0]; ++t) {+ auto y = std::distance(+ static_cast<const float *>(in),+ std::max_element(+ static_cast<const float *>(in),+ static_cast<const float *>(in) + vocab_size));+ in += vocab_size;++ if (y != blank_id && y != prev_id) {+ tokens.push_back(y);+ }+ prev_id = y;+ }+ string str_lang = "";+ string str_emo = "";+ string str_event = "";+ string str_itn = "";+ if(tokens.size() >=3){+ str_lang = vocab->Id2String(tokens[0]);+ str_emo = vocab->Id2String(tokens[1]);+ str_event = vocab->Id2String(tokens[2]);+ str_itn = vocab->Id2String(tokens[3]);+ }++ for(int32_t i = 4; i < tokens.size(); ++i){+ string word = vocab->Id2String(tokens[i]);+ size_t found = word.find(unicodeChar);+ if(found != std::string::npos){+ text += " " + word.substr(3);+ }else{+ text += word;+ }+ }+ if(str_itn == "<|withitn|>"){+ if(str_lang == "<|zh|>"){+ text += "。";+ }else{+ text += ".";+ }+ }++ return str_lang + str_emo + str_event + " " + text;+}++void SenseVoiceSmall::LfrCmvn(std::vector<std::vector<float>> &asr_feats) {++ std::vector<std::vector<float>> out_feats;+ int T = asr_feats.size();+ int T_lrf = ceil(1.0 * T / lfr_n);++ // Pad frames at start(copy first frame)+ for (int i = 0; i < (lfr_m - 1) / 2; i++) {+ asr_feats.insert(asr_feats.begin(), asr_feats[0]);+ }+ // Merge lfr_m frames as one,lfr_n frames per window+ T = T + (lfr_m - 1) / 2;+ std::vector<float> p;+ for (int i = 0; i < T_lrf; i++) {+ if (lfr_m <= T - i * lfr_n) {+ for (int j = 0; j < lfr_m; j++) {+ p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());+ }+ out_feats.emplace_back(p);+ p.clear();+ } else {+ // Fill to lfr_m frames at last window if less than lfr_m frames (copy last frame)+ int num_padding = lfr_m - (T - i * lfr_n);+ for (int j = 0; j < (asr_feats.size() - i * lfr_n); j++) {+ p.insert(p.end(), asr_feats[i * lfr_n + j].begin(), asr_feats[i * lfr_n + j].end());+ }+ for (int j = 0; j < num_padding; j++) {+ p.insert(p.end(), asr_feats[asr_feats.size() - 1].begin(), asr_feats[asr_feats.size() - 1].end());+ }+ out_feats.emplace_back(p);+ p.clear();+ }+ }+ // Apply cmvn+ for (auto &out_feat: out_feats) {+ for (int j = 0; j < means_list_.size(); j++) {+ out_feat[j] = (out_feat[j] + means_list_[j]) * vars_list_[j];+ }+ }+ asr_feats = out_feats;+}++std::vector<std::vector<float>> SenseVoiceSmall::CompileHotwordEmbedding(std::string &hotwords) {+ int embedding_dim = encoder_size;+ std::vector<std::vector<float>> hw_emb;+ std::vector<float> vec(embedding_dim, 0);+ hw_emb.push_back(vec);+ return hw_emb;+}++std::vector<std::string> SenseVoiceSmall::Forward(float** din, int* len, bool input_finished, std::string svs_lang, bool svs_itn, int batch_in)+{+ std::vector<std::string> results;+ string result="";+ int32_t in_feat_dim = fbank_opts_.mel_opts.num_bins;++ if(batch_in != 1){+ results.push_back(result);+ return results;+ }++ std::vector<std::vector<float>> asr_feats;+ FbankKaldi(asr_sample_rate, din[0], len[0], asr_feats);+ if(asr_feats.size() == 0){+ results.push_back(result);+ return results;+ }+ LfrCmvn(asr_feats);+ int32_t feat_dim = lfr_m*in_feat_dim;+ int32_t num_frames = asr_feats.size();++ std::vector<float> wav_feats;+ for (const auto &frame_feat: asr_feats) {+ wav_feats.insert(wav_feats.end(), frame_feat.begin(), frame_feat.end());+ }++ //lid textnorm+ int svs_lid = 0;+ int svs_itnid = 15;+ if(lid_map.find(svs_lang) != lid_map.end()){+ svs_lid = lid_map[svs_lang];+ }+ if(svs_itn){+ svs_itnid = 14;+ }++#ifdef _WIN_X86+ Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);+#else+ Ort::MemoryInfo m_memoryInfo = Ort::MemoryInfo::CreateCpu(OrtArenaAllocator, OrtMemTypeDefault);+#endif++ const int64_t input_shape_[3] = {1, num_frames, feat_dim};+ Ort::Value onnx_feats = Ort::Value::CreateTensor<float>(m_memoryInfo,+ wav_feats.data(),+ wav_feats.size(),+ input_shape_,+ 3);++ const int64_t paraformer_length_shape[1] = {1};+ std::vector<int32_t> paraformer_length;+ paraformer_length.emplace_back(num_frames);+ Ort::Value onnx_feats_len = Ort::Value::CreateTensor<int32_t>(+ m_memoryInfo, paraformer_length.data(), paraformer_length.size(), paraformer_length_shape, 1);++ const int64_t lid_shape[1] = {1};+ std::vector<int32_t> lid_length;+ lid_length.emplace_back(svs_lid);+ Ort::Value onnx_lid = Ort::Value::CreateTensor<int32_t>(+ m_memoryInfo, lid_length.data(), lid_length.size(), lid_shape, 1);++ const int64_t textnorm_shape[1] = {1};+ std::vector<int32_t> textnorm_length;+ textnorm_length.emplace_back(svs_itnid);+ Ort::Value onnx_itn = Ort::Value::CreateTensor<int32_t>(+ m_memoryInfo, textnorm_length.data(), textnorm_length.size(), textnorm_shape, 1);++ std::vector<Ort::Value> input_onnx;+ input_onnx.emplace_back(std::move(onnx_feats));+ input_onnx.emplace_back(std::move(onnx_feats_len));+ input_onnx.emplace_back(std::move(onnx_lid));+ input_onnx.emplace_back(std::move(onnx_itn));++ try {+ auto outputTensor = m_session_->Run(Ort::RunOptions{nullptr}, m_szInputNames.data(), input_onnx.data(), input_onnx.size(), m_szOutputNames.data(), m_szOutputNames.size());+ float* floatData = outputTensor[0].GetTensorMutableData<float>();+ std::vector<int64_t> outputShape = outputTensor[0].GetTensorTypeAndShapeInfo().GetShape();++ result = CTCSearch(floatData, paraformer_length, outputShape);+ }+ catch (std::exception const &e)+ {+ LOG(ERROR)<<e.what();+ }++ results.push_back(result);+ return results;+}++string SenseVoiceSmall::Rescoring()+{+ LOG(ERROR)<<"Not Imp!!!!!!";+ return "";+}+} // namespace funasrdiff --git a/runtime/onnxruntime/src/sensevoice-small.h b/runtime/onnxruntime/src/sensevoice-small.hnew file mode 100644index 000000000..f987f3805--- /dev/null+++ b/runtime/onnxruntime/src/sensevoice-small.h@@ -0,0 +1,116 @@+/**+ * Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.+ * MIT License (https://opensource.org/licenses/MIT)+*/+#pragma once++#include "precomp.h"+#include "phone-set.h"++namespace funasr {++ class SenseVoiceSmall : public Model {+ private:+ Vocab* vocab = nullptr;+ Vocab* lm_vocab = nullptr;+ SegDict* seg_dict = nullptr;+ PhoneSet* phone_set_ = nullptr;+ const float scale = 1.0;++ void LoadConfigFromYaml(const char* filename);+ void LoadCmvn(const char *filename);+ void LfrCmvn(std::vector<std::vector<float>> &asr_feats);++ std::shared_ptr<Ort::Session> hw_m_session = nullptr;+ Ort::Env hw_env_;+ Ort::SessionOptions hw_session_options;+ vector<string> hw_m_strInputNames, hw_m_strOutputNames;+ vector<const char*> hw_m_szInputNames;+ vector<const char*> hw_m_szOutputNames;+ bool use_hotword;++ public:+ SenseVoiceSmall();+ ~SenseVoiceSmall();+ void InitAsr(const std::string &am_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);+ // online+ // void InitAsr(const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);+ // 2pass+ // void InitAsr(const std::string &am_model, const std::string &en_model, const std::string &de_model, const std::string &am_cmvn, const std::string &am_config, const std::string &token_file, int thread_num);+ // void InitHwCompiler(const std::string &hw_model, int thread_num);+ // void InitSegDict(const std::string &seg_dict_model);+ std::vector<std::vector<float>> CompileHotwordEmbedding(std::string &hotwords);+ void Reset();+ void FbankKaldi(float sample_rate, const float* waves, int len, std::vector<std::vector<float>> &asr_feats);+ std::vector<std::string> Forward(float** din, int* len, bool input_finished=true, std::string svs_lang="auto", bool svs_itn=true, int batch_in=1);+ string CTCSearch( float * in, std::vector<int32_t> paraformer_length, std::vector<int64_t> outputShape);++ string Rescoring();+ string GetLang(){return language;};+ int GetAsrSampleRate() { return asr_sample_rate; };+ int GetBatchSize() {return batch_size_;};+ void StartUtterance();+ void EndUtterance();+ // void InitLm(const std::string &lm_file, const std::string &lm_cfg_file, const std::string &lex_file);+ // string BeamSearch(WfstDecoder* &wfst_decoder, float* in, int n_len, int64_t token_nums);+ // string FinalizeDecode(WfstDecoder* &wfst_decoder,+ // bool is_stamp=false, std::vector<float> us_alphas={0}, std::vector<float> us_cif_peak={0});+ // Vocab* GetVocab();+ // Vocab* GetLmVocab();+ // PhoneSet* GetPhoneSet();++ knf::FbankOptions fbank_opts_;+ vector<float> means_list_;+ vector<float> vars_list_;+ int lfr_m = PARA_LFR_M;+ int lfr_n = PARA_LFR_N;++ // paraformer-offline+ std::shared_ptr<Ort::Session> m_session_ = nullptr;+ Ort::Env env_;+ Ort::SessionOptions session_options_;++ vector<string> m_strInputNames, m_strOutputNames;+ vector<const char*> m_szInputNames;+ vector<const char*> m_szOutputNames;++ std::string language="zh-cn";++ // paraformer-online+ std::shared_ptr<Ort::Session> encoder_session_ = nullptr;+ std::shared_ptr<Ort::Session> decoder_session_ = nullptr;+ vector<string> en_strInputNames, en_strOutputNames;+ vector<const char*> en_szInputNames_;+ vector<const char*> en_szOutputNames_;+ vector<string> de_strInputNames, de_strOutputNames;+ vector<const char*> de_szInputNames_;+ vector<const char*> de_szOutputNames_;++ // lm+ std::shared_ptr<fst::Fst<fst::StdArc>> lm_ = nullptr;++ string window_type = "hamming";+ int frame_length = 25;+ int frame_shift = 10;+ int n_mels = 80;+ int encoder_size = 512;+ int fsmn_layers = 16;+ int fsmn_lorder = 10;+ int fsmn_dims = 512;+ int asr_sample_rate = MODEL_SAMPLE_RATE;+ int batch_size_ = 1;+ int blank_id = 0;+ //dict+ std::map<std::string, int> lid_map = {+ {"auto", 0},+ {"zh", 3},+ {"en", 4},+ {"yue", 7},+ {"ja", 11},+ {"ko", 12},+ {"nospeech", 13}+ };++ };++} // namespace funasr