Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
feat(runtime): add sentence-level timestamps to WebSocket 2pass server (#2969)
psk-githuba3fb72e9cbdb
diff --git a/runtime/onnxruntime/include/audio.h b/runtime/onnxruntime/include/audio.hindex 3011050ed..b14d3b85d 100644--- a/runtime/onnxruntime/include/audio.h+++ b/runtime/onnxruntime/include/audio.h@@ -100,6 +100,8 @@ class DLLAPI Audio {int offset = 0;int speech_start=-1, speech_end=0;int speech_offline_start=-1;+ int64_t start = 0;+ int64_t end = 0;int seg_sample = MODEL_SAMPLE_RATE/1000;bool LoadPcmwavOnline(const char* buf, int n_file_len, int32_t* sampling_rate);diff --git a/runtime/onnxruntime/include/funasrruntime.h b/runtime/onnxruntime/include/funasrruntime.hindex 685c0241f..1a3cff607 100644--- a/runtime/onnxruntime/include/funasrruntime.h+++ b/runtime/onnxruntime/include/funasrruntime.h@@ -70,6 +70,8 @@ _FUNASRAPI const char* FunASRGetResult(FUNASR_RESULT result,int n_index);_FUNASRAPI const char* FunASRGetStamp(FUNASR_RESULT result);_FUNASRAPI const char* FunASRGetStampSents(FUNASR_RESULT result);_FUNASRAPI const char* FunASRGetTpassResult(FUNASR_RESULT result,int n_index);+_FUNASRAPI const int64_t FunASRGetTpassStart(FUNASR_RESULT result);+_FUNASRAPI const int64_t FunASRGetTpassEnd(FUNASR_RESULT result);_FUNASRAPI const int FunASRGetRetNumber(FUNASR_RESULT result);_FUNASRAPI void FunASRFreeResult(FUNASR_RESULT result);_FUNASRAPI void FunASRUninit(FUNASR_HANDLE handle);diff --git a/runtime/onnxruntime/src/audio.cpp b/runtime/onnxruntime/src/audio.cppindex 98899170a..ba6045f8a 100644--- a/runtime/onnxruntime/src/audio.cpp+++ b/runtime/onnxruntime/src/audio.cpp@@ -1291,6 +1291,10 @@ void Audio::Split(VadModel* vad_obj, int chunk_len, bool input_finished, ASR_TYP}}}else{++ int sample_rate = 16000; // sample_rate 是音频的采样率 这里固定为16000 Hz+ float segment_duration = (static_cast<float>(seg_sample) / sample_rate) * 1000; // 每个分段的持续时间(毫秒)+for(auto vad_segment: vad_segments){int speech_start_i=-1, speech_end_i=-1;if(vad_segment[0] != -1){@@ -1327,6 +1331,12 @@ void Audio::Split(VadModel* vad_obj, int chunk_len, bool input_finished, ASR_TYPframe = nullptr;}+ //设置开始时间和结束时间+ float start_time = speech_start_i * segment_duration; // 开始时间(毫秒)+ float end_time = speech_end_i * segment_duration; // 结束时间(毫秒)+ // 转换为 int64_t 类型并赋值给类的成员变量+ this->start = static_cast<int64_t>(start_time);+ this->end = static_cast<int64_t>(end_time);speech_start = -1;speech_offline_start = -1;// [70, -1]@@ -1352,6 +1362,8 @@ void Audio::Split(VadModel* vad_obj, int chunk_len, bool input_finished, ASR_TYP}}+ float start_time = speech_start_i * segment_duration; // 仅有开始时间+ this->start = static_cast<int64_t>(start_time);}else if(speech_end_i != -1){ // [-1,100]if(speech_start == -1 || speech_offline_start == -1){LOG(ERROR) <<"Vad start is null while vad end is available. Set vad start 0" ;@@ -1401,6 +1413,8 @@ void Audio::Split(VadModel* vad_obj, int chunk_len, bool input_finished, ASR_TYPframe = nullptr;}}+ float end_time = speech_end_i * segment_duration; // 仅有结束时间+ this->end = static_cast<int64_t>(end_time);speech_start = -1;speech_offline_start = -1;}diff --git a/runtime/onnxruntime/src/commonfunc.h b/runtime/onnxruntime/src/commonfunc.hindex 6fd553fe0..81fa2422e 100644--- a/runtime/onnxruntime/src/commonfunc.h+++ b/runtime/onnxruntime/src/commonfunc.h@@ -12,6 +12,8 @@ typedef structstd::string stamp_sents;std::string tpass_msg;float snippet_time;+ int64_t start = 0;+ int64_t end = 0;}FUNASR_RECOG_RESULT;typedef structdiff --git a/runtime/onnxruntime/src/funasrruntime.cpp b/runtime/onnxruntime/src/funasrruntime.cppindex 580c93422..fbcc34bd4 100644--- a/runtime/onnxruntime/src/funasrruntime.cpp+++ b/runtime/onnxruntime/src/funasrruntime.cpp@@ -523,6 +523,8 @@p_result->snippet_time = audio->GetTimeLen();audio->Split(vad_online_handle, chunk_len, input_finished, mode);+ p_result->start = audio->start;+ p_result->end = audio->end;funasr::AudioFrame* frame = nullptr;while(audio->FetchChunck(frame) > 0){@@ -695,6 +697,23 @@return p_result->tpass_msg.c_str();}+ _FUNASRAPI const int64_t FunASRGetTpassStart(FUNASR_RESULT result)+ {+ funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;+ if(!p_result)+ return 0;++ return p_result->start;+ }+ _FUNASRAPI const int64_t FunASRGetTpassEnd(FUNASR_RESULT result)+ {+ funasr::FUNASR_RECOG_RESULT * p_result = (funasr::FUNASR_RECOG_RESULT*)result;+ if(!p_result)+ return 0;++ return p_result->end;+ }+_FUNASRAPI const char* CTTransformerGetResult(FUNASR_RESULT result,int n_index){funasr::FUNASR_PUNC_RESULT * p_result = (funasr::FUNASR_PUNC_RESULT*)result;diff --git a/runtime/websocket/bin/websocket-server-2pass.cpp b/runtime/websocket/bin/websocket-server-2pass.cppindex ff23e9d41..45c016c19 100644--- a/runtime/websocket/bin/websocket-server-2pass.cpp+++ b/runtime/websocket/bin/websocket-server-2pass.cpp@@ -15,11 +15,19 @@#include <thread>#include <utility>#include <vector>+#include <iostream>+#include <chrono>extern std::unordered_map<std::string, int> hws_map_;extern int fst_inc_wts_;extern float global_beam_, lattice_beam_, am_scale_;+int64_t getCurrentTimeMillis() {+ auto now = std::chrono::system_clock::now();+ auto millis = std::chrono::duration_cast<std::chrono::milliseconds>(now.time_since_epoch()).count();+ return millis;+}+context_ptr WebSocketServer::on_tls_init(tls_mode mode,websocketpp::connection_hdl hdl,std::string& s_certfile,@@ -57,22 +65,50 @@ context_ptr WebSocketServer::on_tls_init(tls_mode mode,return ctx;}-nlohmann::json handle_result(FUNASR_RESULT result) {+nlohmann::json handle_result(FUNASR_RESULT result, websocketpp::connection_hdl& hdl, std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,std::owner_less<websocketpp::connection_hdl>>& data_map) {+ std::shared_ptr<FUNASR_MESSAGE> data_msg = nullptr;+ auto it = data_map.find(hdl);+ if (it != data_map.end()) {+ data_msg = it->second;+ }+websocketpp::lib::error_code ec;nlohmann::json jsonresult;jsonresult["text"] = "";std::string tmp_online_msg = FunASRGetResult(result, 0);if (tmp_online_msg != "") {- LOG(INFO) << "online_res :" << tmp_online_msg;+ LOG(INFO) << "wav_name: " << data_msg->msg["wav_name"].get<std::string>() << " | online_res :" << tmp_online_msg;jsonresult["text"] = tmp_online_msg;jsonresult["mode"] = "2pass-online";+ jsonresult["slice_type"] = 1;+ jsonresult["index"] = data_msg->index;++ // 如果是第一句话的第一个实时结果或新的句子开始+ if (!data_msg->is_sentence_started) {+ data_msg->start_time = FunASRGetTpassStart(result); // 记录句子的开始时间+ jsonresult["slice_type"] = 0; //0:一段话开始识别; 1:一段话识别中; 2:一段话识别结束+ data_msg->is_sentence_started = true;+ }}++ data_msg->end_time = FunASRGetTpassEnd(result); // 记录句子的结束时间+ jsonresult["timestamp"] = data_msg->timestamp;+std::string tmp_tpass_msg = FunASRGetTpassResult(result, 0);if (tmp_tpass_msg != "") {- LOG(INFO) << "offline results : " << tmp_tpass_msg;+ LOG(INFO) << "wav_name: " << data_msg->msg["wav_name"].get<std::string>() << " | offline results : " << tmp_tpass_msg;jsonresult["text"] = tmp_tpass_msg;jsonresult["mode"] = "2pass-offline";++ // 句子结束,记录结束时间+ jsonresult["start_time"] = data_msg->start_time;+ jsonresult["end_time"] = data_msg->end_time;+ jsonresult["slice_type"] = 2;+ jsonresult["index"] = data_msg->index;++ data_msg->index++; //句子序号+ data_msg->is_sentence_started = false; // 重置句子状态}std::string tmp_stamp_msg = FunASRGetStamp(result);@@ -98,6 +134,7 @@ nlohmann::json handle_result(FUNASR_RESULT result) {}// feed buffer to asr engine for decodervoid WebSocketServer::do_decoder(+ std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,std::owner_less<websocketpp::connection_hdl>>& data_map,std::vector<char>& buffer,websocketpp::connection_hdl& hdl,nlohmann::json& msg,@@ -158,10 +195,11 @@ void WebSocketServer::do_decoder(}if (Result) {websocketpp::lib::error_code ec;- nlohmann::json jsonresult = handle_result(Result);+ nlohmann::json jsonresult = handle_result(Result, hdl, data_map);jsonresult["wav_name"] = wav_name;jsonresult["is_final"] = false;if (jsonresult["text"] != "") {+ //LOG(INFO) << "jsonresult: " << jsonresult.dump(4);if (is_ssl) {wss_server_->send(hdl, jsonresult.dump(),websocketpp::frame::opcode::text, ec);@@ -200,9 +238,10 @@ void WebSocketServer::do_decoder(}if (Result) {websocketpp::lib::error_code ec;- nlohmann::json jsonresult = handle_result(Result);+ nlohmann::json jsonresult = handle_result(Result, hdl, data_map);jsonresult["wav_name"] = wav_name;jsonresult["is_final"] = true;+ //LOG(INFO) << "jsonresult: " << jsonresult.dump(4);if (is_ssl) {wss_server_->send(hdl, jsonresult.dump(),websocketpp::frame::opcode::text, ec);@@ -263,6 +302,10 @@ void WebSocketServer::on_open(websocketpp::connection_hdl hdl) {std::make_shared<std::vector<std::vector<std::string>>>(2);data_msg->strand_ = std::make_shared<asio::io_context::strand>(io_decoder_);+ data_msg->is_sentence_started = false;++ data_msg->timestamp = getCurrentTimeMillis();+data_map.emplace(hdl, data_msg);}catch (std::exception const& e) {std::cerr << "Error: " << e.what() << std::endl;@@ -501,6 +544,7 @@ void WebSocketServer::on_message(websocketpp::connection_hdl hdl,std::vector<std::vector<float>> hotwords_embedding_(*(msg_data->hotwords_embedding));msg_data->strand_->post(std::bind(&WebSocketServer::do_decoder, this,+ data_map,std::move(*(sample_data_p.get())), std::move(hdl),std::ref(msg_data->msg), std::ref(*(punc_cache_p.get())),std::move(hotwords_embedding_),@@ -550,6 +594,7 @@ void WebSocketServer::on_message(websocketpp::connection_hdl hdl,std::vector<std::vector<float>> hotwords_embedding_(*(msg_data->hotwords_embedding));msg_data->strand_->post(std::bind(&WebSocketServer::do_decoder, this,+ data_map,std::move(subvector), std::move(hdl),std::ref(msg_data->msg),std::ref(*(punc_cache_p.get())),diff --git a/runtime/websocket/bin/websocket-server-2pass.h b/runtime/websocket/bin/websocket-server-2pass.hindex e61a93b2d..3ba63089c 100644--- a/runtime/websocket/bin/websocket-server-2pass.h+++ b/runtime/websocket/bin/websocket-server-2pass.h@@ -61,7 +61,13 @@ typedef struct {std::string online_res = "";std::string tpass_res = "";std::shared_ptr<asio::io_context::strand> strand_; // for data execute in order- FUNASR_DEC_HANDLE decoder_handle=nullptr;+ FUNASR_DEC_HANDLE decoder_handle=nullptr;++ bool is_sentence_started = false;+ int64_t start_time = 0;+ int64_t end_time = 0;+ int64_t index = 0;+ int64_t timestamp = 0;} FUNASR_MESSAGE;// See https://wiki.mozilla.org/Security/Server_Side_TLS for more details about@@ -114,7 +120,9 @@ class WebSocketServer {server_->clear_access_channels(websocketpp::log::alevel::all);}}- void do_decoder(std::vector<char>& buffer, websocketpp::connection_hdl& hdl,+ void do_decoder(std::map<websocketpp::connection_hdl, std::shared_ptr<FUNASR_MESSAGE>,std::owner_less<websocketpp::connection_hdl>>& data_map,+ std::vector<char>& buffer,+ websocketpp::connection_hdl& hdl,nlohmann::json& msg,std::vector<std::vector<std::string>>& punc_cache,std::vector<std::vector<float>> &hotwords_embedding,