Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
/**
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
* MIT License (https://opensource.org/licenses/MIT)
*/
#pragma once
#include "precomp.h"
namespace funasr {
class ParaformerOnline : public Model {
/**
* Author: Speech Lab of DAMO Academy, Alibaba Group
* ParaformerOnline: Fast and Accurate Parallel Transformer for Non-autoregressive End-to-End Speech Recognition
* https://arxiv.org/pdf/2206.08317.pdf
*/
private:
void FbankKaldi(float sample_rate, std::vector<std::vector<float>> &wav_feats,
std::vector<float> &waves);
int OnlineLfrCmvn(vector<vector<float>> &wav_feats, bool input_finished);
void GetPosEmb(std::vector<std::vector<float>> &wav_feats, int timesteps, int feat_dim);
void CifSearch(std::vector<std::vector<float>> hidden, std::vector<float> alphas, bool is_final, std::vector<std::vector<float>> &list_frame);
static int ComputeFrameNum(int sample_length, int frame_sample_length, int frame_shift_sample_length) {
int frame_num = static_cast<int>((sample_length - frame_sample_length) / frame_shift_sample_length + 1);
if (frame_num >= 1 && sample_length >= frame_sample_length)
return frame_num;
else
return 0;
}
void InitOnline(
knf::FbankOptions &fbank_opts,
std::shared_ptr<Ort::Session> &encoder_session,
std::shared_ptr<Ort::Session> &decoder_session,
vector<const char*> &en_szInputNames,
vector<const char*> &en_szOutputNames,
vector<const char*> &de_szInputNames,
vector<const char*> &de_szOutputNames,
vector<float> &means_list,
vector<float> &vars_list,
int frame_length_,
int frame_shift_,
int n_mels_,
int lfr_m_,
int lfr_n_,
int encoder_size_,
int fsmn_layers_,
int fsmn_lorder_,
int fsmn_dims_,
float cif_threshold_,
float tail_alphas_);
void StartUtterance()
{
}
void EndUtterance()
{
}
Model* offline_handle_ = nullptr;
// from offline_handle_
knf::FbankOptions fbank_opts_;
std::shared_ptr<Ort::Session> encoder_session_ = nullptr;
std::shared_ptr<Ort::Session> decoder_session_ = nullptr;
Ort::SessionOptions session_options_;
vector<const char*> en_szInputNames_;
vector<const char*> en_szOutputNames_;
vector<const char*> de_szInputNames_;
vector<const char*> de_szOutputNames_;
vector<float> means_list_;
vector<float> vars_list_;
// configs from offline_handle_
int frame_length = 25;
int frame_shift = 10;
int n_mels = 80;
int lfr_m = PARA_LFR_M;
int lfr_n = PARA_LFR_N;
int encoder_size = 512;
int fsmn_layers = 16;
int fsmn_lorder = 10;
int fsmn_dims = 512;
float cif_threshold = 1.0;
float tail_alphas = 0.45;
// configs
int feat_dims = lfr_m*n_mels;
std::vector<int> chunk_size = {5,10,5};
int frame_sample_length_ = MODEL_SAMPLE_RATE / 1000 * frame_length;
int frame_shift_sample_length_ = MODEL_SAMPLE_RATE / 1000 * frame_shift;
// The reserved waveforms by fbank
std::vector<float> reserve_waveforms_;
// waveforms reserved after last shift position
std::vector<float> input_cache_;
// lfr reserved cache
std::vector<std::vector<float>> lfr_splice_cache_;
// position index cache
int start_idx_cache_ = 0;
// cif alpha
std::vector<float> alphas_cache_;
std::vector<std::vector<float>> hidden_cache_;
std::vector<std::vector<float>> feats_cache_;
// fsmn init caches
std::vector<float> fsmn_init_cache_;
std::vector<Ort::Value> decoder_onnx;
bool is_first_chunk = true;
bool is_last_chunk = false;
double sqrt_factor;
public:
ParaformerOnline(Model* offline_handle, std::vector<int> chunk_size, std::string model_type=MODEL_PARA);
~ParaformerOnline();
void Reset();
void ResetCache();
void InitCache();
void ExtractFeats(float sample_rate, vector<vector<float>> &wav_feats, vector<float> &waves, bool input_finished);
void AddOverlapChunk(std::vector<std::vector<float>> &wav_feats, bool input_finished);
string ForwardChunk(std::vector<std::vector<float>> &wav_feats, bool input_finished);
string Forward(float* din, int len, bool input_finished, const std::vector<std::vector<float>> &hw_emb={{0.0}}, void* wfst_decoder=nullptr);
string Rescoring();
int GetAsrSampleRate() { return offline_handle_->GetAsrSampleRate(); };
// 2pass
std::string online_res;
int chunk_len;
};
} // namespace funasr