Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
Improving FBank computation and implementing the dispose method (#1497)
manyeyesa2d6575d89c2
diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.csindex f42bfb12e..672eac236 100644--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVad.cs@@ -1,19 +1,16 @@-using System.Linq;-using System.Text;-using System.Threading.Tasks;-using Microsoft.ML;+using AliFsmnVadSharp.Model;+using AliFsmnVadSharp.Utils;+using Microsoft.Extensions.Logging;using Microsoft.ML.OnnxRuntime;using Microsoft.ML.OnnxRuntime.Tensors;-using Microsoft.Extensions.Logging;-using AliFsmnVadSharp.Model;-using AliFsmnVadSharp.Utils;namespace AliFsmnVadSharp{- public class AliFsmnVad+ public class AliFsmnVad : IDisposable{+ private bool _disposed;private InferenceSession _onnxSession;- private readonly ILogger<AliFsmnVad> _logger;+ private readonly ILogger _logger;private string _frontend;private WavFrontend _wavFrontend;private int _batchSize = 1;@@ -23,7 +20,7 @@ namespace AliFsmnVadSharppublic AliFsmnVad(string modelFilePath, string configFilePath, string mvnFilePath, int batchSize = 1){- Microsoft.ML.OnnxRuntime.SessionOptions options = new Microsoft.ML.OnnxRuntime.SessionOptions();+ SessionOptions options = new SessionOptions();options.AppendExecutionProvider_CPU(0);options.InterOpNumThreads = 1;_onnxSession = new InferenceSession(modelFilePath, options);@@ -371,17 +368,41 @@ namespace AliFsmnVadSharp}return speech;}+ protected virtual void Dispose(bool disposing)+ {+ if (!_disposed)+ {+ if (disposing)+ {+ if (_onnxSession != null)+ {+ _onnxSession.Dispose();+ }+ if (_wavFrontend != null)+ {+ _wavFrontend.Dispose();+ }+ if (_encoderConfEntity != null)+ {+ _encoderConfEntity = null;+ }+ if (_vad_post_conf != null)+ {+ _vad_post_conf = null;+ }+ }+ _disposed = true;+ }+ }-----------+ public void Dispose()+ {+ Dispose(disposing: true);+ GC.SuppressFinalize(this);+ }+ ~AliFsmnVad()+ {+ Dispose(_disposed);+ }}}\ No newline at end of filediff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csprojindex 49915173e..769bf0c0f 100644--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/AliFsmnVadSharp.csproj@@ -7,16 +7,13 @@</PropertyGroup><ItemGroup>+ <PackageReference Include="KaldiNativeFbankSharp" Version="1.1.2" /><PackageReference Include="Microsoft.Extensions.Logging" Version="7.0.0" /><PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.15.0" /><PackageReference Include="YamlDotNet" Version="13.1.0" /></ItemGroup><ItemGroup>- <None Update="Lib\kaldi-native-fbank-dll.dll">- <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>- <TargetPath>kaldi-native-fbank-dll.dll</TargetPath>- </None><None Update="speech_fsmn_vad_zh-cn-16k-common-pytorch\example\0.wav"><CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory></None>diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.csdeleted file mode 100644index af0ad3664..000000000--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KaldiNativeFbank.cs+++ /dev/null@@ -1,40 +0,0 @@-using System;-using System.Collections.Generic;-using System.Linq;-using System.Text;-using System.Threading.Tasks;-using System.Runtime.InteropServices;-using AliFsmnVadSharp.Struct;--namespace AliFsmnVadSharp.DLL-{- public static class KaldiNativeFbank- {- private const string dllName = @"kaldi-native-fbank-dll";-- [DllImport(dllName, EntryPoint = "GetFbankOptions", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern IntPtr GetFbankOptions(float dither, bool snip_edges, float sample_rate, int num_bins, float frame_shift = 10.0f, float frame_length = 25.0f, float energy_floor = 0.0f, bool debug_mel = false, string window_type = "hamming");-- [DllImport(dllName, EntryPoint = "GetOnlineFbank", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern KnfOnlineFbank GetOnlineFbank(IntPtr opts);-- [DllImport(dllName, EntryPoint = "AcceptWaveform", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern void AcceptWaveform(KnfOnlineFbank knfOnlineFbank, float sample_rate, float[] samples, int samples_size);-- [DllImport(dllName, EntryPoint = "InputFinished", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern void InputFinished(KnfOnlineFbank knfOnlineFbank);-- [DllImport(dllName, EntryPoint = "GetNumFramesReady", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern int GetNumFramesReady(KnfOnlineFbank knfOnlineFbank);-- [DllImport(dllName, EntryPoint = "AcceptWaveformxxx", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern FbankDatas AcceptWaveformxxx(KnfOnlineFbank knfOnlineFbank, float sample_rate, float[] samples, int samples_size);-- [DllImport(dllName, EntryPoint = "GetFbank", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern void GetFbank(KnfOnlineFbank knfOnlineFbank,int frame, ref FbankData pData);-- [DllImport(dllName, EntryPoint = "GetFbanks", CharSet = CharSet.Ansi, CallingConvention = CallingConvention.Cdecl)]- internal static extern void GetFbanks(KnfOnlineFbank knfOnlineFbank, int framesNum, ref FbankDatas fbankDatas);-- }-}diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.csdeleted file mode 100644index 45549b2bf..000000000--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/DLL/KnfOnlineFbank.cs+++ /dev/null@@ -1,26 +0,0 @@-using System;-using System.Collections.Generic;-using System.Linq;-using System.Runtime.InteropServices;-using System.Text;-using System.Threading.Tasks;--namespace AliFsmnVadSharp.DLL-{- internal struct FbankData- {- public IntPtr data;- public int data_length;- };-- internal struct FbankDatas- {- public IntPtr data;- public int data_length;- };-- internal struct KnfOnlineFbank- {- public IntPtr impl;- };-}diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.csindex ce519b1e9..38d3c9072 100644--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/E2EVadModel.cs@@ -503,20 +503,27 @@ namespace AliFsmnVadSharp{return 0;}- for (int i = _vad_opts.nn_eval_block_size - 1; i > -1; i += -1)+ try{- FrameState frame_state = FrameState.kFrameStateInvalid;- frame_state = GetFrameState(_frm_cnt - 1 - i);- if (i != 0)- {- DetectOneFrame(frame_state, _frm_cnt - 1 - i, false);- }- else+ for (int i = _vad_opts.nn_eval_block_size - 1; i > -1; i += -1){- DetectOneFrame(frame_state, _frm_cnt - 1, true);- }+ FrameState frame_state = FrameState.kFrameStateInvalid;+ frame_state = GetFrameState(_frm_cnt - 1 - i);+ if (i != 0)+ {+ DetectOneFrame(frame_state, _frm_cnt - 1 - i, false);+ }+ else+ {+ DetectOneFrame(frame_state, _frm_cnt - 1, true);+ }+ }+ }+ catch (Exception e)+ {+ //}return 0;diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dlldeleted file mode 100644index cddc94074..000000000Binary files a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Lib/kaldi-native-fbank-dll.dll and /dev/null differdiff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.csdeleted file mode 100644index bbad3dc95..000000000--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/Struct/FbankData.cs+++ /dev/null@@ -1,6 +0,0 @@-using System.Runtime.InteropServices;--namespace AliFsmnVadSharp.Struct-{--}diff --git a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.csindex 2c5b50fbb..6ffb79f52 100644--- a/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs+++ b/runtime/csharp/AliFsmnVad/AliFsmnVadSharp/WavFrontend.cs@@ -1,30 +1,23 @@-using System;-using System.Collections.Generic;-using System.Linq;-using System.Text;-using System.Threading.Tasks;-using AliFsmnVadSharp.Model;-using AliFsmnVadSharp.DLL;-using AliFsmnVadSharp.Struct;-using System.Runtime.InteropServices;+using AliFsmnVadSharp.Model;+using KaldiNativeFbankSharp;namespace AliFsmnVadSharp{- internal class WavFrontend+ internal class WavFrontend : IDisposable{- private string _mvnFilePath;+ private bool _disposed;+private FrontendConfEntity _frontendConfEntity;- IntPtr _opts = IntPtr.Zero;+ OnlineFbank _onlineFbank;private CmvnEntity _cmvnEntity;private static int _fbank_beg_idx = 0;public WavFrontend(string mvnFilePath, FrontendConfEntity frontendConfEntity){- _mvnFilePath = mvnFilePath;_frontendConfEntity = frontendConfEntity;_fbank_beg_idx = 0;- _opts = KaldiNativeFbank.GetFbankOptions(+ _onlineFbank = new OnlineFbank(dither: _frontendConfEntity.dither,snip_edges: true,sample_rate: _frontendConfEntity.fs,@@ -37,28 +30,7 @@ namespace AliFsmnVadSharp{float sample_rate = _frontendConfEntity.fs;samples = samples.Select((float x) => x * 32768f).ToArray();- // method1- //FbankDatas fbankDatas = new FbankDatas();- //KaldiNativeFbank.GetFbanks(_knfOnlineFbank, framesNum,ref fbankDatas);- // method2- KnfOnlineFbank _knfOnlineFbank = KaldiNativeFbank.GetOnlineFbank(_opts);- KaldiNativeFbank.AcceptWaveform(_knfOnlineFbank, sample_rate, samples, samples.Length);- KaldiNativeFbank.InputFinished(_knfOnlineFbank);- int framesNum = KaldiNativeFbank.GetNumFramesReady(_knfOnlineFbank);- float[] fbanks = new float[framesNum * 80];- for (int i = 0; i < framesNum; i++)- {- FbankData fbankData = new FbankData();- KaldiNativeFbank.GetFbank(_knfOnlineFbank, i, ref fbankData);- float[] _fbankData = new float[fbankData.data_length];- Marshal.Copy(fbankData.data, _fbankData, 0, fbankData.data_length);- Array.Copy(_fbankData, 0, fbanks, i * 80, _fbankData.Length);- fbankData.data = IntPtr.Zero;- _fbankData = null;- }-- samples = null;- GC.Collect();+ float[] fbanks = _onlineFbank.GetFbank(samples);return fbanks;}@@ -180,6 +152,29 @@ namespace AliFsmnVadSharpcmvnEntity.Vars = vars_list;return cmvnEntity;}+ protected virtual void Dispose(bool disposing)+ {+ if (!_disposed)+ {+ if (disposing)+ {+ if (_onlineFbank != null)+ {+ _onlineFbank.Dispose();+ }+ }+ _disposed = true;+ }+ }+ public void Dispose()+ {+ Dispose(disposing: true);+ GC.SuppressFinalize(this);+ }+ ~WavFrontend()+ {+ Dispose(_disposed);+ }}}