Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
/**
* Copyright FunASR (https://github.com/alibaba-damo-academy/FunASR). All Rights Reserved.
* MIT License (https://opensource.org/licenses/MIT)
*/
#include "encode_converter.h"
#include <assert.h>
namespace funasr {
using namespace std;
U16CHAR_T UTF16[8];
U8CHAR_T UTF8[8];
size_t MyUtf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16);
size_t MyUtf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8);
void EncodeConverter::SwapEndian(U16CHAR_T* pbuf, size_t len)
{
for (size_t i = 0; i < len; i++) {
pbuf[i] = ((pbuf[i] >> 8) | (pbuf[i] << 8));
}
}
size_t MyUtf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8)
{
size_t n = 0;
if (pu16[0] <= 0x007F)
{
pu8[0] = (pu16[0] & 0x7F);
n = 1;
}
else if (pu16[0] >= 0x0080 && pu16[0] <= 0x07FF)
{
pu8[1] = (0x80 | (pu16[0] & 0x003F));
pu8[0] = (0xC0 | ((pu16[0] >> 6) & 0x001F));
n = 2;
}
else if (pu16[0] >= 0x0800)
{
pu8[2] = (0x80 | (pu16[0] & 0x003F));
pu8[1] = (0x80 | ((pu16[0] >> 6) & 0x003F));
pu8[0] = (0xE0 | ((pu16[0] >> 12) & 0x000F));
n = 3;
}
return n;
}
#define is2ByteUtf16(u16) ( (u16) >= 0x0080 && (u16) <= 0x07FF )
#define is3ByteUtf16(u16) ( (u16) >= 0x0800 )
size_t EncodeConverter::Utf16ToUtf8(const U16CHAR_T* pu16, U8CHAR_T* pu8)
{
size_t n = 0;
if (pu16[0] <= 0x007F)
{
pu8[0] = (pu16[0] & 0x7F);
n = 1;
}
else if (pu16[0] >= 0x0080 && pu16[0] <= 0x07FF)
{
pu8[1] = (0x80 | (pu16[0] & 0x003F));
pu8[0] = (0xC0 | ((pu16[0] >> 6) & 0x001F));
n = 2;
}
else if (pu16[0] >= 0x0800)
{
pu8[2] = (0x80 | (pu16[0] & 0x003F));
pu8[1] = (0x80 | ((pu16[0] >> 6) & 0x003F));
pu8[0] = (0xE0 | ((pu16[0] >> 12) & 0x000F));
n = 3;
}
return n;
}
size_t EncodeConverter::Utf16ToUtf8(const U16CHAR_T* pu16, size_t ilen,
U8CHAR_T* pu8, size_t olen)
{
size_t offset = 0;
size_t sz = 0;
/*
for (size_t i = 0; i < ilen && offset < static_cast<int>(olen) - 3; i++) {
sz = utf16ToUtf8(pu16 + i, pu8 + offset);
offset += sz;
}
*/
for (size_t i = 0; i < ilen && static_cast<int>(offset) < static_cast<int>(olen); i++) {
sz = Utf16ToUtf8(pu16 + i, pu8 + offset);
if (static_cast<int>(offset + static_cast<int>(sz)) <= static_cast<int>(olen))
offset += sz;
}
// pu8[offset] = '\0';
return offset;
}
u8string EncodeConverter::Utf16ToUtf8(const u16string& u16str)
{
size_t buflen = u16str.length()*3 + 1;
U8CHAR_T* pu8 = new U8CHAR_T[buflen];
size_t len = Utf16ToUtf8(u16str.data(), u16str.length(),
pu8, buflen);
u8string u8str(pu8, len);
delete [] pu8;
return u8str;
}
size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, U16CHAR_T* pu16)
{
size_t n = 0;
if ((pu8[0] & 0xF0) == 0xE0)
{
if ((pu8[1] & 0xC0) == 0x80 &&
(pu8[2] & 0xC0) == 0x80)
{
pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
pu16[0] <<= 8;
pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
}
else
{
pu16[0] = defUniChar;
}
n = 3;
}
else if ((pu8[0] & 0xE0) == 0xC0)
{
if ((pu8[1] & 0xC0) == 0x80)
{
pu16[0] = ((pu8[0] & 0x1C) >> 2);
pu16[0] <<= 8;
pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
}
else
{
pu16[0] = defUniChar;
}
n = 2;
}
else if ((pu8[0] & 0x80) == 0x00)
{
pu16[0] = pu8[0];
n = 1;
}
return n;
}
size_t MyUtf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16)
{
size_t n = 0;
if ((pu8[0] & 0xF0) == 0xE0 && ilen >= 3)
{
if ((pu8[1] & 0xC0) == 0x80 &&
(pu8[2] & 0xC0) == 0x80)
{
pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
pu16[0] <<= 8;
pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
n = 3;
}
else
{
pu16[0] = 0x0000;
n = 1;
}
}
else if ((pu8[0] & 0xE0) == 0xC0 && ilen >= 2)
{
if ((pu8[1] & 0xC0) == 0x80)
{
pu16[0] = ((pu8[0] & 0x1C) >> 2);
pu16[0] <<= 8;
pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
n = 2;
}
else
{
pu16[0] = 0x0000;
n = 1;
}
}
else if ((pu8[0] & 0x80) == 0x00)
{
pu16[0] = pu8[0];
n = 1;
}
else
{
pu16[0] = 0x0000;
n = 1;
}
return n;
}
size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen, U16CHAR_T* pu16)
{
size_t n = 0;
if ((pu8[0] & 0xF0) == 0xE0 && ilen >= 3)
{
if ((pu8[1] & 0xC0) == 0x80 &&
(pu8[2] & 0xC0) == 0x80)
{
pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
pu16[0] <<= 8;
pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
n = 3;
if( !is3ByteUtf16(pu16[0]) )
{
pu16[0] = 0x0000;
n = 1;
}
}
else
{
pu16[0] = 0x0000;
n = 1;
}
}
else if ((pu8[0] & 0xE0) == 0xC0 && ilen >= 2)
{
if ((pu8[1] & 0xC0) == 0x80)
{
pu16[0] = ((pu8[0] & 0x1C) >> 2);
pu16[0] <<= 8;
pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
n = 2;
if( !is2ByteUtf16(pu16[0]) )
{
pu16[0] = 0x0000;
n = 1;
}
}
else
{
pu16[0] = 0x0000;
n = 1;
}
}
else if ((pu8[0] & 0x80) == 0x00)
{
pu16[0] = pu8[0];
n = 1;
}
else
{
pu16[0] = 0x0000;
n = 1;
}
return n;
/*
size_t n = 0;
if ((pu8[0] & 0xF0) == 0xE0)
{
if (ilen >= 3 && (pu8[1] & 0xC0) == 0x80 &&
(pu8[2] & 0xC0) == 0x80)
{
pu16[0] = (((pu8[0] & 0x0F) << 4) | ((pu8[1] & 0x3C) >> 2));
pu16[0] <<= 8;
pu16[0] |= (((pu8[1] & 0x03) << 6) | (pu8[2] & 0x3F));
}
else
{
pu16[0] = defUniChar;
}
n = 3;
}
else if ((pu8[0] & 0xE0) == 0xC0)
{
if( ilen >= 2 && (pu8[1] & 0xC0) == 0x80)
{
pu16[0] = ((pu8[0] & 0x1C) >> 2);
pu16[0] <<= 8;
pu16[0] |= (((pu8[0] & 0x03) << 6) | (pu8[1] & 0x3F));
}
else
{
pu16[0] = defUniChar;
}
n = 2;
}
else if ((pu8[0] & 0x80) == 0x00)
{
pu16[0] = pu8[0];
n = 1;
}
else
{
pu16[0] = defUniChar;
n = 1;
for (size_t i = 1; i < ilen; i++)
{
if ((pu8[i] & 0xF0) == 0xE0 || (pu8[i] & 0xE0) == 0xC0 || (pu8[i] & 0x80) == 0x00)
break;
n++;
}
}
return n;
*/
}
size_t EncodeConverter::Utf8ToUtf16(const U8CHAR_T* pu8, size_t ilen,
U16CHAR_T* pu16, size_t olen)
{
int offset = 0;
size_t sz = 0;
for (size_t i = 0; i < ilen && offset < static_cast<int>(olen); offset ++)
{
sz = Utf8ToUtf16(pu8 + i, ilen - i, pu16 + offset);
i += sz;
if (sz == 0) {
// failed
// assert(sz != 0);
break;
}
}
// pu16[offset] = '\0';
return offset;
}
u16string EncodeConverter::Utf8ToUtf16(const u8string& u8str)
{
U16CHAR_T* p16 = new U16CHAR_T[u8str.length() + 1];
size_t len = Utf8ToUtf16(u8str.data(), u8str.length(),
p16, u8str.length() + 1);
u16string u16str(p16, len);
delete[] p16;
return u16str;
}
bool EncodeConverter::IsUTF8(const U8CHAR_T* pu8, size_t ilen)
{
size_t i;
size_t n = 0;
for (i = 0; i < ilen; i += n)
{
if ((pu8[i] & 0xF0) == 0xE0 &&
(pu8[i + 1] & 0xC0) == 0x80 &&
(pu8[i + 2] & 0xC0) == 0x80)
{
n = 3;
}
else if ((pu8[i] & 0xE0) == 0xC0 &&
(pu8[i + 1] & 0xC0) == 0x80)
{
n = 2;
}
else if ((pu8[i] & 0x80) == 0x00)
{
n = 1;
}
else
{
break;
}
}
return i == ilen;
}
bool EncodeConverter::IsUTF8(const u8string& u8str)
{
return IsUTF8(u8str.data(), u8str.length());
}
size_t EncodeConverter::GetUTF8Len(const U8CHAR_T* pu8, size_t ilen)
{
size_t i;
size_t n = 0;
size_t rlen = 0;
for (i = 0; i < ilen; i += n, rlen ++)
{
if ((pu8[i] & 0xF0) == 0xE0 &&
(pu8[i + 1] & 0xC0) == 0x80 &&
(pu8[i + 2] & 0xC0) == 0x80)
{
n = 3;
}
else if ((pu8[i] & 0xE0) == 0xC0 &&
(pu8[i + 1] & 0xC0) == 0x80)
{
n = 2;
}
else if ((pu8[i] & 0x80) == 0x00)
{
n = 1;
}
else
{
break;
}
}
if (i == ilen)
return 0;
else
return rlen;
}
size_t EncodeConverter::GetUTF8Len(const u8string& u8str)
{
return GetUTF8Len(u8str.data(), u8str.length());
}
size_t EncodeConverter::Utf16ToUtf8Len(const U16CHAR_T* pu16, size_t ilen)
{
int offset = 0;
for (size_t i = 0; i < ilen ; i++) {
if (pu16[i] <= 0x007F)
{
offset += 1;
}
else if (pu16[i] >= 0x0080 && pu16[i] <= 0x07FF)
{
offset += 2;
}
else if (pu16[i] >= 0x0800)
{
offset += 3;
}
}
return offset;
}
uint16_t EncodeConverter::ToUni(const char* sc, int &len)
{
uint16_t wide[2];
len = (int)Utf8ToUtf16((const U8CHAR_T*)sc, wide);
return wide[0];
}
bool EncodeConverter::IsAllChineseCharactor(const U8CHAR_T* pu8, size_t ilen) {
if (pu8 == nullptr || ilen <= 0) {
return false;
}
U16CHAR_T* p16 = new U16CHAR_T[ilen + 1];
size_t len = Utf8ToUtf16(pu8, ilen, p16, ilen + 1);
for (size_t i = 0; i < len; i++) {
if (p16[i] < 0x4e00 || p16[i] > 0x9fff) {
delete[] p16;
return false;
}
}
delete[] p16;
return true;
}
bool EncodeConverter::HasAlpha(const U8CHAR_T* pu8, size_t ilen) {
if (pu8 == nullptr || ilen <= 0) {
return false;
}
for (size_t i = 0; i < ilen; i++) {
if (pu8[i]> 0 && isalpha(pu8[i])){
return true;
}
}
return false;
}
bool EncodeConverter::IsAllAlpha(const U8CHAR_T* pu8, size_t ilen) {
if (pu8 == nullptr || ilen <= 0) {
return false;
}
for (size_t i = 0; i < ilen; i++) {
if (!(pu8[i]> 0 && isalpha(pu8[i]))){
return false;
}
}
return true;
}
bool EncodeConverter::IsAllAlphaAndPunct(const U8CHAR_T* pu8, size_t ilen) {
if (pu8 == nullptr || ilen <= 0) {
return false;
}
bool flag1 = HasAlpha(pu8, ilen);
if (flag1 == false) {
return false;
}
for (size_t i = 0; i < ilen; i++) {
if (!(pu8[i]> 0 && (isalpha(pu8[i]) || (ispunct(pu8[i]))))){
return false;
}
}
return true;
}
bool EncodeConverter::IsAllAlphaAndDigit(const U8CHAR_T* pu8, size_t ilen) {
if (pu8 == nullptr || ilen <= 0) {
return false;
}
bool flag1 = HasAlpha(pu8, ilen);
if (flag1 == false) {
return false;
}
for (size_t i = 0; i < ilen; i++) {
if (!(pu8[i]> 0 && (isalnum(pu8[i]) || isalpha(pu8[i]) || pu8[i] == '\''))){
return false;
}
}
return true;
}
bool EncodeConverter::IsAllAlphaAndDigitAndBlank(const U8CHAR_T* pu8, size_t ilen) {
if (pu8 == nullptr || ilen <= 0) {
return false;
}
for (size_t i = 0; i < ilen; i++) {
if (!(pu8[i]> 0 && (isalnum(pu8[i]) || isalpha(pu8[i]) || isblank(pu8[i]) || pu8[i] == '\''))){
return false;
}
}
return true;
}
bool EncodeConverter::NeedAddTailBlank(std::string str) {
U8CHAR_T *pu8 = (U8CHAR_T*)str.data();
size_t ilen = str.size();
if (pu8 == nullptr || ilen <= 0) {
return false;
}
if (IsAllAlpha(pu8, ilen) || IsAllAlphaAndPunct(pu8, ilen) || IsAllAlphaAndDigit(pu8, ilen)) {
return true;
} else {
return false;
}
}
std::vector<std::string> EncodeConverter::MergeEnglishWord(std::vector<std::string> &str_vec_input,
std::vector<int> &merge_mask) {
std::vector<std::string> output;
for (int i = 0; i < merge_mask.size(); i++) {
if (merge_mask[i] == 1 && i > 0) {
output[output.size() - 1] += str_vec_input[i];
} else {
output.push_back(str_vec_input[i]);
}
}
str_vec_input.swap(output);
return str_vec_input;
}
size_t EncodeConverter::Utf8ToCharset(const std::string &input, std::vector<std::string> &output) {
std::string ch;
for (size_t i = 0, len = 0; i != input.length(); i += len) {
unsigned char byte = (unsigned)input[i];
if (byte >= 0xFC) // lenght 6
len = 6;
else if (byte >= 0xF8)
len = 5;
else if (byte >= 0xF0)
len = 4;
else if (byte >= 0xE0)
len = 3;
else if (byte >= 0xC0)
len = 2;
else
len = 1;
ch = input.substr(i, len);
output.push_back(ch);
}
return output.size();
}
}