Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
// Acknowledgement: this code is adapted from
// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/processor/token_parser.cc
// Retrieved in Aug 2023.
// Copyright (c) 2022 Zhendong Peng ([email protected])
// 2023 Jing Du ([email protected])
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "itn-token-parser.h"
#include <glog/logging.h>
#include "utf8-string.h"
namespace funasr {
const std::string EOS = "<EOS>";
const std::set<std::string> UTF8_WHITESPACE = {" ", "\t", "\n", "\r",
"\x0b\x0c"};
const std::set<std::string> ASCII_LETTERS = {
"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
"o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B",
"C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P",
"Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "_"};
const std::unordered_map<std::string, std::vector<std::string>> TN_ORDERS = {
{"date", {"year", "month", "day"}},
{"fraction", {"denominator", "numerator"}},
{"measure", {"denominator", "numerator", "value"}},
{"money", {"value", "currency"}},
{"time", {"noon", "hour", "minute", "second"}}};
const std::unordered_map<std::string, std::vector<std::string>> ITN_ORDERS = {
{"date", {"year", "month", "day"}},
{"fraction", {"sign", "numerator", "denominator"}},
{"measure", {"numerator", "denominator", "value"}},
{"money", {"currency", "value"}},
{"time", {"hour", "minute", "second", "noon"}}};
TokenParser::TokenParser(ParseType type) {
if (type == ParseType::kTN) {
orders = TN_ORDERS;
} else {
orders = ITN_ORDERS;
}
}
void TokenParser::load(const std::string& input) {
string2chars(input, &text);
CHECK_GT(text.size(), 0);
index = 0;
ch = text[0];
}
bool TokenParser::read() {
if (index < text.size() - 1) {
index += 1;
ch = text[index];
return true;
}
ch = EOS;
return false;
}
bool TokenParser::parse_ws() {
bool not_eos = ch != EOS;
while (not_eos && ch == " ") {
not_eos = read();
}
return not_eos;
}
bool TokenParser::parse_char(const std::string& exp) {
if (ch == exp) {
read();
return true;
}
return false;
}
bool TokenParser::parse_chars(const std::string& exp) {
bool ok = false;
std::vector<std::string> chars;
string2chars(exp, &chars);
for (const auto& x : chars) {
ok |= parse_char(x);
}
return ok;
}
std::string TokenParser::parse_key() {
CHECK_NE(ch, EOS);
CHECK_EQ(UTF8_WHITESPACE.count(ch), 0);
std::string key = "";
while (ASCII_LETTERS.count(ch) > 0) {
key += ch;
read();
}
return key;
}
std::string TokenParser::parse_value() {
CHECK_NE(ch, EOS);
bool escape = false;
std::string value = "";
while (ch != "\"") {
value += ch;
escape = ch == "\\" && !escape;
read();
if (escape) {
value += ch;
read();
}
}
return value;
}
void TokenParser::parse(const std::string& input) {
load(input);
while (parse_ws()) {
std::string name = parse_key();
parse_chars(" { ");
Token token(name);
while (parse_ws()) {
if (ch == "}") {
parse_char("}");
break;
}
std::string key = parse_key();
parse_chars(": \"");
std::string value = parse_value();
parse_char("\"");
token.append(key, value);
}
tokens.emplace_back(token);
}
}
std::string TokenParser::reorder(const std::string& input) {
parse(input);
std::string output = "";
for (auto& token : tokens) {
output += token.string(orders) + " ";
}
return trim(output);
}
} // namespace funasr