Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

4.1 KB
Raw Blame
// Acknowledgement: this code is adapted from 
// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/processor/token_parser.cc
// Retrieved in Aug 2023.

// Copyright (c) 2022 Zhendong Peng ([email protected])
//               2023 Jing Du ([email protected])
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "itn-token-parser.h"
#include <glog/logging.h>
#include "utf8-string.h"

namespace funasr {
const std::string EOS = "<EOS>";
const std::set<std::string> UTF8_WHITESPACE = {" ", "\t", "\n", "\r",
                                               "\x0b\x0c"};
const std::set<std::string> ASCII_LETTERS = {
    "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n",
    "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", "A", "B",
    "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", "P",
    "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", "_"};
const std::unordered_map<std::string, std::vector<std::string>> TN_ORDERS = {
    {"date", {"year", "month", "day"}},
    {"fraction", {"denominator", "numerator"}},
    {"measure", {"denominator", "numerator", "value"}},
    {"money", {"value", "currency"}},
    {"time", {"noon", "hour", "minute", "second"}}};
const std::unordered_map<std::string, std::vector<std::string>> ITN_ORDERS = {
    {"date", {"year", "month", "day"}},
    {"fraction", {"sign", "numerator", "denominator"}},
    {"measure", {"numerator", "denominator", "value"}},
    {"money", {"currency", "value"}},
    {"time", {"hour", "minute", "second", "noon"}}};

TokenParser::TokenParser(ParseType type) {
  if (type == ParseType::kTN) {
    orders = TN_ORDERS;
  } else {
    orders = ITN_ORDERS;
  }
}

void TokenParser::load(const std::string& input) {
  string2chars(input, &text);
  CHECK_GT(text.size(), 0);
  index = 0;
  ch = text[0];
}

bool TokenParser::read() {
  if (index < text.size() - 1) {
    index += 1;
    ch = text[index];
    return true;
  }
  ch = EOS;
  return false;
}

bool TokenParser::parse_ws() {
  bool not_eos = ch != EOS;
  while (not_eos && ch == " ") {
    not_eos = read();
  }
  return not_eos;
}

bool TokenParser::parse_char(const std::string& exp) {
  if (ch == exp) {
    read();
    return true;
  }
  return false;
}

bool TokenParser::parse_chars(const std::string& exp) {
  bool ok = false;
  std::vector<std::string> chars;
  string2chars(exp, &chars);
  for (const auto& x : chars) {
    ok |= parse_char(x);
  }
  return ok;
}

std::string TokenParser::parse_key() {
  CHECK_NE(ch, EOS);
  CHECK_EQ(UTF8_WHITESPACE.count(ch), 0);

  std::string key = "";
  while (ASCII_LETTERS.count(ch) > 0) {
    key += ch;
    read();
  }
  return key;
}

std::string TokenParser::parse_value() {
  CHECK_NE(ch, EOS);
  bool escape = false;

  std::string value = "";
  while (ch != "\"") {
    value += ch;
    escape = ch == "\\" && !escape;
    read();
    if (escape) {
      value += ch;
      read();
    }
  }
  return value;
}

void TokenParser::parse(const std::string& input) {
  load(input);
  while (parse_ws()) {
    std::string name = parse_key();
    parse_chars(" { ");

    Token token(name);
    while (parse_ws()) {
      if (ch == "}") {
        parse_char("}");
        break;
      }
      std::string key = parse_key();
      parse_chars(": \"");
      std::string value = parse_value();
      parse_char("\"");
      token.append(key, value);
    }
    tokens.emplace_back(token);
  }
}

std::string TokenParser::reorder(const std::string& input) {
  parse(input);
  std::string output = "";
  for (auto& token : tokens) {
    output += token.string(orders) + " ";
  }
  return trim(output);
}

}  // namespace funasr