Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

2.9 KB
Raw Blame
// Acknowledgement: this code is adapted from 
// https://github.com/wenet-e2e/WeTextProcessing/blob/master/runtime/processor/processor.cc 
// Retrieved in Aug 2023.

// Copyright (c) 2022 Zhendong Peng ([email protected])
//               2023 Jing Du ([email protected])
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "itn-processor.h"

using fst::StringTokenType;

namespace funasr {
ITNProcessor::ITNProcessor(){};
ITNProcessor::~ITNProcessor(){};

void  ITNProcessor::InitITN(const std::string& tagger_path,
                     const std::string& verbalizer_path, 
                     int thread_num) {
  try{
    tagger_.reset(StdVectorFst::Read(tagger_path));
    LOG(INFO) << "Successfully load model from " << tagger_path;
    verbalizer_.reset(StdVectorFst::Read(verbalizer_path));
    LOG(INFO) << "Successfully load model from " << verbalizer_path;
  }catch(exception const &e){
    LOG(ERROR) << "Error loading itn models";
    exit(-1);
  }
  compiler_ = std::make_shared<StringCompiler<StdArc>>(StringTokenType::BYTE);
  printer_ = std::make_shared<StringPrinter<StdArc>>(StringTokenType::BYTE);

  if (tagger_path.find("_tn_") != tagger_path.npos) {
    parse_type_ = ParseType::kTN;
  } else if (tagger_path.find("_itn_") != tagger_path.npos) {
    parse_type_ = ParseType::kITN;
  } else {
    LOG(FATAL) << "Invalid fst prefix, prefix should contain"
               << " either \"_tn_\" or \"_itn_\".";
  }
}

std::string ITNProcessor::shortest_path(const StdVectorFst& lattice) {
  StdVectorFst shortest_path;
  fst::ShortestPath(lattice, &shortest_path, 1, true);

  std::string output;
  printer_->operator()(shortest_path, &output);
  return output;
}

std::string ITNProcessor::compose(const std::string& input,
                               const StdVectorFst* fst) {
  StdVectorFst input_fst;
  compiler_->operator()(input, &input_fst);

  StdVectorFst lattice;
  fst::Compose(input_fst, *fst, &lattice);
  return shortest_path(lattice);
}

std::string ITNProcessor::tag(const std::string& input) {
  return compose(input, tagger_.get());
}

std::string ITNProcessor::verbalize(const std::string& input) {
  if (input.empty()) {
    return "";
  }
  TokenParser parser(parse_type_);
  std::string output = parser.reorder(input);
  return compose(output, verbalizer_.get());
}

std::string ITNProcessor::Normalize(const std::string& input) {
  return verbalize(tag(input));
}

}  // namespace funasr