Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
import pynini
from fun_text_processing.text_normalization.zh.graph_utils import (
FUN_ALPHA,
FUN_DIGIT,
FUN_PUNCT,
FUN_SIGMA,
FUN_WHITE_SPACE,
GraphFst,
)
from fun_text_processing.text_normalization.zh.utils import get_abs_path
from pynini.lib import pynutil, utf8
class PostProcessor(GraphFst):
"""
Postprocessing of TN, now contains:
1. punctuation removal
2. letter case conversion
3. oov tagger
"""
def __init__(
self,
remove_puncts: bool = False,
to_upper: bool = False,
to_lower: bool = False,
tag_oov: bool = False,
):
super().__init__(name="PostProcessor", kind="processor")
graph = pynini.cdrewrite("", "", "", FUN_SIGMA)
if remove_puncts:
remove_puncts_graph = pynutil.delete(
pynini.union(
FUN_PUNCT, pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv"))
)
)
graph @= pynini.cdrewrite(remove_puncts_graph, "", "", FUN_SIGMA).optimize()
if to_upper or to_lower:
if to_upper:
conv_cases_graph = pynini.inverse(
pynini.string_file(get_abs_path("data/char/upper_to_lower.tsv"))
)
else:
conv_cases_graph = pynini.string_file(get_abs_path("data/char/upper_to_lower.tsv"))
graph @= pynini.cdrewrite(conv_cases_graph, "", "", FUN_SIGMA).optimize()
if tag_oov:
zh_charset_std = pynini.string_file(
get_abs_path("data/char/charset_national_standard_2013_8105.tsv")
)
zh_charset_ext = pynini.string_file(get_abs_path("data/char/charset_extension.tsv"))
zh_charset = (
zh_charset_std
| zh_charset_ext
| pynini.string_file(get_abs_path("data/char/punctuations_zh.tsv"))
)
en_charset = FUN_DIGIT | FUN_ALPHA | FUN_PUNCT | FUN_WHITE_SPACE
charset = zh_charset | en_charset
with open(get_abs_path("data/char/oov_tags.tsv"), "r") as f:
tags = f.readline().strip().split("\t")
assert len(tags) == 2
ltag, rtag = tags
oov_charset = pynini.difference(utf8.VALID_UTF8_CHAR, charset)
tag_oov_graph = pynutil.insert(ltag) + oov_charset + pynutil.insert(rtag)
graph @= pynini.cdrewrite(tag_oov_graph, "", "", FUN_SIGMA).optimize()
self.fst = graph.optimize()