Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

1.5 KB
Raw Blame
import os
import json
import torch
import logging
import concurrent.futures
import librosa
import torch.distributed as dist
from typing import Collection
import torch
try:
    import torchaudio
except ImportError:
    torchaudio = None
from torch import nn
import random
import re
import string
from funasr.tokenizer.cleaner import TextCleaner
from funasr.register import tables


@tables.register("preprocessor_classes", "TextPreprocessRemovePunctuation")
class TextPreprocessRemovePunctuation(nn.Module):
    def __init__(self, **kwargs):
        """Initialize TextPreprocessRemovePunctuation.
        
            Args:
                **kwargs: Additional keyword arguments.
            """
        super().__init__()

    def forward(self, text, **kwargs):
        # 定义英文标点符号
        """Forward pass for training.
        
            Args:
                text: Text tensor or string input.
                **kwargs: Additional keyword arguments.
            """
        en_punct = string.punctuation
        # 定义中文标点符号(部分常用的)
        cn_punct = "。?!,、;:“”‘’()《》【】…—~·"
        # 合并英文和中文标点符号
        all_punct = en_punct + cn_punct
        # 创建正则表达式模式,匹配任何在all_punct中的字符
        punct_pattern = re.compile("[{}]".format(re.escape(all_punct)))
        # 使用正则表达式的sub方法替换掉这些字符
        return punct_pattern.sub("", text)