Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
from funasr.register import tables
@tables.register("tokenizer_classes", "HuggingfaceTokenizer")
def HuggingfaceTokenizer(init_param_path, **kwargs):
"""Huggingfacetokenizer.
Args:
init_param_path: TODO.
**kwargs: Additional keyword arguments.
"""
try:
from transformers import AutoTokenizer
except Exception as e:
raise ImportError(
"HuggingfaceTokenizer requires 'transformers'. "
"Please install it with: pip install -U transformers"
) from e
tokenizer = AutoTokenizer.from_pretrained(init_param_path)
return tokenizer