Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
from pathlib import Path
from typing import Iterable
from typing import List
from typing import Union
import sentencepiece as spm
from funasr.tokenizer.abs_tokenizer import BaseTokenizer
from funasr.register import tables
@tables.register("tokenizer_classes", "SentencepiecesTokenizer")
class SentencepiecesTokenizer(BaseTokenizer):
def __init__(self, bpemodel: Union[Path, str], **kwargs):
"""Initialize SentencepiecesTokenizer.
Args:
bpemodel: TODO.
**kwargs: Additional keyword arguments.
"""
super().__init__(**kwargs)
self.bpemodel = str(bpemodel)
# NOTE(kamo):
# Don't build SentencePieceProcessor in __init__()
# because it's not picklable and it may cause following error,
# "TypeError: can't pickle SwigPyObject objects",
# when giving it as argument of "multiprocessing.Process()".
self.sp = None
self._build_sentence_piece_processor()
def __repr__(self):
"""Internal: repr ."""
return f'{self.__class__.__name__}(model="{self.bpemodel}")'
def _build_sentence_piece_processor(self):
# Build SentencePieceProcessor lazily.
"""Internal: build sentence piece processor."""
if self.sp is None:
self.sp = spm.SentencePieceProcessor()
self.sp.load(self.bpemodel)
def text2tokens(self, line: str) -> List[str]:
"""Text2tokens.
Args:
line: TODO.
"""
self._build_sentence_piece_processor()
return self.sp.EncodeAsPieces(line)
def tokens2text(self, tokens: Iterable[str]) -> str:
"""Tokens2text.
Args:
tokens: TODO.
"""
self._build_sentence_piece_processor()
return self.sp.DecodePieces(list(tokens))
def encode(self, line: str, **kwargs) -> List[int]:
"""Encode.
Args:
line: TODO.
**kwargs: Additional keyword arguments.
"""
self._build_sentence_piece_processor()
return self.sp.EncodeAsIds(line)
def decode(self, line: List[int], **kwargs):
"""Decode.
Args:
line: TODO.
**kwargs: Additional keyword arguments.
"""
self._build_sentence_piece_processor()
return self.sp.DecodeIds(line)
def get_vocab_size(self):
"""Get vocab size."""
return self.sp.GetPieceSize()
def ids2tokens(self, line, **kwargs):
"""Return model-vocabulary pieces, or legacy decoded text without a token list."""
if getattr(self, "token_list", None) is not None:
return super().ids2tokens(line)
return self.decode(line, **kwargs)
def tokens2ids(self, *args, **kwargs):
"""Tokens2ids.
Args:
*args: Variable positional arguments.
**kwargs: Additional keyword arguments.
"""
return self.encode(*args, **kwargs)