Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(tokenizer): respect supplied vocabulary in ids2tokens

carey-bked5685c9becc
diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml
index 07f491091..aa0cc070d 100644
--- a/.github/workflows/test-numpy-compatibility.yml
+++ b/.github/workflows/test-numpy-compatibility.yml
@@ -8,6 +8,8 @@ on:
- "funasr/cli.py"
- "tests/test_cli.py"
- "funasr/auto/auto_model.py"
+ - "funasr/tokenizer/sentencepiece_tokenizer.py"
+ - "tests/test_sentencepiece_tokenizer.py"
- "funasr/utils/load_utils.py"
- "funasr/utils/timestamp_tools.py"
- "funasr/models/contextual_paraformer/model.py"
@@ -36,6 +38,8 @@ on:
- "funasr/cli.py"
- "tests/test_cli.py"
- "funasr/auto/auto_model.py"
+ - "funasr/tokenizer/sentencepiece_tokenizer.py"
+ - "tests/test_sentencepiece_tokenizer.py"
- "funasr/utils/load_utils.py"
- "funasr/utils/timestamp_tools.py"
- "funasr/models/contextual_paraformer/model.py"
@@ -105,6 +109,7 @@ jobs:
tests/test_eend_feature_compatibility.py \
tests/test_pypi_metadata.py \
tests/test_sensevoice_word_timestamps.py \
+ tests/test_sentencepiece_tokenizer.py \
tests/test_pcm_input_format.py \
tests/test_load_audio_bytes.py \
tests/test_timestamp_tools.py \
diff --git a/funasr/tokenizer/sentencepiece_tokenizer.py b/funasr/tokenizer/sentencepiece_tokenizer.py
index acba22c2a..1c76c8984 100644
--- a/funasr/tokenizer/sentencepiece_tokenizer.py
+++ b/funasr/tokenizer/sentencepiece_tokenizer.py
@@ -13,11 +13,11 @@ from funasr.register import tables
class SentencepiecesTokenizer(BaseTokenizer):
def __init__(self, bpemodel: Union[Path, str], **kwargs):
"""Initialize SentencepiecesTokenizer.
-
- Args:
- bpemodel: TODO.
- **kwargs: Additional keyword arguments.
- """
+
+ Args:
+ bpemodel: TODO.
+ **kwargs: Additional keyword arguments.
+ """
super().__init__(**kwargs)
self.bpemodel = str(bpemodel)
# NOTE(kamo):
@@ -41,39 +41,39 @@ class SentencepiecesTokenizer(BaseTokenizer):
def text2tokens(self, line: str) -> List[str]:
"""Text2tokens.
-
- Args:
- line: TODO.
- """
+
+ Args:
+ line: TODO.
+ """
self._build_sentence_piece_processor()
return self.sp.EncodeAsPieces(line)
def tokens2text(self, tokens: Iterable[str]) -> str:
"""Tokens2text.
-
- Args:
- tokens: TODO.
- """
+
+ Args:
+ tokens: TODO.
+ """
self._build_sentence_piece_processor()
return self.sp.DecodePieces(list(tokens))
def encode(self, line: str, **kwargs) -> List[int]:
"""Encode.
-
- Args:
- line: TODO.
- **kwargs: Additional keyword arguments.
- """
+
+ Args:
+ line: TODO.
+ **kwargs: Additional keyword arguments.
+ """
self._build_sentence_piece_processor()
return self.sp.EncodeAsIds(line)
def decode(self, line: List[int], **kwargs):
"""Decode.
-
- Args:
- line: TODO.
- **kwargs: Additional keyword arguments.
- """
+
+ Args:
+ line: TODO.
+ **kwargs: Additional keyword arguments.
+ """
self._build_sentence_piece_processor()
return self.sp.DecodeIds(line)
@@ -81,20 +81,17 @@ class SentencepiecesTokenizer(BaseTokenizer):
"""Get vocab size."""
return self.sp.GetPieceSize()
- def ids2tokens(self, *args, **kwargs):
- """Ids2tokens.
-
- Args:
- *args: Variable positional arguments.
- **kwargs: Additional keyword arguments.
- """
- return self.decode(*args, **kwargs)
+ def ids2tokens(self, line, **kwargs):
+ """Return model-vocabulary pieces, or legacy decoded text without a token list."""
+ if getattr(self, "token_list", None) is not None:
+ return super().ids2tokens(line)
+ return self.decode(line, **kwargs)
def tokens2ids(self, *args, **kwargs):
"""Tokens2ids.
-
- Args:
- *args: Variable positional arguments.
- **kwargs: Additional keyword arguments.
- """
+
+ Args:
+ *args: Variable positional arguments.
+ **kwargs: Additional keyword arguments.
+ """
return self.encode(*args, **kwargs)
diff --git a/tests/test_sentencepiece_tokenizer.py b/tests/test_sentencepiece_tokenizer.py
new file mode 100644
index 000000000..f3a4349ea
--- /dev/null
+++ b/tests/test_sentencepiece_tokenizer.py
@@ -0,0 +1,162 @@
+"""Model vocabulary IDs must reach callers as pieces, not BPE-decoded text."""
+
+import json
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import sentencepiece as spm
+import torch
+
+from funasr.models.paraformer.model import Paraformer
+from funasr.tokenizer.build_tokenizer import build_tokenizer
+from funasr.tokenizer.sentencepiece_tokenizer import SentencepiecesTokenizer
+
+
[email protected](scope="module")
+def bpe_model(tmp_path_factory):
+ prefix = tmp_path_factory.mktemp("sentencepiece") / "tiny"
+ spm.SentencePieceTrainer.train(
+ sentence_iterator=iter(
+ ["hello world", "hello there", "he tried to think how it could be", "你好 world"] * 20
+ ),
+ model_prefix=str(prefix),
+ model_type="bpe",
+ vocab_size=48,
+ user_defined_symbols=["<|en|>"],
+ shuffle_input_sentence=False,
+ num_threads=1,
+ minloglevel=2,
+ )
+ return str(prefix.with_suffix(".model"))
+
+
+def vocabulary(bpe_model, reordered=True):
+ sp = spm.SentencePieceProcessor(model_file=bpe_model)
+ pieces = [sp.id_to_piece(i) for i in range(sp.get_piece_size())]
+ return list(reversed(pieces)) if reordered else pieces
+
+
[email protected]("storage", ["list", "tuple", "json", "txt"])
+def test_reordered_vocabulary_decodes_expected_text(bpe_model, tmp_path, storage):
+ tokens = vocabulary(bpe_model)
+ token_list = tokens
+ if storage == "tuple":
+ token_list = tuple(tokens)
+ elif storage in ("json", "txt"):
+ path = tmp_path / ("tokens." + storage)
+ content = json.dumps(tokens) if storage == "json" else "\n".join(tokens) + "\n"
+ path.write_text(content, encoding="utf-8")
+ token_list = str(path)
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=token_list)
+ pieces = tokenizer.text2tokens("hello world")
+ model_ids = [tokens.index(piece) for piece in pieces]
+
+ # Establish that the two vocabularies really assign different meanings to IDs.
+ assert tokenizer.decode(model_ids) != "hello world"
+ result = tokenizer.ids2tokens(model_ids)
+
+ assert result == pieces
+ assert tokenizer.tokens2text(result) == "hello world"
+
+
[email protected]("text", ["hello world", "你好 world", "<|en|> hello world", ""])
+def test_matching_vocabulary_preserves_text_and_piece_boundaries(bpe_model, text):
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model, False))
+ ids = tokenizer.encode(text)
+
+ pieces = tokenizer.ids2tokens(ids)
+
+ assert pieces == tokenizer.text2tokens(text)
+ assert tokenizer.tokens2text(pieces) == tokenizer.decode(ids) == text
+
+
[email protected]("container", [list, tuple, iter, np.asarray])
+def test_model_ids_accept_base_tokenizer_containers(bpe_model, container):
+ tokens = vocabulary(bpe_model)
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens)
+ ids = [tokens.index(piece) for piece in tokenizer.text2tokens("hello world")]
+
+ assert tokenizer.tokens2text(tokenizer.ids2tokens(container(ids))) == "hello world"
+
+
+def test_special_tokens_and_model_only_ids_are_preserved(bpe_model):
+ tokens = vocabulary(bpe_model) + ["<blank>"]
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens)
+ pieces = ["<s>", "<unk>", "<|en|>", "▁", "▁", "</s>", "<blank>"]
+
+ assert tokenizer.ids2tokens([tokens.index(piece) for piece in pieces]) == pieces
+ assert tokenizer.ids2tokens([]) == []
+
+
+def test_model_ids_require_one_dimensional_numpy_array(bpe_model):
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model))
+
+ with pytest.raises(ValueError, match="Must be 1 dim ndarray"):
+ tokenizer.ids2tokens(np.array([[1, 2]]))
+
+
[email protected]("construction", ["omitted", "none", "build_tokenizer"])
+def test_without_vocabulary_preserves_legacy_text_decode(bpe_model, construction):
+ if construction == "build_tokenizer":
+ tokenizer = build_tokenizer("bpe", bpemodel=bpe_model)
+ elif construction == "none":
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=None)
+ else:
+ tokenizer = SentencepiecesTokenizer(bpe_model)
+ ids = tokenizer.encode("hello world")
+
+ assert tokenizer.ids2tokens(ids) == "hello world"
+ assert tokenizer.ids2tokens(line=ids) == "hello world"
+ assert tokenizer.ids2tokens([ids, ids]) == ["hello world", "hello world"]
+ assert tokenizer.ids2tokens([]) == ""
+
+
+def test_encode_decode_and_legacy_keyword_remain_sentencepiece_operations(bpe_model):
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=vocabulary(bpe_model))
+ ids = tokenizer.encode("hello world")
+ model_ids = [tokenizer.token_list.index(p) for p in tokenizer.text2tokens("hello world")]
+
+ assert tokenizer.tokens2ids("hello world") == ids
+ assert tokenizer.decode(ids) == "hello world"
+ assert tokenizer.tokens2text(tokenizer.ids2tokens(line=model_ids, unused=True)) == "hello world"
+
+
+def test_paraformer_inference_uses_model_pieces_and_filters_control_ids(bpe_model, tmp_path):
+ tokens = ["<blank>", "<s>", "</s>"] + [
+ p for p in vocabulary(bpe_model) if p not in ("<s>", "</s>")
+ ]
+ tokenizer = SentencepiecesTokenizer(bpe_model, token_list=tokens)
+ pieces = tokenizer.text2tokens("hello world")
+ ids = [0, 1] + [tokens.index(p) for p in pieces] + [2, 0]
+ scores = torch.full((1, len(ids), len(tokens)), -10.0)
+ scores[0, torch.arange(len(ids)), ids] = 10.0
+ lengths = torch.tensor([len(ids)])
+ speech = torch.zeros(1, len(ids), 2)
+ # Only the acoustic stages are controlled; the actual inference method,
+ # tokenizer, special-ID filtering, text conversion and writer all run.
+ state = SimpleNamespace(
+ beam_search=None,
+ sos=1,
+ eos=2,
+ blank_id=0,
+ encode=lambda *args: (speech, lengths),
+ calc_predictor=lambda *args: (speech, lengths.float(), None, None),
+ cal_decoder_with_predictor=lambda *args: (scores, lengths),
+ )
+
+ results, _ = Paraformer.inference(
+ state,
+ speech,
+ data_lengths=lengths,
+ key=["sample"],
+ tokenizer=tokenizer,
+ data_type="fbank",
+ device="cpu",
+ output_dir=str(tmp_path),
+ )
+
+ assert results == [{"key": "sample", "text": "hello world"}]
+ assert (tmp_path / "1best_recog" / "token").read_text().strip() == (
+ "sample " + " ".join(pieces)
+ )