Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(paraformer): skip blank lines and read hotword files as UTF-8 (#3761)
Mohammad Hijjawi8f8931dc9a84
diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.pyindex 4c4df739b..a8f23e926 100644--- a/funasr/models/contextual_paraformer/model.py+++ b/funasr/models/contextual_paraformer/model.py@@ -7,7 +7,6 @@ import osimport reimport timeimport torch-import codecsimport loggingimport tempfileimport requests@@ -596,9 +595,11 @@ class ContextualParaformer(Paraformer):logging.info("Attempting to parse hotwords from local txt...")hotword_list = []hotword_str_list = []- with codecs.open(hotword_list_or_file, "r") as fin:+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:for line in fin.readlines():hw = line.strip()+ if not hw:+ continuehw_list = hw.split()if seg_dict is not None:hw_list = seg_tokenize(hw_list, seg_dict)@@ -623,9 +624,11 @@ class ContextualParaformer(Paraformer):hotword_list_or_file = text_file_pathhotword_list = []hotword_str_list = []- with codecs.open(hotword_list_or_file, "r") as fin:+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:for line in fin.readlines():hw = line.strip()+ if not hw:+ continuehw_list = hw.split()if seg_dict is not None:hw_list = seg_tokenize(hw_list, seg_dict)diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.pyindex efa4745a3..6a5787530 100644--- a/funasr/models/seaco_paraformer/model.py+++ b/funasr/models/seaco_paraformer/model.py@@ -8,7 +8,6 @@ import reimport timeimport copyimport torch-import codecsimport loggingimport tempfileimport requests@@ -629,9 +628,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):logging.info("Attempting to parse hotwords from local txt...")hotword_list = []hotword_str_list = []- with codecs.open(hotword_list_or_file, "r") as fin:+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:for line in fin.readlines():hw = line.strip()+ if not hw:+ continuehw_list = hw.split()if seg_dict is not None:hw_list = seg_tokenize(hw_list, seg_dict)@@ -656,9 +657,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):hotword_list_or_file = text_file_pathhotword_list = []hotword_str_list = []- with codecs.open(hotword_list_or_file, "r") as fin:+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:for line in fin.readlines():hw = line.strip()+ if not hw:+ continuehw_list = hw.split()if seg_dict is not None:hw_list = seg_tokenize(hw_list, seg_dict)diff --git a/tests/test_paraformer_hotword_file.py b/tests/test_paraformer_hotword_file.pynew file mode 100644index 000000000..c12ff4557--- /dev/null+++ b/tests/test_paraformer_hotword_file.py@@ -0,0 +1,84 @@+"""Hotword ``.txt`` parsing for SeACo-Paraformer and Contextual Paraformer.++``generate_hotwords_list`` turns every line of a hotword file into a list of+token ids. A blank line (for example an extra newline at the end of the file)+used to become an empty hotword, and the zero length then made+``pack_padded_sequence`` raise during decoding. The file was also read with the+platform's locale encoding, so a UTF-8 Chinese hotword file decoded into+``<unk>`` tokens (or raised) on Windows.+"""++import builtins+import os+import types++import pytest+import torch++from funasr.models.contextual_paraformer.model import ContextualParaformer+from funasr.models.seaco_paraformer.model import SeacoParaformer++SOS = 1+VOCAB = {"<unk>": 0, "<s>": SOS, "阿": 2, "里": 3, "巴": 4, "云": 5}+++class _Tokenizer:+ def tokens2ids(self, tokens):+ return [VOCAB.get(t, VOCAB["<unk>"]) for t in tokens]+++def _generate(model_cls, tmp_path, content):+ # Released Paraformer models ship a seg_dict next to am.mvn; it splits+ # Chinese words into characters.+ (tmp_path / "seg_dict").write_text("阿 阿\n里 里\n巴 巴\n云 云\n", encoding="utf-8")+ hotword_file = tmp_path / "hotwords.txt"+ hotword_file.write_text(content, encoding="utf-8")+ frontend = types.SimpleNamespace(cmvn_file=os.path.join(str(tmp_path), "am.mvn"))+ return model_cls.generate_hotwords_list(+ types.SimpleNamespace(sos=SOS),+ str(hotword_file),+ tokenizer=_Tokenizer(),+ frontend=frontend,+ )++[email protected](autouse=True)+def _fresh_seg_dict_cache():+ from funasr.models.seaco_paraformer import model as seaco_model++ seaco_model.load_seg_dict.cache_clear()+ yield+ seaco_model.load_seg_dict.cache_clear()++[email protected]("model_cls", [SeacoParaformer, ContextualParaformer])+def test_blank_lines_in_hotword_file_are_skipped(model_cls, tmp_path):+ hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n\n \n阿里云\n\n")++ assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]+ # Every hotword must be packable by the bias encoder.+ lengths = torch.tensor([len(hw) for hw in hotword_list])+ torch.nn.utils.rnn.pack_padded_sequence(+ torch.zeros(len(hotword_list), int(lengths.max()), 4),+ lengths,+ batch_first=True,+ enforce_sorted=False,+ )++[email protected]("model_cls", [SeacoParaformer, ContextualParaformer])+def test_hotword_file_is_read_as_utf8_regardless_of_locale(model_cls, tmp_path, monkeypatch):+ real_open = builtins.open++ def cp1252_default_open(file, mode="r", *args, **kwargs):+ # Simulate a Windows locale: text mode without an explicit encoding.+ if "b" not in mode and kwargs.get("encoding") is None and len(args) < 2:+ kwargs["encoding"] = "cp1252"+ kwargs.setdefault("errors", "replace")+ return real_open(file, mode, *args, **kwargs)++ monkeypatch.setattr(builtins, "open", cp1252_default_open)++ hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n阿里云\n")++ assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]