Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(paraformer): skip blank lines and read hotword files as UTF-8 (#3761)

Mohammad Hijjawi8f8931dc9a84
diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.py
index 4c4df739b..a8f23e926 100644
--- a/funasr/models/contextual_paraformer/model.py
+++ b/funasr/models/contextual_paraformer/model.py
@@ -7,7 +7,6 @@ import os
import re
import time
import torch
-import codecs
import logging
import tempfile
import requests
@@ -596,9 +595,11 @@ class ContextualParaformer(Paraformer):
logging.info("Attempting to parse hotwords from local txt...")
hotword_list = []
hotword_str_list = []
- with codecs.open(hotword_list_or_file, "r") as fin:
+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
for line in fin.readlines():
hw = line.strip()
+ if not hw:
+ continue
hw_list = hw.split()
if seg_dict is not None:
hw_list = seg_tokenize(hw_list, seg_dict)
@@ -623,9 +624,11 @@ class ContextualParaformer(Paraformer):
hotword_list_or_file = text_file_path
hotword_list = []
hotword_str_list = []
- with codecs.open(hotword_list_or_file, "r") as fin:
+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
for line in fin.readlines():
hw = line.strip()
+ if not hw:
+ continue
hw_list = hw.split()
if seg_dict is not None:
hw_list = seg_tokenize(hw_list, seg_dict)
diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.py
index efa4745a3..6a5787530 100644
--- a/funasr/models/seaco_paraformer/model.py
+++ b/funasr/models/seaco_paraformer/model.py
@@ -8,7 +8,6 @@ import re
import time
import copy
import torch
-import codecs
import logging
import tempfile
import requests
@@ -629,9 +628,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
logging.info("Attempting to parse hotwords from local txt...")
hotword_list = []
hotword_str_list = []
- with codecs.open(hotword_list_or_file, "r") as fin:
+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
for line in fin.readlines():
hw = line.strip()
+ if not hw:
+ continue
hw_list = hw.split()
if seg_dict is not None:
hw_list = seg_tokenize(hw_list, seg_dict)
@@ -656,9 +657,11 @@ class SeacoParaformer(BiCifParaformer, Paraformer):
hotword_list_or_file = text_file_path
hotword_list = []
hotword_str_list = []
- with codecs.open(hotword_list_or_file, "r") as fin:
+ with open(hotword_list_or_file, "r", encoding="utf-8") as fin:
for line in fin.readlines():
hw = line.strip()
+ if not hw:
+ continue
hw_list = hw.split()
if seg_dict is not None:
hw_list = seg_tokenize(hw_list, seg_dict)
diff --git a/tests/test_paraformer_hotword_file.py b/tests/test_paraformer_hotword_file.py
new file mode 100644
index 000000000..c12ff4557
--- /dev/null
+++ b/tests/test_paraformer_hotword_file.py
@@ -0,0 +1,84 @@
+"""Hotword ``.txt`` parsing for SeACo-Paraformer and Contextual Paraformer.
+
+``generate_hotwords_list`` turns every line of a hotword file into a list of
+token ids. A blank line (for example an extra newline at the end of the file)
+used to become an empty hotword, and the zero length then made
+``pack_padded_sequence`` raise during decoding. The file was also read with the
+platform's locale encoding, so a UTF-8 Chinese hotword file decoded into
+``<unk>`` tokens (or raised) on Windows.
+"""
+
+import builtins
+import os
+import types
+
+import pytest
+import torch
+
+from funasr.models.contextual_paraformer.model import ContextualParaformer
+from funasr.models.seaco_paraformer.model import SeacoParaformer
+
+SOS = 1
+VOCAB = {"<unk>": 0, "<s>": SOS, "阿": 2, "里": 3, "巴": 4, "云": 5}
+
+
+class _Tokenizer:
+ def tokens2ids(self, tokens):
+ return [VOCAB.get(t, VOCAB["<unk>"]) for t in tokens]
+
+
+def _generate(model_cls, tmp_path, content):
+ # Released Paraformer models ship a seg_dict next to am.mvn; it splits
+ # Chinese words into characters.
+ (tmp_path / "seg_dict").write_text("阿 阿\n里 里\n巴 巴\n云 云\n", encoding="utf-8")
+ hotword_file = tmp_path / "hotwords.txt"
+ hotword_file.write_text(content, encoding="utf-8")
+ frontend = types.SimpleNamespace(cmvn_file=os.path.join(str(tmp_path), "am.mvn"))
+ return model_cls.generate_hotwords_list(
+ types.SimpleNamespace(sos=SOS),
+ str(hotword_file),
+ tokenizer=_Tokenizer(),
+ frontend=frontend,
+ )
+
+
[email protected](autouse=True)
+def _fresh_seg_dict_cache():
+ from funasr.models.seaco_paraformer import model as seaco_model
+
+ seaco_model.load_seg_dict.cache_clear()
+ yield
+ seaco_model.load_seg_dict.cache_clear()
+
+
[email protected]("model_cls", [SeacoParaformer, ContextualParaformer])
+def test_blank_lines_in_hotword_file_are_skipped(model_cls, tmp_path):
+ hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n\n \n阿里云\n\n")
+
+ assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]
+ # Every hotword must be packable by the bias encoder.
+ lengths = torch.tensor([len(hw) for hw in hotword_list])
+ torch.nn.utils.rnn.pack_padded_sequence(
+ torch.zeros(len(hotword_list), int(lengths.max()), 4),
+ lengths,
+ batch_first=True,
+ enforce_sorted=False,
+ )
+
+
[email protected]("model_cls", [SeacoParaformer, ContextualParaformer])
+def test_hotword_file_is_read_as_utf8_regardless_of_locale(model_cls, tmp_path, monkeypatch):
+ real_open = builtins.open
+
+ def cp1252_default_open(file, mode="r", *args, **kwargs):
+ # Simulate a Windows locale: text mode without an explicit encoding.
+ if "b" not in mode and kwargs.get("encoding") is None and len(args) < 2:
+ kwargs["encoding"] = "cp1252"
+ kwargs.setdefault("errors", "replace")
+ return real_open(file, mode, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "open", cp1252_default_open)
+
+ hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n阿里云\n")
+
+ assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]