Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
fix(data): read and write scp2jsonl/jsonl2scp lists as UTF-8 (#3774)
Mohammad Hijjawi28a889cba5a0
diff --git a/funasr/datasets/audio_datasets/jsonl2scp.py b/funasr/datasets/audio_datasets/jsonl2scp.pyindex cdd2f5e65..5e732596b 100644--- a/funasr/datasets/audio_datasets/jsonl2scp.py+++ b/funasr/datasets/audio_datasets/jsonl2scp.py@@ -19,8 +19,8 @@ def gen_scp_from_jsonl(jsonl_file, data_type_list, wav_scp_file, text_file):wav_scp_file: TODO.text_file: TODO."""- wav_f = open(wav_scp_file, "w")- text_f = open(text_file, "w")+ wav_f = open(wav_scp_file, "w", encoding="utf-8")+ text_f = open(text_file, "w", encoding="utf-8")with open(jsonl_file, encoding="utf-8") as fin:for line in fin:data = json.loads(line.strip())diff --git a/funasr/datasets/audio_datasets/scp2jsonl.py b/funasr/datasets/audio_datasets/scp2jsonl.pyindex f1f2f710f..1fbb34e41 100644--- a/funasr/datasets/audio_datasets/scp2jsonl.py+++ b/funasr/datasets/audio_datasets/scp2jsonl.py@@ -34,7 +34,7 @@ def gen_jsonl_from_wav_text_list(json_dict = {}for data_type, data_file in zip(data_type_list, path):json_dict[data_type] = {}- with open(data_file, "r") as f:+ with open(data_file, "r", encoding="utf-8") as f:data_file_lists = f.readlines()lines_for_each_th = (len(data_file_lists) - 1) // cpu_cores + 1@@ -62,7 +62,7 @@ def gen_jsonl_from_wav_text_list(res = parse_context_length(data_file_lists, data_type)json_dict[data_type].update(res)- with open(jsonl_file_out, "w") as f:+ with open(jsonl_file_out, "w", encoding="utf-8") as f:for key in json_dict[data_type_list[0]].keys():jsonl_line = {"key": key}for data_file in data_type_list:diff --git a/funasr/datasets/audio_datasets/sensevoice2jsonl.py b/funasr/datasets/audio_datasets/sensevoice2jsonl.pyindex e4f72366b..f9c40128e 100644--- a/funasr/datasets/audio_datasets/sensevoice2jsonl.py+++ b/funasr/datasets/audio_datasets/sensevoice2jsonl.py@@ -37,7 +37,7 @@ def gen_jsonl_from_wav_text_list(json_dict = {}for data_type, data_file in zip(data_type_list, path):json_dict[data_type] = {}- with open(data_file, "r") as f:+ with open(data_file, "r", encoding="utf-8") as f:data_file_lists = f.readlines()lines_for_each_th = (len(data_file_lists) - 1) // cpu_cores + 1@@ -112,7 +112,7 @@ def gen_jsonl_from_wav_text_list(json_dict["event_target"][key] = {}json_dict["event_target"][key]["event_target"] = rich_dict[key][2]- with open(jsonl_file_out, "w") as f:+ with open(jsonl_file_out, "w", encoding="utf-8") as f:for key in json_dict[data_type_list[0]].keys():jsonl_line = {"key": key}for data_file in data_type_list:diff --git a/tests/test_dataset_list_tools_encoding.py b/tests/test_dataset_list_tools_encoding.pynew file mode 100644index 000000000..3d1c74973--- /dev/null+++ b/tests/test_dataset_list_tools_encoding.py@@ -0,0 +1,106 @@+"""``scp2jsonl`` / ``sensevoice2jsonl`` / ``jsonl2scp`` text encoding.++The training data loader reads jsonl lists as UTF-8, but the conversion tools+opened ``wav.scp``/``text.txt`` and their outputs with the platform's locale+encoding. On Windows a UTF-8 Chinese transcript then failed to decode (cp1252)+or was rewritten in GBK (cp936), which the loader could not read back.+"""++import builtins+import json++import numpy as np+import pytest+import soundfile++from funasr.datasets.audio_datasets import jsonl2scp, scp2jsonl, sensevoice2jsonl++TEXT = "欢迎大家来体验达摩院推出的语音识别模型"+++def gbk_locale(monkeypatch):+ real_open = builtins.open++ def gbk_default_open(file, mode="r", *args, **kwargs):+ # Simulate a Chinese Windows locale: text mode without an explicit encoding.+ if "b" not in mode and kwargs.get("encoding") is None and len(args) < 2:+ kwargs["encoding"] = "cp936"+ kwargs.setdefault("errors", "replace")+ return real_open(file, mode, *args, **kwargs)++ monkeypatch.setattr(builtins, "open", gbk_default_open)+++def _write_wav(path):+ soundfile.write(str(path), np.zeros(1600, dtype="float32"), 16000)+++def _read_jsonl(path):+ with open(path, encoding="utf-8") as f:+ return [json.loads(line) for line in f]+++def test_scp2jsonl_reads_and_writes_utf8(tmp_path, gbk_locale):+ wav = tmp_path / "utt1.wav"+ _write_wav(wav)+ wav_scp = tmp_path / "wav.scp"+ wav_scp.write_text(f"utt1 {wav}\n", encoding="utf-8")+ text = tmp_path / "text.txt"+ text.write_text(f"utt1 {TEXT}\n", encoding="utf-8")+ out = tmp_path / "train.jsonl"++ scp2jsonl.gen_jsonl_from_wav_text_list(+ [str(wav_scp), str(text)], ("source", "target"), jsonl_file_out=str(out)+ )++ assert _read_jsonl(out) == [+ {+ "key": "utt1",+ "source": str(wav),+ "source_len": 10,+ "target": TEXT,+ "target_len": len(TEXT),+ }+ ]+++def test_sensevoice2jsonl_reads_and_writes_utf8(tmp_path, gbk_locale):+ wav = tmp_path / "utt1.wav"+ _write_wav(wav)+ files = {+ "source": f"utt1 {wav}\n",+ "target": f"utt1 {TEXT}\n",+ "text_language": "utt1 <|zh|>\n",+ "emo_target": "utt1 <|NEUTRAL|>\n",+ "event_target": "utt1 <|Speech|>\n",+ }+ paths = []+ for name, content in files.items():+ path = tmp_path / f"{name}.txt"+ path.write_text(content, encoding="utf-8")+ paths.append(str(path))+ out = tmp_path / "train.jsonl"++ # All rich labels are given, so no model is loaded.+ sensevoice2jsonl.gen_jsonl_from_wav_text_list(paths, list(files), jsonl_file_out=str(out))++ [row] = _read_jsonl(out)+ assert row["target"] == TEXT+ assert row["text_language"] == "<|zh|>"+++def test_jsonl2scp_writes_utf8(tmp_path, gbk_locale):+ jsonl = tmp_path / "train.jsonl"+ jsonl.write_text(+ json.dumps({"key": "utt1", "source": "/data/utt1.wav", "target": TEXT}, ensure_ascii=False)+ + "\n",+ encoding="utf-8",+ )+ wav_scp = tmp_path / "wav.scp"+ text = tmp_path / "text.txt"++ jsonl2scp.gen_scp_from_jsonl(str(jsonl), ("source", "target"), str(wav_scp), str(text))++ assert wav_scp.read_text(encoding="utf-8") == "utt1\t/data/utt1.wav\n"+ assert text.read_text(encoding="utf-8") == f"utt1\t{TEXT}\n"