Skip to content

Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.

fix(data): read and write scp2jsonl/jsonl2scp lists as UTF-8 (#3774)

Mohammad Hijjawi28a889cba5a0
diff --git a/funasr/datasets/audio_datasets/jsonl2scp.py b/funasr/datasets/audio_datasets/jsonl2scp.py
index cdd2f5e65..5e732596b 100644
--- a/funasr/datasets/audio_datasets/jsonl2scp.py
+++ b/funasr/datasets/audio_datasets/jsonl2scp.py
@@ -19,8 +19,8 @@ def gen_scp_from_jsonl(jsonl_file, data_type_list, wav_scp_file, text_file):
wav_scp_file: TODO.
text_file: TODO.
"""
- wav_f = open(wav_scp_file, "w")
- text_f = open(text_file, "w")
+ wav_f = open(wav_scp_file, "w", encoding="utf-8")
+ text_f = open(text_file, "w", encoding="utf-8")
with open(jsonl_file, encoding="utf-8") as fin:
for line in fin:
data = json.loads(line.strip())
diff --git a/funasr/datasets/audio_datasets/scp2jsonl.py b/funasr/datasets/audio_datasets/scp2jsonl.py
index f1f2f710f..1fbb34e41 100644
--- a/funasr/datasets/audio_datasets/scp2jsonl.py
+++ b/funasr/datasets/audio_datasets/scp2jsonl.py
@@ -34,7 +34,7 @@ def gen_jsonl_from_wav_text_list(
json_dict = {}
for data_type, data_file in zip(data_type_list, path):
json_dict[data_type] = {}
- with open(data_file, "r") as f:
+ with open(data_file, "r", encoding="utf-8") as f:
data_file_lists = f.readlines()
lines_for_each_th = (len(data_file_lists) - 1) // cpu_cores + 1
@@ -62,7 +62,7 @@ def gen_jsonl_from_wav_text_list(
res = parse_context_length(data_file_lists, data_type)
json_dict[data_type].update(res)
- with open(jsonl_file_out, "w") as f:
+ with open(jsonl_file_out, "w", encoding="utf-8") as f:
for key in json_dict[data_type_list[0]].keys():
jsonl_line = {"key": key}
for data_file in data_type_list:
diff --git a/funasr/datasets/audio_datasets/sensevoice2jsonl.py b/funasr/datasets/audio_datasets/sensevoice2jsonl.py
index e4f72366b..f9c40128e 100644
--- a/funasr/datasets/audio_datasets/sensevoice2jsonl.py
+++ b/funasr/datasets/audio_datasets/sensevoice2jsonl.py
@@ -37,7 +37,7 @@ def gen_jsonl_from_wav_text_list(
json_dict = {}
for data_type, data_file in zip(data_type_list, path):
json_dict[data_type] = {}
- with open(data_file, "r") as f:
+ with open(data_file, "r", encoding="utf-8") as f:
data_file_lists = f.readlines()
lines_for_each_th = (len(data_file_lists) - 1) // cpu_cores + 1
@@ -112,7 +112,7 @@ def gen_jsonl_from_wav_text_list(
json_dict["event_target"][key] = {}
json_dict["event_target"][key]["event_target"] = rich_dict[key][2]
- with open(jsonl_file_out, "w") as f:
+ with open(jsonl_file_out, "w", encoding="utf-8") as f:
for key in json_dict[data_type_list[0]].keys():
jsonl_line = {"key": key}
for data_file in data_type_list:
diff --git a/tests/test_dataset_list_tools_encoding.py b/tests/test_dataset_list_tools_encoding.py
new file mode 100644
index 000000000..3d1c74973
--- /dev/null
+++ b/tests/test_dataset_list_tools_encoding.py
@@ -0,0 +1,106 @@
+"""``scp2jsonl`` / ``sensevoice2jsonl`` / ``jsonl2scp`` text encoding.
+
+The training data loader reads jsonl lists as UTF-8, but the conversion tools
+opened ``wav.scp``/``text.txt`` and their outputs with the platform's locale
+encoding. On Windows a UTF-8 Chinese transcript then failed to decode (cp1252)
+or was rewritten in GBK (cp936), which the loader could not read back.
+"""
+
+import builtins
+import json
+
+import numpy as np
+import pytest
+import soundfile
+
+from funasr.datasets.audio_datasets import jsonl2scp, scp2jsonl, sensevoice2jsonl
+
+TEXT = "欢迎大家来体验达摩院推出的语音识别模型"
+
+
+def gbk_locale(monkeypatch):
+ real_open = builtins.open
+
+ def gbk_default_open(file, mode="r", *args, **kwargs):
+ # Simulate a Chinese Windows locale: text mode without an explicit encoding.
+ if "b" not in mode and kwargs.get("encoding") is None and len(args) < 2:
+ kwargs["encoding"] = "cp936"
+ kwargs.setdefault("errors", "replace")
+ return real_open(file, mode, *args, **kwargs)
+
+ monkeypatch.setattr(builtins, "open", gbk_default_open)
+
+
+def _write_wav(path):
+ soundfile.write(str(path), np.zeros(1600, dtype="float32"), 16000)
+
+
+def _read_jsonl(path):
+ with open(path, encoding="utf-8") as f:
+ return [json.loads(line) for line in f]
+
+
+def test_scp2jsonl_reads_and_writes_utf8(tmp_path, gbk_locale):
+ wav = tmp_path / "utt1.wav"
+ _write_wav(wav)
+ wav_scp = tmp_path / "wav.scp"
+ wav_scp.write_text(f"utt1 {wav}\n", encoding="utf-8")
+ text = tmp_path / "text.txt"
+ text.write_text(f"utt1 {TEXT}\n", encoding="utf-8")
+ out = tmp_path / "train.jsonl"
+
+ scp2jsonl.gen_jsonl_from_wav_text_list(
+ [str(wav_scp), str(text)], ("source", "target"), jsonl_file_out=str(out)
+ )
+
+ assert _read_jsonl(out) == [
+ {
+ "key": "utt1",
+ "source": str(wav),
+ "source_len": 10,
+ "target": TEXT,
+ "target_len": len(TEXT),
+ }
+ ]
+
+
+def test_sensevoice2jsonl_reads_and_writes_utf8(tmp_path, gbk_locale):
+ wav = tmp_path / "utt1.wav"
+ _write_wav(wav)
+ files = {
+ "source": f"utt1 {wav}\n",
+ "target": f"utt1 {TEXT}\n",
+ "text_language": "utt1 <|zh|>\n",
+ "emo_target": "utt1 <|NEUTRAL|>\n",
+ "event_target": "utt1 <|Speech|>\n",
+ }
+ paths = []
+ for name, content in files.items():
+ path = tmp_path / f"{name}.txt"
+ path.write_text(content, encoding="utf-8")
+ paths.append(str(path))
+ out = tmp_path / "train.jsonl"
+
+ # All rich labels are given, so no model is loaded.
+ sensevoice2jsonl.gen_jsonl_from_wav_text_list(paths, list(files), jsonl_file_out=str(out))
+
+ [row] = _read_jsonl(out)
+ assert row["target"] == TEXT
+ assert row["text_language"] == "<|zh|>"
+
+
+def test_jsonl2scp_writes_utf8(tmp_path, gbk_locale):
+ jsonl = tmp_path / "train.jsonl"
+ jsonl.write_text(
+ json.dumps({"key": "utt1", "source": "/data/utt1.wav", "target": TEXT}, ensure_ascii=False)
+ + "\n",
+ encoding="utf-8",
+ )
+ wav_scp = tmp_path / "wav.scp"
+ text = tmp_path / "text.txt"
+
+ jsonl2scp.gen_scp_from_jsonl(str(jsonl), ("source", "target"), str(wav_scp), str(text))
+
+ assert wav_scp.read_text(encoding="utf-8") == "utt1\t/data/utt1.wav\n"
+ assert text.read_text(encoding="utf-8") == f"utt1\t{TEXT}\n"