Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): preserve Unicode separators in S3 JSON Lines transcripts (#16725)

Max-Reisinger1b20b9e83cec
diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.py
index c358bf9c911..2e9fbcb708d 100644
--- a/studio/backend/core/training/s3_dataset.py
+++ b/studio/backend/core/training/s3_dataset.py
@@ -282,7 +282,8 @@ def _rewrite_json_manifest(manifest_path: str, resolve) -> None:
except json.JSONDecodeError:
data = None # JSON Lines in a .json file
if data is None:
- rows = [json.loads(line) for line in text.splitlines() if line.strip()]
+ # Text mode normalizes CRLF; Unicode separators inside strings are data.
+ rows = [json.loads(line) for line in text.split("\n") if line.strip()]
for row in rows:
_rewrite_row(row, resolve)
with open(manifest_path, "w", encoding = "utf-8") as f:
diff --git a/studio/backend/tests/test_s3_audio_dataset.py b/studio/backend/tests/test_s3_audio_dataset.py
index e7040390401..db6beff6a89 100644
--- a/studio/backend/tests/test_s3_audio_dataset.py
+++ b/studio/backend/tests/test_s3_audio_dataset.py
@@ -58,7 +58,7 @@ class _FakeS3Client:
callback = kwargs.get("Callback")
if callback is not None:
callback(1)
- with open(local_path, "w", encoding = "utf-8") as f:
+ with open(local_path, "w", encoding = "utf-8", newline = "") as f:
f.write(self._contents.get(key, f"content-of:{key}"))
@@ -137,6 +137,24 @@ def test_audio_with_no_manifest_names_whats_missing(monkeypatch, tmp_path):
s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
[email protected]("suffix", [".jsonl", ".json"])
[email protected]("newline", ["\n", "\r\n"], ids = ["lf", "crlf"])
+def test_s3_json_lines_preserves_unicode_transcript(monkeypatch, tmp_path, suffix, newline):
+ rows = [
+ {"audio": "audio/a.wav", "text": "first\u0085second\u2028third\u2029fourth"},
+ {"audio": "audio/a.wav", "text": "last"},
+ ]
+ manifest = "".join(json.dumps(row, ensure_ascii = False) + newline for row in rows)
+ key = f"datasets/train{suffix}"
+ _install(monkeypatch, [key, "datasets/audio/a.wav"], contents = {key: manifest})
+
+ result = s3_dataset.prepare_s3_dataset_download(_cfg(), dest_dir = str(tmp_path))
+
+ audio_path = tmp_path / "audio" / "a.wav"
+ assert audio_path.is_file()
+ assert _rows(result.files[0]) == [{**row, "audio": str(audio_path)} for row in rows]
+
+
def test_jsonl_audio_references_are_rewritten_to_local_paths(monkeypatch, tmp_path):
manifest = "\n".join(
[