Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): preserve Unicode separators in S3 JSON Lines transcripts (#16725)
Max-Reisinger1b20b9e83cec
diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.pyindex c358bf9c911..2e9fbcb708d 100644--- a/studio/backend/core/training/s3_dataset.py+++ b/studio/backend/core/training/s3_dataset.py@@ -282,7 +282,8 @@ def _rewrite_json_manifest(manifest_path: str, resolve) -> None:except json.JSONDecodeError:data = None # JSON Lines in a .json fileif data is None:- rows = [json.loads(line) for line in text.splitlines() if line.strip()]+ # Text mode normalizes CRLF; Unicode separators inside strings are data.+ rows = [json.loads(line) for line in text.split("\n") if line.strip()]for row in rows:_rewrite_row(row, resolve)with open(manifest_path, "w", encoding = "utf-8") as f:diff --git a/studio/backend/tests/test_s3_audio_dataset.py b/studio/backend/tests/test_s3_audio_dataset.pyindex e7040390401..db6beff6a89 100644--- a/studio/backend/tests/test_s3_audio_dataset.py+++ b/studio/backend/tests/test_s3_audio_dataset.py@@ -58,7 +58,7 @@ class _FakeS3Client:callback = kwargs.get("Callback")if callback is not None:callback(1)- with open(local_path, "w", encoding = "utf-8") as f:+ with open(local_path, "w", encoding = "utf-8", newline = "") as f:f.write(self._contents.get(key, f"content-of:{key}"))@@ -137,6 +137,24 @@ def test_audio_with_no_manifest_names_whats_missing(monkeypatch, tmp_path):s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))[email protected]("suffix", [".jsonl", ".json"])[email protected]("newline", ["\n", "\r\n"], ids = ["lf", "crlf"])+def test_s3_json_lines_preserves_unicode_transcript(monkeypatch, tmp_path, suffix, newline):+ rows = [+ {"audio": "audio/a.wav", "text": "first\u0085second\u2028third\u2029fourth"},+ {"audio": "audio/a.wav", "text": "last"},+ ]+ manifest = "".join(json.dumps(row, ensure_ascii = False) + newline for row in rows)+ key = f"datasets/train{suffix}"+ _install(monkeypatch, [key, "datasets/audio/a.wav"], contents = {key: manifest})++ result = s3_dataset.prepare_s3_dataset_download(_cfg(), dest_dir = str(tmp_path))++ audio_path = tmp_path / "audio" / "a.wav"+ assert audio_path.is_file()+ assert _rows(result.files[0]) == [{**row, "audio": str(audio_path)} for row in rows]++def test_jsonl_audio_references_are_rewritten_to_local_paths(monkeypatch, tmp_path):manifest = "\n".join([