Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: keep diffusion captions that hold U+2028, U+2029 or U+0085 (#17436)
Vineeth Sai Varikuntla5541af5d1f7a
diff --git a/studio/backend/core/training/diffusion_h3_clips.py b/studio/backend/core/training/diffusion_h3_clips.pyindex a7ee47008ca..81f47440e64 100644--- a/studio/backend/core/training/diffusion_h3_clips.py+++ b/studio/backend/core/training/diffusion_h3_clips.py@@ -149,7 +149,7 @@ def discover_clip_caption_pairs(if not meta_path.is_file():continuetry:- meta_lines = meta_path.read_text(encoding = "utf-8").splitlines()+ meta_lines = meta_path.read_text(encoding = "utf-8").split("\n")except (OSError, UnicodeError):continuefor line in meta_lines:diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex 5aafcd3dda6..efcd5f40ca9 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -1469,8 +1469,9 @@ def discover_image_caption_pairs(if not meta_path.is_file():continue# Skip a bad upload (invalid UTF-8 or non-object JSON) so the instance_prompt fallback still applies.+ # split("\n"), not splitlines(): JSON keeps U+2028, U+2029 and U+0085 raw inside a string.try:- meta_lines = meta_path.read_text(encoding = "utf-8").splitlines()+ meta_lines = meta_path.read_text(encoding = "utf-8").split("\n")except (OSError, UnicodeError):continuefor line in meta_lines:diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.pyindex 5e99e375b9f..7110a50895b 100644--- a/studio/backend/routes/training.py+++ b/studio/backend/routes/training.py@@ -4772,8 +4772,9 @@ def _load_metadata_captions(folder: Path) -> dict[str, str]:if not meta_path.is_file():continue# Tolerate a bad upload (invalid UTF-8, or non-object JSON): skip the record so these endpoints do not 500.+ # split("\n"), not splitlines(): JSON keeps U+2028, U+2029 and U+0085 raw inside a string.try:- lines = meta_path.read_text(encoding = "utf-8").splitlines()+ lines = meta_path.read_text(encoding = "utf-8").split("\n")except (OSError, UnicodeError):continuefor line in lines:@@ -5252,7 +5253,7 @@ def _materialize_imagefolder_jsonl(entry: dict, dest: Path, cap: int) -> int:captions: dict[str, str] = {}# read_text on a companion raises and fails the whole import.for jf in drop_appledouble_metadata(sorted(snap.rglob("*.jsonl"))):- for line in jf.read_text(encoding = "utf-8").splitlines():+ for line in jf.read_text(encoding = "utf-8").split("\n"):line = line.strip()if not line:continuediff --git a/studio/backend/tests/test_diffusion_dataset_clips.py b/studio/backend/tests/test_diffusion_dataset_clips.pyindex 9de05e377fa..f34e45de610 100644--- a/studio/backend/tests/test_diffusion_dataset_clips.py+++ b/studio/backend/tests/test_diffusion_dataset_clips.py@@ -399,6 +399,31 @@ def test_the_summary_reads_a_video_keyed_metadata_caption(tmp_path):assert len(discover_clip_caption_pairs(folder)) == 2[email protected]("separator", ["\u2028", "\u2029", "\u0085"])+def test_a_caption_holding_a_unicode_line_separator_is_not_dropped(tmp_path, separator):+ """splitlines() cut these raw in-string separators mid-row, silently dropping the caption."""+ from core.training.diffusion_h3_clips import discover_clip_caption_pairs+ from core.training.diffusion_train_common import discover_image_caption_pairs+ from routes.training import _load_metadata_captions++ folder = tmp_path / "separators"+ folder.mkdir()+ _write_png(folder / "a.png")+ (folder / "b.mp4").write_bytes(_MP4_BYTES)+ caption = f"line one{separator}line two"+ rows = [+ {"file_name": "a.png", "text": caption},+ {"file_name": "b.mp4", "text": caption},+ ]+ (folder / "metadata.jsonl").write_text(+ "\n".join(json.dumps(row, ensure_ascii = False) for row in rows) + "\n", encoding = "utf-8"+ )++ assert _load_metadata_captions(folder) == {"a.png": caption, "b.mp4": caption}+ assert [c for _, c in discover_image_caption_pairs(folder)] == [caption]+ assert [c for _, c in discover_clip_caption_pairs(folder)] == [caption]++def test_start_leaves_an_image_only_folder_alone(tmp_path):"""No clips, no refusal: the ordinary uncaptioned-folder error still speaks for itself."""from routes.training import _clip_dataset_refusal