Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep diffusion captions that hold U+2028, U+2029 or U+0085 (#17436)

Vineeth Sai Varikuntla5541af5d1f7a
diff --git a/studio/backend/core/training/diffusion_h3_clips.py b/studio/backend/core/training/diffusion_h3_clips.py
index a7ee47008ca..81f47440e64 100644
--- a/studio/backend/core/training/diffusion_h3_clips.py
+++ b/studio/backend/core/training/diffusion_h3_clips.py
@@ -149,7 +149,7 @@ def discover_clip_caption_pairs(
if not meta_path.is_file():
continue
try:
- meta_lines = meta_path.read_text(encoding = "utf-8").splitlines()
+ meta_lines = meta_path.read_text(encoding = "utf-8").split("\n")
except (OSError, UnicodeError):
continue
for line in meta_lines:
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index 5aafcd3dda6..efcd5f40ca9 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -1469,8 +1469,9 @@ def discover_image_caption_pairs(
if not meta_path.is_file():
continue
# Skip a bad upload (invalid UTF-8 or non-object JSON) so the instance_prompt fallback still applies.
+ # split("\n"), not splitlines(): JSON keeps U+2028, U+2029 and U+0085 raw inside a string.
try:
- meta_lines = meta_path.read_text(encoding = "utf-8").splitlines()
+ meta_lines = meta_path.read_text(encoding = "utf-8").split("\n")
except (OSError, UnicodeError):
continue
for line in meta_lines:
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 5e99e375b9f..7110a50895b 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -4772,8 +4772,9 @@ def _load_metadata_captions(folder: Path) -> dict[str, str]:
if not meta_path.is_file():
continue
# Tolerate a bad upload (invalid UTF-8, or non-object JSON): skip the record so these endpoints do not 500.
+ # split("\n"), not splitlines(): JSON keeps U+2028, U+2029 and U+0085 raw inside a string.
try:
- lines = meta_path.read_text(encoding = "utf-8").splitlines()
+ lines = meta_path.read_text(encoding = "utf-8").split("\n")
except (OSError, UnicodeError):
continue
for line in lines:
@@ -5252,7 +5253,7 @@ def _materialize_imagefolder_jsonl(entry: dict, dest: Path, cap: int) -> int:
captions: dict[str, str] = {}
# read_text on a companion raises and fails the whole import.
for jf in drop_appledouble_metadata(sorted(snap.rglob("*.jsonl"))):
- for line in jf.read_text(encoding = "utf-8").splitlines():
+ for line in jf.read_text(encoding = "utf-8").split("\n"):
line = line.strip()
if not line:
continue
diff --git a/studio/backend/tests/test_diffusion_dataset_clips.py b/studio/backend/tests/test_diffusion_dataset_clips.py
index 9de05e377fa..f34e45de610 100644
--- a/studio/backend/tests/test_diffusion_dataset_clips.py
+++ b/studio/backend/tests/test_diffusion_dataset_clips.py
@@ -399,6 +399,31 @@ def test_the_summary_reads_a_video_keyed_metadata_caption(tmp_path):
assert len(discover_clip_caption_pairs(folder)) == 2
[email protected]("separator", ["\u2028", "\u2029", "\u0085"])
+def test_a_caption_holding_a_unicode_line_separator_is_not_dropped(tmp_path, separator):
+ """splitlines() cut these raw in-string separators mid-row, silently dropping the caption."""
+ from core.training.diffusion_h3_clips import discover_clip_caption_pairs
+ from core.training.diffusion_train_common import discover_image_caption_pairs
+ from routes.training import _load_metadata_captions
+
+ folder = tmp_path / "separators"
+ folder.mkdir()
+ _write_png(folder / "a.png")
+ (folder / "b.mp4").write_bytes(_MP4_BYTES)
+ caption = f"line one{separator}line two"
+ rows = [
+ {"file_name": "a.png", "text": caption},
+ {"file_name": "b.mp4", "text": caption},
+ ]
+ (folder / "metadata.jsonl").write_text(
+ "\n".join(json.dumps(row, ensure_ascii = False) for row in rows) + "\n", encoding = "utf-8"
+ )
+
+ assert _load_metadata_captions(folder) == {"a.png": caption, "b.mp4": caption}
+ assert [c for _, c in discover_image_caption_pairs(folder)] == [caption]
+ assert [c for _, c in discover_clip_caption_pairs(folder)] == [caption]
+
+
def test_start_leaves_an_image_only_folder_alone(tmp_path):
"""No clips, no refusal: the ordinary uncaptioned-folder error still speaks for itself."""
from routes.training import _clip_dataset_refusal