Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: preserve S3 dataset files with colliding names (#16830)
DimitriChrysafis41da0b13f25d
diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.pyindex 2e9fbcb708d..c7d8417f868 100644--- a/studio/backend/core/training/s3_dataset.py+++ b/studio/backend/core/training/s3_dataset.py@@ -19,6 +19,7 @@ import loggingimport osimport shutilimport tempfile+import unicodedatafrom importlib.util import find_specfrom typing import Callable, Optionalfrom utils.datasets.format_detection import _AUDIO_EXTENSIONS@@ -158,15 +159,19 @@ def _contained_local_path(return pathmod.join(target_dir, *parts)+def _local_path_key(path: str) -> str:+ return unicodedata.normalize("NFC", os.path.normpath(path)).casefold()++def _unique_local_path(target_dir: str, filename: str, used_paths: set[str]) -> str:- """Return an unused flattened path for an S3 object basename."""+ """Return an unused local path for an S3 object basename."""stem, ext = os.path.splitext(filename)candidate = _contained_local_path(target_dir, [filename])suffix = 1- while candidate in used_paths or os.path.exists(candidate):+ while _local_path_key(candidate) in used_paths or os.path.exists(candidate):candidate = _contained_local_path(target_dir, [f"{stem}_{suffix}{ext}"])suffix += 1- used_paths.add(candidate)+ used_paths.add(_local_path_key(candidate))return candidate@@ -192,8 +197,9 @@ def _key_relative_to_prefix(key: str, prefix: Optional[str]) -> str:def _structured_local_paths(prefix: Optional[str], keys: list[str], target_dir: str) -> dict[str, str]:- """Map ``keys`` to local paths mirroring their prefix-relative layout."""+ """Map keys to distinct local paths under their prefix-relative directories."""local_by_key: dict[str, str] = {}+ used_paths: set[str] = set()for key in keys:relative = _key_relative_to_prefix(key, prefix)parts = [part for part in relative.split("/") if part not in ("", ".")]@@ -203,7 +209,10 @@ def _structured_local_paths(f"S3 key {key!r} contains '..' or empty path segments and cannot ""be downloaded into the dataset directory.")- local_by_key[key] = _contained_local_path(target_dir, parts)+ path = _contained_local_path(target_dir, parts)+ local_by_key[key] = _unique_local_path(+ os.path.dirname(path), os.path.basename(path), used_paths+ )return local_by_key@@ -369,8 +378,9 @@ def prepare_s3_dataset_download(local_files: list[str] = []if audio_keys:- manifest_local_by_key = _structured_local_paths(prefix, keys, target_dir)- audio_local_by_key = _structured_local_paths(prefix, audio_keys, target_dir)+ local_by_key = _structured_local_paths(prefix, keys + audio_keys, target_dir)+ manifest_local_by_key = {key: local_by_key[key] for key in keys}+ audio_local_by_key = {key: local_by_key[key] for key in audio_keys}_download_structured(client, bucket, manifest_local_by_key, cancel_callback)# Rewrite before the audio download so a bad manifest fails without fetching it._rewrite_audio_references(manifest_local_by_key, audio_local_by_key, bucket, prefix)diff --git a/studio/backend/tests/test_s3_audio_dataset.py b/studio/backend/tests/test_s3_audio_dataset.pyindex db6beff6a89..0578a4ca268 100644--- a/studio/backend/tests/test_s3_audio_dataset.py+++ b/studio/backend/tests/test_s3_audio_dataset.py@@ -6,8 +6,10 @@ boto3 is faked as in test_s3_dataset.py."""import csvimport importlib.util+import ioimport jsonimport os+import unicodedatafrom pathlib import Pathimport pytest@@ -378,3 +380,165 @@ def test_ordinary_keys_stay_inside_the_download_dir(parts, tmp_path):if "\\" not in parts[-1]:win = s3_dataset._contained_local_path("C:\\data\\download", parts, pathmod = ntpath)assert win == ntpath.join("C:\\data\\download", *parts)++[email protected]("manifest_format", ["jsonl", "json", "csv"])+ "names",+ [+ ("audio/clip.wav", "audio//clip.wav"),+ ("audio/clip.wav", "audio/./clip.wav"),+ ("audio/A.wav", "audio/a.wav"),+ ("audio/caf\u00e9.wav", "audio/cafe\u0301.wav"),+ ("Audio/clip.wav", "audio/clip.wav"),+ ("caf\u00e9/clip.wav", "cafe\u0301/clip.wav"),+ ],+)+def test_audio_aliases_keep_each_recording_and_its_transcript(+ monkeypatch, tmp_path, names, manifest_format+):+ keys = [f"datasets/{name}" for name in names]+ records = [+ {"audio": f"s3://my-bucket/{key}", "text": f"transcript-{i}, heard"}+ for i, key in enumerate(keys)+ ]+ if manifest_format == "csv":+ handle = io.StringIO()+ writer = csv.DictWriter(handle, fieldnames = ["audio", "text"])+ writer.writeheader()+ writer.writerows(records)+ manifest = handle.getvalue()+ elif manifest_format == "json":+ manifest = json.dumps(records)+ else:+ manifest = "\n".join(map(json.dumps, records))+ manifest_key = f"datasets/train.{manifest_format}"+ _install(+ monkeypatch,+ [manifest_key, *keys],+ contents = {+ manifest_key: manifest + "\n",+ keys[0]: "FIRST",+ keys[1]: "SECOND",+ },+ )+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ if manifest_format == "csv":+ with open(files[0], encoding = "utf-8", newline = "") as handle:+ rows = list(csv.DictReader(handle))+ elif manifest_format == "json":+ rows = json.loads(Path(files[0]).read_text())+ else:+ rows = _rows(files[0])+ paths = [row["audio"] for row in rows]+ assert len({unicodedata.normalize("NFC", path).casefold() for path in paths}) == 2+ assert [Path(path).read_text() for path in paths] == ["FIRST", "SECOND"]+ assert [row["text"] for row in rows] == ["transcript-0, heard", "transcript-1, heard"]+++ "names",+ [("train.jsonl", "./train.jsonl"), ("TRAIN.jsonl", "train.jsonl")],+)+def test_manifest_aliases_keep_all_training_rows(monkeypatch, tmp_path, names):+ keys = [f"datasets/{name}" for name in names]+ _install(+ monkeypatch,+ [*keys, "datasets/audio/clip.wav"],+ contents = {+ key: json.dumps({"audio": "audio/clip.wav", "text": f"transcript-{i}"}) + "\n"+ for i, key in enumerate(keys)+ },+ )+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ assert len({unicodedata.normalize("NFC", path).casefold() for path in files}) == 2+ assert [row["text"] for path in files for row in _rows(path)] == [+ "transcript-0",+ "transcript-1",+ ]+ assert all(Path(_rows(path)[0]["audio"]).is_file() for path in files)+++def test_audio_alias_suffixes_do_not_overwrite_another_object(monkeypatch, tmp_path):+ keys = ["datasets/audio/clip.wav", "datasets/audio//clip.wav", "datasets/audio/clip_1.wav"]+ _install(+ monkeypatch,+ ["datasets/metadata.jsonl", *keys],+ contents = {+ "datasets/metadata.jsonl": "\n".join(+ json.dumps({"audio": f"s3://my-bucket/{key}"}) for key in keys+ )+ + "\n",+ **{key: f"recording-{i}" for i, key in enumerate(keys)},+ },+ )+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ assert [Path(row["audio"]).read_text() for row in _rows(files[0])] == [+ "recording-0",+ "recording-1",+ "recording-2",+ ]++[email protected]("existing_name", ["metadata.jsonl", "audio/clip.wav"])+def test_structured_download_keeps_existing_local_files(monkeypatch, tmp_path, existing_name):+ existing = tmp_path / existing_name+ existing.parent.mkdir(parents = True, exist_ok = True)+ existing.write_text("KEEP")+ _install(+ monkeypatch,+ ["datasets/metadata.jsonl", "datasets/audio/clip.wav"],+ contents = {+ "datasets/metadata.jsonl": '{"audio":"audio/clip.wav","text":"new sample"}\n',+ "datasets/audio/clip.wav": "NEW",+ },+ )+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ assert existing.read_text() == "KEEP"+ assert Path(_rows(files[0])[0]["audio"]).read_text() == "NEW"++[email protected]("names", [("clip.wav", "/clip.wav"), ("A.wav", "a.wav")])+def test_parquet_audio_aliases_download_every_object(monkeypatch, tmp_path, names):+ keys = [f"datasets/audio/{name}" for name in names]+ _install(+ monkeypatch,+ ["datasets/train.parquet", *keys],+ contents = {keys[0]: "FIRST", keys[1]: "SECOND"},+ )+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ assert files == [str(tmp_path / "train.parquet")]+ audio = sorted(path.read_text() for path in (tmp_path / "audio").iterdir())+ assert audio == ["FIRST", "SECOND"]+++def test_parquet_download_keeps_existing_audio(monkeypatch, tmp_path):+ existing = tmp_path / "clip.wav"+ existing.write_text("KEEP")+ _install(monkeypatch, ["datasets/train.parquet", "datasets/clip.wav"])+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ assert files == [str(tmp_path / "train.parquet")]+ assert existing.read_text() == "KEEP"+ assert (tmp_path / "clip_1.wav").read_text() == "content-of:datasets/clip.wav"++[email protected]("second_key", ["./train.parquet", "TRAIN.parquet"])+def test_parquet_manifest_aliases_keep_both_shards(monkeypatch, tmp_path, second_key):+ _install(+ monkeypatch,+ ["datasets/train.parquet", f"datasets/{second_key}", "datasets/clip.wav"],+ contents = {+ "datasets/train.parquet": "FIRST",+ f"datasets/{second_key}": "SECOND",+ },+ )+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ assert [Path(path).read_text() for path in files] == ["FIRST", "SECOND"]+ assert (tmp_path / "clip.wav").is_file()+++def test_parquet_audio_without_collisions_keeps_its_paths(monkeypatch, tmp_path):+ _install(monkeypatch, ["datasets/train.parquet", "datasets/audio/clip.wav"])+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))+ assert files == [str(tmp_path / "train.parquet")]+ assert (tmp_path / "audio" / "clip.wav").read_text() == "content-of:datasets/audio/clip.wav"diff --git a/studio/backend/tests/test_s3_dataset.py b/studio/backend/tests/test_s3_dataset.pyindex dec1cc07c33..48a9da4b539 100644--- a/studio/backend/tests/test_s3_dataset.py+++ b/studio/backend/tests/test_s3_dataset.py@@ -261,3 +261,19 @@ def test_s3config_requires_credentials_or_iam():def test_s3config_iam_role_needs_no_keys():cfg = S3Config.model_validate({"bucket": "b", "useIamRole": True})assert cfg.use_iam_role is True+++ "names",+ [+ ("Train.parquet", "train.parquet"),+ ("caf\u00e9.parquet", "cafe\u0301.parquet"),+ ],+)+def test_local_paths_are_reserved_before_downloads_create_files(tmp_path, names):+ used_paths = set()+ first = s3_dataset._unique_local_path(str(tmp_path), names[0], used_paths)+ second = s3_dataset._unique_local_path(str(tmp_path), names[1], used_paths)+ assert os.path.basename(second).endswith("_1.parquet")+ assert not Path(first).exists()+ assert not Path(second).exists()