Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: preserve S3 dataset files with colliding names (#16830)

DimitriChrysafis41da0b13f25d
diff --git a/studio/backend/core/training/s3_dataset.py b/studio/backend/core/training/s3_dataset.py
index 2e9fbcb708d..c7d8417f868 100644
--- a/studio/backend/core/training/s3_dataset.py
+++ b/studio/backend/core/training/s3_dataset.py
@@ -19,6 +19,7 @@ import logging
import os
import shutil
import tempfile
+import unicodedata
from importlib.util import find_spec
from typing import Callable, Optional
from utils.datasets.format_detection import _AUDIO_EXTENSIONS
@@ -158,15 +159,19 @@ def _contained_local_path(
return pathmod.join(target_dir, *parts)
+def _local_path_key(path: str) -> str:
+ return unicodedata.normalize("NFC", os.path.normpath(path)).casefold()
+
+
def _unique_local_path(target_dir: str, filename: str, used_paths: set[str]) -> str:
- """Return an unused flattened path for an S3 object basename."""
+ """Return an unused local path for an S3 object basename."""
stem, ext = os.path.splitext(filename)
candidate = _contained_local_path(target_dir, [filename])
suffix = 1
- while candidate in used_paths or os.path.exists(candidate):
+ while _local_path_key(candidate) in used_paths or os.path.exists(candidate):
candidate = _contained_local_path(target_dir, [f"{stem}_{suffix}{ext}"])
suffix += 1
- used_paths.add(candidate)
+ used_paths.add(_local_path_key(candidate))
return candidate
@@ -192,8 +197,9 @@ def _key_relative_to_prefix(key: str, prefix: Optional[str]) -> str:
def _structured_local_paths(
prefix: Optional[str], keys: list[str], target_dir: str
) -> dict[str, str]:
- """Map ``keys`` to local paths mirroring their prefix-relative layout."""
+ """Map keys to distinct local paths under their prefix-relative directories."""
local_by_key: dict[str, str] = {}
+ used_paths: set[str] = set()
for key in keys:
relative = _key_relative_to_prefix(key, prefix)
parts = [part for part in relative.split("/") if part not in ("", ".")]
@@ -203,7 +209,10 @@ def _structured_local_paths(
f"S3 key {key!r} contains '..' or empty path segments and cannot "
"be downloaded into the dataset directory."
)
- local_by_key[key] = _contained_local_path(target_dir, parts)
+ path = _contained_local_path(target_dir, parts)
+ local_by_key[key] = _unique_local_path(
+ os.path.dirname(path), os.path.basename(path), used_paths
+ )
return local_by_key
@@ -369,8 +378,9 @@ def prepare_s3_dataset_download(
local_files: list[str] = []
if audio_keys:
- manifest_local_by_key = _structured_local_paths(prefix, keys, target_dir)
- audio_local_by_key = _structured_local_paths(prefix, audio_keys, target_dir)
+ local_by_key = _structured_local_paths(prefix, keys + audio_keys, target_dir)
+ manifest_local_by_key = {key: local_by_key[key] for key in keys}
+ audio_local_by_key = {key: local_by_key[key] for key in audio_keys}
_download_structured(client, bucket, manifest_local_by_key, cancel_callback)
# Rewrite before the audio download so a bad manifest fails without fetching it.
_rewrite_audio_references(manifest_local_by_key, audio_local_by_key, bucket, prefix)
diff --git a/studio/backend/tests/test_s3_audio_dataset.py b/studio/backend/tests/test_s3_audio_dataset.py
index db6beff6a89..0578a4ca268 100644
--- a/studio/backend/tests/test_s3_audio_dataset.py
+++ b/studio/backend/tests/test_s3_audio_dataset.py
@@ -6,8 +6,10 @@ boto3 is faked as in test_s3_dataset.py."""
import csv
import importlib.util
+import io
import json
import os
+import unicodedata
from pathlib import Path
import pytest
@@ -378,3 +380,165 @@ def test_ordinary_keys_stay_inside_the_download_dir(parts, tmp_path):
if "\\" not in parts[-1]:
win = s3_dataset._contained_local_path("C:\\data\\download", parts, pathmod = ntpath)
assert win == ntpath.join("C:\\data\\download", *parts)
+
+
[email protected]("manifest_format", ["jsonl", "json", "csv"])
+ "names",
+ [
+ ("audio/clip.wav", "audio//clip.wav"),
+ ("audio/clip.wav", "audio/./clip.wav"),
+ ("audio/A.wav", "audio/a.wav"),
+ ("audio/caf\u00e9.wav", "audio/cafe\u0301.wav"),
+ ("Audio/clip.wav", "audio/clip.wav"),
+ ("caf\u00e9/clip.wav", "cafe\u0301/clip.wav"),
+ ],
+)
+def test_audio_aliases_keep_each_recording_and_its_transcript(
+ monkeypatch, tmp_path, names, manifest_format
+):
+ keys = [f"datasets/{name}" for name in names]
+ records = [
+ {"audio": f"s3://my-bucket/{key}", "text": f"transcript-{i}, heard"}
+ for i, key in enumerate(keys)
+ ]
+ if manifest_format == "csv":
+ handle = io.StringIO()
+ writer = csv.DictWriter(handle, fieldnames = ["audio", "text"])
+ writer.writeheader()
+ writer.writerows(records)
+ manifest = handle.getvalue()
+ elif manifest_format == "json":
+ manifest = json.dumps(records)
+ else:
+ manifest = "\n".join(map(json.dumps, records))
+ manifest_key = f"datasets/train.{manifest_format}"
+ _install(
+ monkeypatch,
+ [manifest_key, *keys],
+ contents = {
+ manifest_key: manifest + "\n",
+ keys[0]: "FIRST",
+ keys[1]: "SECOND",
+ },
+ )
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ if manifest_format == "csv":
+ with open(files[0], encoding = "utf-8", newline = "") as handle:
+ rows = list(csv.DictReader(handle))
+ elif manifest_format == "json":
+ rows = json.loads(Path(files[0]).read_text())
+ else:
+ rows = _rows(files[0])
+ paths = [row["audio"] for row in rows]
+ assert len({unicodedata.normalize("NFC", path).casefold() for path in paths}) == 2
+ assert [Path(path).read_text() for path in paths] == ["FIRST", "SECOND"]
+ assert [row["text"] for row in rows] == ["transcript-0, heard", "transcript-1, heard"]
+
+
+ "names",
+ [("train.jsonl", "./train.jsonl"), ("TRAIN.jsonl", "train.jsonl")],
+)
+def test_manifest_aliases_keep_all_training_rows(monkeypatch, tmp_path, names):
+ keys = [f"datasets/{name}" for name in names]
+ _install(
+ monkeypatch,
+ [*keys, "datasets/audio/clip.wav"],
+ contents = {
+ key: json.dumps({"audio": "audio/clip.wav", "text": f"transcript-{i}"}) + "\n"
+ for i, key in enumerate(keys)
+ },
+ )
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ assert len({unicodedata.normalize("NFC", path).casefold() for path in files}) == 2
+ assert [row["text"] for path in files for row in _rows(path)] == [
+ "transcript-0",
+ "transcript-1",
+ ]
+ assert all(Path(_rows(path)[0]["audio"]).is_file() for path in files)
+
+
+def test_audio_alias_suffixes_do_not_overwrite_another_object(monkeypatch, tmp_path):
+ keys = ["datasets/audio/clip.wav", "datasets/audio//clip.wav", "datasets/audio/clip_1.wav"]
+ _install(
+ monkeypatch,
+ ["datasets/metadata.jsonl", *keys],
+ contents = {
+ "datasets/metadata.jsonl": "\n".join(
+ json.dumps({"audio": f"s3://my-bucket/{key}"}) for key in keys
+ )
+ + "\n",
+ **{key: f"recording-{i}" for i, key in enumerate(keys)},
+ },
+ )
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ assert [Path(row["audio"]).read_text() for row in _rows(files[0])] == [
+ "recording-0",
+ "recording-1",
+ "recording-2",
+ ]
+
+
[email protected]("existing_name", ["metadata.jsonl", "audio/clip.wav"])
+def test_structured_download_keeps_existing_local_files(monkeypatch, tmp_path, existing_name):
+ existing = tmp_path / existing_name
+ existing.parent.mkdir(parents = True, exist_ok = True)
+ existing.write_text("KEEP")
+ _install(
+ monkeypatch,
+ ["datasets/metadata.jsonl", "datasets/audio/clip.wav"],
+ contents = {
+ "datasets/metadata.jsonl": '{"audio":"audio/clip.wav","text":"new sample"}\n',
+ "datasets/audio/clip.wav": "NEW",
+ },
+ )
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ assert existing.read_text() == "KEEP"
+ assert Path(_rows(files[0])[0]["audio"]).read_text() == "NEW"
+
+
[email protected]("names", [("clip.wav", "/clip.wav"), ("A.wav", "a.wav")])
+def test_parquet_audio_aliases_download_every_object(monkeypatch, tmp_path, names):
+ keys = [f"datasets/audio/{name}" for name in names]
+ _install(
+ monkeypatch,
+ ["datasets/train.parquet", *keys],
+ contents = {keys[0]: "FIRST", keys[1]: "SECOND"},
+ )
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ assert files == [str(tmp_path / "train.parquet")]
+ audio = sorted(path.read_text() for path in (tmp_path / "audio").iterdir())
+ assert audio == ["FIRST", "SECOND"]
+
+
+def test_parquet_download_keeps_existing_audio(monkeypatch, tmp_path):
+ existing = tmp_path / "clip.wav"
+ existing.write_text("KEEP")
+ _install(monkeypatch, ["datasets/train.parquet", "datasets/clip.wav"])
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ assert files == [str(tmp_path / "train.parquet")]
+ assert existing.read_text() == "KEEP"
+ assert (tmp_path / "clip_1.wav").read_text() == "content-of:datasets/clip.wav"
+
+
[email protected]("second_key", ["./train.parquet", "TRAIN.parquet"])
+def test_parquet_manifest_aliases_keep_both_shards(monkeypatch, tmp_path, second_key):
+ _install(
+ monkeypatch,
+ ["datasets/train.parquet", f"datasets/{second_key}", "datasets/clip.wav"],
+ contents = {
+ "datasets/train.parquet": "FIRST",
+ f"datasets/{second_key}": "SECOND",
+ },
+ )
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ assert [Path(path).read_text() for path in files] == ["FIRST", "SECOND"]
+ assert (tmp_path / "clip.wav").is_file()
+
+
+def test_parquet_audio_without_collisions_keeps_its_paths(monkeypatch, tmp_path):
+ _install(monkeypatch, ["datasets/train.parquet", "datasets/audio/clip.wav"])
+ files = s3_dataset.download_s3_dataset(_cfg(), dest_dir = str(tmp_path))
+ assert files == [str(tmp_path / "train.parquet")]
+ assert (tmp_path / "audio" / "clip.wav").read_text() == "content-of:datasets/audio/clip.wav"
diff --git a/studio/backend/tests/test_s3_dataset.py b/studio/backend/tests/test_s3_dataset.py
index dec1cc07c33..48a9da4b539 100644
--- a/studio/backend/tests/test_s3_dataset.py
+++ b/studio/backend/tests/test_s3_dataset.py
@@ -261,3 +261,19 @@ def test_s3config_requires_credentials_or_iam():
def test_s3config_iam_role_needs_no_keys():
cfg = S3Config.model_validate({"bucket": "b", "useIamRole": True})
assert cfg.use_iam_role is True
+
+
+ "names",
+ [
+ ("Train.parquet", "train.parquet"),
+ ("caf\u00e9.parquet", "cafe\u0301.parquet"),
+ ],
+)
+def test_local_paths_are_reserved_before_downloads_create_files(tmp_path, names):
+ used_paths = set()
+ first = s3_dataset._unique_local_path(str(tmp_path), names[0], used_paths)
+ second = s3_dataset._unique_local_path(str(tmp_path), names[1], used_paths)
+ assert os.path.basename(second).endswith("_1.parquet")
+ assert not Path(first).exists()
+ assert not Path(second).exists()