Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: keep new image sets from merging into an existing one (#12987)

Nilayacf89c27f60c
diff --git a/studio/backend/models/training.py b/studio/backend/models/training.py
index 6a4a2282766..4bf17b78e6c 100644
--- a/studio/backend/models/training.py
+++ b/studio/backend/models/training.py
@@ -1185,25 +1185,27 @@ class DiffusionTrainableFamily(BaseModel):
# When set, a LoRA trained on this family previews on this repo instead of the training base (Krea
# trains on Raw, runs on Turbo).
deploy_base: Optional[str] = None
- # Variant-specific training-base to inference-base pairs, including public mirror ids.
+ # maps each training base, including public mirrors, to its inference base.
deploy_bases: Dict[str, str] = Field(default_factory = dict)
- # Per-checkpoint facts that overlay the family-level params/VRAM guidance.
+ # overlays checkpoint-specific parameter and VRAM guidance on family defaults.
base_specs: Dict[str, dict] = Field(default_factory = dict)
class DiffusionTrainingInfoResponse(BaseModel):
- """Where diffusion training reads/writes on this Unsloth, plus usable datasets and the
- trainable model families (so the UI can offer a base picker with realistic guidance)."""
+ """lists paths, usable datasets, and UI-facing trainable families for this Unsloth instance."""
datasets_root: str
outputs_root: str
datasets: List[DiffusionDatasetSummary]
+ # includes every occupied folder name, even captions-only folders.
+ dataset_names: List[str] = Field(default_factory = list)
+ # occupied, unlisted folders that this upload form may safely continue.
+ continuation_dataset_names: List[str] = Field(default_factory = list)
families: List[DiffusionTrainableFamily] = Field(default_factory = list)
class DiffusionDatasetUploadResponse(BaseModel):
- """Result of uploading images/clips/captions into a named dataset folder. Counts are
- for the whole folder after the upload, so repeat uploads show the running total."""
+ """counts cover the whole folder after the upload, including earlier uploads."""
name: str
path: str
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 6a76ce16713..4dca2b1cf35 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -3632,6 +3632,25 @@ _DIFFUSION_DATASET_MEDIA_EXTS = _DIFFUSION_DATASET_IMAGE_EXTS | _DIFFUSION_DATAS
_DIFFUSION_DATASET_TEXT_EXTS = {".txt", ".caption", ".jsonl"}
+def _reserved_diffusion_dataset_names() -> frozenset[str]:
+ """Names below the dataset root owned by other Studio dataset workflows."""
+ from utils.paths import (
+ dataset_uploads_root,
+ recipe_datasets_root,
+ seed_uploads_root,
+ unstructured_uploads_root,
+ )
+ return frozenset(
+ path.name.casefold()
+ for path in (
+ dataset_uploads_root(),
+ recipe_datasets_root(),
+ seed_uploads_root(),
+ unstructured_uploads_root(),
+ )
+ )
+
+
def _resolve_dataset_caption(
folder: Path, image_path: Path, meta_captions: dict[str, str]
) -> Optional[str]:
@@ -3669,11 +3688,8 @@ _DATASET_IMPORT_LOCKS_GUARD = threading.Lock()
def _dataset_import_lock(folder: Path) -> "threading.Lock":
- """One lock per dataset folder, so two imports cannot fill the same empty name at once. Keyed by
- the resolved path (one folder can be reached by different names) and kept for the process
- lifetime: there are a handful of folders and a Lock is tiny, while dropping one while another
- thread holds it would defeat the point."""
- key = str(folder.resolve(strict = False))
+ """serialize imports by case-folded resolved path; process-lifetime locks cannot disappear while held."""
+ key = str(folder.resolve(strict = False)).casefold()
with _DATASET_IMPORT_LOCKS_GUARD:
lock = _DATASET_IMPORT_LOCKS.get(key)
if lock is None:
@@ -3839,6 +3855,8 @@ async def diffusion_training_info(current_subject: str = Depends(get_current_sub
def scan() -> DiffusionTrainingInfoResponse:
root = datasets_root()
found: list[DiffusionDatasetSummary] = []
+ continuations: list[str] = []
+ reserved_names = _reserved_diffusion_dataset_names()
try:
# Skip hidden dirs: never user datasets, and an in-progress example import stages into a dot-prefixed
# sibling.
@@ -3856,14 +3874,28 @@ async def diffusion_training_info(current_subject: str = Depends(get_current_sub
summary = _diffusion_dataset_summary(child)
except OSError:
continue
- # A clip-only folder is a real dataset for the video families, so admit on either count.
+ # clip-only folders are valid for video families.
if summary.image_count > 0 or summary.clip_count > 0:
found.append(summary)
+ elif child.name.casefold() not in reserved_names:
+ # captions-only, or emptied by deleting its last item (hidden .thumbs may remain).
+ try:
+ visible = [e for e in child.iterdir() if not e.name.startswith(".")]
+ continuable = not visible or any(
+ e.is_file() and e.suffix.lower() in _DIFFUSION_DATASET_TEXT_EXTS
+ for e in visible
+ )
+ except OSError:
+ continuable = False
+ if continuable:
+ continuations.append(child.name)
families = [DiffusionTrainableFamily(**info) for info in _ui_trainable_families(found)]
return DiffusionTrainingInfoResponse(
datasets_root = str(root),
outputs_root = str(outputs_root()),
datasets = found,
+ dataset_names = [p.name for p in children],
+ continuation_dataset_names = continuations,
families = families,
)
@@ -3873,8 +3905,7 @@ async def diffusion_training_info(current_subject: str = Depends(get_current_sub
_DATASET_NAME_RE = None
-# Reserved in EVERY directory on Windows, with or without an extension (NUL.txt is NUL). The superscript COM/LPT digits
-# count as digits to Win32 and are reserved too.
+# Windows reserves these names with any extension; superscript COM/LPT digits also count as digits in Win32.
_WINDOWS_RESERVED_NAMES = frozenset(
{"con", "prn", "aux", "nul"}
| {f"com{d}" for d in "123456789¹²³"}
@@ -3947,13 +3978,11 @@ def _clean_diffusion_dataset_name(name: str) -> str:
async def upload_diffusion_dataset(
name: str = Form(...),
files: list[UploadFile] = File(...),
+ create_only: bool = Form(False),
current_subject: str = Depends(get_current_subject),
_interlock: None = Depends(diffusion_dataset_interlock),
):
- """Upload training images (and optional caption .txt / metadata.jsonl files) into a
- named folder under the Unsloth datasets root, creating it if needed. Repeat uploads
- into the same name accumulate, so large datasets can arrive in batches. The returned
- name can be passed directly as ``data_dir`` to /diffusion/start."""
+ """repeat uploads accumulate in one Unsloth dataset folder; pass its name as ``data_dir`` to /diffusion/start."""
import os
import tempfile
@@ -3961,12 +3990,13 @@ async def upload_diffusion_dataset(
_require_diffusion_dataset_mutable()
cleaned = _clean_diffusion_dataset_name(name)
- # Run the same symlink + root-containment check as the read/caption/delete endpoints before any write, so a
- # symlinked name cannot make the upload write outside root.
+ if cleaned.casefold() in _reserved_diffusion_dataset_names():
+ raise HTTPException(
+ status_code = 400,
+ detail = f"'{cleaned}' is reserved for Studio's internal dataset storage.",
+ )
folder = _resolve_dataset_folder(name, must_exist = False)
- folder.mkdir(parents = True, exist_ok = True)
- # Serialize against a concurrent import into the SAME folder: the training interlock counts
- # mutations rather than excluding them. The duplicate-stem check below is inside the lock.
+ # serialize same-folder imports because the training interlock permits mutations; duplicate checks run here
_lock = _dataset_import_lock(folder)
if not _lock.acquire(blocking = False):
raise HTTPException(
@@ -3976,17 +4006,49 @@ async def upload_diffusion_dataset(
"then upload again."
),
)
+ created_folder = False
try:
+ if create_only:
+ occupied = (
+ next(
+ (
+ p
+ for p in folder.parent.iterdir()
+ if p.is_dir()
+ and not p.is_symlink()
+ and p.name.casefold() == cleaned.casefold()
+ ),
+ None,
+ )
+ if folder.parent.is_dir()
+ else None
+ )
+ if occupied is not None:
+ raise HTTPException(
+ status_code = 409,
+ detail = (
+ f"Dataset '{occupied.name}' already exists. Pick it in the dataset list "
+ "to add files, or choose another name."
+ ),
+ )
+ try:
+ folder.mkdir(parents = True, exist_ok = not create_only)
+ created_folder = create_only
+ except FileExistsError:
+ raise HTTPException(
+ status_code = 409,
+ detail = (
+ f"Dataset '{cleaned}' already exists. Pick it in the dataset list to add "
+ "files, or choose another name."
+ ),
+ )
limit_bytes = get_upload_limit_bytes()
total_bytes = 0
uploaded = 0
allowed = _DIFFUSION_DATASET_MEDIA_EXTS | _DIFFUSION_DATASET_TEXT_EXTS
- # Validate every filename up front so a valid file ahead of a bad one is not left on disk when the 400 fires;
- # the upload is all-or-nothing.
+ # validate all filenames first so an invalid name cannot leave earlier files on disk
names: list[str] = []
- # Indexes over `names` so the three batch-local duplicate checks below are hash
- # lookups, not scans over every earlier filename (O(N^2) at the 1000-file cap). First
- # / insertion order is kept, so each error still names the filename the scans picked.
+ # use insertion-ordered indexes to avoid O(N^2) scans while preserving the filename reported in errors
seen_names: set = set()
first_name_by_casefold: dict = {}
media_names_by_stem_cf: dict = {}
@@ -4159,6 +4221,13 @@ async def upload_diffusion_dataset(
caption_count = summary.caption_count,
uploaded = uploaded,
)
+ except BaseException:
+ if created_folder:
+ try:
+ folder.rmdir()
+ except OSError:
+ pass
+ raise
finally:
_lock.release()
diff --git a/studio/backend/tests/test_diffusion_dataset_api.py b/studio/backend/tests/test_diffusion_dataset_api.py
index eef1880fe24..8194523cd5b 100644
--- a/studio/backend/tests/test_diffusion_dataset_api.py
+++ b/studio/backend/tests/test_diffusion_dataset_api.py
@@ -661,7 +661,7 @@ def test_upload_rejects_extension_case_variant_sidecar_collision(client, ds_root
def test_upload_allows_exact_name_overwrite_and_caption_sidecar(client, ds_root):
- # Re-uploading the EXACT same name is an allowed overwrite, and a .txt caption for the same stem is the kohya flow.
+ # exact-name overwrites are allowed, and same-stem .txt files are Kohya captions
assert (
_upload(client, "styleset", [("sample.png", _png_bytes((10, 20, 30)))]).status_code == 200
)
@@ -674,13 +674,62 @@ def test_upload_allows_exact_name_overwrite_and_caption_sidecar(client, ds_root)
assert (folder / "sample.txt").read_text(encoding = "utf-8") == "a caption"
+def test_create_only_upload_refuses_every_occupied_spelling(client, ds_root):
+ folder = ds_root / "StyleSet"
+ folder.mkdir()
+ original = _png_bytes((10, 20, 30))
+ (folder / "sample.png").write_bytes(original)
+
+ for name in ("StyleSet", "styleset"):
+ r = client.post(
+ "/api/train/diffusion/dataset",
+ data = {"name": name, "create_only": "true"},
+ files = [("files", ("sample.png", _png_bytes((90, 90, 90)), "image/png"))],
+ )
+ assert r.status_code == 409, r.text
+ assert "already exists" in r.json()["detail"]
+
+ assert (folder / "sample.png").read_bytes() == original
+ created = client.post(
+ "/api/train/diffusion/dataset",
+ data = {"name": "fresh", "create_only": "true"},
+ files = [("files", ("new.png", _png_bytes(), "image/png"))],
+ )
+ assert created.status_code == 200, created.text
+ assert (ds_root / "fresh" / "new.png").is_file()
+
+
+def test_upload_refuses_internal_dataset_folders(client, ds_root):
+ for name in ("uploads", "RECIPES", "seed-uploads", "unstructured-uploads"):
+ r = _upload(client, name, [("sample.png", _png_bytes())])
+ assert r.status_code == 400, r.text
+ assert "internal dataset storage" in r.json()["detail"]
+
+
+def test_create_only_upload_removes_a_new_folder_after_validation_failure(client, ds_root):
+ rejected = client.post(
+ "/api/train/diffusion/dataset",
+ data = {"name": "retryable", "create_only": "true"},
+ files = [("files", ("bad.exe", b"not a dataset file", "application/octet-stream"))],
+ )
+ assert rejected.status_code == 400, rejected.text
+ assert not (ds_root / "retryable").exists()
+
+ retried = client.post(
+ "/api/train/diffusion/dataset",
+ data = {"name": "retryable", "create_only": "true"},
+ files = [("files", ("sample.png", _png_bytes(), "image/png"))],
+ )
+ assert retried.status_code == 200, retried.text
+ assert (ds_root / "retryable" / "sample.png").is_file()
+
+
# ── import: promotion is all-or-nothing ──────────────────────────────────────
def test_import_promotion_leaves_no_partial_dataset_on_failure(ds_root, monkeypatch):
- # The staging dir is promoted in one atomic rename. If it fails, the folder must be left with NO images rather than a
- # half-filled dataset the image_count>0 idempotency check would accept. Simulate the failure and assert a clean retry.
+ # atomic promotion prevents a failed import from satisfying the image_count idempotency check
import os
- # A client that returns the 500 (as production does) instead of re-raising.
+ # match production by returning 500 responses instead of reraising server exceptions
app = FastAPI()
app.include_router(training_router, prefix = "/api/train")
app.dependency_overrides[get_current_subject] = lambda: "test-user"
diff --git a/studio/backend/tests/test_diffusion_dataset_clips.py b/studio/backend/tests/test_diffusion_dataset_clips.py
index a5462816795..b8cbd96f111 100644
--- a/studio/backend/tests/test_diffusion_dataset_clips.py
+++ b/studio/backend/tests/test_diffusion_dataset_clips.py
@@ -214,7 +214,7 @@ def test_info_lists_a_clip_only_dataset(client, ds_root):
def test_info_still_skips_a_folder_holding_neither(client, ds_root):
- """Admission widened to "an image OR a clip", not to "any folder"."""
+ """admission requires an image or a clip, not merely a folder"""
folder = ds_root / "captions-only"
folder.mkdir()
(folder / "metadata.jsonl").write_text(
@@ -225,11 +225,38 @@ def test_info_still_skips_a_folder_holding_neither(client, ds_root):
r = client.get("/api/train/diffusion/info")
assert r.status_code == 200, r.text
assert [d["name"] for d in r.json()["datasets"]] == []
+ assert r.json()["dataset_names"] == ["captions-only"]
+ assert r.json()["continuation_dataset_names"] == ["captions-only"]
+
+
+def test_info_offers_an_emptied_dataset_folder_as_a_continuation(client, ds_root):
+ emptied = ds_root / "emptied"
+ (emptied / ".thumbs").mkdir(parents = True)
+ other = ds_root / "other-files"
+ other.mkdir()
+ (other / "notes.bin").write_bytes(b"x")
+
+ r = client.get("/api/train/diffusion/info")
+ assert r.status_code == 200, r.text
+ assert r.json()["continuation_dataset_names"] == ["emptied"]
+
+
+def test_info_never_offers_internal_dataset_folders_as_continuations(client, ds_root):
+ for name in ("uploads", "recipes", "seed-uploads", "unstructured-uploads"):
+ folder = ds_root / name
+ folder.mkdir()
+ (folder / "metadata.jsonl").write_text(
+ json.dumps({"file_name": "pending.png", "text": "x"}) + "\n",
+ encoding = "utf-8",
+ )
+
+ r = client.get("/api/train/diffusion/info")
+ assert r.status_code == 200, r.text
+ assert r.json()["continuation_dataset_names"] == []
def test_list_images_marks_clips_and_leaves_images_unchanged(client, ds_root):
- """Clips list so a caller can see every name holding a stem-keyed sidecar open, but they
- carry no pixel dimensions and the grid filters them out on ``kind``."""
+ """clips expose stem-keyed sidecars but have zero dimensions and are filtered by ``kind``"""
folder = ds_root / "both"
folder.mkdir()
_write_png(folder / "still.png", size = (12, 9))
diff --git a/studio/frontend/src/features/images/api.ts b/studio/frontend/src/features/images/api.ts
index 89350d82d6a..953bc16bfac 100644
--- a/studio/frontend/src/features/images/api.ts
+++ b/studio/frontend/src/features/images/api.ts
@@ -836,6 +836,10 @@ export interface DiffusionTrainingInfo {
datasets_root: string;
outputs_root: string;
datasets: DiffusionDatasetSummary[];
+ // includes occupied folders without trainable media, which `datasets` omits.
+ dataset_names?: string[];
+ // occupied captions-only folders that the diffusion uploader may safely continue.
+ continuation_dataset_names?: string[];
// Added by the multi-family trainer backend; tolerate its absence.
families?: DiffusionTrainableFamily[];
}
@@ -853,9 +857,11 @@ export interface DiffusionDatasetUploadResult extends DiffusionDatasetSummary {
export async function uploadDiffusionDataset(
name: string,
files: File[],
+ createOnly = false,
): Promise<DiffusionDatasetUploadResult> {
const form = new FormData();
form.append("name", name);
+ form.append("create_only", createOnly ? "true" : "false");
for (const f of files) form.append("files", f);
return parseJson(
await authFetch("/api/train/diffusion/dataset", { method: "POST", body: form }),
diff --git a/studio/frontend/src/features/images/train/dataset-files.ts b/studio/frontend/src/features/images/train/dataset-files.ts
index 5478a09482e..9a305513fd9 100644
--- a/studio/frontend/src/features/images/train/dataset-files.ts
+++ b/studio/frontend/src/features/images/train/dataset-files.ts
@@ -228,6 +228,32 @@ export function existingStemClash(files: File[], existing: string[]): DatasetCol
return null;
}
+export function existingDatasetName(name: string, datasets: { name: string }[]): string | null {
+ const folded = name.trim().toLowerCase();
+ return datasets.find((d) => d.name.toLowerCase() === folded)?.name ?? null;
+}
+
+export function datasetNamesForCreation(info: {
+ datasets: { name: string }[];
+ dataset_names?: string[];
+} | null): { name: string }[] {
+ return info?.dataset_names?.map((name) => ({ name })) ?? info?.datasets ?? [];
+}
+
+// exact spelling: the upload sends the typed name, and a case variant is a separate folder on a
+// case-sensitive filesystem.
+export function isDatasetContinuation(name: string, continuationName: string | null): boolean {
+ return continuationName !== null && name.trim() === continuationName;
+}
+
+export function freeDatasetName(datasets: { name: string }[]): string {
+ let name = "my-images";
+ for (let i = 2; existingDatasetName(name, datasets); i += 1) {
+ name = `my-images-${i}`;
+ }
+ return name;
+}
+
// readEntries yields at most 100 entries per call and ends with an empty batch.
function readAllEntries(reader: FileSystemDirectoryReader): Promise<FileSystemEntry[]> {
return new Promise((resolve, reject) => {
diff --git a/studio/frontend/src/features/images/train/diffusion-train-panel.tsx b/studio/frontend/src/features/images/train/diffusion-train-panel.tsx
index 898fa1e269a..75c3dde2316 100644
--- a/studio/frontend/src/features/images/train/diffusion-train-panel.tsx
+++ b/studio/frontend/src/features/images/train/diffusion-train-panel.tsx
@@ -84,8 +84,12 @@ import {
DATASET_FILE_ACCEPT,
DATASET_IMAGE_EXTS,
chunkDatasetUpload,
+ datasetNamesForCreation,
+ existingDatasetName,
existingStemClash,
filesFromDataTransfer,
+ freeDatasetName,
+ isDatasetContinuation,
metadataKeyedOnSubfolders,
oversizedChunk,
selectDatasetFiles,
@@ -398,6 +402,9 @@ export function DiffusionTrainPanel({
}) => void;
}) {
const [info, setInfo] = useState<DiffusionTrainingInfo | null>(null);
+ const [infoLoadState, setInfoLoadState] = useState<
+ "idle" | "loading" | "loaded" | "failed"
+ >("idle");
const families = useMemo(() => mergeFamilies(info?.families), [info?.families]);
const setFamilyName = onFamilyNameChange;
@@ -453,6 +460,9 @@ export function DiffusionTrainPanel({
const [dataset, setDataset] = useState<string>(UPLOAD_DATASET);
const [uploadName, setUploadName] = useState("my-images");
+ const [continuationDatasetName, setContinuationDatasetName] = useState<string | null>(null);
+ // a typed name is the user's: a taken one shows the note instead of being replaced.
+ const uploadNameEdited = useRef(false);
const [uploading, setUploading] = useState(false);
const fileInputRef = useRef<HTMLInputElement | null>(null);
// Adds to the selected set; the other input creates a new one.
@@ -460,6 +470,7 @@ export function DiffusionTrainPanel({
// one folder picker serves both targets, so the destination is captured when it opens.
const folderInputRef = useRef<HTMLInputElement | null>(null);
const folderTarget = useRef("");
+ const folderCreatesDataset = useRef(false);
const [dropActive, setDropActive] = useState(false);
// the authoritative in-flight guard: `uploading` is render state and reads stale in a closure.
const uploadInFlight = useRef(false);
@@ -535,12 +546,20 @@ export function DiffusionTrainPanel({
// The run whose Resume request is in flight, so its button alone shows the pending label.
const [resumingJobId, setResumingJobId] = useState<string | null>(null);
+ // only the newest request may set state: an older one settling late must not undo it.
+ const infoRequestId = useRef(0);
const refreshInfo = useCallback(async (): Promise<DiffusionTrainingInfo | null> => {
+ const requestId = ++infoRequestId.current;
+ setInfoLoadState("loading");
try {
const i = await getDiffusionTrainingInfo();
- setInfo(i);
+ if (requestId === infoRequestId.current) {
+ setInfo(i);
+ setInfoLoadState("loaded");
+ }
return i;
} catch {
+ if (requestId === infoRequestId.current) setInfoLoadState("failed");
return null;
}
}, []);
@@ -761,6 +780,28 @@ export function DiffusionTrainPanel({
dataset !== UPLOAD_DATASET ? info?.datasets.find((d) => d.name === dataset) : undefined;
// A deleted dataset leaves a name that no longer resolves; fall back to the upload form.
const uploadMode = dataset === UPLOAD_DATASET || (info !== null && !selectedDataset);
+ const namesLoading =
+ uploadMode && (infoLoadState === "idle" || infoLoadState === "loading");
+ const namesUnavailable = uploadMode && infoLoadState === "failed";
+ const occupiedDatasets = useMemo(() => datasetNamesForCreation(info), [info]);
+ const continuingUploadName = isDatasetContinuation(uploadName, continuationDatasetName);
+ const createsDataset = uploadMode && !continuingUploadName;
+ const takenName = createsDataset ? existingDatasetName(uploadName, occupiedDatasets) : null;
+ // A captions-only folder is not in the picker, so the backend explicitly marks safe continuations.
+ // Other unlisted names include Studio's internal dataset storage and must stay blocked.
+ const takenNameUnlisted =
+ takenName !== null && (info?.continuation_dataset_names ?? []).includes(takenName);
+ const takenNameMessage = takenNameUnlisted
+ ? `A folder named "${takenName}" already exists but holds no images or clips yet, so it is not in the list. Add to it, or choose another name.`
+ : `A set named "${takenName}" already exists. Pick it in the list above to add to it, or choose another name.`;
+ useEffect(() => {
+ if (!uploadMode || continuingUploadName || uploadNameEdited.current) return;
+ setUploadName((current) =>
+ existingDatasetName(current, occupiedDatasets)
+ ? freeDatasetName(occupiedDatasets)
+ : current,
+ );
+ }, [uploadMode, continuingUploadName, occupiedDatasets]);
// Trainable items in the picked dataset, images and clips alike. caption_count is the folder
// total over both kinds, so every ratio must be against this and not image_count.
const selectedItemCount = selectedDataset ? datasetItemCount(selectedDataset) : 0;
@@ -833,13 +874,13 @@ export function DiffusionTrainPanel({
// Uploads accumulate, so the same call both creates a folder and adds to an existing one.
// `picked` can come from a file pick, a folder pick or a drop, so it is filtered here.
const uploadTo = useCallback(
- async (name: string, picked: File[]) => {
+ async (name: string, picked: File[], createOnly = false) => {
if (picked.length === 0) return; // the picker was cancelled
if (!name) {
toast.error("Give the dataset a folder name, e.g. my-style-photos.");
return;
}
- const { files, imageCount, clipCount, skipped, collisions } = selectDatasetFiles(picked);
+ const { files, skipped, collisions } = selectDatasetFiles(picked);
if (collisions.length > 0) {
const { kind, first, second } = collisions[0];
const more =
@@ -863,10 +904,6 @@ export function DiffusionTrainPanel({
);
return;
}
- // /diffusion/info only lists folders holding a trainable item, so say why the set will not appear
- // yet rather than refusing a captions-first upload.
- const newCaptionsOnly =
- imageCount === 0 && clipCount === 0 && !(info?.datasets ?? []).some((d) => d.name === name);
if (uploadInFlight.current) {
toast.error("An upload is already running. Wait for it to finish, then try again.");
return;
@@ -939,7 +976,7 @@ export function DiffusionTrainPanel({
}
}
}
- let res = await uploadDiffusionDataset(name, chunks[0]);
+ let res = await uploadDiffusionDataset(name, chunks[0], createOnly);
let sent = res.uploaded;
let stopped: string | null = null;
for (const chunk of chunks.slice(1)) {
@@ -970,11 +1007,15 @@ export function DiffusionTrainPanel({
`${skipped === 1 ? "was" : "were"} neither an image, a clip, nor a caption.`,
);
}
- if (newCaptionsOnly) {
+ const resultCaptionsOnly = res.image_count === 0 && (res.clip_count ?? 0) === 0;
+ if (resultCaptionsOnly) {
+ setContinuationDatasetName(res.name);
toast.info(
`"${res.name}" holds captions but no images or clips yet, so it stays out of the ` +
"dataset picker until you add some.",
);
+ } else {
+ setContinuationDatasetName(null);
}
if (misKeyed) {
toast.info(
@@ -992,11 +1033,12 @@ export function DiffusionTrainPanel({
setUploading(false);
}
},
- [info, refreshInfo],
+ [refreshInfo],
);
- const pickFolder = useCallback((name: string) => {
+ const pickFolder = useCallback((name: string, createOnly = false) => {
folderTarget.current = name;
+ folderCreatesDataset.current = createOnly;
folderInputRef.current?.click();
}, []);
@@ -1009,6 +1051,18 @@ export function DiffusionTrainPanel({
if (!event.dataTransfer.types.includes("Files")) return;
event.preventDefault();
setDropActive(false);
+ if (namesLoading) {
+ toast.error("Your image sets are still loading. Drop again in a moment.");
+ return;
+ }
+ if (namesUnavailable) {
+ toast.error("Could not load your image sets. Retry before dropping files.");
+ return;
+ }
+ if (takenName) {
+ toast.error(takenNameMessage);
+ return;
+ }
if (uploadInFlight.current) {
toast.error("An upload is already running. Wait for it to finish, then drop again.");
return;
@@ -1034,9 +1088,17 @@ export function DiffusionTrainPanel({
toast.error("That drop had no files in it.");
return;
}
- await uploadTo(dropTarget, dropped);
+ await uploadTo(dropTarget, dropped, createsDataset);
},
- [dropTarget, uploadTo],
+ [
+ dropTarget,
+ uploadTo,
+ createsDataset,
+ namesLoading,
+ namesUnavailable,
+ takenName,
+ takenNameMessage,
+ ],
);
const onStart = useCallback(async () => {
@@ -1639,6 +1701,12 @@ export function DiffusionTrainPanel({
if (ex) void importExample(ex);
return; // the controlled value stays put while the import runs
}
+ // any explicit pick ends a continuation; "Add to it" is the way back in.
+ setContinuationDatasetName(null);
+ if (v === UPLOAD_DATASET && existingDatasetName(uploadName, occupiedDatasets)) {
+ uploadNameEdited.current = false;
+ setUploadName(freeDatasetName(occupiedDatasets));
+ }
setDataset(v);
setGridOpen(false);
}}
@@ -1709,7 +1777,7 @@ export function DiffusionTrainPanel({
onChange={(e) => {
const files = Array.from(e.target.files ?? []);
e.target.value = "";
- void uploadTo(folderTarget.current, files);
+ void uploadTo(folderTarget.current, files, folderCreatesDataset.current);
}}
/>
{importingId && (
@@ -1728,7 +1796,10 @@ export function DiffusionTrainPanel({
value={uploadName}
placeholder="my-photos"
spellCheck={false}
- onChange={(e) => setUploadName(e.target.value)}
+ onChange={(e) => {
+ uploadNameEdited.current = true;
+ setUploadName(e.target.value);
+ }}
className="h-8 min-w-0 flex-1 text-xs"
aria-label="New dataset name"
/>
@@ -1744,7 +1815,7 @@ export function DiffusionTrainPanel({
onChange={(e) => {
const files = Array.from(e.target.files ?? []);
e.target.value = "";
- void uploadTo(uploadName.trim(), files);
+ void uploadTo(uploadName.trim(), files, createsDataset);
}}
/>
{/* The pick is the confirmation, so it uploads without a second click. */}
@@ -1760,22 +1831,55 @@ export function DiffusionTrainPanel({
}
fileInputRef.current?.click();
}}
- disabled={uploading}
+ disabled={uploading || namesLoading || namesUnavailable || takenName !== null}
>
<HugeiconsIcon icon={Upload01Icon} className="size-3.5" />
{uploading ? "Uploading..." : "Upload"}
</Button>
<FolderPickButton
- disabled={uploading}
+ disabled={uploading || namesLoading || namesUnavailable || takenName !== null}
onPick={() => {
if (!uploadName.trim()) {
toast.error("Give the dataset a folder name, e.g. my-style-photos.");
return;
}
- pickFolder(uploadName.trim());
+ pickFolder(uploadName.trim(), createsDataset);
}}
/>
</div>
+ {takenName && (
+ <div className="flex items-center gap-2 text-ui-11 text-destructive">
+ <p className="leading-snug">{takenNameMessage}</p>
+ {takenNameUnlisted && (
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ className="h-6 shrink-0 px-2 text-ui-11"
+ onClick={() => {
+ setUploadName(takenName);
+ setContinuationDatasetName(takenName);
+ }}
+ >
+ Add to it
+ </Button>
+ )}
+ </div>
+ )}
+ {namesUnavailable && (
+ <div className="flex items-center gap-2 text-ui-11 text-destructive">
+ <span>Could not load existing image sets.</span>
+ <Button
+ type="button"
+ size="sm"
+ variant="ghost"
+ className="h-6 px-2 text-ui-11"
+ onClick={() => void refreshInfo()}
+ >
+ Retry
+ </Button>
+ </div>
+ )}
<p className="text-ui-11 leading-snug text-muted-foreground">
{isTauri ? "Pick files or a folder." : "Pick files or a folder, or drop them here."}{" "}
Images, or clips for the video families. A caption file beside one (cat.png and
diff --git a/studio/frontend/tests/diffusion-upload-new-set-name.test.ts b/studio/frontend/tests/diffusion-upload-new-set-name.test.ts
new file mode 100644
index 00000000000..328c44c8063
--- /dev/null
+++ b/studio/frontend/tests/diffusion-upload-new-set-name.test.ts
@@ -0,0 +1,174 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+// Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+ datasetNamesForCreation,
+ existingDatasetName,
+ freeDatasetName,
+ isDatasetContinuation,
+} from "../src/features/images/train/dataset-files.ts";
+
+import { readSrcAsync } from "./helpers/kit.ts";
+
+const source = await readSrcAsync(
+ "features/images/train/diffusion-train-panel.tsx",
+);
+const apiSource = await readSrcAsync("features/images/api.ts");
+
+const datasets = [{ name: "my-images" }, { name: "Dogs" }];
+
+test("a new set name that an existing set already uses is caught, whatever its case", () => {
+ assert.equal(existingDatasetName("my-images", datasets), "my-images");
+ assert.equal(existingDatasetName(" dogs ", datasets), "Dogs");
+ assert.equal(existingDatasetName("cats", datasets), null);
+ assert.equal(existingDatasetName("my-images", []), null);
+});
+
+test("the prefilled name for a new set is one no set uses yet", () => {
+ assert.equal(freeDatasetName([]), "my-images");
+ assert.equal(freeDatasetName(datasets), "my-images-2");
+ assert.equal(
+ freeDatasetName([...datasets, { name: "MY-IMAGES-2" }]),
+ "my-images-3",
+ );
+});
+
+test("names of occupied but untrainable folders still reserve a new-set name", () => {
+ assert.deepEqual(
+ datasetNamesForCreation({
+ datasets: [],
+ dataset_names: ["captions-only"],
+ }),
+ [{ name: "captions-only" }],
+ );
+ assert.deepEqual(
+ datasetNamesForCreation({ datasets }),
+ datasets,
+ );
+});
+
+test("a captions-only set created in this form remains available for follow-up files", () => {
+ assert.equal(isDatasetContinuation("captions-only", "captions-only"), true);
+ assert.equal(isDatasetContinuation(" captions-only ", "captions-only"), true);
+ assert.equal(isDatasetContinuation("CAPTIONS-ONLY", "captions-only"), false);
+ assert.equal(isDatasetContinuation("another-set", "captions-only"), false);
+ assert.equal(isDatasetContinuation("captions-only", null), false);
+ assert.match(source, /setContinuationDatasetName\(res\.name\)/);
+ assert.match(source, /const createsDataset = uploadMode && !continuingUploadName;/);
+ assert.match(
+ source,
+ /const resultCaptionsOnly = res\.image_count === 0 && \(res\.clip_count \?\? 0\) === 0;/,
+ );
+ assert.match(
+ source,
+ /\(info\?\.continuation_dataset_names \?\? \[\]\)\.includes\(takenName\)/,
+ );
+ assert.match(source, /\{takenNameUnlisted && \(/);
+});
+
+test("the new-set form waits for the set list before it uploads", () => {
+ assert.match(
+ source,
+ /uploadMode && \(infoLoadState === "idle" \|\| infoLoadState === "loading"\)/,
+ );
+ assert.match(source, /const namesUnavailable = uploadMode && infoLoadState === "failed";/);
+});
+
+test("a failed initial or stale set-list request can be retried without enabling uploads", () => {
+ const refresh = source.slice(
+ source.indexOf("const infoRequestId"),
+ source.indexOf("// On first activation"),
+ );
+ assert.match(refresh, /setInfoLoadState\("loading"\)/);
+ assert.match(refresh, /setInfoLoadState\("loaded"\)/);
+ assert.match(
+ refresh,
+ /catch \{\s+if \(requestId === infoRequestId\.current\) setInfoLoadState\("failed"\)/,
+ );
+ assert.match(refresh, /if \(requestId === infoRequestId\.current\) \{\s+setInfo\(i\);/);
+
+ const form = source.slice(source.indexOf("{uploadMode ? ("));
+ const newSet = form.slice(0, form.indexOf(") : ("));
+ assert.match(newSet, /\{namesUnavailable && \(/);
+ assert.match(newSet, /onClick=\{\(\) => void refreshInfo\(\)\}/);
+ assert.match(newSet, />\s*Retry\s*<\/Button>/);
+});
+
+test("the initial upload form replaces an occupied default name after inventory loads", () => {
+ assert.match(
+ source,
+ /setUploadName\(\(current\) =>\s+existingDatasetName\(current, occupiedDatasets\)\s+\? freeDatasetName\(occupiedDatasets\)\s+: current/,
+ );
+ assert.match(source, /\[uploadMode, continuingUploadName, occupiedDatasets\]/);
+});
+
+test("the new-set form does not upload into a set that already exists", () => {
+ const form = source.slice(source.indexOf("{uploadMode ? ("));
+ const newSet = form.slice(0, form.indexOf(") : ("));
+ assert.equal(
+ newSet.match(
+ /disabled=\{uploading \|\| namesLoading \|\| namesUnavailable \|\| takenName !== null\}/g,
+ )?.length,
+ 2,
+ );
+ assert.match(newSet, /\{takenName && \(/);
+
+ const drop = source.slice(
+ source.indexOf("const onDrop"),
+ source.indexOf("const onStart"),
+ );
+ assert.ok(drop.indexOf("if (takenName)") >= 0);
+ assert.ok(drop.indexOf("if (takenName)") < drop.indexOf("await uploadTo("));
+ assert.ok(drop.indexOf("if (namesLoading)") >= 0);
+ assert.ok(drop.indexOf("if (namesLoading)") < drop.indexOf("await uploadTo("));
+ assert.ok(drop.indexOf("if (namesUnavailable)") >= 0);
+ assert.ok(drop.indexOf("if (namesUnavailable)") < drop.indexOf("await uploadTo("));
+ assert.match(drop, /await uploadTo\(dropTarget, dropped, createsDataset\)/);
+
+ assert.match(newSet, /void uploadTo\(uploadName\.trim\(\), files, createsDataset\)/);
+ assert.match(newSet, /pickFolder\(uploadName\.trim\(\), createsDataset\)/);
+ assert.match(source, /uploadDiffusionDataset\(name, chunks\[0\], createOnly\)/);
+ assert.match(apiSource, /form\.append\("create_only", createOnly \? "true" : "false"\)/);
+});
+
+test("adding to the selected set still uploads into it", () => {
+ const add = source.slice(
+ source.indexOf("{!uploadMode && selectedDataset && ("),
+ );
+ const buttons = add.slice(0, add.indexOf("</>"));
+ assert.match(buttons, /void uploadTo\(dataset, files\)/);
+ assert.doesNotMatch(buttons, /takenName|namesLoading/);
+});
+
+test("a safe captions-only folder can be continued from the new-set form", () => {
+ assert.match(
+ source,
+ /const takenNameUnlisted =\s+takenName !== null && \(info\?\.continuation_dataset_names \?\? \[\]\)\.includes\(takenName\);/,
+ );
+ const form = source.slice(source.indexOf("{uploadMode ? ("));
+ const newSet = form.slice(0, form.indexOf(") : ("));
+ assert.match(
+ newSet,
+ /\{takenNameUnlisted && \([\s\S]*?setUploadName\(takenName\);\s+setContinuationDatasetName\(takenName\);[\s\S]*?Add to it/,
+ );
+});
+
+test("picking any entry in the dataset list ends a continuation", () => {
+ const change = source.slice(source.indexOf("onValueChange={(v) => {"));
+ const body = change.slice(0, change.indexOf("setDataset(v);"));
+ assert.match(body, /setContinuationDatasetName\(null\);/);
+ assert.ok(body.indexOf("return;") < body.indexOf("setContinuationDatasetName(null)"));
+});
+
+test("a name the user typed is never replaced by a generated one", () => {
+ assert.match(
+ source,
+ /if \(!uploadMode \|\| continuingUploadName \|\| uploadNameEdited\.current\) return;/,
+ );
+ const form = source.slice(source.indexOf("{uploadMode ? ("));
+ const newSet = form.slice(0, form.indexOf(") : ("));
+ assert.match(newSet, /uploadNameEdited\.current = true;\s+setUploadName\(e\.target\.value\);/);
+});