Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Dataset label and example-import routes, via TestClient on a temp datasets root; no network."""
from __future__ import annotations
import io
import json
import pytest
from fastapi import FastAPI, HTTPException
from fastapi.testclient import TestClient
from PIL import Image
from auth.authentication import get_current_subject
from routes.training import router as training_router
def _png_bytes(color = (200, 100, 50), size = (8, 8)) -> bytes:
buf = io.BytesIO()
Image.new("RGB", size, color).save(buf, format = "PNG")
return buf.getvalue()
def _write_png(
path,
color = (200, 100, 50),
size = (8, 8),
) -> None:
Image.new("RGB", size, color).save(path, format = "PNG")
@pytest.fixture
def client():
app = FastAPI()
app.include_router(training_router, prefix = "/api/train")
app.dependency_overrides[get_current_subject] = lambda: "test-user"
return TestClient(app)
@pytest.fixture
def ds_root(monkeypatch, tmp_path):
import utils.paths as up
root = tmp_path / "assets" / "datasets"
root.mkdir(parents = True)
monkeypatch.setattr(up, "datasets_root", lambda: root)
return root
def test_list_images_caption_precedence(client, ds_root):
folder = ds_root / "styleset"
folder.mkdir()
_write_png(folder / "a.png")
_write_png(folder / "b.png")
_write_png(folder / "c.png")
(folder / "metadata.jsonl").write_text(
json.dumps({"file_name": "a.png", "text": "from metadata"})
+ "\n"
+ json.dumps({"file_name": "b.png", "text": "from metadata"})
+ "\n",
encoding = "utf-8",
)
(folder / "a.txt").write_text("edited sidecar", encoding = "utf-8")
r = client.get("/api/train/diffusion/dataset/styleset/images")
assert r.status_code == 200, r.text
body = r.json()
assert body["name"] == "styleset"
recs = {rec["filename"]: rec for rec in body["images"]}
assert set(recs) == {"a.png", "b.png", "c.png"}
assert recs["a.png"]["caption"] == "edited sidecar"
assert recs["a.png"]["caption_source"] == "sidecar"
assert recs["b.png"]["caption"] == "from metadata"
assert recs["b.png"]["caption_source"] == "metadata"
assert recs["c.png"]["caption"] is None
assert recs["c.png"]["caption_source"] == "none"
assert recs["a.png"]["width"] == 8 and recs["a.png"]["height"] == 8
def test_list_images_tolerates_invalid_utf8_sidecar(client, ds_root):
# Sidecars are raw bytes; read_text can raise UnicodeDecodeError (a ValueError, not OSError).
folder = ds_root / "badutf8"
folder.mkdir()
_write_png(folder / "a.png")
_write_png(folder / "b.png")
(folder / "a.txt").write_bytes(b"\xff\xfe not valid utf-8")
(folder / "b.txt").write_text("cap b", encoding = "utf-8")
r = client.get("/api/train/diffusion/dataset/badutf8/images")
assert r.status_code == 200, r.text
recs = {rec["filename"]: rec for rec in r.json()["images"]}
assert set(recs) == {"a.png", "b.png"}
assert recs["a.png"]["caption"] in (None, "")
assert recs["b.png"]["caption"] == "cap b"
put = client.put(
"/api/train/diffusion/dataset/badutf8/caption/a.png", json = {"caption": "fixed"}
)
assert put.status_code == 200, put.text
assert put.json()["caption"] == "fixed"
def test_list_images_missing_dataset_404(client, ds_root):
assert client.get("/api/train/diffusion/dataset/nope/images").status_code == 404
def test_get_image_and_thumbnail_excluded_from_listing(client, ds_root):
folder = ds_root / "pics"
folder.mkdir()
_write_png(folder / "one.png", size = (64, 48))
full = client.get("/api/train/diffusion/dataset/pics/image/one.png")
assert full.status_code == 200, full.text
thumb = client.get("/api/train/diffusion/dataset/pics/image/one.png?thumb=32")
assert thumb.status_code == 200
assert thumb.headers["content-type"] == "image/jpeg"
assert (folder / ".thumbs").is_dir()
listing = client.get("/api/train/diffusion/dataset/pics/images").json()
assert [rec["filename"] for rec in listing["images"]] == ["one.png"]
def test_get_image_missing_404(client, ds_root):
(ds_root / "pics").mkdir()
assert client.get("/api/train/diffusion/dataset/pics/image/ghost.png").status_code == 404
def test_put_caption_roundtrip_and_clear(client, ds_root):
folder = ds_root / "cap"
folder.mkdir()
_write_png(folder / "x.png")
r = client.put(
"/api/train/diffusion/dataset/cap/caption/x.png", json = {"caption": "a red apple"}
)
assert r.status_code == 200, r.text
assert r.json()["caption"] == "a red apple"
assert r.json()["caption_source"] == "sidecar"
assert (folder / "x.txt").read_text(encoding = "utf-8") == "a red apple"
r = client.put("/api/train/diffusion/dataset/cap/caption/x.png", json = {"caption": " "})
assert r.status_code == 200
assert r.json()["caption"] is None
assert r.json()["caption_source"] == "none"
assert not (folder / "x.txt").exists()
def test_put_caption_overrides_metadata_row(client, ds_root):
folder = ds_root / "cap"
folder.mkdir()
_write_png(folder / "x.png")
(folder / "metadata.jsonl").write_text(
json.dumps({"file_name": "x.png", "text": "from metadata"}) + "\n", encoding = "utf-8"
)
r = client.put(
"/api/train/diffusion/dataset/cap/caption/x.png", json = {"caption": "edited caption"}
)
assert r.status_code == 200, r.text
assert r.json()["caption"] == "edited caption"
assert r.json()["caption_source"] == "sidecar"
def test_put_caption_missing_image_404(client, ds_root):
(ds_root / "cap").mkdir()
r = client.put("/api/train/diffusion/dataset/cap/caption/ghost.png", json = {"caption": "hi"})
assert r.status_code == 404
def test_put_caption_too_long_400(client, ds_root):
folder = ds_root / "cap"
folder.mkdir()
_write_png(folder / "x.png")
r = client.put("/api/train/diffusion/dataset/cap/caption/x.png", json = {"caption": "z" * 2001})
assert r.status_code == 400
def test_delete_image_cleans_sidecar_and_thumb(client, ds_root):
folder = ds_root / "d"
folder.mkdir()
_write_png(folder / "x.png")
(folder / "x.txt").write_text("cap", encoding = "utf-8")
client.get("/api/train/diffusion/dataset/d/image/x.png?thumb=32")
assert list((folder / ".thumbs").glob("x.png_*.jpg"))
r = client.delete("/api/train/diffusion/dataset/d/image/x.png")
assert r.status_code == 200, r.text
assert not (folder / "x.png").exists()
assert not (folder / "x.txt").exists()
assert not list((folder / ".thumbs").glob("x.png_*.jpg"))
def test_delete_keeps_a_caption_a_same_stem_sibling_still_uses(client, ds_root):
# cat.jpg and cat.png share cat.txt, so deleting one image must not strip the survivor's caption.
folder = ds_root / "d"
folder.mkdir()
_write_png(folder / "cat.png")
Image.new("RGB", (8, 8), (9, 9, 9)).save(folder / "cat.jpg", format = "JPEG")
(folder / "cat.txt").write_text("a cat", encoding = "utf-8")
r = client.delete("/api/train/diffusion/dataset/d/image/cat.jpg")
assert r.status_code == 200, r.text
assert not (folder / "cat.jpg").exists()
assert (folder / "cat.txt").read_text(encoding = "utf-8") == "a cat"
info = client.get("/api/train/diffusion/info").json()
row = [d for d in info["datasets"] if d["name"] == "d"][0]
assert (row["image_count"], row["caption_count"]) == (1, 1)
r = client.delete("/api/train/diffusion/dataset/d/image/cat.png")
assert r.status_code == 200, r.text
assert not (folder / "cat.txt").exists()
def test_delete_removes_a_caption_no_other_image_shares(client, ds_root):
folder = ds_root / "d"
folder.mkdir()
_write_png(folder / "cat.png")
(folder / "cat.txt").write_text("a cat", encoding = "utf-8")
(folder / "cat.caption").write_text("also a cat", encoding = "utf-8")
(folder / "cat.json").write_text("{}", encoding = "utf-8")
r = client.delete("/api/train/diffusion/dataset/d/image/cat.png")
assert r.status_code == 200, r.text
assert not (folder / "cat.txt").exists()
assert not (folder / "cat.caption").exists()
assert (folder / "cat.json").exists()
def test_thumb_cache_key_distinguishes_same_stem_extensions(client, ds_root):
folder = ds_root / "d"
folder.mkdir()
Image.new("RGB", (8, 8), (10, 20, 30)).save(folder / "sample.png", format = "PNG")
Image.new("RGB", (8, 8), (200, 210, 220)).save(folder / "sample.jpg", format = "JPEG")
client.get("/api/train/diffusion/dataset/d/image/sample.png?thumb=32")
client.get("/api/train/diffusion/dataset/d/image/sample.jpg?thumb=32")
thumbs = sorted(p.name for p in (folder / ".thumbs").glob("*.jpg"))
assert thumbs == ["sample.jpg_32_w.jpg", "sample.png_32_w.jpg"]
def test_thumbnail_of_transparent_image_shows_white_background(client, ds_root):
folder = ds_root / "d"
folder.mkdir()
sticker = Image.new("RGBA", (64, 64), (0, 0, 0, 0))
sticker.paste((255, 220, 0, 255), (16, 16, 48, 48))
sticker.save(folder / "sticker.png", format = "PNG")
r = client.get("/api/train/diffusion/dataset/d/image/sticker.png?thumb=64")
assert r.status_code == 200
thumb = Image.open(io.BytesIO(r.content)).convert("RGB")
assert min(thumb.getpixel((2, 2))) > 245
def test_thumbnail_cached_before_white_flattening_is_not_served(client, ds_root):
folder = ds_root / "d"
(folder / ".thumbs").mkdir(parents = True)
Image.new("RGBA", (64, 64), (0, 0, 0, 0)).save(folder / "sticker.png", format = "PNG")
Image.new("RGB", (64, 64), (0, 0, 0)).save(
folder / ".thumbs" / "sticker.png_64.jpg", format = "JPEG"
)
r = client.get("/api/train/diffusion/dataset/d/image/sticker.png?thumb=64")
assert r.status_code == 200
assert min(Image.open(io.BytesIO(r.content)).convert("RGB").getpixel((2, 2))) > 245
def test_dataset_name_traversal_rejected_over_http(client, ds_root):
assert client.get("/api/train/diffusion/dataset/bad name!/images").status_code == 400
def test_image_filename_validation_rejects_traversal():
from pathlib import Path
from routes.training import _safe_dataset_image_path
folder = Path("/tmp/some-dataset")
for bad in ("../../etc/passwd", "/etc/passwd", "..", "sub/dir.png", "notimage.txt"):
with pytest.raises(HTTPException) as exc:
_safe_dataset_image_path(folder, bad)
assert exc.value.status_code == 400
assert _safe_dataset_image_path(folder, "ok.png") == folder / "ok.png"
def test_clean_dataset_name_rejects_dotdot():
from routes.training import _clean_diffusion_dataset_name
for bad in ("../x", "a/b", "..", " "):
with pytest.raises(HTTPException) as exc:
_clean_diffusion_dataset_name(bad)
assert exc.value.status_code == 400
def test_list_dataset_examples(client, ds_root):
r = client.get("/api/train/diffusion/dataset-examples")
assert r.status_code == 200, r.text
ids = {e["id"] for e in r.json()["examples"]}
assert {
"dreambooth-dog",
"tuxemon",
"tarot-1920",
"smithsonian-butterflies",
"pixel-nouns",
} <= ids
dog = next(e for e in r.json()["examples"] if e["id"] == "dreambooth-dog")
assert dog["suggested_trigger"] == "a photo of sks dog"
assert dog["license"]
def test_list_dataset_examples_large_sets(client, ds_root):
r = client.get("/api/train/diffusion/dataset-examples")
examples = {e["id"]: e for e in r.json()["examples"]}
butterflies = examples["smithsonian-butterflies"]
assert butterflies["image_cap"] == 100
assert butterflies["suggested_trigger"] == "a photo of a sks butterfly"
assert "CC0" in butterflies["license"]
nouns = examples["pixel-nouns"]
assert nouns["image_cap"] == 100
assert nouns["suggested_trigger"] is None
assert nouns["license"] == "cc0-1.0"
class _FakeImageFeature:
# Mimics datasets.Image so _detect_image_column matches by class name.
pass
_FakeImageFeature.__name__ = "Image"
class _FakeDS:
def __init__(self, rows, features):
self._rows = rows
self.features = features
def __iter__(self):
return iter(self._rows)
def _install_fake_load_dataset(
monkeypatch,
n_rows,
features = "default",
streamable = True,
):
calls = {"count": 0, "streaming": [], "features": features}
rows = [
{"image": Image.new("RGB", (8, 8), (i * 30 % 255, 60, 90)), "prompt": f"caption {i}"}
for i in range(n_rows)
]
if features == "default":
features = {"image": _FakeImageFeature(), "prompt": object()}
def fake_load(repo, **kwargs):
calls["count"] += 1
calls["streaming"].append(bool(kwargs.get("streaming")))
assert kwargs.get("split") == "train"
if kwargs.get("streaming") and not streamable:
raise ValueError("Loading a dataset cached in a LocalFileSystem is not supported")
return _FakeDS(rows, features)
import datasets
monkeypatch.setattr(datasets, "load_dataset", fake_load)
return calls
def test_import_example_writes_images_and_captions(client, ds_root, monkeypatch):
calls = _install_fake_load_dataset(monkeypatch, n_rows = 3)
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r.status_code == 200, r.text
body = r.json()
assert body["name"] == "my-tux"
assert body["imported"] == 3
assert body["image_count"] == 3
assert body["caption_count"] == 3
assert body["license"] == "cc-by-sa-3.0"
assert body["source_repo"] == "linoyts/Tuxemon"
folder = ds_root / "my-tux"
assert sorted(p.name for p in folder.glob("*.png")) == [f"img_{i:04d}.png" for i in range(3)]
assert (folder / "img_0000.txt").read_text(encoding = "utf-8") == "caption 0"
r2 = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r2.status_code == 200
assert r2.json()["imported"] == 0
assert r2.json()["image_count"] == 3
assert calls["count"] == 1
def test_import_example_respects_cap(client, ds_root, monkeypatch):
_install_fake_load_dataset(monkeypatch, n_rows = 5)
entry = next(
e
for e in __import__("routes.training", fromlist = ["_DATASET_EXAMPLES"])._DATASET_EXAMPLES
if e["id"] == "tuxemon"
)
monkeypatch.setitem(entry, "image_cap", 2)
r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "tuxemon"})
assert r.status_code == 200, r.text
assert r.json()["imported"] == 2
assert r.json()["image_count"] == 2
def test_import_example_streams_instead_of_preparing_the_whole_split(client, ds_root, monkeypatch):
calls = _install_fake_load_dataset(monkeypatch, n_rows = 3)
r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "tuxemon"})
assert r.status_code == 200, r.text
assert r.json()["imported"] == 3
assert calls["streaming"] == [True]
def test_import_example_falls_back_when_the_repo_cannot_stream(client, ds_root, monkeypatch):
calls = _install_fake_load_dataset(monkeypatch, n_rows = 3, streamable = False)
r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "tuxemon"})
assert r.status_code == 200, r.text
assert r.json()["imported"] == 3
assert calls["streaming"] == [True, False]
def test_import_example_resolves_columns_from_the_first_row_without_features(
client, ds_root, monkeypatch
):
_install_fake_load_dataset(monkeypatch, n_rows = 2, features = None)
r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "tuxemon"})
assert r.status_code == 200, r.text
assert r.json()["imported"] == 2
assert r.json()["caption_count"] == 2
def test_import_example_without_an_image_column_maps_to_502(client, ds_root, monkeypatch):
_install_fake_load_dataset(monkeypatch, n_rows = 2, features = {"prompt": object()})
r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "tuxemon"})
assert r.status_code == 502
def _seed_non_image_dataset_files(folder):
"""A folder that holds no images but is not empty: captions, metadata and a thumb cache."""
folder.mkdir(parents = True, exist_ok = True)
(folder / "metadata.jsonl").write_text('{"file_name": "a.png"}\n', encoding = "utf-8")
(folder / "notes.txt").write_text("caption one", encoding = "utf-8")
(folder / ".thumbs").mkdir()
(folder / ".thumbs" / "cached").write_text("x", encoding = "utf-8")
return {"metadata.jsonl", "notes.txt", ".thumbs"}
def test_import_example_keeps_the_dataset_when_rmdir_fails(client, ds_root, monkeypatch):
import os
_install_fake_load_dataset(monkeypatch, n_rows = 2)
folder = ds_root / "keepme"
before = _seed_non_image_dataset_files(folder)
real_rmdir = os.rmdir
def failing_rmdir(path, *args, **kwargs):
if os.path.abspath(path) == os.path.abspath(str(folder)):
raise OSError(39, "Directory not empty")
return real_rmdir(path, *args, **kwargs)
monkeypatch.setattr(os, "rmdir", failing_rmdir)
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "keepme"},
)
assert r.status_code == 409, r.text
assert "Nothing was written" in r.json()["detail"]
assert folder.is_dir()
assert before <= {p.name for p in folder.iterdir()}
assert (folder / "notes.txt").read_text(encoding = "utf-8") == "caption one"
assert (folder / ".thumbs" / "cached").read_text(encoding = "utf-8") == "x"
def test_import_example_keeps_the_dataset_when_the_rename_fails(client, ds_root, monkeypatch):
import os
_install_fake_load_dataset(monkeypatch, n_rows = 2)
folder = ds_root / "keepme2"
before = _seed_non_image_dataset_files(folder)
def failing_replace(src, dst, *args, **kwargs):
raise OSError(13, "Permission denied")
monkeypatch.setattr(os, "replace", failing_replace)
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "keepme2"},
)
assert r.status_code == 409, r.text
assert folder.is_dir()
assert before <= {p.name for p in folder.iterdir()}
assert (folder / "notes.txt").read_text(encoding = "utf-8") == "caption one"
def test_import_example_unknown_id_404(client, ds_root):
r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "does-not-exist"})
assert r.status_code == 404
def test_import_example_load_failure_maps_to_502(client, ds_root, monkeypatch):
import datasets
def boom(repo, **kwargs):
raise RuntimeError("network down")
monkeypatch.setattr(datasets, "load_dataset", boom)
r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "tuxemon"})
assert r.status_code == 502
assert "Could not import" in r.json()["detail"]
def _jpg_bytes(color = (30, 120, 200), size = (8, 8)) -> bytes:
buf = io.BytesIO()
Image.new("RGB", size, color).save(buf, format = "JPEG")
return buf.getvalue()
def _upload(client, name, files):
parts = [("files", (fn, data, "application/octet-stream")) for fn, data in files]
return client.post("/api/train/diffusion/dataset", data = {"name": name}, files = parts)
def test_upload_rejects_same_stem_different_extension(client, ds_root):
assert _upload(client, "styleset", [("sample.png", _png_bytes())]).status_code == 200
dup = _upload(client, "styleset", [("sample.jpg", _jpg_bytes())])
assert dup.status_code == 400
assert "Duplicate image name" in dup.json()["detail"]
folder = ds_root / "styleset"
assert sorted(p.name for p in folder.iterdir() if p.suffix != ".txt") == ["sample.png"]
def test_upload_same_stem_collision_within_one_batch(client, ds_root):
r = _upload(client, "styleset", [("sample.png", _png_bytes()), ("sample.jpg", _jpg_bytes())])
assert r.status_code == 400
assert "Duplicate image name" in r.json()["detail"]
def test_upload_rejects_exact_duplicate_name_within_one_batch(client, ds_root):
r = _upload(
client,
"styleset",
[("sample.png", _png_bytes((10, 20, 30))), ("sample.png", _png_bytes((90, 90, 90)))],
)
assert r.status_code == 400
assert "more than once" in r.json()["detail"]
assert not (ds_root / "styleset" / "sample.png").exists()
r = _upload(client, "styleset", [("sample.txt", b"a"), ("sample.txt", b"b")])
assert r.status_code == 400
assert "more than once" in r.json()["detail"]
r = _upload(client, "styleset", [("Cat.png", _png_bytes()), ("cat.png", _png_bytes())])
assert r.status_code == 200
def test_upload_rejects_case_only_duplicate_on_a_case_insensitive_filesystem(
client, ds_root, monkeypatch
):
# Cat.png and cat.png collide on NTFS and APFS, so the rejection is gated on a real probe.
import routes.training as training
monkeypatch.setattr(training, "_dataset_folder_is_case_insensitive", lambda folder: True)
r = _upload(
client,
"styleset",
[("Cat.png", _png_bytes((10, 20, 30))), ("cat.png", _png_bytes((90, 90, 90)))],
)
assert r.status_code == 400
assert "only by letter case" in r.json()["detail"]
assert "Cat.png" in r.json()["detail"]
folder = ds_root / "styleset"
assert not any(p.suffix.lower() == ".png" for p in folder.iterdir())
r = _upload(client, "styleset", [("Cat.txt", b"caption A"), ("cat.txt", b"caption B")])
assert r.status_code == 400
assert "only by letter case" in r.json()["detail"]
assert not any(p.suffix.lower() == ".txt" for p in folder.iterdir())
assert _upload(client, "styleset", [("Cat.png", _png_bytes())]).status_code == 200
assert _upload(client, "styleset", [("cat.png", _png_bytes())]).status_code == 200
def test_case_insensitivity_probe_matches_this_filesystem(client, ds_root):
# The probe is what gates the rejection above, so pin it against the real root: Linux CI is case-sensitive.
import routes.training as training
folder = ds_root / "probeset"
folder.mkdir()
training._DATASETS_CASE_INSENSITIVE = None
try:
(folder / "probe.tmp").write_text("x")
expected = (folder / "PROBE.TMP").exists()
assert training._dataset_folder_is_case_insensitive(folder) is expected
finally:
training._DATASETS_CASE_INSENSITIVE = None
def test_upload_rejects_extension_case_variant_sidecar_collision(client, ds_root):
r = _upload(client, "styleset", [("dog.PNG", _png_bytes()), ("dog.png", _png_bytes())])
assert r.status_code == 400
assert "Duplicate image name" in r.json()["detail"]
folder = ds_root / "styleset"
assert not any(p.suffix.lower() == ".png" for p in folder.iterdir())
assert _upload(client, "styleset", [("dog.png", _png_bytes())]).status_code == 200
dup = _upload(client, "styleset", [("dog.PNG", _png_bytes())])
assert dup.status_code == 400
assert "Duplicate image name" in dup.json()["detail"]
assert sorted(p.name for p in folder.iterdir() if p.suffix != ".txt") == ["dog.png"]
def test_upload_allows_exact_name_overwrite_and_caption_sidecar(client, ds_root):
# exact-name overwrites are allowed, and same-stem .txt files are Kohya captions
assert (
_upload(client, "styleset", [("sample.png", _png_bytes((10, 20, 30)))]).status_code == 200
)
assert (
_upload(client, "styleset", [("sample.png", _png_bytes((90, 90, 90)))]).status_code == 200
)
assert _upload(client, "styleset", [("sample.txt", b"a caption")]).status_code == 200
folder = ds_root / "styleset"
assert (folder / "sample.png").is_file()
assert (folder / "sample.txt").read_text(encoding = "utf-8") == "a caption"
def test_create_only_upload_refuses_every_occupied_spelling(client, ds_root):
folder = ds_root / "StyleSet"
folder.mkdir()
original = _png_bytes((10, 20, 30))
(folder / "sample.png").write_bytes(original)
for name in ("StyleSet", "styleset"):
r = client.post(
"/api/train/diffusion/dataset",
data = {"name": name, "create_only": "true"},
files = [("files", ("sample.png", _png_bytes((90, 90, 90)), "image/png"))],
)
assert r.status_code == 409, r.text
assert "already exists" in r.json()["detail"]
assert (folder / "sample.png").read_bytes() == original
created = client.post(
"/api/train/diffusion/dataset",
data = {"name": "fresh", "create_only": "true"},
files = [("files", ("new.png", _png_bytes(), "image/png"))],
)
assert created.status_code == 200, created.text
assert (ds_root / "fresh" / "new.png").is_file()
def test_upload_refuses_internal_dataset_folders(client, ds_root):
for name in ("uploads", "RECIPES", "seed-uploads", "unstructured-uploads"):
r = _upload(client, name, [("sample.png", _png_bytes())])
assert r.status_code == 400, r.text
assert "internal dataset storage" in r.json()["detail"]
def test_create_only_upload_removes_a_new_folder_after_validation_failure(client, ds_root):
rejected = client.post(
"/api/train/diffusion/dataset",
data = {"name": "retryable", "create_only": "true"},
files = [("files", ("bad.exe", b"not a dataset file", "application/octet-stream"))],
)
assert rejected.status_code == 400, rejected.text
assert not (ds_root / "retryable").exists()
retried = client.post(
"/api/train/diffusion/dataset",
data = {"name": "retryable", "create_only": "true"},
files = [("files", ("sample.png", _png_bytes(), "image/png"))],
)
assert retried.status_code == 200, retried.text
assert (ds_root / "retryable" / "sample.png").is_file()
# ── import: promotion is all-or-nothing ──────────────────────────────────────
def test_import_promotion_leaves_no_partial_dataset_on_failure(ds_root, monkeypatch):
# atomic promotion prevents a failed import from satisfying the image_count idempotency check
import os
# match production by returning 500 responses instead of reraising server exceptions
app = FastAPI()
app.include_router(training_router, prefix = "/api/train")
app.dependency_overrides[get_current_subject] = lambda: "test-user"
noraise = TestClient(app, raise_server_exceptions = False)
calls = _install_fake_load_dataset(monkeypatch, n_rows = 3)
folder = ds_root / "my-tux"
real_replace = os.replace
def flaky_replace(src, dst, *a, **k):
if str(dst) == str(folder):
raise OSError("simulated crash during promotion")
return real_replace(src, dst, *a, **k)
monkeypatch.setattr(os, "replace", flaky_replace)
r = noraise.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r.status_code == 409
assert list(ds_root.glob("my-tux/*.png")) == []
assert not any(p.name.startswith(".my-tux.import-") for p in ds_root.iterdir())
assert not any(p.name.startswith(".my-tux.rescue-") for p in ds_root.iterdir())
monkeypatch.setattr(os, "replace", real_replace)
r2 = noraise.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r2.status_code == 200, r2.text
assert r2.json()["imported"] == 3
assert calls["count"] == 2
def test_upload_rolls_back_when_a_later_promotion_fails(ds_root, monkeypatch):
from pathlib import Path
app = FastAPI()
app.include_router(training_router, prefix = "/api/train")
app.dependency_overrides[get_current_subject] = lambda: "test-user"
noraise = TestClient(app, raise_server_exceptions = False)
folder = ds_root / "styleset"
folder.mkdir()
(folder / "a.txt").write_bytes(b"ORIGINAL-A")
(folder / "b.txt").write_bytes(b"ORIGINAL-B")
real_replace = Path.replace
state = {"failed": False}
def flaky_replace(self, target, *a, **k):
if (
not state["failed"]
and str(target).endswith("b.txt")
and self.name.startswith(".upload-")
and not self.name.startswith(".upload-backup-")
):
state["failed"] = True
raise OSError("simulated second commit failure")
return real_replace(self, target, *a, **k)
monkeypatch.setattr(Path, "replace", flaky_replace)
parts = [
("files", ("a.txt", b"NEW-A", "application/octet-stream")),
("files", ("b.txt", b"NEW-B", "application/octet-stream")),
]
r = noraise.post("/api/train/diffusion/dataset", data = {"name": "styleset"}, files = parts)
assert r.status_code == 500
monkeypatch.setattr(Path, "replace", real_replace)
assert (folder / "a.txt").read_bytes() == b"ORIGINAL-A"
assert (folder / "b.txt").read_bytes() == b"ORIGINAL-B"
assert not list(folder.glob(".upload-*.part"))
assert not list(folder.glob(".upload-backup-*.part"))
def test_upload_rechecks_training_state_before_commit(ds_root, monkeypatch):
import routes.training as tr
folder = ds_root / "styleset"
folder.mkdir()
(folder / "a.png").write_bytes(_png_bytes())
calls = {"n": 0}
def fake_active():
calls["n"] += 1
return calls["n"] >= 2
monkeypatch.setattr(tr, "_diffusion_training_active", fake_active)
app = FastAPI()
app.include_router(training_router, prefix = "/api/train")
app.dependency_overrides[get_current_subject] = lambda: "test-user"
noraise = TestClient(app, raise_server_exceptions = False)
parts = [("files", ("a.png", _png_bytes(color = (1, 2, 3)), "image/png"))]
r = noraise.post("/api/train/diffusion/dataset", data = {"name": "styleset"}, files = parts)
assert r.status_code == 409
assert calls["n"] >= 2
assert (folder / "a.png").read_bytes() == _png_bytes()
assert not list(folder.glob(".upload-*.part"))
assert not list(folder.glob(".upload-backup-*.part"))
def test_resolve_dataset_folder_rejects_symlink(ds_root, tmp_path):
# A dataset dir that is a symlink outside the datasets root must be rejected, else delete / caption / read reach external files.
from routes.training import _resolve_dataset_folder
external = tmp_path / "external"
external.mkdir()
(external / "victim.png").write_bytes(_png_bytes())
(ds_root / "linked").symlink_to(external, target_is_directory = True)
with pytest.raises(HTTPException) as exc:
_resolve_dataset_folder("linked")
assert exc.value.status_code == 400
def test_upload_through_symlinked_dataset_cannot_escape_root(client, ds_root, tmp_path):
external = tmp_path / "external"
external.mkdir()
(ds_root / "linked").symlink_to(external, target_is_directory = True)
r = _upload(client, "linked", [("intruder.png", _png_bytes())])
assert r.status_code == 400
assert "symbolic link" in r.json()["detail"]
assert not (external / "intruder.png").exists()
assert not any(external.iterdir())
def test_delete_through_symlinked_dataset_cannot_escape_root(client, ds_root, tmp_path):
external = tmp_path / "external"
external.mkdir()
victim = external / "victim.png"
_write_png(victim)
(ds_root / "linked").symlink_to(external, target_is_directory = True)
r = client.delete("/api/train/diffusion/dataset/linked/image/victim.png")
assert r.status_code == 400
assert victim.exists()
def test_delete_image_with_glob_chars_only_removes_own_thumbs(client, ds_root):
from urllib.parse import quote
folder = ds_root / "d"
folder.mkdir()
_write_png(folder / "[ab].png")
_write_png(folder / "a.png")
thumbs = folder / ".thumbs"
thumbs.mkdir()
(thumbs / "[ab].png_32.jpg").write_bytes(b"own")
(thumbs / "a.png_32.jpg").write_bytes(b"sibling")
r = client.delete("/api/train/diffusion/dataset/d/image/" + quote("[ab].png", safe = ""))
assert r.status_code == 200, r.text
assert not (thumbs / "[ab].png_32.jpg").exists()
assert (thumbs / "a.png_32.jpg").exists()
assert not (folder / "[ab].png").exists()
assert (folder / "a.png").exists()
def test_import_preserves_unrelated_files_when_folder_not_empty(client, ds_root, monkeypatch):
_install_fake_load_dataset(monkeypatch, n_rows = 3)
folder = ds_root / "my-tux"
folder.mkdir(parents = True)
(folder / "notes.md").write_text("keep me", encoding = "utf-8")
# The promoted folder is the staging dir renamed into place, so its inode changes; a per-file move would keep the original.
inode_before = folder.stat().st_ino
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r.status_code == 200, r.text
assert r.json()["imported"] == 3
assert sorted(p.name for p in folder.glob("*.png")) == [f"img_{i:04d}.png" for i in range(3)]
assert (folder / "notes.md").read_text(encoding = "utf-8") == "keep me"
assert folder.stat().st_ino != inode_before
assert [d.name for d in ds_root.glob(".my-tux.import-*")] == []
def test_import_promotes_atomically_over_a_thumbs_cache(client, ds_root, monkeypatch):
_install_fake_load_dataset(monkeypatch, n_rows = 2)
folder = ds_root / "my-tux"
(folder / ".thumbs").mkdir(parents = True)
(folder / ".thumbs" / "old.png_32.jpg").write_bytes(b"stale")
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r.status_code == 200, r.text
assert r.json()["imported"] == 2
assert sorted(p.name for p in folder.glob("*.png")) == ["img_0000.png", "img_0001.png"]
assert (folder / ".thumbs" / "old.png_32.jpg").read_bytes() == b"stale"
assert [d.name for d in ds_root.glob(".my-tux.import-*")] == []
def test_import_replaces_a_pre_existing_file_the_import_also_writes(client, ds_root, monkeypatch):
_install_fake_load_dataset(monkeypatch, n_rows = 2)
folder = ds_root / "my-tux"
folder.mkdir(parents = True)
(folder / "img_0000.txt").write_text("stale caption", encoding = "utf-8")
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r.status_code == 200, r.text
assert r.json()["imported"] == 2
assert (folder / "img_0000.txt").read_text(encoding = "utf-8") == "caption 0"
assert [d.name for d in ds_root.glob(".my-tux.import-*")] == []
def test_a_second_concurrent_import_of_the_same_name_is_refused(client, ds_root, monkeypatch):
"""The training interlock counts mutations rather than excluding them, so two imports into one
empty name both passed and merged; the second is refused."""
import routes.training as training_route
_install_fake_load_dataset(monkeypatch, n_rows = 3)
folder = ds_root / "my-tux"
folder.mkdir(parents = True)
held = training_route._dataset_import_lock(folder)
assert held.acquire(blocking = False)
try:
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
finally:
held.release()
assert r.status_code == 409, r.text
assert "already running" in r.json()["detail"]
assert list(folder.glob("*.png")) == []
ok = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert ok.status_code == 200, ok.text
assert ok.json()["imported"] == 3
def test_an_import_into_a_folder_filled_meanwhile_does_not_merge(client, ds_root, monkeypatch):
"""The re-check under the lock: a request that waited while another import promoted its
staging dir must return the folder as-is, not move its own files on top."""
import routes.training as training_route
_install_fake_load_dataset(monkeypatch, n_rows = 3)
folder = ds_root / "my-tux"
folder.mkdir(parents = True)
(folder / "img_0000.png").write_bytes(b"\x89PNG\r\n\x1a\n" + b"0" * 16)
(folder / "img_0000.txt").write_text("from the first import", encoding = "utf-8")
r = client.post(
"/api/train/diffusion/dataset/import-example",
json = {"id": "tuxemon", "name": "my-tux"},
)
assert r.status_code == 200, r.text
assert r.json()["imported"] == 0
assert (folder / "img_0000.txt").read_text(encoding = "utf-8") == "from the first import"
assert not training_route._dataset_import_lock(folder).locked()
def test_an_unreadable_sidecar_shadows_the_metadata_caption(client, ds_root):
"""Any existing sidecar, even unreadable, shadows metadata captions, as the trainer treats it."""
folder = ds_root / "tombstone"
folder.mkdir()
_write_png(folder / "a.png")
_write_png(folder / "b.png")
(folder / "a.txt").write_bytes(b"\xff\xfe not valid utf-8")
(folder / "metadata.jsonl").write_text(
'{"file_name": "a.png", "text": "metadata caption for a"}\n'
'{"file_name": "b.png", "text": "metadata caption for b"}\n',
encoding = "utf-8",
)
recs = {
rec["filename"]: rec
for rec in client.get("/api/train/diffusion/dataset/tombstone/images").json()["images"]
}
assert recs["a.png"]["caption"] in (None, "")
assert recs["b.png"]["caption"] == "metadata caption for b"
info = client.get("/api/train/diffusion/info").json()
summary = next(d for d in info["datasets"] if d["name"] == "tombstone")
assert summary["image_count"] == 2
assert summary["caption_count"] == 1
@pytest.mark.parametrize("bad", ["CON", "nul", "COM1", "lpt9", "NUL.txt", "aux.images"])
def test_upload_rejects_windows_reserved_dataset_names(client, ds_root, bad):
# Reserved in every directory on Windows (NUL.txt is NUL), so mkdir dies there. Rejected on every platform, since datasets travel.
resp = _upload(client, bad, [("sample.png", _png_bytes())])
assert resp.status_code == 400, resp.text
assert "reserved" in resp.json()["detail"].lower()
def test_upload_rejects_a_trailing_period_dataset_name(client, ds_root):
# Win32 strips a trailing period, so 'photos.' opens the existing 'photos' dataset and an upload would modify the wrong one.
assert _upload(client, "photos", [("sample.png", _png_bytes())]).status_code == 200
resp = _upload(client, "photos.", [("other.png", _png_bytes())])
assert resp.status_code == 400, resp.text
assert "period" in resp.json()["detail"].lower()
assert sorted(p.name for p in (ds_root / "photos").iterdir()) == ["sample.png"]
def test_upload_refuses_while_an_import_holds_the_same_folder(client, ds_root):
from routes.training import _dataset_import_lock
folder = ds_root / "shared-name"
folder.mkdir(parents = True, exist_ok = True)
lock = _dataset_import_lock(folder)
assert lock.acquire(blocking = False)
try:
resp = _upload(client, "shared-name", [("sample.png", _png_bytes())])
assert resp.status_code == 409, resp.text
assert "import" in resp.json()["detail"].lower()
assert not list(folder.glob("*.png"))
finally:
lock.release()
assert _upload(client, "shared-name", [("sample.png", _png_bytes())]).status_code == 200