Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Video clips in dataset routes; the clip and image sets stay disjoint and share caption rules."""
from __future__ import annotations
import io
import json
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from PIL import Image
from auth.authentication import get_current_subject
from core.training.diffusion_clip_formats import CLIP_EXTS
from routes.training import (
_DIFFUSION_DATASET_CLIP_EXTS,
_DIFFUSION_DATASET_IMAGE_EXTS,
_DIFFUSION_DATASET_MEDIA_EXTS,
_diffusion_dataset_summary,
_resolve_dataset_caption,
)
from routes.training import router as training_router
def _write_png(
path,
color = (200, 100, 50),
size = (8, 8),
) -> None:
Image.new("RGB", size, color).save(path, format = "PNG")
def _png_bytes(color = (200, 100, 50), size = (8, 8)) -> bytes:
buf = io.BytesIO()
Image.new("RGB", size, color).save(buf, format = "PNG")
return buf.getvalue()
# Never decoded here, only counted and moved, so an ftyp box stands in for an mp4.
_MP4_BYTES = b"\x00\x00\x00\x18ftypmp42" + b"\x00" * 64
@pytest.fixture
def client():
app = FastAPI()
app.include_router(training_router, prefix = "/api/train")
app.dependency_overrides[get_current_subject] = lambda: "test-user"
return TestClient(app)
@pytest.fixture
def ds_root(monkeypatch, tmp_path):
import utils.paths as up
root = tmp_path / "assets" / "datasets"
root.mkdir(parents = True)
monkeypatch.setattr(up, "datasets_root", lambda: root)
return root
def test_image_and_clip_extension_sets_are_disjoint():
"""An extension in both sets would count as an image, so the kind counts would stop summing."""
assert _DIFFUSION_DATASET_IMAGE_EXTS & _DIFFUSION_DATASET_CLIP_EXTS == set()
assert _DIFFUSION_DATASET_MEDIA_EXTS == (
_DIFFUSION_DATASET_IMAGE_EXTS | _DIFFUSION_DATASET_CLIP_EXTS
)
assert len(_DIFFUSION_DATASET_MEDIA_EXTS) == len(_DIFFUSION_DATASET_IMAGE_EXTS) + len(
_DIFFUSION_DATASET_CLIP_EXTS
)
# Every extension is lowercase and dotted, since every test site compares Path.suffix.lower().
for ext in _DIFFUSION_DATASET_MEDIA_EXTS:
assert ext == ext.lower() and ext.startswith(".") and len(ext) > 1
def test_routes_read_the_shared_clip_definition():
"""The routes must not carry their own copy: a container the upload accepts but the
trainer's discovery does not read is a dataset that uploads fine and trains on nothing."""
assert _DIFFUSION_DATASET_CLIP_EXTS == set(CLIP_EXTS)
def test_trainer_clip_discovery_uses_the_same_set_when_it_is_present():
"""Trainer clip discovery must use the shared extension set, or a clip is counted but never trained."""
clips = pytest.importorskip("core.training.diffusion_h3_clips")
assert set(clips._VIDEO_EXTS) == set(CLIP_EXTS)
def test_clip_captions_resolve_through_the_same_rules_as_images(tmp_path):
"""Identical caption rules, asserted by running BOTH kinds through the one resolver over the
same four cases. If any row ever differs, clip support has stopped being additive."""
folder = tmp_path / "mixed"
folder.mkdir()
meta = {
"sidecar.png": "from metadata",
"sidecar.mp4": "from metadata",
"meta.png": "from metadata",
"meta.mp4": "from metadata",
"tombstone.png": "from metadata",
"tombstone.mp4": "from metadata",
}
for stem in ("sidecar", "meta", "tombstone", "bare"):
_write_png(folder / f"{stem}.png")
(folder / f"{stem}.mp4").write_bytes(_MP4_BYTES)
(folder / "sidecar.txt").write_text("edited sidecar", encoding = "utf-8")
(folder / "tombstone.txt").write_text(" ", encoding = "utf-8")
# sidecar.txt is shared by sidecar.png and sidecar.mp4 on purpose.
for stem, expected in (
("sidecar", "edited sidecar"),
("meta", "from metadata"),
("tombstone", ""),
("bare", None),
):
image = _resolve_dataset_caption(folder, folder / f"{stem}.png", meta)
clip = _resolve_dataset_caption(folder, folder / f"{stem}.mp4", meta)
assert image == expected, stem
assert clip == expected, stem
assert image == clip, stem
def test_an_image_only_dataset_summarises_exactly_as_before(tmp_path):
"""The regression guard on the additive claim: nothing about an image dataset moves."""
folder = tmp_path / "styleset"
folder.mkdir()
_write_png(folder / "a.png")
_write_png(folder / "b.jpg")
_write_png(folder / "c.webp")
(folder / "a.txt").write_text("a cat", encoding = "utf-8")
(folder / "metadata.jsonl").write_text(
json.dumps({"file_name": "b.jpg", "text": "a dog"}) + "\n", encoding = "utf-8"
)
(folder / "notes.pdf").write_bytes(b"not a dataset file")
summary = _diffusion_dataset_summary(folder)
assert summary.image_count == 3
assert summary.clip_count == 0
assert summary.caption_count == 2
def test_summary_counts_clips_and_their_captions(tmp_path):
folder = tmp_path / "clipset"
folder.mkdir()
for name in ("one.mp4", "two.MOV", "three.webm"):
(folder / name).write_bytes(_MP4_BYTES)
(folder / "one.txt").write_text("a waterfall", encoding = "utf-8")
(folder / "captions.jsonl").write_text(
json.dumps({"file_name": "two.MOV", "text": "a train"}) + "\n", encoding = "utf-8"
)
summary = _diffusion_dataset_summary(folder)
assert summary.image_count == 0
assert summary.clip_count == 3
assert summary.caption_count == 2
def test_summary_counts_a_mixed_folder_by_kind(tmp_path):
folder = tmp_path / "both"
folder.mkdir()
_write_png(folder / "still.png")
(folder / "moving.mp4").write_bytes(_MP4_BYTES)
summary = _diffusion_dataset_summary(folder)
assert (summary.image_count, summary.clip_count) == (1, 1)
def test_info_lists_a_clip_only_dataset(client, ds_root):
"""Clip-only folders were missing from the picker, so video training had no dataset."""
folder = ds_root / "clipset"
folder.mkdir()
(folder / "one.mp4").write_bytes(_MP4_BYTES)
(folder / "one.txt").write_text("a waterfall", encoding = "utf-8")
r = client.get("/api/train/diffusion/info")
assert r.status_code == 200, r.text
datasets = {d["name"]: d for d in r.json()["datasets"]}
assert "clipset" in datasets
assert datasets["clipset"]["clip_count"] == 1
assert datasets["clipset"]["image_count"] == 0
assert datasets["clipset"]["caption_count"] == 1
def test_info_still_skips_a_folder_holding_neither(client, ds_root):
"""admission requires an image or a clip, not merely a folder"""
folder = ds_root / "captions-only"
folder.mkdir()
(folder / "metadata.jsonl").write_text(
json.dumps({"file_name": "gone.png", "text": "x"}) + "\n", encoding = "utf-8"
)
(folder / "readme.txt").write_text("nothing to train on", encoding = "utf-8")
r = client.get("/api/train/diffusion/info")
assert r.status_code == 200, r.text
assert [d["name"] for d in r.json()["datasets"]] == []
assert r.json()["dataset_names"] == ["captions-only"]
assert r.json()["continuation_dataset_names"] == ["captions-only"]
def test_info_offers_an_emptied_dataset_folder_as_a_continuation(client, ds_root):
emptied = ds_root / "emptied"
(emptied / ".thumbs").mkdir(parents = True)
other = ds_root / "other-files"
other.mkdir()
(other / "notes.bin").write_bytes(b"x")
r = client.get("/api/train/diffusion/info")
assert r.status_code == 200, r.text
assert r.json()["continuation_dataset_names"] == ["emptied"]
def test_info_never_offers_internal_dataset_folders_as_continuations(client, ds_root):
for name in ("uploads", "recipes", "seed-uploads", "unstructured-uploads"):
folder = ds_root / name
folder.mkdir()
(folder / "metadata.jsonl").write_text(
json.dumps({"file_name": "pending.png", "text": "x"}) + "\n",
encoding = "utf-8",
)
r = client.get("/api/train/diffusion/info")
assert r.status_code == 200, r.text
assert r.json()["continuation_dataset_names"] == []
def test_list_images_marks_clips_and_leaves_images_unchanged(client, ds_root):
"""clips expose stem-keyed sidecars but have zero dimensions and are filtered by ``kind``"""
folder = ds_root / "both"
folder.mkdir()
_write_png(folder / "still.png", size = (12, 9))
(folder / "moving.mp4").write_bytes(_MP4_BYTES)
(folder / "moving.txt").write_text("a train", encoding = "utf-8")
r = client.get("/api/train/diffusion/dataset/both/images")
assert r.status_code == 200, r.text
recs = {rec["filename"]: rec for rec in r.json()["images"]}
assert recs["still.png"]["kind"] == "image"
assert (recs["still.png"]["width"], recs["still.png"]["height"]) == (12, 9)
assert recs["moving.mp4"]["kind"] == "clip"
assert (recs["moving.mp4"]["width"], recs["moving.mp4"]["height"]) == (0, 0)
assert recs["moving.mp4"]["caption"] == "a train"
assert recs["moving.mp4"]["caption_source"] == "sidecar"
def test_upload_accepts_a_clip_and_its_sidecar_as_a_pair(client, ds_root):
r = client.post(
"/api/train/diffusion/dataset",
data = {"name": "clipset"},
files = [
("files", ("one.mp4", _MP4_BYTES, "video/mp4")),
("files", ("one.txt", b"a waterfall", "text/plain")),
("files", ("two.webm", _MP4_BYTES, "video/webm")),
],
)
assert r.status_code == 200, r.text
body = r.json()
assert body["uploaded"] == 3
assert body["clip_count"] == 2
assert body["image_count"] == 0
assert body["caption_count"] == 1
folder = ds_root / "clipset"
assert (folder / "one.mp4").read_bytes() == _MP4_BYTES
assert (folder / "one.txt").read_text(encoding = "utf-8") == "a waterfall"
def test_upload_still_refuses_an_extension_of_neither_kind(client, ds_root):
r = client.post(
"/api/train/diffusion/dataset",
data = {"name": "clipset"},
files = [("files", ("notes.pdf", b"%PDF-1.4", "application/pdf"))],
)
assert r.status_code == 400, r.text
assert not (ds_root / "clipset" / "notes.pdf").exists()
def test_upload_refuses_an_image_and_a_clip_sharing_a_stem(client, ds_root):
"""cat.png and cat.mp4 share cat.txt, as two images do, so the refusal must cover both."""
r = client.post(
"/api/train/diffusion/dataset",
data = {"name": "mixed"},
files = [
("files", ("cat.png", _png_bytes(), "image/png")),
("files", ("cat.mp4", _MP4_BYTES, "video/mp4")),
],
)
assert r.status_code == 400, r.text
assert "Duplicate clip name 'cat'" in r.json()["detail"]
assert "cat.png" in r.json()["detail"]
assert list((ds_root / "mixed").iterdir()) == []
def test_upload_refuses_a_clip_whose_stem_is_already_in_the_folder(client, ds_root):
folder = ds_root / "mixed"
folder.mkdir()
_write_png(folder / "cat.png")
r = client.post(
"/api/train/diffusion/dataset",
data = {"name": "mixed"},
files = [("files", ("cat.mp4", _MP4_BYTES, "video/mp4"))],
)
assert r.status_code == 400, r.text
assert not (folder / "cat.mp4").exists()
def test_upload_does_not_decode_a_clip_as_an_image(client, ds_root):
"""Decompression-bomb check is about pixels; a clip Pillow cannot open must not be a 400."""
r = client.post(
"/api/train/diffusion/dataset",
data = {"name": "clipset"},
files = [("files", ("one.mkv", b"\x1a\x45\xdf\xa3" + b"\x00" * 32, "video/x-matroska"))],
)
assert r.status_code == 200, r.text
assert r.json()["clip_count"] == 1
def test_deleting_an_image_keeps_a_clip_of_the_same_stem_captioned(client, ds_root):
"""Legacy folders can hold cat.png beside cat.mp4 (the upload refuses new ones). Deleting
the image must not strip the sidecar the surviving clip still reads."""
folder = ds_root / "legacy"
folder.mkdir()
_write_png(folder / "cat.png")
(folder / "cat.mp4").write_bytes(_MP4_BYTES)
(folder / "cat.txt").write_text("a cat", encoding = "utf-8")
r = client.delete("/api/train/diffusion/dataset/legacy/image/cat.png")
assert r.status_code == 200, r.text
assert not (folder / "cat.png").exists()
assert (folder / "cat.txt").read_text(encoding = "utf-8") == "a cat"
assert _diffusion_dataset_summary(folder).caption_count == 1
def test_start_refuses_a_clip_only_folder_by_naming_the_clips(tmp_path):
"""A clip folder is listed now, so a user can pick it and press Start. Discovery reads
stills, so the raw refusal is "No captioned images found ... provide per-image .txt
captions" -- wrong twice over on a folder where every clip has one."""
from core.training.diffusion_train_common import discover_image_caption_pairs
from routes.training import _clip_dataset_refusal
folder = tmp_path / "clip-style"
folder.mkdir()
for stem in ("a", "b"):
(folder / f"{stem}.mp4").write_bytes(_MP4_BYTES)
(folder / f"{stem}.txt").write_text(f"clip {stem}", encoding = "utf-8")
assert _diffusion_dataset_summary(folder).caption_count == 2
with pytest.raises(ValueError, match = "No captioned images found"):
discover_image_caption_pairs(folder)
detail = _clip_dataset_refusal(str(folder))
assert detail is not None
assert "2 video clips" in detail
assert "not supported yet" in detail
assert ".txt" not in detail
def test_start_refuses_a_mixed_folder_rather_than_training_the_images_alone(tmp_path):
"""The sharper case: discovery SUCCEEDS on the images, so nothing raises and the run
trains on the still subset while the picker counted the clips as trainable items."""
from core.training.diffusion_train_common import discover_image_caption_pairs
from routes.training import _clip_dataset_refusal
folder = tmp_path / "mixed"
folder.mkdir()
_write_png(folder / "still.png")
(folder / "still.txt").write_text("a still", encoding = "utf-8")
(folder / "clip.mp4").write_bytes(_MP4_BYTES)
(folder / "clip.txt").write_text("a clip", encoding = "utf-8")
assert _diffusion_dataset_summary(folder).caption_count == 2
assert len(discover_image_caption_pairs(folder)) == 1
detail = _clip_dataset_refusal(str(folder))
assert detail is not None
assert "1 video clip" in detail and "1 video clips" not in detail
def test_the_summary_reads_a_video_keyed_metadata_caption(tmp_path):
"""The summary must read video-keyed metadata captions, or clip datasets report zero captions."""
import json
folder = tmp_path / "video-keyed"
folder.mkdir()
for stem in ("a", "b"):
(folder / f"{stem}.mp4").write_bytes(_MP4_BYTES)
(folder / "metadata.jsonl").write_text(
"\n".join(
json.dumps({"video": f"{stem}.mp4", "text": f"clip {stem}"}) for stem in ("a", "b")
),
encoding = "utf-8",
)
summary = _diffusion_dataset_summary(folder)
assert summary.clip_count == 2
assert summary.caption_count == 2
from core.training.diffusion_h3_clips import discover_clip_caption_pairs
assert len(discover_clip_caption_pairs(folder)) == 2
@pytest.mark.parametrize("separator", ["\u2028", "\u2029", "\u0085"])
def test_a_caption_holding_a_unicode_line_separator_is_not_dropped(tmp_path, separator):
"""splitlines() cut these raw in-string separators mid-row, silently dropping the caption."""
from core.training.diffusion_h3_clips import discover_clip_caption_pairs
from core.training.diffusion_train_common import discover_image_caption_pairs
from routes.training import _load_metadata_captions
folder = tmp_path / "separators"
folder.mkdir()
_write_png(folder / "a.png")
(folder / "b.mp4").write_bytes(_MP4_BYTES)
caption = f"line one{separator}line two"
rows = [
{"file_name": "a.png", "text": caption},
{"file_name": "b.mp4", "text": caption},
]
(folder / "metadata.jsonl").write_text(
"\n".join(json.dumps(row, ensure_ascii = False) for row in rows) + "\n", encoding = "utf-8"
)
assert _load_metadata_captions(folder) == {"a.png": caption, "b.mp4": caption}
assert [c for _, c in discover_image_caption_pairs(folder)] == [caption]
assert [c for _, c in discover_clip_caption_pairs(folder)] == [caption]
def test_start_leaves_an_image_only_folder_alone(tmp_path):
"""No clips, no refusal: the ordinary uncaptioned-folder error still speaks for itself."""
from routes.training import _clip_dataset_refusal
folder = tmp_path / "photo-style"
folder.mkdir()
_write_png(folder / "one.png")
assert _clip_dataset_refusal(str(folder)) is None
assert _clip_dataset_refusal(str(tmp_path / "does-not-exist")) is None
def test_deleting_an_image_keeps_a_case_folded_clips_caption(client, ds_root, monkeypatch):
"""Case-folding filesystems: CAT.mp4's sidecar is cat.txt, so an exact stem compare would delete
the caption the clip reads; uploads refuse such pairs, but folders can hold one."""
import routes.training as tr
monkeypatch.setattr(tr, "_dataset_folder_is_case_insensitive", lambda folder: True)
folder = ds_root / "legacy-case"
folder.mkdir()
_write_png(folder / "cat.png")
(folder / "CAT.mp4").write_bytes(_MP4_BYTES)
(folder / "cat.txt").write_text("a cat", encoding = "utf-8")
r = client.delete("/api/train/diffusion/dataset/legacy-case/image/cat.png")
assert r.status_code == 200, r.text
assert not (folder / "cat.png").exists()
assert (folder / "cat.txt").read_text(encoding = "utf-8") == "a cat"
def test_deleting_an_image_still_drops_its_sidecar_where_case_is_kept(client, ds_root, monkeypatch):
"""Here CAT.mp4 reads CAT.txt, so cat.txt belongs to the image alone and goes with it, or the
next cat.png inherits a caption written for something else."""
import routes.training as tr
folder = ds_root / "sensitive-case"
folder.mkdir()
# cat.txt and CAT.txt must be two files; on a case-insensitive filesystem (the macOS and
# Windows defaults) they are one, so the case-kept layout cannot be built there.
(folder / "probe.txt").write_text("", encoding = "utf-8")
if (folder / "PROBE.txt").exists():
pytest.skip("needs a case-sensitive filesystem to hold cat.txt and CAT.txt apart")
(folder / "probe.txt").unlink()
monkeypatch.setattr(tr, "_dataset_folder_is_case_insensitive", lambda folder: False)
_write_png(folder / "cat.png")
(folder / "CAT.mp4").write_bytes(_MP4_BYTES)
(folder / "cat.txt").write_text("a cat", encoding = "utf-8")
(folder / "CAT.txt").write_text("a CAT", encoding = "utf-8")
r = client.delete("/api/train/diffusion/dataset/sensitive-case/image/cat.png")
assert r.status_code == 200, r.text
assert not (folder / "cat.txt").exists()
assert (folder / "CAT.txt").read_text(encoding = "utf-8") == "a CAT"