Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

101 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Diffusion LoRA training service and routes, driven with in-thread fakes and a mocked service."""

from __future__ import annotations

import json
import queue as _queue
import threading
import time

import contextlib
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient

from auth.authentication import authenticated_via_api_key, get_current_subject
from core.training.diffusion_training_service import DiffusionTrainingService
from routes.training import router as training_router


class _FakeQueue:
    def __init__(self) -> None:
        self._q: _queue.Queue = _queue.Queue()

    def put(self, x):
        self._q.put(x)

    def get(self, timeout = None):
        return self._q.get(timeout = timeout)

    def get_nowait(self):
        return self._q.get_nowait()

    def empty(self):
        return self._q.empty()


class _FakeProc:
    def __init__(self, target, kwargs, daemon):
        self._target = target
        self._kwargs = kwargs
        self._thread: threading.Thread | None = None
        self.pid = 4321

    def start(self):
        self._thread = threading.Thread(target = self._target, kwargs = self._kwargs, daemon = True)
        self._thread.start()

    def is_alive(self):
        return self._thread is not None and self._thread.is_alive()


class _FakeCtx:
    def Queue(self):
        return _FakeQueue()

    def Process(self, target, kwargs, daemon):
        return _FakeProc(target, kwargs, daemon)


@pytest.fixture(autouse = True)
def _healthy_diffusers(healthy_diffusers):
    """Every test here is about the route or the config, not about the runner's diffusers."""


@pytest.fixture(autouse = True)
def _isolated_runs_dir(monkeypatch, tmp_path):
    """Terminal service events persist a run record; point the runs dir at tmp so tests
    never write into a real studio home. Yields the dir for the history tests."""
    import core.training.diffusion_training_service as dts

    d = tmp_path / "runs" / "diffusion"
    d.mkdir(parents = True, exist_ok = True)
    monkeypatch.setattr(dts, "_runs_dir", lambda: d)
    pumps = []
    real_start = dts.DiffusionTrainingService.start

    def start(self, *args, **kwargs):
        job_id = real_start(self, *args, **kwargs)
        pumps.append(self._pump)
        return job_id

    monkeypatch.setattr(dts.DiffusionTrainingService, "start", start)

    yield d

    # The pump resolves _runs_dir() at write time; wait so a late write cannot leak into another test.
    for pump in pumps:
        pump.join(timeout = 30)
        assert not pump.is_alive(), (
            "a diffusion run this test started is still running at teardown, so its record "
            "would land in a later test's runs dir; stop it or wait for it in the test"
        )


def _happy_target(*, event_queue, stop_queue, config):
    event_queue.put({"type": "model_load_started", "num_images": 3})
    event_queue.put({"type": "model_load_completed"})
    event_queue.put(
        {
            "type": "progress",
            "step": 1,
            "total_steps": 2,
            "loss": 0.5,
            "avg_loss": 0.5,
            "learning_rate": 1e-4,
        }
    )
    event_queue.put(
        {
            "type": "progress",
            "step": 2,
            "total_steps": 2,
            "loss": 0.4,
            "avg_loss": 0.45,
            "learning_rate": 1e-4,
        }
    )
    event_queue.put(
        {
            "type": "complete",
            "output_dir": config["output_dir"],
            "lora_path": config["output_dir"] + "/pytorch_lora_weights.safetensors",
            "stopped": False,
        }
    )


def _stoppable_target(*, event_queue, stop_queue, config):
    event_queue.put({"type": "model_load_completed"})
    stop_queue.get(timeout = 5.0)
    event_queue.put(
        {"type": "complete", "output_dir": config["output_dir"], "lora_path": "x", "stopped": True}
    )


def _crashing_target(*, event_queue, stop_queue, config):
    event_queue.put({"type": "model_load_started"})


def test_default_target_activates_native_tls_before_diffusion_trainer(monkeypatch):
    """TLS activation runs inside the scrub wrapper, and before diffusers imports."""
    import os
    import sys
    import types
    import core.training.diffusion_training_service as service
    import utils.native_path_leases as leases

    calls = []
    monkeypatch.setenv(leases.LEASE_SECRET_ENV, "secret")

    def _activate():
        calls.append("native_tls")
        assert leases.LEASE_SECRET_ENV not in os.environ

    monkeypatch.setattr("utils.native_tls.activate_native_tls", _activate)

    trainer = types.ModuleType("core.training.diffusion_lora_trainer")
    trainer.run_diffusion_training_process = lambda **kwargs: calls.append("trainer")
    monkeypatch.setitem(sys.modules, "core.training.diffusion_lora_trainer", trainer)

    def _run_without_native_path_secret(target, **kwargs):
        calls.append("native_path_secret")
        assert target is service._run_diffusion_child
        assert kwargs == {
            "event_queue": "events",
            "stop_queue": "stop",
            "config": {"base_model": "example/model"},
        }
        os.environ.pop(leases.LEASE_SECRET_ENV, None)
        return target(**kwargs)

    monkeypatch.setattr(
        "utils.native_path_leases.run_without_native_path_secret",
        _run_without_native_path_secret,
    )

    service._default_target(
        event_queue = "events", stop_queue = "stop", config = {"base_model": "example/model"}
    )

    assert calls == ["native_path_secret", "native_tls", "trainer"]


_CFG = {"base_model": "b", "data_dir": "d", "output_dir": "/tmp/out", "train_steps": 2}


def _wait_status(
    svc,
    *terminal,
    timeout = 3.0,
):
    end = time.time() + timeout
    while time.time() < end:
        st = svc.status()
        if st["status"] in terminal:
            return st
        time.sleep(0.02)
    return svc.status()


def _wait_record(
    runs_dir,
    job_id,
    timeout = 5.0,
):
    """Wait for the record file itself: the pump persists it after publishing the terminal status."""
    import json

    path = runs_dir / f"{job_id}.json"
    end = time.time() + timeout
    while time.time() < end:
        try:
            return json.loads(path.read_text(encoding = "utf-8"))
        except (FileNotFoundError, json.JSONDecodeError):
            time.sleep(0.01)
    raise AssertionError(f"the pump never persisted {path} within {timeout}s")


def test_service_happy_path():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    job_id = svc.start(dict(_CFG))
    assert job_id
    st = _wait_status(svc, "completed")
    assert st["status"] == "completed"
    assert st["step"] == 2 and st["total_steps"] == 2
    assert st["num_images"] == 3
    assert st["loss"] == 0.4 and st["avg_loss"] == 0.45
    assert st["lora_path"].endswith("pytorch_lora_weights.safetensors")
    assert st["active"] is False


def test_service_rejects_bad_config_before_spawn():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    with pytest.raises(ValueError):
        svc.start({**_CFG, "train_steps": 0})
    assert svc.status()["status"] == "idle"


def test_service_rejects_second_concurrent_job():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _stoppable_target)
    svc.start(dict(_CFG))
    _wait_status(svc, "running")
    with pytest.raises(RuntimeError):
        svc.start(dict(_CFG))
    assert svc.stop() is True
    _wait_status(svc, "stopped")


def test_service_stop_marks_stopped():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _stoppable_target)
    svc.start(dict(_CFG))
    _wait_status(svc, "running")
    assert svc.stop() is True
    st = _wait_status(svc, "stopped")
    assert st["status"] == "stopped"
    assert st["active"] is False
    assert svc.stop() is False


def test_service_stop_for_shutdown_saves_and_waits():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _stoppable_target)
    svc.start(dict(_CFG))
    _wait_status(svc, "running")
    assert svc.stop_for_shutdown(timeout = 5) is True
    st = svc.status()
    assert st["status"] == "stopped"
    assert st["active"] is False


def test_service_stop_for_shutdown_is_immediate_when_idle():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    t0 = time.monotonic()
    assert svc.stop_for_shutdown(timeout = 5) is True
    assert time.monotonic() - t0 < 1


class _ExitedProc:
    def is_alive(self):
        return False


def test_service_stop_for_shutdown_waits_for_the_pump_after_the_worker_exits():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    written = threading.Event()
    svc._proc = _ExitedProc()
    svc._pump = threading.Thread(target = lambda: (time.sleep(0.5), written.set()), daemon = True)
    svc._pump.start()
    assert svc.stop_for_shutdown(timeout = 5) is True
    assert written.is_set()


def _ignores_stop_target(*, event_queue, stop_queue, config):
    event_queue.put({"type": "model_load_completed"})
    time.sleep(3)


def test_service_stop_for_shutdown_wait_is_bounded():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _ignores_stop_target)
    svc.start(dict(_CFG))
    _wait_status(svc, "running")
    t0 = time.monotonic()
    assert svc.stop_for_shutdown(timeout = 0.5) is False
    assert 0.4 < time.monotonic() - t0 < 2


def test_service_crash_without_terminal_event_is_error():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _crashing_target)
    svc.start(dict(_CFG))
    st = _wait_status(svc, "error")
    assert st["status"] == "error"
    assert "unexpectedly" in st["message"]


def test_apply_event_transitions():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "model_load_started", "num_images": 5})
    assert svc.status()["in_model_load"] is True and svc.status()["num_images"] == 5
    svc._apply_event({"type": "model_load_completed"})
    assert svc.status()["in_model_load"] is False
    svc._apply_event({"type": "error", "message": "boom"})
    assert svc.status()["status"] == "error" and svc.status()["message"] == "boom"


def test_preparation_counts_survive_status_polling_and_clear_for_training():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 48, "total": 60})
    assert svc.status()["preparation"] == {"stage": "cache_latents", "done": 48, "total": 60}
    assert svc.status()["in_model_load"] is True
    svc._apply_event({"type": "model_load_completed"})
    assert svc.status()["preparation"] is None
    svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 60, "total": 60})
    svc._apply_event({"type": "progress", "step": 1, "total_steps": 30, "loss": 0.4})
    assert svc.status()["preparation"] is None
    assert svc.status()["in_model_load"] is False


@pytest.mark.parametrize("event", [{"type": "complete"}, {"type": "error", "message": "boom"}])
def test_terminal_events_clear_preparation_counts(event):
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 48, "total": 60})
    svc._apply_event(event)
    assert svc.status()["preparation"] is None


def _dies_while_preparing(*, event_queue, stop_queue, config):
    event_queue.put({"type": "model_load_started", "num_images": 60})
    event_queue.put({"type": "preparing", "stage": "cache_latents", "done": 12, "total": 60})


def test_worker_dying_mid_preparation_clears_preparation_counts():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _dies_while_preparing)
    svc.start(dict(_CFG))
    st = _wait_status(svc, "error")
    assert st["status"] == "error"
    assert st["preparation"] is None
    assert st["in_model_load"] is False


def test_the_joint_losses_reach_status_and_history():
    """video_loss and audio_loss must reach status and history, index-aligned with steps."""
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event(
        {
            "type": "progress",
            "step": 1,
            "total_steps": 2,
            "loss": 0.5,
            "video_loss": 0.4,
            "audio_loss": 0.1,
        }
    )
    st = svc.status()
    assert st["video_loss"] == 0.4 and st["audio_loss"] == 0.1
    assert st["metric_video_loss"] == [0.4] and st["metric_audio_loss"] == [0.1]

    svc._apply_event({"type": "progress", "step": 2, "total_steps": 2, "loss": 0.4})
    st = svc.status()
    assert st["metric_steps"] == [1, 2]
    assert len(st["metric_video_loss"]) == len(st["metric_steps"])
    assert len(st["metric_audio_loss"]) == len(st["metric_steps"])

    # All-null lets the chart tell "not a joint run" from a joint run with zero audio loss.
    solo = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    solo._apply_event({"type": "progress", "step": 1, "total_steps": 1, "loss": 0.5})
    st = solo.status()
    assert st["video_loss"] is None and st["audio_loss"] is None
    assert st["metric_video_loss"] == [None] and st["metric_audio_loss"] == [None]


def test_progress_nulls_non_finite_floats_for_strict_json():
    # Strict JSON forbids NaN/Infinity, so the service must null divergent metrics.
    import json
    import math

    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event(
        {
            "type": "progress",
            "step": 3,
            "total_steps": 10,
            "loss": float("nan"),
            "avg_loss": float("inf"),
            "learning_rate": float("-inf"),
            "grad_norm": float("inf"),
        }
    )
    snap = svc.status()
    assert snap["loss"] is None
    assert snap["avg_loss"] is None
    assert snap["learning_rate"] is None
    assert snap["grad_norm"] is None
    assert snap["metric_loss"] == []
    assert snap["metric_steps"] == []
    json.dumps(snap, allow_nan = False)

    svc._apply_event(
        {"type": "progress", "step": 4, "total_steps": 10, "loss": 0.5, "learning_rate": 1e-4}
    )
    snap2 = svc.status()
    assert snap2["loss"] == 0.5
    assert snap2["metric_loss"] == [0.5] and snap2["metric_steps"] == [4]
    assert math.isfinite(snap2["learning_rate"])
    json.dumps(snap2, allow_nan = False)


def test_terminal_events_clear_model_load_flag():
    # Stop/error during load skips model_load_completed, so the terminal update must reset in_model_load.
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "model_load_started"})
    assert svc.status()["in_model_load"] is True
    svc._apply_event({"type": "complete", "stopped": True})
    assert svc.status()["in_model_load"] is False and svc.status()["status"] == "stopped"

    svc2 = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc2._apply_event({"type": "model_load_started"})
    svc2._apply_event({"type": "error", "message": "load failed"})
    assert svc2.status()["in_model_load"] is False and svc2.status()["status"] == "error"


def test_complete_event_keeps_the_ema_adapter_path():
    # EMA runs write a SECOND adapter as ema_path; dropping the field makes it undiscoverable.
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event(
        {
            "type": "complete",
            "output_dir": "/o",
            "lora_path": "/o/a.safetensors",
            "ema_path": "/o/ema/a.safetensors",
        }
    )
    snap = svc.status()
    assert snap["lora_path"] == "/o/a.safetensors"
    assert snap["ema_path"] == "/o/ema/a.safetensors"
    svc._apply_event({"type": "complete", "output_dir": "/o2", "lora_path": "/o2/a.safetensors"})
    assert svc.status()["ema_path"] is None


class _FakeService:
    def __init__(self):
        self._running = False
        self._reserved = False
        self.started_with = None
        self.stopped_with_save = None
        self.calls: list = []
        self.status_extra: dict = {}

    def reserve(self):
        self._reserved = True
        self.calls.append("reserve")

    def unreserve(self):
        self._reserved = False
        self.calls.append("unreserve")

    def is_active(self):
        return self._reserved or self._running

    @contextlib.contextmanager
    def dataset_mutation(self):
        # Mirrors the real interlock: refuse while a run owns the dataset, and register the mutation.
        from core.training.diffusion_training_service import TrainingActiveError

        if self.is_active():
            raise TrainingActiveError(
                "Training images cannot be changed while diffusion training is active."
            )
        self.calls.append("dataset_mutation")
        yield

    def start(self, config):
        self.started_with = config
        self._running = True
        self.calls.append("start")
        return "job-123"

    def stop(self, save = True):
        self.stopped_with_save = save
        was = self._running
        self._running = False
        return was

    def status(self):
        return {
            "active": self._running,
            "job_id": "job-123" if self._running else None,
            "status": "running" if self._running else "idle",
            "message": "",
            "step": 1,
            "total_steps": 2,
            "loss": 0.5,
            "avg_loss": 0.5,
            "learning_rate": 1e-4,
            "num_images": 3,
            "in_model_load": False,
            "output_dir": None,
            "lora_path": None,
            "started_at": None,
            "updated_at": None,
            **self.status_extra,
        }


class _FakeLLMBackend:
    def __init__(self, active = False):
        self._active = active

    def is_training_active(self):
        return self._active


@pytest.fixture
def client(monkeypatch):
    fake = _FakeService()
    monkeypatch.setattr(
        "core.training.diffusion_training_service.get_diffusion_training_service", lambda: fake
    )
    # Neutralize the LLM interlock + GPU-free; the route imports get_training_backend at module scope.
    import routes.training as tr

    monkeypatch.setattr(tr, "get_training_backend", lambda: _FakeLLMBackend(active = False))
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: None)
    monkeypatch.setattr(
        "core.training.diffusion_train_common.discover_image_caption_pairs",
        lambda data_dir, **kw: [("img.png", "caption")],
    )
    app = FastAPI()
    app.include_router(training_router, prefix = "/api/train")
    app.dependency_overrides[get_current_subject] = lambda: "test-user"
    # Session (UI) auth: the API-key inference-in-flight guard is a no-op there.
    app.dependency_overrides[authenticated_via_api_key] = lambda: False
    c = TestClient(app)
    c._fake = fake  # type: ignore[attr-defined]
    c._app = app  # type: ignore[attr-defined]
    return c


_BODY = {
    "base_model": "stabilityai/sdxl-turbo",
    "data_dir": "uploads/my-images",
    "output_dir": "my-lora-run",
    "train_steps": 10,
}


def test_route_start_ok(client):
    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200, r.text
    assert r.json() == {"job_id": "job-123", "status": "running"}
    assert client._fake.started_with["base_model"] == "stabilityai/sdxl-turbo"
    from pathlib import Path

    assert Path(client._fake.started_with["data_dir"]).is_absolute()
    assert Path(client._fake.started_with["output_dir"]).is_absolute()


def test_route_start_frees_gpu_off_the_coroutine_thread(client, monkeypatch):
    # GPU cleanup can block for seconds, so the async route must offload it via asyncio.to_thread.
    import threading

    import routes.training as tr

    threads: dict = {}

    def _record_cleanup():
        threads["cleanup"] = threading.current_thread()

    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", _record_cleanup)

    orig_start = client._fake.start

    def _record_start(config):
        threads["inline"] = threading.current_thread()
        return orig_start(config)

    monkeypatch.setattr(client._fake, "start", _record_start)

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200, r.text
    assert threads["cleanup"] is not threads["inline"]


def test_route_start_reserves_before_freeing_gpu(client, monkeypatch):
    # Reserve BEFORE freeing GPU models so a concurrent load guard refuses in the free-then-spawn window.
    import routes.training as tr

    order: list = []

    def _record_free():
        order.append("free")
        order.append(f"active={client._fake.is_active()}")

    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", _record_free)

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200, r.text
    assert client._fake.calls[0] == "reserve"
    assert client._fake.calls.index("reserve") < client._fake.calls.index("start")
    assert order == ["free", "active=True"]
    assert "unreserve" in client._fake.calls


def test_route_start_reserves_before_scanning_dataset(client, monkeypatch):
    # The scan decode-probes every image, so it must run after reserving or a concurrent edit races it.
    order: list = []

    def _record_scan(data_dir, **kw):
        client._fake.calls.append("scan")
        order.append(f"scan_active={client._fake.is_active()}")
        return [("img.png", "caption")]

    monkeypatch.setattr(
        "core.training.diffusion_train_common.discover_image_caption_pairs", _record_scan
    )

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200, r.text
    assert client._fake.calls.index("reserve") < client._fake.calls.index("scan")
    assert order == ["scan_active=True"]
    assert "unreserve" in client._fake.calls


def test_route_start_unreserves_when_dataset_preflight_fails(client, monkeypatch):
    # A preflight failure after reserving must roll back, or training stays "active" forever.
    def _bad_scan(data_dir, **kw):
        raise ValueError("no captioned images found")

    monkeypatch.setattr(
        "core.training.diffusion_train_common.discover_image_caption_pairs", _bad_scan
    )

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 400
    assert "no captioned images" in r.json()["detail"]
    assert "reserve" in client._fake.calls and "unreserve" in client._fake.calls
    assert "start" not in client._fake.calls
    assert client._fake.is_active() is False


def test_service_reserve_marks_active_and_rolls_back():
    from core.training.diffusion_training_service import DiffusionTrainingService

    svc = DiffusionTrainingService()
    assert svc.is_active() is False
    svc.reserve()
    assert svc.is_active() is True
    svc.unreserve()
    assert svc.is_active() is False


def test_service_reserve_is_compare_and_set():
    # reserve() must compare-and-set: two starts can interleave between is_active() and reserving.
    from core.training.diffusion_training_service import DiffusionTrainingService

    svc = DiffusionTrainingService()
    svc.reserve()
    with pytest.raises(RuntimeError, match = "already running"):
        svc.reserve()
    assert svc.is_active() is True
    svc.unreserve()
    svc.reserve()
    assert svc.is_active() is True


def test_route_start_preflights_gated_base_off_the_coroutine_thread(client, monkeypatch):
    # _preflight_gated_base does a blocking urlopen HEAD (up to 5s), so the route must offload it.
    import threading

    import routes.training as tr

    threads: dict = {}

    def _record_preflight(base_model, hf_token):
        threads["preflight"] = threading.current_thread()

    monkeypatch.setattr(tr, "_preflight_gated_base", _record_preflight)

    orig_start = client._fake.start

    def _record_start(config):
        threads["inline"] = threading.current_thread()
        return orig_start(config)

    monkeypatch.setattr(client._fake, "start", _record_start)

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200, r.text
    assert threads["preflight"] is not threads["inline"]


def test_a_clip_trained_family_is_not_turned_away_by_the_clip_refusal(
    client, monkeypatch, dit_train_host
):
    """The clip refusal must not outrank clip families, or every valid MiniMax-H3 request gets a 400."""
    import routes.training as tr
    from core.training import diffusion_train_common as _dtc

    consulted: list[str] = []

    def _refusal(data_dir):
        consulted.append(str(data_dir))
        return "'clips' holds 2 video clips. Training from clips is not supported yet."

    monkeypatch.setattr(tr, "_clip_dataset_refusal", _refusal)
    monkeypatch.setattr(
        _dtc, "discover_training_pairs", lambda family, data_dir, **kw: [("a.mp4", "a rabbit")]
    )

    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": "MiniMaxAI/MiniMax-H3", "instance_prompt": "p"},
    )
    assert r.status_code == 200, r.text
    assert consulted == [], "the clip refusal was consulted for a clip-trained family"

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 400
    assert "not supported yet" in r.json()["detail"]
    assert len(consulted) == 1


def test_a_clip_family_still_refuses_a_folder_holding_stills(client, monkeypatch, dit_train_host):
    """Clip families still refuse mixed folders: discover_clip_caption_pairs silently drops the stills."""
    import routes.training as tr
    from core.training import diffusion_train_common as _dtc

    monkeypatch.setattr(
        tr,
        "_image_dataset_refusal",
        lambda data_dir: "'d' holds 3 still images alongside its clips.",
    )
    monkeypatch.setattr(tr, "_clip_dataset_refusal", lambda data_dir: None)
    monkeypatch.setattr(
        _dtc, "discover_training_pairs", lambda family, data_dir, **kw: [("a.mp4", "a rabbit")]
    )

    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": "MiniMaxAI/MiniMax-H3", "instance_prompt": "p"},
    )
    assert r.status_code == 400
    assert "alongside its clips" in r.json()["detail"]

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200, r.text


@pytest.mark.parametrize(
    ("hf_token", "authorization"),
    [(None, None), ("  hf_test  ", "Bearer hf_test")],
)
def test_route_start_preflights_the_normalized_fetch_mirror(
    client, monkeypatch, dit_train_host, healthy_diffusers, hf_token, authorization
):
    import urllib.error
    import urllib.request

    from core.inference import diffusion_families

    source = "black-forest-labs/FLUX.2-klein-base-9B"
    mirror = "unsloth/FLUX.2-klein-base-9B"
    monkeypatch.setattr(
        diffusion_families,
        "prefer_ungated_mirror",
        lambda base, token = None: mirror if base.lower() == source.lower() else base,
    )
    requests = []

    def _fake_urlopen(req, timeout = None):
        requests.append(req)
        if source in req.full_url:
            raise urllib.error.HTTPError(req.full_url, 401, "Unauthorized", {}, None)
        assert mirror in req.full_url
        return object()

    monkeypatch.setattr("utils.utils.auth_safe_open", _fake_urlopen)
    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": source, "hf_token": hf_token},
    )

    assert r.status_code == 200, r.text
    assert len(requests) == 1
    assert requests[0].get_header("Authorization") == authorization
    # The fetch-only mirror must not replace the canonical id persisted with the run.
    assert client._fake.started_with["base_model"] == source


@pytest.mark.parametrize("no_mirror_env", [False, True])
def test_a_tokenless_run_takes_the_mirror_even_with_the_vendor_repo_cached(
    monkeypatch, no_mirror_env
):
    """A token-less run takes the mirror even if a gated vendor repo has any leftover cached shard."""
    from core.inference import diffusion_families
    from core.training.diffusion_train_common import DiffusionLoraConfig

    source = "black-forest-labs/FLUX.1-dev"
    mirror = "unsloth/FLUX.1-dev"
    monkeypatch.setattr(diffusion_families, "prefer_ungated_mirror", lambda base, token = None: base)
    if no_mirror_env:
        monkeypatch.setenv("UNSLOTH_DIFFUSION_NO_MIRROR", "1")
    else:
        monkeypatch.delenv("UNSLOTH_DIFFUSION_NO_MIRROR", raising = False)

    def _cfg(token):
        return DiffusionLoraConfig(
            base_model = source, data_dir = "d", output_dir = "o", hf_token = token
        ).normalized()

    expected = source if no_mirror_env else mirror
    assert _cfg(None).fetch_base_model == expected
    assert _cfg("   ").fetch_base_model == expected
    assert _cfg("hf_realtoken").fetch_base_model == source
    assert _cfg(None).base_model == source


@pytest.mark.parametrize("no_mirror_env", [False, True])
def test_a_tokenless_run_keeps_a_cached_ungated_base(monkeypatch, no_mirror_env):
    """The override is only for gated repos; ungated mirrors keep a complete cached snapshot."""
    from core.inference import diffusion_families
    from core.training.diffusion_train_common import DiffusionLoraConfig

    source = "black-forest-labs/FLUX.2-klein-base-4B"
    assert diffusion_families.mirror_repo(source), "precondition: this base is mirrored"
    assert not diffusion_families.upstream_is_gated(source), "precondition: and it is ungated"
    monkeypatch.setattr(diffusion_families, "prefer_ungated_mirror", lambda base, token = None: base)
    if no_mirror_env:
        monkeypatch.setenv("UNSLOTH_DIFFUSION_NO_MIRROR", "1")
    else:
        monkeypatch.delenv("UNSLOTH_DIFFUSION_NO_MIRROR", raising = False)

    def _cfg(token):
        return DiffusionLoraConfig(
            base_model = source, data_dir = "d", output_dir = "o", hf_token = token
        ).normalized()

    assert _cfg(None).fetch_base_model == source
    assert _cfg("   ").fetch_base_model == source
    assert _cfg("hf_realtoken").fetch_base_model == source


def test_the_start_preflight_never_heads_the_hub_for_a_local_clone(monkeypatch, tmp_path):
    """A relative clone named like a vendor repo must not trigger a token-less Hub HEAD."""
    import urllib.request

    from routes.training import _preflight_gated_base

    local = "black-forest-labs/FLUX.1-dev"
    monkeypatch.chdir(tmp_path)
    (tmp_path / local).mkdir(parents = True)

    def _explode(*a, **k):
        pytest.fail("a local clone must never be probed over the network")

    monkeypatch.setattr("utils.utils.auth_safe_open", _explode)

    _preflight_gated_base(local, None)


def test_a_tokenless_run_keeps_a_local_clone_named_like_a_gated_base(monkeypatch, tmp_path):
    """A local directory named like a gated base is kept, never rewritten to the mirror."""
    from core.inference import diffusion_families
    from core.training.diffusion_train_common import DiffusionLoraConfig

    source = "black-forest-labs/FLUX.1-dev"
    assert diffusion_families.upstream_is_gated(source), "precondition: this base is gated"
    monkeypatch.chdir(tmp_path)
    (tmp_path / source).mkdir(parents = True)
    monkeypatch.delenv("UNSLOTH_DIFFUSION_NO_MIRROR", raising = False)

    cfg = DiffusionLoraConfig(
        base_model = source, data_dir = "d", output_dir = "o", hf_token = None
    ).normalized()

    assert cfg.fetch_base_model == source
    assert cfg.base_model == source


def test_route_start_forwards_extra_training_knobs(client):
    body = {**_BODY, "max_grad_norm": 0.5, "lora_target_modules": ["to_q", "to_v"]}
    r = client.post("/api/train/diffusion/start", json = body)
    assert r.status_code == 200, r.text
    assert client._fake.started_with["max_grad_norm"] == 0.5
    assert client._fake.started_with["lora_target_modules"] == ["to_q", "to_v"]


def test_route_start_forwards_num_epochs(client):
    body = {k: v for k, v in _BODY.items() if k != "train_steps"}
    r = client.post("/api/train/diffusion/start", json = {**body, "num_epochs": 8})
    assert r.status_code == 200, r.text
    assert client._fake.started_with["num_epochs"] == 8


def test_route_start_forwards_dit_loss_knobs(client):
    body = {
        **_BODY,
        "ema_decay": 0.99,
        "cfg_dropout": 0.1,
        "weighting_scheme": "bell",
        "flow_shift": 3.0,
    }
    r = client.post("/api/train/diffusion/start", json = body)
    assert r.status_code == 200, r.text
    started = client._fake.started_with
    assert started["ema_decay"] == 0.99 and started["cfg_dropout"] == 0.1
    assert started["weighting_scheme"] == "bell" and started["flow_shift"] == 3.0


def test_request_model_dit_loss_knob_bounds():
    # Bounds mirror DiffusionLoraConfig.normalized(); flow_shift also accepts "auto".
    from pydantic import ValidationError

    from models.training import DiffusionTrainingStartRequest

    base = {"base_model": "b", "data_dir": "d", "output_dir": "o"}
    defaults = DiffusionTrainingStartRequest(**base)
    assert (defaults.ema_decay, defaults.cfg_dropout) == (0.0, 0.0)
    assert defaults.weighting_scheme == "none" and defaults.flow_shift is None
    assert DiffusionTrainingStartRequest(**base, flow_shift = "auto").flow_shift == "auto"
    for bad in ({"ema_decay": 1.0}, {"cfg_dropout": 1.5}, {"weighting_scheme": "bogus"}):
        with pytest.raises(ValidationError):
            DiffusionTrainingStartRequest(**base, **bad)


def test_request_model_rejects_lora_dropout_of_one():
    # lora_dropout = 1.0 zeroes the LoRA branch's gradients, silently saving an untrained adapter.
    from pydantic import ValidationError

    from models.training import DiffusionTrainingStartRequest

    base = {"base_model": "b", "data_dir": "d", "output_dir": "o"}
    assert DiffusionTrainingStartRequest(**base).lora_dropout == 0.0
    assert DiffusionTrainingStartRequest(**base, lora_dropout = 0.99).lora_dropout == 0.99
    for bad in (1.0, 1.5, -0.1):
        with pytest.raises(ValidationError):
            DiffusionTrainingStartRequest(**base, lora_dropout = bad)


def test_request_model_num_epochs_bounds():
    # Mirrors DiffusionLoraConfig's 0..1000 num_epochs range.
    from pydantic import ValidationError

    from models.training import DiffusionTrainingStartRequest

    base = {"base_model": "b", "data_dir": "d", "output_dir": "o"}
    assert DiffusionTrainingStartRequest(**base).num_epochs == 0  # default = use train_steps
    assert DiffusionTrainingStartRequest(**base, num_epochs = 1000).num_epochs == 1000
    for bad in (-1, 1001):
        with pytest.raises(ValidationError):
            DiffusionTrainingStartRequest(**base, num_epochs = bad)


def test_request_model_base_precision_accepts_mxfp8():
    from pydantic import ValidationError

    from models.training import DiffusionTrainingStartRequest

    base = {"base_model": "b", "data_dir": "d", "output_dir": "o"}
    assert DiffusionTrainingStartRequest(**base).base_precision == "nf4"
    assert DiffusionTrainingStartRequest(**base, base_precision = "mxfp8").base_precision == "mxfp8"
    with pytest.raises(ValidationError):
        DiffusionTrainingStartRequest(**base, base_precision = "bogus")


def test_config_from_dict_epoch_mode_drops_max_steps_sentinel():
    # Epoch-mode payloads send max_steps: 0 as a sentinel that normalized() would reject, so it is dropped.
    from core.training.diffusion_train_common import DiffusionLoraConfig, _config_from_dict

    cfg = _config_from_dict(
        {
            "base_model": "stabilityai/stable-diffusion-xl-base-1.0",
            "data_dir": "d",
            "output_dir": "o",
            "max_steps": 0,
            "num_epochs": 2,
        }
    )
    assert cfg.train_steps == DiffusionLoraConfig.train_steps
    assert cfg.num_epochs == 2
    norm = cfg.normalized()
    assert norm.num_epochs == 2

    cfg_explicit = _config_from_dict(
        {
            "base_model": "stabilityai/stable-diffusion-xl-base-1.0",
            "data_dir": "d",
            "output_dir": "o",
            "max_steps": 25,
            "num_epochs": 2,
        }
    )
    assert cfg_explicit.train_steps == 25


def test_permutation_sampler_covers_dataset_once_per_cycle():
    # Each index appears once per cycle before any repeat, so a short run never leaves images unseen.
    import random

    from core.training.diffusion_train_common import PermutationBatchSampler

    n = 100
    sampler = PermutationBatchSampler(n, random.Random(0))

    drawn: list[int] = []
    while len(drawn) < n:
        drawn.extend(sampler.next_batch(3))
    first_cycle = drawn[:n]
    assert sorted(first_cycle) == list(range(n))

    fresh = PermutationBatchSampler(n, random.Random(0))
    cycle_a = fresh.next_batch(n)
    cycle_b = fresh.next_batch(n)
    assert sorted(cycle_a) == list(range(n))
    assert sorted(cycle_b) == list(range(n))
    assert cycle_a != cycle_b

    replay = PermutationBatchSampler(n, random.Random(0))
    assert replay.next_batch(n) == cycle_a

    big = PermutationBatchSampler(4, random.Random(1))
    batch = big.next_batch(10)
    assert len(batch) == 10
    assert set(batch) == {0, 1, 2, 3}


def test_permutation_sampler_honors_batch_on_tiny_dataset():
    import random

    from core.training.diffusion_train_common import PermutationBatchSampler

    sampler = PermutationBatchSampler(2, random.Random(0))
    batch = sampler.next_batch(8)  # train_batch_size = 8, not clamped to 2
    assert len(batch) == 8
    assert set(batch) == {0, 1}
    assert batch.count(0) == 4 and batch.count(1) == 4


def test_route_start_accepts_zero_max_grad_norm(client):
    # 0 means disable clipping (trainer skips clip_grad_norm_), so it must be accepted.
    r = client.post("/api/train/diffusion/start", json = {**_BODY, "max_grad_norm": 0.0})
    assert r.status_code == 200, r.text
    assert client._fake.started_with["max_grad_norm"] == 0.0


def test_route_start_rejects_nonpositive_snr_gamma(client):
    # A gamma <= 0 zeroes/inverts the min-SNR weight; null is the disable value.
    r = client.post("/api/train/diffusion/start", json = {**_BODY, "snr_gamma": 0})
    assert r.status_code == 422


def test_route_start_rejects_uncontained_paths(client):
    r = client.post("/api/train/diffusion/start", json = {**_BODY, "data_dir": "/etc"})
    assert r.status_code == 400


def test_route_start_resolves_bare_name_under_image_dataset_root(client, monkeypatch, tmp_path):
    # resolve_dataset_path searches LLM roots first, so a bare name must prefer the image dataset root.
    import utils.paths as up

    ds_root = tmp_path / "assets" / "datasets"
    img_ds = ds_root / "my-photos"
    img_ds.mkdir(parents = True)
    (img_ds / "a.png").write_bytes(b"x")
    (ds_root / "uploads").mkdir()
    (ds_root / "uploads" / "my-photos").write_text("an LLM dataset upload, not a folder")
    (ds_root / "recipes" / "my-photos").mkdir(parents = True)
    monkeypatch.setattr(up, "datasets_root", lambda: ds_root)

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "data_dir": "my-photos"})
    assert r.status_code == 200, r.text
    assert client._fake.started_with["data_dir"] == str(img_ds)


def test_route_start_refuses_a_dataset_holding_clips(client, monkeypatch, tmp_path):
    """No trainer reads clips; without the route's own check a mixed folder starts and trains on
    stills alone while the picker counted clips."""
    import utils.paths as up

    ds_root = tmp_path / "assets" / "datasets"
    folder = ds_root / "mixed-set"
    folder.mkdir(parents = True)
    (folder / "a.png").write_bytes(b"x")
    (folder / "b.mp4").write_bytes(b"x")
    monkeypatch.setattr(up, "datasets_root", lambda: ds_root)

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "data_dir": "mixed-set"})
    assert r.status_code == 400, r.text
    assert "1 video clip" in r.json()["detail"]
    assert client._fake.started_with is None


def test_route_start_still_accepts_an_image_only_dataset(client, monkeypatch, tmp_path):
    """The twin of the above: the clip check must not stand in the way of a normal folder."""
    import utils.paths as up

    ds_root = tmp_path / "assets" / "datasets"
    folder = ds_root / "photo-set"
    folder.mkdir(parents = True)
    (folder / "a.png").write_bytes(b"x")
    monkeypatch.setattr(up, "datasets_root", lambda: ds_root)

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "data_dir": "photo-set"})
    assert r.status_code == 200, r.text


def test_route_start_blocked_by_active_llm_training(client, monkeypatch):
    import routes.training as tr

    monkeypatch.setattr(tr, "get_training_backend", lambda: _FakeLLMBackend(active = True))
    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 409
    assert "LLM training" in r.json()["detail"]


def test_route_start_missing_required_is_422(client):
    r = client.post("/api/train/diffusion/start", json = {"base_model": "x"})
    assert r.status_code == 422


def test_route_start_bad_config_maps_to_400(client, monkeypatch):
    def _raise(_cfg):
        raise ValueError("resolution must be a multiple of 8")

    client._fake.start = _raise  # type: ignore[assignment]
    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 400
    assert "multiple of 8" in r.json()["detail"]


def test_route_start_conflict_maps_to_409(client):
    def _raise(_cfg):
        raise RuntimeError("A diffusion training job is already running.")

    client._fake.start = _raise  # type: ignore[assignment]
    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 409


def test_route_start_over_api_with_inference_in_flight_is_409(client, monkeypatch):
    client._app.dependency_overrides[authenticated_via_api_key] = lambda: True
    monkeypatch.setattr(
        "core.inference.llama_keepwarm.other_inference_request_count",
        lambda current_request_counted = False: 1,
    )
    freed = {"called": False}
    import routes.training as tr

    monkeypatch.setattr(
        tr, "_free_gpu_for_diffusion_training", lambda: freed.__setitem__("called", True)
    )
    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 409
    assert freed["called"] is False


def test_route_start_over_api_without_inference_proceeds(client, monkeypatch):
    client._app.dependency_overrides[authenticated_via_api_key] = lambda: True
    monkeypatch.setattr(
        "core.inference.llama_keepwarm.other_inference_request_count",
        lambda current_request_counted = False: 0,
    )
    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200


def test_route_status_and_stop(client):
    client.post("/api/train/diffusion/start", json = _BODY)
    s = client.get("/api/train/diffusion/status")
    assert s.status_code == 200 and s.json()["status"] == "running"
    st = client.post("/api/train/diffusion/stop")
    assert st.status_code == 200 and st.json()["status"] == "stopping"
    st2 = client.post("/api/train/diffusion/stop")
    assert st2.json()["status"] == "idle"


def test_service_restart_after_completion():
    # The finished pump is joined OUTSIDE the lock (it needs the lock for final writes), else deadlock.
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc.start(dict(_CFG))
    _wait_status(svc, "completed")
    t0 = time.time()
    job2 = svc.start(dict(_CFG))
    assert job2
    assert time.time() - t0 < 4.0
    st = _wait_status(svc, "completed")
    assert st["status"] == "completed"


def test_stale_pump_events_cannot_corrupt_new_job():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc.start(dict(_CFG))
    _wait_status(svc, "completed")
    current = svc._proc
    svc._apply_event({"type": "error", "message": "stale boom"}, proc = object())
    assert svc.status()["message"] != "stale boom"
    svc._apply_event({"type": "progress", "step": 9}, proc = current)
    assert svc.status()["step"] == 9


@pytest.fixture
def dataset_roots(client, monkeypatch, tmp_path):
    # The endpoints import these lazily per-request, so patching the package attr works.
    import utils.paths as up

    ds_root = tmp_path / "assets" / "datasets"
    out_root = tmp_path / "outputs"
    ds_root.mkdir(parents = True)
    out_root.mkdir(parents = True)
    monkeypatch.setattr(up, "datasets_root", lambda: ds_root)
    monkeypatch.setattr(up, "outputs_root", lambda: out_root)
    return ds_root, out_root


def test_diffusion_info_lists_image_dataset_folders(client, dataset_roots):
    ds_root, out_root = dataset_roots
    good = ds_root / "cat-photos"
    good.mkdir()
    (good / "a.png").write_bytes(b"x")
    (good / "b.jpg").write_bytes(b"x")
    (good / "a.txt").write_text("a cat")
    (ds_root / "empty-dir").mkdir()
    (ds_root / "stray.txt").write_text("not a folder")

    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    body = r.json()
    assert body["datasets_root"] == str(ds_root)
    assert body["outputs_root"] == str(out_root)
    assert [d["name"] for d in body["datasets"]] == ["cat-photos"]
    assert body["datasets"][0]["image_count"] == 2
    assert body["datasets"][0]["caption_count"] == 1


def test_diffusion_info_counts_metadata_captions(client, dataset_roots):
    ds_root, _ = dataset_roots
    folder = ds_root / "meta-captioned"
    folder.mkdir()
    (folder / "a.png").write_bytes(b"x")
    (folder / "b.png").write_bytes(b"x")
    (folder / "c.png").write_bytes(b"x")
    # a.png + b.png via metadata; a.png also has a sidecar (must count once); c.png none.
    (folder / "metadata.jsonl").write_text(
        json.dumps({"file_name": "a.png", "text": "cap a"})
        + "\n"
        + json.dumps({"file_name": "b.png", "text": "cap b"})
        + "\n",
        encoding = "utf-8",
    )
    (folder / "a.txt").write_text("edited a", encoding = "utf-8")

    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    summary = next(d for d in r.json()["datasets"] if d["name"] == "meta-captioned")
    assert summary["image_count"] == 3
    assert summary["caption_count"] == 2


def test_diffusion_dataset_upload_accumulates(client, dataset_roots):
    ds_root, _ = dataset_roots
    files = [
        ("files", ("a.png", b"png-bytes", "image/png")),
        ("files", ("b.JPG", b"jpg-bytes", "image/jpeg")),
        ("files", ("a.txt", b"a caption", "text/plain")),
    ]
    r = client.post("/api/train/diffusion/dataset", data = {"name": "my style"}, files = files)
    assert r.status_code == 200, r.text
    body = r.json()
    assert body["name"] == "my style"
    assert body["uploaded"] == 3
    assert body["image_count"] == 2
    assert body["caption_count"] == 1
    assert (ds_root / "my style" / "a.png").read_bytes() == b"png-bytes"

    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "my style"},
        files = [("files", ("c.webp", b"w", "image/webp"))],
    )
    assert r.status_code == 200, r.text
    assert r.json()["uploaded"] == 1
    assert r.json()["image_count"] == 3


def test_diffusion_dataset_upload_normalizes_windows_and_rejects_dotdot(client, dataset_roots):
    ds_root, _ = dataset_roots
    # POSIX Path.name does not split a Windows backslash filename, so it must be folded to the basename.
    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "winset"},
        files = [("files", ("C:\\Users\\me\\pics\\cat.png", b"png-bytes", "image/png"))],
    )
    assert r.status_code == 200, r.text
    assert (ds_root / "winset" / "cat.png").read_bytes() == b"png-bytes"
    recs = client.get("/api/train/diffusion/dataset/winset/images").json()["images"]
    assert any(rec["filename"] == "cat.png" for rec in recs)
    assert client.get("/api/train/diffusion/dataset/winset/image/cat.png").status_code == 200

    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "winset"},
        files = [("files", ("a..b.png", b"x", "image/png"))],
    )
    assert r.status_code == 400 and "Unsupported file" in r.json()["detail"]


def test_diffusion_dataset_upload_rejects_case_insensitive_stem_clash(client, dataset_roots):
    # Stems differing only by case share a sidecar on case-insensitive filesystems, so compare casefolded.
    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "caseset"},
        files = [
            ("files", ("sample.png", b"p", "image/png")),
            ("files", ("Sample.jpg", b"j", "image/jpeg")),
        ],
    )
    assert r.status_code == 400 and "Duplicate image name" in r.json()["detail"]

    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "caseset2"},
        files = [("files", ("photo.png", b"p", "image/png"))],
    )
    assert r.status_code == 200, r.text
    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "caseset2"},
        files = [("files", ("PHOTO.webp", b"w", "image/webp"))],
    )
    assert r.status_code == 400 and "Duplicate image name" in r.json()["detail"]

    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "caseset3"},
        files = [
            ("files", ("pic.png", b"a", "image/png")),
            ("files", ("Pic.png", b"b", "image/png")),
        ],
    )
    assert r.status_code == 200, r.text


def test_diffusion_dataset_upload_over_cap_keeps_existing_example(
    client, dataset_roots, monkeypatch
):
    import utils.upload_limits as ul

    ds_root, _ = dataset_roots
    folder = ds_root / "my style"
    folder.mkdir()
    (folder / "cat.png").write_bytes(b"ORIGINAL-CAT-BYTES")

    monkeypatch.setattr(ul, "get_upload_limit_bytes", lambda: 8)
    monkeypatch.setattr(ul, "get_upload_limit_label", lambda: "8B")

    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "my style"},
        files = [("files", ("cat.png", b"x" * 64, "image/png"))],
    )
    assert r.status_code == 413, r.text
    assert (folder / "cat.png").read_bytes() == b"ORIGINAL-CAT-BYTES"
    assert sorted(p.name for p in folder.iterdir()) == ["cat.png"]


def test_diffusion_info_empty_sidecar_shadows_metadata_caption(client, dataset_roots):
    # An empty .txt sidecar is a tombstone that shadows the metadata row; the trainer skips the image.
    ds_root, _ = dataset_roots
    folder = ds_root / "tombstoned"
    folder.mkdir()
    (folder / "a.png").write_bytes(b"x")
    (folder / "b.png").write_bytes(b"x")
    (folder / "c.png").write_bytes(b"x")
    (folder / "metadata.jsonl").write_text(
        json.dumps({"file_name": "a.png", "text": "cap a"})
        + "\n"
        + json.dumps({"file_name": "c.png", "text": "cap c"})
        + "\n",
        encoding = "utf-8",
    )
    (folder / "a.txt").write_text("   ", encoding = "utf-8")
    (folder / "b.txt").write_text("cap b", encoding = "utf-8")

    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    summary = next(d for d in r.json()["datasets"] if d["name"] == "tombstoned")
    assert summary["image_count"] == 3
    assert summary["caption_count"] == 2


def _png_bytes(width: int, height: int) -> bytes:
    import io

    pytest.importorskip("PIL")
    from PIL import Image

    buf = io.BytesIO()
    Image.new("RGB", (width, height), (1, 2, 3)).save(buf, format = "PNG")
    return buf.getvalue()


def test_diffusion_dataset_upload_rejects_oversized_image(client, dataset_roots):
    # Decompression bomb: small file, huge dimensions; passes the byte limit but would OOM the trainer.
    pytest.importorskip("PIL")
    big = _png_bytes(5000, 64)  # > 4096 per side
    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "bomb"},
        files = [("files", ("huge.png", big, "image/png"))],
    )
    assert r.status_code == 400, r.text
    assert "too large" in r.json()["detail"]
    ok = _png_bytes(64, 64)
    r2 = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "bomb"},
        files = [("files", ("ok.png", ok, "image/png"))],
    )
    assert r2.status_code == 200, r2.text


def test_diffusion_dataset_upload_maps_pillow_bomb_to_400(client, dataset_roots, monkeypatch):
    # DecompressionBombError derives from Exception, so the dimension guard missed it; shrink the limit.
    pytest.importorskip("PIL")
    from PIL import Image

    monkeypatch.setattr(Image, "MAX_IMAGE_PIXELS", 8)  # 8x8 = 64 pixels > 2 x 8
    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "bomb-hard"},
        files = [("files", ("huge.png", _png_bytes(8, 8), "image/png"))],
    )
    assert r.status_code == 400, r.text
    assert "too large" in r.json()["detail"]
    assert not (dataset_roots[0] / "bomb-hard" / "huge.png").exists()


def test_diffusion_info_tolerates_non_object_jsonl(client, dataset_roots):
    ds_root, _ = dataset_roots
    folder = ds_root / "weird-meta"
    folder.mkdir()
    (folder / "a.png").write_bytes(b"x")
    (folder / "metadata.jsonl").write_bytes(
        b"[]\n"
        b"null\n"
        b'"just a string"\n'
        b"123\n"
        b"{not json\n" + json.dumps({"file_name": "a.png", "text": "cap a"}).encode("utf-8") + b"\n"
    )
    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    summary = next(d for d in r.json()["datasets"] if d["name"] == "weird-meta")
    assert summary["caption_count"] == 1


def test_diffusion_info_skips_null_metadata_captions(client, dataset_roots):
    # str(None) would store the literal "None" and train on that text.
    ds_root, _ = dataset_roots
    folder = ds_root / "null-caption"
    folder.mkdir()
    (folder / "a.png").write_bytes(b"x")
    (folder / "b.png").write_bytes(b"x")
    (folder / "metadata.jsonl").write_text(
        json.dumps({"file_name": "a.png", "text": None})
        + "\n"
        + json.dumps({"file_name": "b.png", "text": "cap b"})
        + "\n",
        encoding = "utf-8",
    )
    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    summary = next(d for d in r.json()["datasets"] if d["name"] == "null-caption")
    assert summary["caption_count"] == 1

    r2 = client.get("/api/train/diffusion/dataset/null-caption/images")
    assert r2.status_code == 200, r2.text
    by_name = {rec["filename"]: rec for rec in r2.json()["images"]}
    assert by_name["a.png"]["caption"] in (None, "")
    assert by_name["b.png"]["caption"] == "cap b"


def test_diffusion_info_tolerates_invalid_utf8_jsonl(client, dataset_roots):
    ds_root, _ = dataset_roots
    folder = ds_root / "bad-utf8-meta"
    folder.mkdir()
    (folder / "a.png").write_bytes(b"x")
    (folder / "metadata.jsonl").write_bytes(b"\xff\xfe not valid utf-8\n")
    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    summary = next(d for d in r.json()["datasets"] if d["name"] == "bad-utf8-meta")
    assert summary["caption_count"] == 0


def test_diffusion_info_tolerates_invalid_utf8_sidecar(client, dataset_roots):
    # read_text raises UnicodeDecodeError, not OSError.
    ds_root, _ = dataset_roots
    folder = ds_root / "bad-utf8-sidecar"
    folder.mkdir()
    (folder / "a.png").write_bytes(b"x")
    (folder / "a.txt").write_bytes(b"\xff\xfe not valid utf-8")
    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    summary = next(d for d in r.json()["datasets"] if d["name"] == "bad-utf8-sidecar")
    assert summary["caption_count"] == 0


def test_diffusion_dataset_mutations_blocked_while_training_active(client, dataset_roots):
    ds_root, _ = dataset_roots
    folder = ds_root / "locked"
    folder.mkdir()
    folder.joinpath("a.png").write_bytes(b"x")
    client._fake._running = True
    up = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "locked"},
        files = [("files", ("b.png", b"x", "image/png"))],
    )
    assert up.status_code == 409, up.text
    cap = client.put("/api/train/diffusion/dataset/locked/caption/a.png", json = {"caption": "hi"})
    assert cap.status_code == 409, cap.text
    dele = client.delete("/api/train/diffusion/dataset/locked/image/a.png")
    assert dele.status_code == 409, dele.text
    imp = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "anything"})
    assert imp.status_code == 409, imp.text


def test_diffusion_info_skips_symlinked_dataset_dir(client, dataset_roots):
    import os

    ds_root, _ = dataset_roots
    outside = ds_root.parent / "outside-images"
    outside.mkdir()
    (outside / "a.png").write_bytes(b"x")
    try:
        os.symlink(outside, ds_root / "linked")
    except OSError:
        pytest.skip("symlinks unavailable")
    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    assert "linked" not in [d["name"] for d in r.json()["datasets"]]


def test_diffusion_dataset_upload_rejects_traversal_names(client, dataset_roots):
    for bad in ("../evil", "a/b", ".hidden", " "):
        r = client.post(
            "/api/train/diffusion/dataset",
            data = {"name": bad},
            files = [("files", ("a.png", b"x", "image/png"))],
        )
        assert r.status_code == 400, f"{bad!r}: {r.status_code}"


def test_diffusion_dataset_upload_rejects_unsupported_files(client, dataset_roots):
    r = client.post(
        "/api/train/diffusion/dataset",
        data = {"name": "ok-name"},
        files = [("files", ("weights.exe", b"mz", "application/octet-stream"))],
    )
    assert r.status_code == 400
    assert "Unsupported file" in r.json()["detail"]


def test_free_gpu_for_diffusion_training_unloads_video(monkeypatch):
    # A resident Video pipeline is owned by the VIDEO arbiter, which the Images teardown does not free.
    import routes.training as tr
    from core.inference import gpu_arbiter

    class _Exp:
        current_checkpoint = None

        def is_export_active(self):
            return False

    class _Diff:
        is_loaded = False

        def unload(self):
            pass

    unloaded = {"video": False}

    class _Vid:
        def status(self):
            return {"loaded": True}

        def unload(self):
            unloaded["video"] = True

    released = []
    monkeypatch.setattr("core.export.get_export_backend", lambda: _Exp())
    monkeypatch.setattr(
        "core.inference.diffusion_engine_router.get_active_diffusion_engine", lambda: _Diff()
    )
    monkeypatch.setattr("core.inference.video.get_video_backend", lambda: _Vid())
    monkeypatch.setattr(gpu_arbiter, "release", lambda owner: released.append(owner))

    tr._free_gpu_for_diffusion_training()

    assert unloaded["video"] is True
    assert gpu_arbiter.VIDEO in released


def test_keepwarm_tracks_image_video_generation_paths():
    # The API-key start guard counts tracked requests, so image/video generation must be tracked.
    from core.inference.llama_keepwarm import _is_inference_path

    assert _is_inference_path("/api/inference/images/generate")
    assert _is_inference_path("/v1/images/generations")
    assert _is_inference_path("/api/inference/video/generate")
    assert not _is_inference_path("/api/inference/images/generate-progress")
    assert not _is_inference_path("/api/inference/video/generate-progress")
    assert not _is_inference_path("/api/inference/video/generate/cancel")
    # Tracking the cancel route would keep the model pinned past the run it cancelled.
    assert not _is_inference_path("/api/inference/images/generate/cancel")


def test_import_example_partial_failure_leaves_no_partial_dataset(
    client, dataset_roots, monkeypatch
):
    import routes.training as tr

    ds_root, _ = dataset_roots

    def _boom(entry, dest, cap):
        (dest / "img_0000.png").write_bytes(b"x")
        raise RuntimeError("transient copy error")

    monkeypatch.setattr(tr, "_materialize_hf_dataset", _boom)
    r = client.post("/api/train/diffusion/dataset/import-example", json = {"id": "dreambooth-dog"})
    assert r.status_code == 502
    folder = ds_root / "dreambooth-dog"
    assert not folder.exists() or not any(folder.iterdir())
    assert not any(p.name.startswith(".dreambooth-dog.import-") for p in ds_root.iterdir())


def test_diffusion_dataset_upload_over_cap_rolls_back_whole_batch(
    client, dataset_roots, monkeypatch
):
    import utils.upload_limits as ul

    monkeypatch.setattr(ul, "get_upload_limit_bytes", lambda: 100)
    ds_root, _ = dataset_roots
    files = [
        ("files", ("small.png", b"x" * 50, "image/png")),
        ("files", ("big.png", b"y" * 200, "image/png")),
    ]
    r = client.post("/api/train/diffusion/dataset", data = {"name": "rollback"}, files = files)
    assert r.status_code == 413, r.text
    folder = ds_root / "rollback"
    assert not (folder / "small.png").exists()
    assert not (folder / "big.png").exists()


def test_diffusion_dataset_upload_over_cap_preserves_existing_file(
    client, dataset_roots, monkeypatch
):
    import utils.upload_limits as ul

    monkeypatch.setattr(ul, "get_upload_limit_bytes", lambda: 100)
    ds_root, _ = dataset_roots
    folder = ds_root / "keep"
    folder.mkdir(parents = True)
    (folder / "existing.png").write_bytes(b"ORIGINAL")
    files = [
        ("files", ("existing.png", b"NEW", "image/png")),
        ("files", ("big.png", b"y" * 200, "image/png")),
    ]
    r = client.post("/api/train/diffusion/dataset", data = {"name": "keep"}, files = files)
    assert r.status_code == 413, r.text
    assert (folder / "existing.png").read_bytes() == b"ORIGINAL"
    assert not (folder / "big.png").exists()
    assert not list(folder.glob(".*.part"))


def test_route_start_refuses_non_sdxl_base_without_freeing_gpu(client, monkeypatch):
    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    r = client.post(
        "/api/train/diffusion/start", json = {**_BODY, "base_model": "unsloth/FLUX.1-dev-GGUF"}
    )
    assert r.status_code == 400
    assert "SDXL" in r.json()["detail"]
    assert freed == []
    assert client._fake.started_with is None


# diffusers>=0.39 only on py>=3.10; a py3.9 host tops out at 0.36.0 (or any older one).
_PIPELINE_TOO_NEW_FOR_0_36 = (
    ("krea/Krea-2-Raw", "Krea2Pipeline"),
    ("black-forest-labs/FLUX.2-klein-4B", "Flux2KleinPipeline"),
)
_PIPELINE_TOO_NEW_FOR_0_35 = _PIPELINE_TOO_NEW_FOR_0_36 + (
    ("Tongyi-MAI/Z-Image-Turbo", "ZImagePipeline"),
    ("black-forest-labs/FLUX.2-dev", "Flux2Pipeline"),
)


def _fake_diffusers(version, *classes):
    """A stand-in ``diffusers`` carrying exactly ``classes``, so the guard sees the attribute
    surface of that release rather than whatever is installed on the runner."""
    import types

    mod = types.ModuleType("diffusers")
    mod.__version__ = version
    for c in classes:
        setattr(mod, c, object)
    return mod


@pytest.mark.parametrize("base_model,pipeline_class", _PIPELINE_TOO_NEW_FOR_0_35)
def test_route_start_refuses_a_family_the_install_has_no_pipeline_for(
    client, monkeypatch, base_model, pipeline_class
):
    # Asserts ORDERING, not just an error: nothing freed and the slot never reserved.
    import sys

    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    # 0.35.2: the last release before any of these four classes existed.
    monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers("0.35.2"))

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": base_model})
    assert r.status_code == 400, r.text
    detail = r.json()["detail"]
    assert pipeline_class in detail
    assert "0.35.2" in detail
    assert freed == []
    assert client._fake.calls == []
    assert client._fake.started_with is None


@pytest.mark.parametrize("base_model,pipeline_class", _PIPELINE_TOO_NEW_FOR_0_36)
def test_route_start_refuses_a_too_new_pipeline_on_the_newest_py39_diffusers(
    client, monkeypatch, base_model, pipeline_class
):
    # 0.36.0 is the newest diffusers on py3.9: it has ZImage/Flux2 but not Krea2 or FLUX.2-klein.
    import sys

    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    monkeypatch.setitem(
        sys.modules,
        "diffusers",
        _fake_diffusers("0.36.0", "ZImagePipeline", "Flux2Pipeline", "StableDiffusionXLPipeline"),
    )

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": base_model})
    assert r.status_code == 400, r.text
    assert pipeline_class in r.json()["detail"]
    assert freed == []
    assert client._fake.calls == []
    assert client._fake.started_with is None


def test_route_start_refuses_a_missing_pipeline_named_by_model_family_too(client, monkeypatch):
    # The explicit model_family override skips detection, so a name-based test cannot reach it.
    import sys

    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers("0.36.0"))

    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": "my-org/some-private-mirror", "model_family": "krea-2"},
    )
    assert r.status_code == 400, r.text
    assert "Krea2Pipeline" in r.json()["detail"]
    assert freed == []
    assert client._fake.calls == []
    assert client._fake.started_with is None


def test_route_start_still_runs_when_the_install_does_have_the_pipeline(
    client, monkeypatch, dit_train_host
):
    import sys
    import urllib.request

    monkeypatch.setattr("utils.utils.auth_safe_open", lambda req, timeout = None: object())
    monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers("0.39.0", "Krea2Pipeline"))

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": "krea/Krea-2-Raw"})
    assert r.status_code == 200, r.text
    assert client._fake.started_with["base_model"] == "krea/Krea-2-Raw"


def test_route_start_refuses_a_diffusers_whose_lazy_submodule_cannot_import(
    client, monkeypatch, dit_train_host
):
    # The lazy attr probe can raise RuntimeError on a broken install; the spawned trainer would hit it too.
    # dit_train_host: the accelerator gate runs first and would answer "needs a GPU" on CI.
    import sys
    import types

    import routes.training as tr

    class _LazyModule(types.ModuleType):
        __version__ = "0.39.0"

        def __getattr__(self, name):
            raise RuntimeError(f"Failed to import diffusers.pipelines.{name.lower()}")

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    monkeypatch.setitem(sys.modules, "diffusers", _LazyModule("diffusers"))

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": "krea/Krea-2-Raw"})
    assert r.status_code == 400, r.text
    detail = r.json()["detail"]
    assert "Krea2Pipeline" in detail
    assert "Failed to import diffusers.pipelines" in detail
    assert freed == []
    assert client._fake.calls == []
    assert client._fake.started_with is None


def test_route_start_refuses_training_when_diffusers_is_absent(client, monkeypatch, dit_train_host):
    # The trainer spawns in the SAME env, so absent diffusers stays absent. dit_train_host: as above.
    import builtins
    import sys

    import routes.training as tr

    real_import = builtins.__import__

    def _no_diffusers(name, *args, **kwargs):
        if name == "diffusers" or name.startswith("diffusers."):
            raise ModuleNotFoundError("No module named 'diffusers'")
        return real_import(name, *args, **kwargs)

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    monkeypatch.delitem(sys.modules, "diffusers", raising = False)
    monkeypatch.setattr(builtins, "__import__", _no_diffusers)

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": "krea/Krea-2-Raw"})
    assert r.status_code == 400, r.text
    assert "No module named 'diffusers'" in r.json()["detail"]
    assert freed == []
    assert client._fake.calls == []


def test_inference_keeps_absorbing_an_unimportable_diffusers(monkeypatch):
    # The default must stay silent, or CPU/Apple hosts using the sd.cpp engine could not load anything.
    import builtins

    from core.inference.diffusion_families import assert_pipeline_class_available

    real_import = builtins.__import__

    def _no_diffusers(name, *args, **kwargs):
        if name == "diffusers" or name.startswith("diffusers."):
            raise ModuleNotFoundError("No module named 'diffusers'")
        return real_import(name, *args, **kwargs)

    monkeypatch.setattr(builtins, "__import__", _no_diffusers)
    assert assert_pipeline_class_available("Krea2Pipeline", "krea-2") is None


@pytest.mark.parametrize(
    "pipeline_class,minimum,needs_py310",
    [
        ("ZImagePipeline", "0.36.0", False),
        ("Flux2Pipeline", "0.36.0", False),
        ("Flux2KleinPipeline", "0.37.0", True),
        ("LTX2Pipeline", "0.37.0", True),
        ("Krea2Pipeline", "0.39.0", True),
        # Below Python 3.10 diffusers is unconstrained, so an ancient install can satisfy the pin.
        ("QwenImagePipeline", "0.35.0", False),
        ("FluxPipeline", "0.30.0", False),
    ],
)
def test_the_refusal_names_the_release_that_family_actually_needs(
    pipeline_class, minimum, needs_py310
):
    # First-export releases read from src/diffusers/__init__.py at upstream tags.
    from core.inference.diffusion_families import (
        _too_old_message,
        pipeline_class_requirement,
    )

    assert pipeline_class_requirement(pipeline_class) == (minimum, needs_py310)
    message = _too_old_message(pipeline_class, "some-family", "0.29.0")
    assert f"diffusers >= {minimum}" in message
    assert "0.29.0" in message
    assert ("Python >= 3.10" in message) is needs_py310


def test_an_unlisted_pipeline_names_no_version_it_cannot_stand_behind():
    # An unlisted class (shipped since before 0.29) gets no version or Python claim at all.
    from core.inference.diffusion_families import (
        _too_old_message,
        pipeline_class_requirement,
    )

    assert pipeline_class_requirement("StableDiffusionXLPipeline") == (None, False)
    message = _too_old_message("StableDiffusionXLPipeline", "sdxl", "0.29.0")
    assert "a newer diffusers" in message and "0.29.0" in message
    assert "0.39" not in message and "3.10" not in message


def test_a_dummy_pipeline_export_is_not_treated_as_importable():
    # Without a backend, diffusers exports DummyObject placeholders; hasattr is True, so look past the name.
    import sys
    import types

    import pytest

    from core.inference.diffusion_families import assert_pipeline_class_available

    dummy = types.new_class("Krea2Pipeline")
    dummy.__module__ = "diffusers.utils.dummy_torch_and_transformers_objects"
    dummy._backends = ["torch", "transformers"]

    stub = types.ModuleType("diffusers")
    stub.__version__ = "0.39.0"
    stub.Krea2Pipeline = dummy
    real = sys.modules.get("diffusers")
    sys.modules["diffusers"] = stub
    try:
        assert assert_pipeline_class_available("Krea2Pipeline", "krea-2") is None
        with pytest.raises(ValueError) as excinfo:
            assert_pipeline_class_available("Krea2Pipeline", "krea-2", strict = True)
    finally:
        if real is not None:
            sys.modules["diffusers"] = real
        else:
            del sys.modules["diffusers"]
    msg = str(excinfo.value)
    assert "placeholder" in msg
    assert "requires: torch, transformers" in msg
    # _backends is the full requirement list, not failed probes: never call a working torch missing.
    assert "missing: torch" not in msg and "pip install -U torch" not in msg


def test_route_start_refuses_ltx2_without_the_pipeline_before_freeing_gpu(client, monkeypatch):
    # LTX-2 is in the VIDEO registry, so it reaches the gate via a different resolve_trainable_family branch.
    import sys

    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    monkeypatch.setitem(sys.modules, "diffusers", _fake_diffusers("0.36.0"))

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": "Lightricks/LTX-2"})
    assert r.status_code == 400, r.text
    assert "LTX2Pipeline" in r.json()["detail"]
    assert freed == []
    assert client._fake.calls == []
    assert client._fake.started_with is None


def test_route_start_refuses_a_component_repo_before_freeing_gpu(client, monkeypatch):
    # unsloth/LTX-2-FP8 is component archives (no model_index.json) yet passes detection, trust and gating.
    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))

    r = client.post("/api/train/diffusion/start", json = {**_BODY, "base_model": "unsloth/LTX-2-FP8"})
    assert r.status_code == 400, r.text
    detail = r.json()["detail"]
    assert "model_index.json" in detail
    assert "Lightricks/LTX-2" in detail
    assert freed == []
    assert client._fake.calls == []
    assert client._fake.started_with is None


def test_route_start_refuses_non_bf16_gpu_without_freeing_gpu(client, monkeypatch):
    # A DiT precision the host cannot run must 400 before the GPU residents are freed
    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    monkeypatch.setattr(
        "core.training.diffusion_train_common.training_precision_preflight_error",
        lambda fam, prec: (
            "This trainer requires a bfloat16-capable GPU (Ampere or newer)."
            if fam != "sdxl"
            else None
        ),
    )
    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": "black-forest-labs/FLUX.1-dev"},
    )
    assert r.status_code == 400
    assert "bfloat16" in r.json()["detail"]
    assert freed == []
    assert client._fake.started_with is None

    # SDXL uses its own mixed_precision path, so the probe returns None for it.
    r2 = client.post("/api/train/diffusion/start", json = _BODY)
    assert r2.status_code == 200, r2.text


def test_apply_event_records_metric_history_and_perf():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event(
        {
            "type": "progress",
            "step": 1,
            "total_steps": 10,
            "loss": 0.5,
            "learning_rate": 1e-4,
            "samples_per_second": 3.2,
            "peak_memory_gb": 7.1,
        }
    )
    svc._apply_event(
        {"type": "progress", "step": 2, "total_steps": 10, "loss": 0.4, "learning_rate": 9e-5}
    )
    st = svc.status()
    assert st["metric_steps"] == [1, 2]
    assert st["metric_loss"] == [0.5, 0.4]
    assert st["metric_lr"] == [1e-4, 9e-5]
    assert st["samples_per_second"] == 3.2
    assert st["peak_memory_gb"] == 7.1


def test_apply_event_metric_history_skips_bad_points():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "progress", "step": 0, "loss": 0.9, "learning_rate": 1e-4})
    svc._apply_event({"type": "progress", "step": 1, "loss": None, "learning_rate": 1e-4})
    svc._apply_event({"type": "progress", "step": 2, "loss": float("nan"), "learning_rate": 1e-4})
    svc._apply_event({"type": "progress", "step": 3, "loss": 0.3, "learning_rate": None})
    st = svc.status()
    assert st["metric_steps"] == [3]
    assert st["metric_loss"] == [0.3]
    assert st["metric_lr"] == [None]  # lr None is retained so the series stays index-aligned


def test_metric_history_decimates_at_cap():
    from core.training import diffusion_training_service as svc_mod

    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "model_load_started"})
    n = svc_mod._METRIC_CAP + 50
    for i in range(1, n + 1):
        svc._apply_event({"type": "progress", "step": i, "loss": 1.0 / i, "learning_rate": 1e-4})
    st = svc.status()
    assert len(st["metric_steps"]) <= svc_mod._METRIC_CAP
    assert len(st["metric_steps"]) == len(st["metric_loss"]) == len(st["metric_lr"])
    assert st["metric_steps"] == sorted(st["metric_steps"])
    assert st["metric_steps"][-1] == n


def test_complete_event_records_family_and_catalog():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event(
        {
            "type": "complete",
            "output_dir": "/o",
            "lora_path": "/o/w.safetensors",
            "catalog_path": "/loras/w.safetensors",
            "family": "sdxl",
            "base_model": "b",
        }
    )
    st = svc.status()
    assert st["status"] == "completed"
    assert st["family"] == "sdxl"
    assert st["base_model"] == "b"
    assert st["catalog_path"] == "/loras/w.safetensors"


def test_status_route_returns_preparation_counts(client):
    # The response model filters unknown keys, so the counts reach the UI only if it declares them.
    client._fake.status_extra = {
        "in_model_load": True,
        "preparation": {"stage": "cache_latents", "done": 48, "total": 60},
    }
    body = client.get("/api/train/diffusion/status").json()
    assert body["preparation"] == {"stage": "cache_latents", "done": 48, "total": 60}
    client._fake.status_extra = {"preparation": None}
    assert client.get("/api/train/diffusion/status").json()["preparation"] is None


def test_status_route_nests_metric_history(client):
    client._fake.status_extra = {
        "metric_steps": [1, 2],
        "metric_loss": [0.5, 0.4],
        "metric_lr": [1e-4, 9e-5],
        "family": "sdxl",
        "samples_per_second": 2.0,
        "peak_memory_gb": 6.0,
    }
    r = client.get("/api/train/diffusion/status")
    assert r.status_code == 200, r.text
    body = r.json()
    assert body["metric_history"]["steps"] == [1, 2]
    assert body["metric_history"]["loss"] == [0.5, 0.4]
    assert body["family"] == "sdxl"
    assert body["samples_per_second"] == 2.0


def test_info_lists_trainable_families(client):
    r = client.get("/api/train/diffusion/info")
    assert r.status_code == 200, r.text
    families = {f["name"]: f for f in r.json()["families"]}
    for fam in ("sdxl", "flux.1", "qwen-image", "z-image"):
        assert fam in families
    assert families["flux.1"]["default_base"] == "black-forest-labs/FLUX.1-dev"
    assert families["z-image"]["defaults"]["resolution"] == 768


def test_start_gated_base_without_access_is_400_and_keeps_gpu(client, monkeypatch, dit_train_host):
    import urllib.error
    import urllib.request

    import routes.training as tr

    freed: list[int] = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))

    def _fake_urlopen(req, timeout = None):
        raise urllib.error.HTTPError(req.full_url, 403, "Forbidden", {}, None)

    monkeypatch.setattr("utils.utils.auth_safe_open", _fake_urlopen)
    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": "black-forest-labs/FLUX.1-dev"},
    )
    assert r.status_code == 400
    assert "gated" in r.json()["detail"].lower()
    assert freed == []
    assert client._fake.started_with is None


def test_route_start_refuses_a_dit_family_without_a_gpu_before_freeing(client, monkeypatch):
    # nf4 is not a CPU fallback: the 4-bit base load needs CUDA/XPU/MPS.
    import torch

    import routes.training as tr

    freed = []
    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: freed.append(1))
    monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
    monkeypatch.setattr(torch.xpu, "is_available", lambda: False)
    monkeypatch.setattr(torch.mps, "is_available", lambda: False)

    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": "black-forest-labs/FLUX.1-dev"},
    )
    assert r.status_code == 400
    assert "GPU" in r.json()["detail"]
    assert freed == []
    assert client._fake.started_with is None

    # SDXL trains fp32 on CPU (its documented fallback), so it is not gated.
    r2 = client.post("/api/train/diffusion/start", json = _BODY)
    assert r2.status_code == 200, r2.text


def test_start_ungated_base_preflight_is_noop(client, monkeypatch):
    import urllib.request

    import routes.training as tr

    monkeypatch.setattr(tr, "_free_gpu_for_diffusion_training", lambda: None)
    # Pin the precision preflight so a GPU-less host does not 400 for another reason.
    monkeypatch.setattr(
        "core.training.diffusion_train_common.training_precision_preflight_error",
        lambda fam, prec: None,
    )
    monkeypatch.setattr("utils.utils.auth_safe_open", lambda req, timeout = None: object())
    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "base_model": "black-forest-labs/FLUX.1-dev"},
    )
    assert r.status_code == 200, r.text
    assert client._fake.started_with["base_model"] == "black-forest-labs/FLUX.1-dev"


def test_run_record_persisted_on_complete(_isolated_runs_dir):
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    job_id = svc.start({**_CFG, "model_family": "z-image", "hf_token": "SECRET"})
    _wait_status(svc, "completed")
    rec = _wait_record(_isolated_runs_dir, job_id)
    assert rec["job_id"] == job_id
    assert rec["status"] == "completed"
    assert rec["saved"] is True
    assert rec["adapter"] == "out"
    assert rec["family"] == "z-image"
    assert rec["step"] == 2 and rec["total_steps"] == 2
    assert rec["avg_loss"] == 0.45
    assert rec["metric_history"]["steps"] == [1, 2]
    assert rec["metric_history"]["loss"] == [0.5, 0.4]
    assert "hf_token" not in rec["config"]
    assert rec["config"]["model_family"] == "z-image"


def test_run_record_no_save_stop_marks_unsaved(_isolated_runs_dir):
    def _cancel_target(*, event_queue, stop_queue, config):
        event_queue.put({"type": "model_load_completed"})
        stop_queue.get(timeout = 5.0)
        event_queue.put(
            {"type": "complete", "output_dir": None, "lora_path": None, "stopped": True}
        )

    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _cancel_target)
    job_id = svc.start(dict(_CFG))
    _wait_status(svc, "running")
    svc.stop(save = False)
    _wait_status(svc, "stopped")
    rec = _wait_record(_isolated_runs_dir, job_id)
    assert rec["status"] == "stopped"
    assert rec["saved"] is False and rec["lora_path"] is None


def test_runs_endpoints_list_and_detail(client, _isolated_runs_dir):
    import json
    import os

    a = {
        "job_id": "a" * 32,
        "status": "completed",
        "adapter": "first",
        "saved": True,
        "step": 10,
        "total_steps": 10,
        "avg_loss": 0.4,
        "config": {"train_steps": 10},
        "metric_history": {"steps": [1], "loss": [0.4], "lr": [1e-4], "grad_norm": [0.2]},
    }
    b = {
        "job_id": "b" * 32,
        "status": "stopped",
        "adapter": "second",
        "saved": False,
        "step": 3,
        "total_steps": 10,
        "avg_loss": 0.6,
        "config": {"train_steps": 10},
        "metric_history": {"steps": [1], "loss": [0.6], "lr": [1e-4], "grad_norm": [0.3]},
    }
    pa = _isolated_runs_dir / f"{a['job_id']}.json"
    pb = _isolated_runs_dir / f"{b['job_id']}.json"
    pa.write_text(json.dumps(a))
    pb.write_text(json.dumps(b))
    os.utime(pa, (1000, 1000))
    os.utime(pb, (2000, 2000))

    r = client.get("/api/train/diffusion/runs")
    assert r.status_code == 200, r.text
    runs = r.json()["runs"]
    assert [x["adapter"] for x in runs] == ["second", "first"]
    assert "config" not in runs[0] and "metric_history" not in runs[0]

    r = client.get(f"/api/train/diffusion/runs/{a['job_id']}")
    assert r.status_code == 200, r.text
    detail = r.json()
    assert detail["adapter"] == "first"
    assert detail["metric_history"]["grad_norm"] == [0.2]
    assert detail["config"] == {"train_steps": 10}

    assert client.get(f"/api/train/diffusion/runs/{'c' * 32}").status_code == 404
    assert client.get("/api/train/diffusion/runs/not-a-job-id").status_code == 404


def test_list_diffusion_runs_skips_wrong_shape_records(_isolated_runs_dir):
    import json

    from core.training.diffusion_training_service import list_diffusion_runs

    good = {"job_id": "a" * 32, "status": "completed", "adapter": "good", "saved": True}
    (_isolated_runs_dir / "good.json").write_text(json.dumps(good))
    (_isolated_runs_dir / "not_a_dict.json").write_text(json.dumps([1, 2, 3]))
    (_isolated_runs_dir / "no_ids.json").write_text(json.dumps({"adapter": "orphan"}))
    (_isolated_runs_dir / "bad_types.json").write_text(
        json.dumps({"job_id": 123, "status": None, "adapter": "typed"})
    )

    runs = list_diffusion_runs()
    adapters = [r.get("adapter") for r in runs]
    assert adapters == ["good"]


def test_runs_route_tolerates_bad_field_record(client, _isolated_runs_dir):
    # Passes the shape check but a wrong-typed field raises ValidationError, so the route catches per record.
    import json

    good = {"job_id": "a" * 32, "status": "completed", "adapter": "good", "saved": True}
    bad = {
        "job_id": "b" * 32,
        "status": "completed",
        "adapter": "bad",
        "avg_loss": "not-a-number",
    }
    (_isolated_runs_dir / f"{good['job_id']}.json").write_text(json.dumps(good))
    (_isolated_runs_dir / f"{bad['job_id']}.json").write_text(json.dumps(bad))

    r = client.get("/api/train/diffusion/runs")
    assert r.status_code == 200, r.text
    adapters = [x["adapter"] for x in r.json()["runs"]]
    assert adapters == ["good"]


def test_run_detail_route_non_object_record_is_404(client, _isolated_runs_dir):
    # A non-object record raises TypeError, not ValidationError, in DiffusionTrainingRunDetail(**rec).
    import json

    job_id = "a" * 32
    (_isolated_runs_dir / f"{job_id}.json").write_text(json.dumps([]))

    r = client.get(f"/api/train/diffusion/runs/{job_id}")
    assert r.status_code == 404, r.text


def test_request_model_rejects_non_finite_learning_rate():
    # 1e309 parses as inf, which satisfies a gt-only bound.
    from pydantic import ValidationError

    from models.training import DiffusionTrainingStartRequest as R

    base = dict(base_model = "b", data_dir = "d", output_dir = "o")
    assert R(**base, learning_rate = 1e-4).learning_rate == 1e-4
    for bad in (float("inf"), 1e309, float("nan"), 1.0, 5.0, 0.0):
        with pytest.raises(ValidationError):
            R(**base, learning_rate = bad)


def test_request_model_rejects_non_finite_clipping_and_snr():
    # inf passed max_grad_norm's ge=0 (no clipping) and snr_gamma's gt=0 (all min-SNR weights 1.0).
    from pydantic import ValidationError

    from models.training import DiffusionTrainingStartRequest as R

    base = dict(base_model = "b", data_dir = "d", output_dir = "o")
    assert R(**base, max_grad_norm = 0.0).max_grad_norm == 0.0
    assert R(**base, max_grad_norm = 1.0).max_grad_norm == 1.0
    assert R(**base, snr_gamma = 5.0).snr_gamma == 5.0
    assert R(**base, snr_gamma = None).snr_gamma is None
    for bad in (float("inf"), float("-inf"), 1e309, float("nan")):
        with pytest.raises(ValidationError):
            R(**base, max_grad_norm = bad)
        with pytest.raises(ValidationError):
            R(**base, snr_gamma = bad)


def test_start_route_never_starts_a_run_with_a_non_finite_knob(client):
    # FastAPI's stdlib json accepts Infinity, so the schema is the only guard; 422 cannot serialise inf.
    from fastapi.testclient import TestClient

    strict = TestClient(client._app, raise_server_exceptions = False)
    for raw in ('"max_grad_norm": 1e309', '"max_grad_norm": Infinity', '"snr_gamma": 1e309'):
        body = json.dumps(_BODY)[:-1] + ", " + raw + "}"
        r = strict.post(
            "/api/train/diffusion/start",
            content = body,
            headers = {"content-type": "application/json"},
        )
        assert r.status_code >= 400, (raw, r.text)
        assert client._fake.started_with is None, raw
    # Positive control, so the assertions above cannot pass vacuously.
    assert strict.post("/api/train/diffusion/start", json = _BODY).status_code == 200
    assert client._fake.started_with is not None


def test_dataset_mutation_and_reserve_refuse_each_other():
    """Dataset mutation and reserve refuse each other under one lock, closing the is_active gap."""
    from core.training.diffusion_training_service import (
        DatasetMutationInFlight,
        DiffusionTrainingService,
        TrainingActiveError,
    )

    svc = DiffusionTrainingService()
    with svc.dataset_mutation():
        with pytest.raises(DatasetMutationInFlight):
            svc.reserve()
    svc.reserve()
    with pytest.raises(TrainingActiveError):
        with svc.dataset_mutation():
            pass
    svc.unreserve()
    with svc.dataset_mutation():
        pass


def test_dataset_mutation_releases_on_failure():
    from core.training.diffusion_training_service import DiffusionTrainingService

    svc = DiffusionTrainingService()
    with pytest.raises(ValueError):
        with svc.dataset_mutation():
            raise ValueError("boom")
    svc.reserve()


def test_diffusion_seed_is_bounded_to_torch_range():
    """torch.manual_seed unpacks int64/uint64, so a wider value raised inside the trainer -- after
    the route had already evicted the resident image/video/chat models."""
    from pydantic import ValidationError

    from core.training.diffusion_lora_trainer import _config_from_dict
    from models.training import DiffusionTrainingStartRequest

    base = {
        "base_model": "unsloth/sdxl-turbo",
        "data_dir": "/tmp/x",
        "output_dir": "/tmp/out",
        "seed": 2**64,
    }
    with pytest.raises(ValueError, match = "seed"):
        _config_from_dict(base).normalized()
    request = {k: v for k, v in base.items() if k != "seed"}
    for bad in (2**64, -(2**63) - 1):
        with pytest.raises(ValidationError):
            DiffusionTrainingStartRequest(**request, seed = bad)
    for good in (2**64 - 1, -(2**63)):
        assert DiffusionTrainingStartRequest(**request, seed = good).seed == good


def test_gpu_load_admission_and_reserve_exclude_each_other():
    # Load guards check is_active() before taking the arbiter; admission closes that race from both sides.
    from core.training.diffusion_training_service import TrainingActiveError

    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)

    with svc.gpu_load_admission():
        with pytest.raises(RuntimeError, match = "loaded onto the GPU"):
            svc.reserve()
    svc.reserve()
    try:
        with pytest.raises(TrainingActiveError, match = "Diffusion training is running"):
            with svc.gpu_load_admission():
                pass
    finally:
        svc.unreserve()

    # Admissions are counted, not boolean: the first exit must not reopen while a second load registers.
    with svc.gpu_load_admission():
        with svc.gpu_load_admission():
            pass
        with pytest.raises(RuntimeError, match = "loaded onto the GPU"):
            svc.reserve()
    svc.reserve()
    svc.unreserve()


def test_route_start_carries_and_contains_the_conditioning_cache_dir(client, dit_train_host):
    # cond_cache_dir must be contained like output_dir; sent against a DiT family.
    from pathlib import Path

    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "model_family": "z-image", "cond_cache_dir": "cond-cache"},
    )
    assert r.status_code == 200, r.text
    resolved = client._fake.started_with["cond_cache_dir"]
    assert Path(resolved).is_absolute()
    assert Path(resolved).name == "cond-cache"

    r = client.post("/api/train/diffusion/start", json = _BODY)
    assert r.status_code == 200, r.text
    assert client._fake.started_with["cond_cache_dir"] is None
    r = client.post("/api/train/diffusion/start", json = {**_BODY, "cond_cache_dir": "   "})
    assert r.status_code == 200, r.text
    assert client._fake.started_with["cond_cache_dir"] is None


def test_route_refuses_an_output_dir_that_is_the_outputs_root(client):
    # "." / "outputs" / "./." resolve to the outputs ROOT, where is_dir()-filtered listings miss the adapter.
    from pathlib import Path

    for name in (".", "./", "./.", "outputs", "outputs/outputs", " . "):
        r = client.post("/api/train/diffusion/start", json = {**_BODY, "output_dir": name})
        assert r.status_code == 400, f"{name!r} -> {r.status_code} {r.text}"
        assert "not a run inside it" in r.json()["detail"]
    r = client.post("/api/train/diffusion/start", json = {**_BODY, "output_dir": "outputs/run-1"})
    assert r.status_code == 200, r.text
    assert Path(client._fake.started_with["output_dir"]).name == "run-1"


def test_route_treats_a_root_cond_cache_dir_as_the_in_memory_cache(client, dit_train_host):
    # Same collapse on the cache side, but it falls back to off rather than the root.
    for name in (".", "./.", "outputs", " . "):
        r = client.post(
            "/api/train/diffusion/start",
            json = {**_BODY, "model_family": "z-image", "cond_cache_dir": name},
        )
        assert r.status_code == 200, r.text
        assert client._fake.started_with["cond_cache_dir"] is None, name


def test_route_rejects_cond_cache_dir_for_sdxl(client, dit_train_host):
    # Only the DiT trainer reads cond_cache_dir, so refuse it for SDXL rather than ignore it.
    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "model_family": "sdxl", "cond_cache_dir": "cond-cache"},
    )
    assert r.status_code == 400, r.text
    detail = r.json()["detail"]
    assert "cond_cache_dir" in detail and "sdxl" in detail
    assert "z-image" in detail

    # The check is on the RESOLVED family, so a detected SDXL base is refused too.
    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "cond_cache_dir": "cond-cache"},
    )
    assert r.status_code == 400, r.text
    assert "cond_cache_dir" in r.json()["detail"]

    r = client.post(
        "/api/train/diffusion/start",
        json = {**_BODY, "model_family": "z-image", "cond_cache_dir": "cond-cache"},
    )
    assert r.status_code == 200, r.text
    from pathlib import Path

    assert Path(client._fake.started_with["cond_cache_dir"]).is_absolute()
    r = client.post("/api/train/diffusion/start", json = {**_BODY, "model_family": "sdxl"})
    assert r.status_code == 200, r.text
    assert client._fake.started_with["cond_cache_dir"] is None


def test_service_reserve_refuses_while_the_llm_trainer_holds_the_gpu(monkeypatch):
    # The route's check runs before network preflights, so reserve() re-tests the LLM backend under its lock.
    import types

    import core.training.diffusion_training_service as dts
    from core.training.diffusion_training_service import DiffusionTrainingService

    svc = DiffusionTrainingService()
    monkeypatch.setattr(dts, "_llm_training_active", lambda: True)
    with pytest.raises(RuntimeError, match = "LLM training job is already running"):
        svc.reserve()
    assert svc.is_active() is False

    monkeypatch.setattr(dts, "_llm_training_active", lambda: False)
    svc.reserve()
    assert svc.is_active() is True


def test_llm_active_probe_fails_open(monkeypatch):
    import sys

    import core.training.diffusion_training_service as dts

    monkeypatch.setitem(sys.modules, "core.training", None)
    assert dts._llm_training_active() is False


def test_llm_start_holds_the_diffusion_admission_across_its_spawn(monkeypatch):
    from core.training.diffusion_training_service import DiffusionTrainingService
    import routes.training as tr

    svc = DiffusionTrainingService()
    monkeypatch.setattr(
        "core.training.diffusion_training_service.get_diffusion_training_service", lambda: svc
    )
    with tr._diffusion_gpu_admission():
        with pytest.raises(RuntimeError, match = "being loaded onto the GPU"):
            svc.reserve()
    svc.reserve()


def test_llm_start_admission_refuses_when_diffusion_is_already_reserved(monkeypatch):
    from core.training.diffusion_training_service import DiffusionTrainingService
    import routes.training as tr

    svc = DiffusionTrainingService()
    monkeypatch.setattr(
        "core.training.diffusion_training_service.get_diffusion_training_service", lambda: svc
    )
    svc.reserve()
    with pytest.raises(tr._DiffusionStartInFlight):
        with tr._diffusion_gpu_admission():
            pass


def test_llm_start_admission_fails_open_without_a_diffusion_stack(monkeypatch):
    import sys

    import routes.training as tr

    monkeypatch.setitem(sys.modules, "core.training.diffusion_training_service", None)
    with tr._diffusion_gpu_admission():
        pass


def test_status_route_preserves_model_download_bytes(client):
    progress = {
        "bytes_done": 4,
        "bytes_total": 10,
        "label": "model.safetensors",
        "scope": "active_files",
    }
    client._fake.status_extra = {"in_model_load": True, "download_progress": progress}
    assert client.get("/api/train/diffusion/status").json()["download_progress"] == progress


def test_model_download_progress_clears_before_preparation():
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "model_load_started"})
    event = {
        "type": "model_download_progress",
        "download_progress": {"bytes_done": 4, "bytes_total": 10},
    }
    svc._apply_event(event)
    assert svc.status()["download_progress"]["bytes_done"] == 4
    svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 3, "total": 10})
    svc._apply_event(event)
    assert svc.status()["download_progress"] is None
    assert svc.status()["preparation"]["done"] == 3
    svc._apply_event({"type": "model_load_started"})
    svc._apply_event(event)
    assert svc.status()["preparation"] is None
    assert svc.status()["download_progress"]["bytes_done"] == 4


@pytest.mark.parametrize(
    "progress", [None, "invalid", {"bytes_done": float("nan"), "bytes_total": 10}]
)
def test_model_download_progress_rejects_unknown_byte_counts(progress):
    svc = DiffusionTrainingService(ctx = _FakeCtx(), target = _happy_target)
    svc._apply_event({"type": "model_load_started"})
    svc._apply_event({"type": "model_download_progress", "download_progress": progress})
    assert svc.status()["download_progress"] is None


def test_model_loading_stage_follows_phase_boundaries():
    svc = DiffusionTrainingService()
    assert svc.status()["model_load_stage"] is None
    svc._apply_event({"type": "model_load_started", "stage": "conditioning"})
    assert svc.status()["model_load_stage"] == "conditioning"
    svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 5, "total": 5})
    assert svc.status()["model_load_stage"] is None
    svc._apply_event({"type": "model_load_started", "stage": "quantizing"})
    assert svc.status()["model_load_stage"] == "quantizing"
    assert svc.status()["preparation"] is None
    svc._apply_event({"type": "model_load_completed"})
    assert svc.status()["model_load_stage"] is None
    assert svc.status()["in_model_load"] is False
    svc._apply_event({"type": "model_load_started", "stage": "transformer"})
    assert svc.status()["model_load_stage"] == "transformer"
    # Trainers that do not emit a stage retain the generic loading fallback.
    svc._apply_event({"type": "model_load_started"})
    assert svc.status()["model_load_stage"] is None


def test_route_preserves_measured_loading_and_preparation_fields(client):
    svc = DiffusionTrainingService()
    client._fake.status = svc.status
    svc._apply_event({"type": "model_load_started", "stage": "quantizing"})
    svc._apply_event(
        {
            "type": "model_load_progress",
            "done": 12,
            "total": 40,
            "unit": "tensors",
            "label": "FluxTransformer2DModel",
        }
    )
    response = client.get("/api/train/diffusion/status")
    assert response.status_code == 200
    status = response.json()
    assert status["model_load_stage"] == "quantizing"
    assert status["model_load_progress"] == {
        "done": 12,
        "total": 40,
        "unit": "tensors",
        "label": "FluxTransformer2DModel",
    }
    svc._apply_event({"type": "preparing", "stage": "cache_latents", "done": 2, "total": 5})
    status = client.get("/api/train/diffusion/status").json()
    assert status["model_load_progress"] is None
    assert status["preparation"] == {"stage": "cache_latents", "done": 2, "total": 5}
    svc._apply_event({"type": "model_load_started", "stage": "transformer"})
    status = client.get("/api/train/diffusion/status").json()
    assert status["preparation"] is None
    assert status["model_load_progress"] is None
    svc._apply_event({"type": "model_load_completed"})
    assert client.get("/api/train/diffusion/status").json()["model_load_stage"] is None


@pytest.mark.parametrize(
    "done,total,unit",
    [
        (-1, 2, "tensors"),
        (3, 2, "tensors"),
        (0, 0, "tensors"),
        (1, 2, "seconds"),
        (float("nan"), 2, "tensors"),
    ],
)
def test_invalid_model_load_counts_are_ignored(done, total, unit):
    svc = DiffusionTrainingService()
    svc._apply_event({"type": "model_load_started"})
    svc._apply_event({"type": "model_load_progress", "done": done, "total": total, "unit": unit})
    assert svc.status()["model_load_progress"] is None


@pytest.mark.parametrize(
    "terminal", ["complete", "error", "progress", "preparing", "model_load_completed"]
)
def test_measured_loading_clears_when_leaving_load_phase(terminal):
    svc = DiffusionTrainingService()
    svc._apply_event({"type": "model_load_started", "stage": "quantizing"})
    event = {"type": "model_load_progress", "done": 1, "total": 2, "unit": "tensors"}
    svc._apply_event(event)
    assert svc.status()["model_load_progress"]["done"] == 1
    svc._apply_event({"type": terminal})
    assert svc.status()["model_load_progress"] is None
    assert svc.status()["model_load_stage"] is None
    svc._apply_event(event)
    assert svc.status()["model_load_progress"] is None