Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Diffusion trainer speed defaults: optimizer factory, nf4 compile gate, checkpointing modes."""
from __future__ import annotations
import importlib.util
import json
import random
import sys
from pathlib import Path
from types import ModuleType
import pytest
import torch
import core.training.diffusion_dit_trainer as dit
import core.training.diffusion_lora_trainer as sdxl
import core.training.diffusion_train_common as dtc
from core.training import diffusion_checkpoint as dc
from core.training.diffusion_train_common import (
DiffusionLoraConfig,
PermutationBatchSampler,
make_lora_optimizer,
recorded_optimizer_class,
restore_resume_state,
write_resume_checkpoint,
)
BNB_KEY = "bitsandbytes.optim.adamw.AdamW8bit"
class _Fake8bit(torch.optim.AdamW):
"""Stands in for bnb.optim.AdamW8bit."""
@pytest.fixture
def fake_bnb(monkeypatch):
mod = ModuleType("bitsandbytes")
optim = ModuleType("bitsandbytes.optim")
optim.AdamW8bit = _Fake8bit
mod.optim = optim
monkeypatch.setitem(sys.modules, "bitsandbytes", mod)
monkeypatch.setitem(sys.modules, "bitsandbytes.optim", optim)
monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: True)
return mod
def _params():
p = torch.nn.Parameter(torch.zeros(2, 2))
p.grad = torch.ones(2, 2)
return [p]
@pytest.mark.parametrize("factory", [dit._make_optimizer, sdxl._make_lora_optimizer])
def test_fresh_run_builds_torch_adamw_even_with_bitsandbytes(fake_bnb, factory):
opt = factory(_params(), 1e-4)
assert type(opt) is torch.optim.AdamW
opt.step()
@pytest.mark.parametrize("factory", [dit._make_optimizer, sdxl._make_lora_optimizer])
def test_resuming_an_8bit_bundle_rebuilds_adamw8bit(fake_bnb, factory):
opt = factory(_params(), 1e-4, BNB_KEY)
assert isinstance(opt, _Fake8bit)
assert type(factory(_params(), 1e-4, "torch.optim.adamw.AdamW")) is torch.optim.AdamW
assert type(factory(_params(), 1e-4, None)) is torch.optim.AdamW
def test_8bit_resume_on_xpu_falls_back_to_torch_adamw(fake_bnb, monkeypatch):
monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: False)
opt = make_lora_optimizer(_params(), 1e-4, BNB_KEY)
assert type(opt) is torch.optim.AdamW
opt.step()
def test_fp32_override_still_forces_plain_adamw(fake_bnb, monkeypatch):
monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")
for resume in (None, BNB_KEY):
opt = make_lora_optimizer(_params(), 1e-4, resume)
assert type(opt) is torch.optim.AdamW
assert not opt.defaults.get("fused")
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")
def test_cuda_default_is_fused(monkeypatch):
monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
p = torch.nn.Parameter(torch.zeros(4, 4, device = "cuda"))
opt = make_lora_optimizer([p], 1e-4)
assert type(opt) is torch.optim.AdamW and opt.defaults.get("fused") is True
def _identity() -> dc.CheckpointIdentity:
return dc.CheckpointIdentity(
family = "sdxl",
base_model = "stabilityai/sdxl-turbo",
lora_target_modules = ("to_k", "to_q", "to_v", "to_out.0"),
lora_rank = 16,
lora_alpha = 16,
precision = "bf16",
base_precision = "nf4",
resolution = 1024,
base_revision = "rev-deadbeef",
dataset_fingerprint = "ds-3-cafe",
)
def _cfg(out: Path, **kw) -> DiffusionLoraConfig:
return DiffusionLoraConfig(
base_model = "stabilityai/sdxl-turbo",
data_dir = "unused",
output_dir = str(out),
train_steps = 500,
**kw,
)
@pytest.fixture
def run_dir():
from utils.paths import outputs_root
d = outputs_root() / "speed-run"
d.mkdir(parents = True, exist_ok = True)
return d
def _write(
out: Path,
model,
optimizer,
step: int = 3,
):
sched = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda s: 1.0)
written, error = write_resume_checkpoint(
_cfg(out),
step = step,
model = model,
optimizer = optimizer,
lr_scheduler = sched,
identity = _identity(),
sampler = PermutationBatchSampler(7, random.Random(0)),
rng_streams = {"loop": random.Random(0), "variant": random.Random(1)},
)
assert error is None and written
return sched
def test_recorded_optimizer_class_reads_the_bundle_being_resumed(run_dir, monkeypatch):
# The resume preflight refuses an 8-bit bundle on a host without bitsandbytes, so pretend it is installed.
real_find_spec = importlib.util.find_spec
monkeypatch.setattr(
importlib.util,
"find_spec",
lambda name, *a, **k: object() if name == "bitsandbytes" else real_find_spec(name, *a, **k),
)
monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
tmp_path = run_dir
model = torch.nn.Linear(4, 4, bias = False)
_write(tmp_path, model, torch.optim.AdamW(model.parameters(), lr = 1e-3))
cfg = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "checkpoint-3"))
assert recorded_optimizer_class(cfg, _identity()) == "torch.optim.adamw.AdamW"
manifest_path = tmp_path / "checkpoint-3" / dc.TRAINER_STATE_FILENAME
manifest = json.loads(manifest_path.read_text(encoding = "utf-8"))
manifest["optimizer_class"] = BNB_KEY
manifest_path.write_text(json.dumps(manifest), encoding = "utf-8")
assert recorded_optimizer_class(cfg, _identity()) == BNB_KEY
assert recorded_optimizer_class(_cfg(tmp_path), _identity()) is None
bogus = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "nope"))
assert recorded_optimizer_class(bogus, _identity()) is None
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")
def test_a_fused_bundle_resumed_under_the_fp32_override_stays_non_fused(run_dir, monkeypatch):
# load_state_dict restores the saved fused flag; the reference-optimizer override must survive it.
model = torch.nn.Linear(4, 4, bias = False).cuda()
fused = torch.optim.AdamW(model.parameters(), lr = 1e-3, fused = True)
model.weight.grad = torch.ones_like(model.weight)
fused.step()
_write(run_dir, model, fused)
monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")
live = torch.nn.Linear(4, 4, bias = False).cuda()
opt = make_lora_optimizer(list(live.parameters()), 1e-3)
assert not opt.param_groups[0].get("fused")
restore_resume_state(
_cfg(run_dir, resume_from_checkpoint = str(run_dir / "checkpoint-3")),
model = live,
optimizer = opt,
lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),
identity = _identity(),
)
assert not opt.param_groups[0].get("fused")
assert opt.state[live.weight]["exp_avg"].abs().sum() > 0
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")
def test_a_non_fused_bundle_resumes_into_fused_adamw(run_dir, monkeypatch):
# A plain AdamW bundle keeps its step on the CPU; the fused kernel needs it on the param's device.
torch.manual_seed(0)
model = torch.nn.Linear(4, 4, bias = False).cuda()
plain = torch.optim.AdamW(model.parameters(), lr = 1e-3)
model.weight.grad = torch.ones_like(model.weight)
plain.step()
_write(run_dir, model, plain)
monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
live = torch.nn.Linear(4, 4, bias = False).cuda()
opt = make_lora_optimizer(list(live.parameters()), 1e-3)
assert opt.param_groups[0].get("fused")
restore_resume_state(
_cfg(run_dir, resume_from_checkpoint = str(run_dir / "checkpoint-3")),
model = live,
optimizer = opt,
lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),
identity = _identity(),
)
assert opt.param_groups[0].get("fused")
live.weight.grad = torch.ones_like(live.weight)
opt.step()
assert opt.state[live.weight]["step"].item() == 2
@pytest.mark.skipif(not torch.cuda.is_available(), reason = "bitsandbytes AdamW8bit needs CUDA")
def test_old_8bit_bundle_resumes_and_continues_identically(run_dir, monkeypatch):
tmp_path = run_dir
bnb = pytest.importorskip("bitsandbytes")
if not hasattr(getattr(bnb, "optim", None), "AdamW8bit"):
pytest.skip("bitsandbytes is stubbed out in this environment")
monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: True)
def _model():
torch.manual_seed(0)
return torch.nn.Linear(128, 64, bias = False).cuda()
def _step(model, opt, i):
opt.zero_grad(set_to_none = True)
model.weight.grad = torch.full_like(model.weight, 0.01 * (i + 1))
opt.step()
ref = _model()
ref_opt = bnb.optim.AdamW8bit(ref.parameters(), lr = 1e-3)
for i in range(3):
_step(ref, ref_opt, i)
_write(tmp_path, ref, ref_opt)
cfg = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "checkpoint-3"))
live = _model()
opt = make_lora_optimizer(
list(live.parameters()), 1e-3, recorded_optimizer_class(cfg, _identity())
)
assert dc.optimizer_key(opt) == BNB_KEY
restored = restore_resume_state(
cfg,
model = live,
optimizer = opt,
lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),
identity = _identity(),
)
assert restored.step == 3
for i in range(3, 6):
_step(ref, ref_opt, i)
_step(live, opt, i)
assert torch.equal(live.weight, ref.weight)
def _ccfg(mode = "auto"):
return DiffusionLoraConfig(
base_model = "Tongyi-MAI/Z-Image",
data_dir = "unused",
output_dir = "unused",
compile_transformer = mode,
)
def test_auto_compiles_nf4_only_when_the_versions_are_supported(monkeypatch):
monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
assert dit._should_compile(_ccfg(), True, "cuda", "nf4") is True
monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: False)
assert dit._should_compile(_ccfg(), True, "cuda", "nf4") is False
assert dit._should_compile(_ccfg("on"), True, "cuda", "nf4") is True
assert dit._should_compile(_ccfg("off"), True, "cuda", "nf4") is False
monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
assert dit._should_compile(_ccfg(), True, "xpu", "nf4") is False
assert dit._should_compile(_ccfg(), True, "cpu", "nf4") is False
def test_auto_keeps_a_short_nf4_run_eager(monkeypatch):
from dataclasses import replace
monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
short = replace(_ccfg(), train_steps = dit.NF4_COMPILE_MIN_STEPS - 1)
assert dit._should_compile(short, True, "cuda", "nf4") is False
assert (
dit._should_compile(replace(short, compile_transformer = "on"), True, "cuda", "nf4") is True
)
assert dit._should_compile(short, False, "cuda", "bf16") is True
at_floor = replace(_ccfg(), train_steps = dit.NF4_COMPILE_MIN_STEPS)
assert dit._should_compile(at_floor, True, "cuda", "nf4") is True
@pytest.mark.parametrize(
"torch_version, bnb_version, rocm, expected",
[
("2.13.0+cu130", "0.50.2", False, True),
("2.10.0+cu128", "0.46.1", False, True),
("2.9.1+cu128", "0.50.2", False, False),
("2.13.0", "0.45.5", False, False),
("2.13.0", "0.46.0", False, False),
("2.13.0+rocm7.0", "0.50.2", True, False),
("2.11.0.dev20260101+cu130", "0.49.0.dev0", False, True),
("2.10.0rc1", "0.50.2", False, False),
("2.13.0", "0.46.1rc1", False, False),
("2.10.0.dev20250901+cu128", "0.50.2", False, False),
],
)
def test_nf4_compile_version_floor(monkeypatch, torch_version, bnb_version, rocm, expected):
import importlib.metadata
monkeypatch.setattr(torch, "__version__", torch_version)
monkeypatch.setattr(dit, "torch_is_rocm", lambda: rocm)
real = importlib.metadata.version
monkeypatch.setattr(
importlib.metadata,
"version",
lambda name: bnb_version if name == "bitsandbytes" else real(name),
)
assert dit._nf4_compile_supported() is expected
def test_nf4_compile_gate_fails_closed_without_bitsandbytes(monkeypatch):
import importlib.metadata
def _missing(name):
raise importlib.metadata.PackageNotFoundError(name)
monkeypatch.setattr(importlib.metadata, "version", _missing)
assert dit._nf4_compile_supported() is False
def _tiny_zimage():
diffusers = pytest.importorskip("diffusers")
peft = pytest.importorskip("peft")
torch.manual_seed(0)
model = diffusers.ZImageTransformer2DModel(
dim = 64,
n_layers = 4,
n_refiner_layers = 1,
n_heads = 2,
n_kv_heads = 2,
cap_feat_dim = 32,
axes_dims = [8, 12, 12],
axes_lens = [64, 32, 32],
)
model.requires_grad_(False)
model.add_adapter(
peft.LoraConfig(r = 4, lora_alpha = 4, init_lora_weights = False, target_modules = ["to_q", "to_v"])
)
return model
def _lora_grads(model):
torch.manual_seed(1)
x, cap, t = [torch.randn(16, 1, 8, 8)], [torch.randn(6, 32)], torch.tensor([0.3])
model.train()
out = model(x, t, cap, return_dict = False)[0]
sum((o**2).mean() for o in out).backward()
return {n: p.grad.clone() for n, p in model.named_parameters() if p.requires_grad}
@pytest.mark.parametrize("mode", ["plain", "partial"])
def test_checkpoint_modes_match_uncheckpointed_grads(mode, monkeypatch):
monkeypatch.delenv(dtc.GC_MODE_ENV, raising = False)
reference = _lora_grads(_tiny_zimage())
model = _tiny_zimage()
calls = []
real_checkpoint = torch.utils.checkpoint.checkpoint
monkeypatch.setattr(
torch.utils.checkpoint,
"checkpoint",
lambda fn, *a, **k: calls.append(k.get("use_reentrant")) or real_checkpoint(fn, *a, **k),
)
assert dtc.enable_diffusion_gradient_checkpointing(model, mode) == mode
grads = _lora_grads(model)
assert grads.keys() == reference.keys()
for name in reference:
torch.testing.assert_close(grads[name], reference[name], rtol = 1e-5, atol = 1e-6)
# 6 ZImageTransformerBlocks (4 layers + 1 noise + 1 context refiner): partial skips every 2nd.
assert calls and set(calls) == {False}
assert len(calls) == (6 if mode == "plain" else 3)
def _fake_zoo_helper(monkeypatch):
import sys
import types
called = []
module = types.ModuleType("unsloth_zoo.block_checkpointing")
module.enable_block_gradient_checkpointing = lambda model, every_n_blocks: called.append(
every_n_blocks
)
monkeypatch.setitem(sys.modules, "unsloth_zoo.block_checkpointing", module)
return called
def test_checkpoint_mode_comes_from_the_env_and_defaults_to_plain(monkeypatch):
called = _fake_zoo_helper(monkeypatch)
monkeypatch.delenv(dtc.GC_MODE_ENV, raising = False)
monkeypatch.delenv(dtc.GC_EVERY_N_ENV, raising = False)
assert dtc.enable_diffusion_gradient_checkpointing(object()) == "plain"
for value, expected in (("partial", "partial"), ("PARTIAL ", "partial"), ("bogus", "plain")):
monkeypatch.setenv(dtc.GC_MODE_ENV, value)
assert dtc.enable_diffusion_gradient_checkpointing(object()) == expected
monkeypatch.setenv(dtc.GC_MODE_ENV, "partial")
monkeypatch.setenv(dtc.GC_EVERY_N_ENV, "x")
assert dtc.enable_diffusion_gradient_checkpointing(object()) == "partial"
monkeypatch.setenv(dtc.GC_EVERY_N_ENV, "4")
assert dtc.enable_diffusion_gradient_checkpointing(object()) == "partial"
assert called == [1, 2, 2, 1, 2, 4]
def test_checkpointing_falls_back_to_every_block_without_the_zoo_helper(monkeypatch):
import sys
seen = {}
class _Model(torch.nn.Module):
def enable_gradient_checkpointing(self, gradient_checkpointing_func = None):
seen["func"] = gradient_checkpointing_func
monkeypatch.setenv(dtc.GC_MODE_ENV, "partial")
monkeypatch.setitem(sys.modules, "unsloth_zoo.block_checkpointing", None)
assert dtc.enable_diffusion_gradient_checkpointing(_Model()) == "plain"
assert seen["func"].keywords == {"use_reentrant": False}
class _FakeAccel:
def __init__(self, peak_bytes):
self.peak_bytes, self.resets = peak_bytes, 0
def reset_peak_memory_stats(self):
self.resets += 1
def max_memory_allocated(self):
return self.peak_bytes
@pytest.mark.parametrize("device", ["cuda", "xpu"])
def test_training_peak_is_reported_on_cuda_and_xpu(device, monkeypatch):
accel = _FakeAccel(3_456_000_000)
monkeypatch.setattr(torch, device, accel, raising = False)
dtc.reset_training_peak_memory(device)
assert accel.resets == 1
assert dtc.training_peak_memory_gb(device) == 3.46
@pytest.mark.parametrize("device", ["cpu", "mps"])
def test_training_peak_is_unmeasured_elsewhere(device):
dtc.reset_training_peak_memory(device)
assert dtc.training_peak_memory_gb(device) == 0.0
def test_a_failing_peak_probe_reports_unmeasured(monkeypatch):
class _Broken(_FakeAccel):
def max_memory_allocated(self):
raise RuntimeError("no device")
def reset_peak_memory_stats(self):
raise RuntimeError("no device")
monkeypatch.setattr(torch, "xpu", _Broken(0), raising = False)
dtc.reset_training_peak_memory("xpu")
assert dtc.training_peak_memory_gb("xpu") == 0.0
def test_a_preview_round_never_reads_as_training_peak(monkeypatch):
class _Accel(_FakeAccel):
def reset_peak_memory_stats(self):
self.resets += 1
self.peak_bytes = 0
accel = _Accel(0)
monkeypatch.setattr(torch, "xpu", accel, raising = False)
peak = dtc.TrainingPeakMemory("xpu")
accel.peak_bytes = 5_000_000_000 # training steps
peak.pause()
accel.peak_bytes = 9_000_000_000 # the preview's VAE decode
peak.resume()
accel.peak_bytes = 4_000_000_000 # training again
assert peak.gb() == 5.0
accel.peak_bytes = 6_000_000_000
assert peak.gb() == 6.0
def test_a_short_resume_keeps_nf4_eager(monkeypatch):
from dataclasses import replace
import core.training.diffusion_checkpoint as ckpt
monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
cfg = replace(_ccfg(), train_steps = 500, resume_from_checkpoint = "/runs/a/checkpoint-450")
monkeypatch.setattr(ckpt, "preflight_resume", lambda *a, **k: ("/runs/a/checkpoint-450", 450))
left = dtc.resume_steps_left(cfg, identity = None)
assert left == 50
assert dit._should_compile(cfg, True, "cuda", "nf4", left) is False
assert dit._should_compile(cfg, True, "cuda", "nf4") is True # a fresh 500-step run compiles
# An unreadable bundle keeps the target (restore_resume_state reports it).
monkeypatch.setattr(
ckpt, "preflight_resume", lambda *a, **k: (_ for _ in ()).throw(OSError("gone"))
)
assert dtc.resume_steps_left(cfg, identity = None) == 500
assert dtc.resume_steps_left(replace(cfg, resume_from_checkpoint = None), None) == 500