Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: faster image LoRA training (fused AdamW, compile nf4 bases) (#13236)

Daniel Han9ef7ef32e379
diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.py
index db77af2ce3e..4ca7af78ca3 100644
--- a/studio/backend/core/training/diffusion_checkpoint.py
+++ b/studio/backend/core/training/diffusion_checkpoint.py
@@ -1817,10 +1817,10 @@ def _join_clauses(items: list[str]) -> str:
def _assert_optimizer_buildable(path: Path, manifest: dict[str, Any]) -> None:
"""Refuse moments this host provably cannot build an optimizer for, BEFORE teardown.
- The trainers choose bitsandbytes AdamW8bit or torch AdamW from the HOST, so a bundle can arrive
- with foreign moments, and the child's own check then fires after the route has evicted the
- resident inference models and loaded a multi-GB base, for a run that is guaranteed to terminate
- without training.
+ Bundles from older builds carry bitsandbytes AdamW8bit moments, which the trainers can only
+ restore into AdamW8bit. Without bitsandbytes the child's own check fires after the route has
+ evicted the resident inference models and loaded a multi-GB base, for a run that is guaranteed
+ to terminate without training.
Deliberately one-directional, and deliberately import-free. ``find_spec`` answers "is
bitsandbytes installed" without creating a CUDA context in the Unsloth process, but it cannot
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 833aa2ed0c1..9cc9d418628 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -21,7 +21,7 @@ the VAE freed: the cache stores the posterior's affine parameters (mean/std fold
family's latent normalisation), so every step still draws a fresh VAE sample --
distribution-identical to encoding in the loop, without keeping the VAE resident or paying a
per-step encode. The transformer trains as a QLoRA (nf4) adapter by default with gradient
-checkpointing and 8-bit AdamW, so only the (small) LoRA params + optimizer state and the frozen
+checkpointing and fused AdamW, so only the (small) LoRA params + optimizer state and the frozen
4-bit base sit in VRAM during the loop.
"""
@@ -46,7 +46,11 @@ from core.training.diffusion_train_common import (
EventCb,
LATENT_CACHE_OVER_BUDGET,
StopCb,
- bitsandbytes_optimizer_supported,
+ enable_diffusion_gradient_checkpointing,
+ TrainingPeakMemory,
+ resume_steps_left,
+ make_lora_optimizer,
+ recorded_optimizer_class,
_apply_perf_flags,
_assert_trusted_base_model,
_emit,
@@ -1883,6 +1887,7 @@ def _should_compile(
base_is_bnb,
device,
base_precision = "nf4",
+ steps_left = None,
) -> bool:
mode = (cfg.compile_transformer or "auto").strip().lower()
if device != "cuda" or mode == "off":
@@ -1893,11 +1898,35 @@ def _should_compile(
return False
if mode == "on":
return True
- # Regional compile is the whole point of the dense modes (2.6x measured on Z-Image bf16) but is fragile over
- # bitsandbytes 4-bit modules, so it stays off for QLoRA.
+ if base_precision == "nf4":
+ steps = cfg.train_steps if steps_left is None else steps_left
+ return steps >= NF4_COMPILE_MIN_STEPS and _nf4_compile_supported()
return base_precision in ("bf16", "fp8", "mxfp8")
+# Lowest stack where bnb 4-bit ops are torch custom ops (0 graph breaks, loss matches eager); ROCm unmeasured, stays eager.
+NF4_COMPILE_MIN_TORCH = "2.10"
+NF4_COMPILE_MIN_BNB = "0.46.1"
+# The nf4 compile only pays back after a few hundred steps, so a short run stays eager.
+NF4_COMPILE_MIN_STEPS = 200
+
+
+def _nf4_compile_supported() -> bool:
+ try:
+ import importlib.metadata
+
+ import torch
+ from packaging.version import Version
+
+ if torch_is_rocm():
+ return False
+ if Version(torch.__version__) < Version(NF4_COMPILE_MIN_TORCH):
+ return False
+ return Version(importlib.metadata.version("bitsandbytes")) >= Version(NF4_COMPILE_MIN_BNB)
+ except Exception: # noqa: BLE001 -- unknown versions keep the historical eager path
+ return False
+
+
def _maybe_compile_transformer(
transformer,
cfg,
@@ -1905,12 +1934,13 @@ def _maybe_compile_transformer(
device,
on_event,
base_precision = "nf4",
+ steps_left = None,
) -> bool:
"""Regionally compile the transformer blocks (diffusers compile_repeated_blocks) after the LoRA
is attached. Never fatal: a wrap failure falls back to eager with a warning event, and
dynamo's suppress_errors keeps a frame that fails to COMPILE at the first step running eager
instead of raising mid-run."""
- if not _should_compile(cfg, base_is_bnb, device, base_precision):
+ if not _should_compile(cfg, base_is_bnb, device, base_precision, steps_left):
if base_precision in ("fp8", "mxfp8"):
_emit(
on_event,
@@ -2242,13 +2272,7 @@ def _train_dit(
)
)
if cfg.gradient_checkpointing:
- # Non-reentrant checkpointing: reentrant recompute of a bnb 4-bit LoRA linear can trip an illegal memory
- # access on the larger FLUX transformer, and it is the recommended mode anyway.
- import functools
- import torch.utils.checkpoint as _ckpt
- transformer.enable_gradient_checkpointing(
- gradient_checkpointing_func = functools.partial(_ckpt.checkpoint, use_reentrant = False)
- )
+ enable_diffusion_gradient_checkpointing(transformer)
cast_training_params(transformer, dtype = torch.float32)
lora_params = [p for p in transformer.parameters() if p.requires_grad]
@@ -2265,8 +2289,15 @@ def _train_dit(
# conversions so they track the final fp32 objects.
ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None
+ # A resume runs only the remaining steps, so the nf4 compile break-even is judged on those.
compiled = _maybe_compile_transformer(
- transformer, cfg, base_is_bnb, device, on_event, base_precision
+ transformer,
+ cfg,
+ base_is_bnb,
+ device,
+ on_event,
+ base_precision,
+ resume_steps_left(cfg, identity),
)
# Compiled Qwen graphs need one fixed text length across steps: pin the pad bucket to the dataset's longest
# caption.
@@ -2274,7 +2305,9 @@ def _train_dit(
if compiled and spec.family == "qwen-image":
qwen_pad_to = max(e[0].shape[1] for e in caption_embeds.values())
- optimizer = _make_optimizer(lora_params, cfg.learning_rate)
+ optimizer = _make_optimizer(
+ lora_params, cfg.learning_rate, recorded_optimizer_class(cfg, identity)
+ )
scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token
)
@@ -2474,6 +2507,7 @@ def _train_dit(
if sample_plan is not None and resumed < cfg.train_steps:
_sample(resumed)
+ train_peak = TrainingPeakMemory(device)
for opt_step in range(resumed, cfg.train_steps):
if stop_latched:
stopped = True
@@ -2562,8 +2596,7 @@ def _train_dit(
# Step 1 pays the one-time costs (cudnn autotune, compile warmup), so the reported rate starts after it.
t_steady = now
if done % cfg.log_every == 0 or done == cfg.train_steps:
- if device == "cuda":
- peak_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)
+ peak_gb = train_peak.gb()
per_step = batch_size * cfg.gradient_accumulation_steps
if t_steady is not None and ran_here > 1:
sps = round((ran_here - 1) * per_step / max(now - t_steady, 1e-6), 3)
@@ -2583,7 +2616,9 @@ def _train_dit(
)
stop_now = _check_stop()
if not stop_now and sample_plan is not None and sample_plan.due(done, cfg.train_steps):
+ train_peak.pause()
_sample(done)
+ train_peak.resume()
stop_now = stop_latched
# Skipped on the final step and when stopping, since the stop path writes one at the exact step.
if (
@@ -2657,31 +2692,12 @@ def _train_dit(
return str(out_dir)
-def _make_optimizer(params, lr):
- """8-bit AdamW (bitsandbytes) when available -- half the optimizer state, no accuracy regression
- for LoRA -- else torch AdamW, fused on CUDA (with a fallback when this build/device lacks the
- fused kernel). UNSLOTH_DIFFUSION_FP32_OPTIM forces plain (non-fused) AdamW, as it does for
- SDXL: the accuracy guard wants the reference optimizer, and a host where the override means
- one thing for one trainer and nothing for the other cannot answer "can this checkpoint be
- resumed here" before the run starts."""
- import torch
-
- if os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM", "") in ("1", "true"):
- return torch.optim.AdamW(params, lr = lr)
- # Checked before construction, not around it: on XPU the 8-bit optimizer builds fine and
- # only dies at the first step(), which the except below would never see.
- if bitsandbytes_optimizer_supported():
- try:
- import bitsandbytes as bnb
- return bnb.optim.AdamW8bit(params, lr = lr)
- except Exception: # noqa: BLE001 -- bnb missing / no CUDA: fall back to torch AdamW
- pass
- if torch.cuda.is_available():
- try:
- return torch.optim.AdamW(params, lr = lr, fused = True)
- except Exception: # noqa: BLE001 -- fused unsupported on this build/device
- pass
- return torch.optim.AdamW(params, lr = lr)
+def _make_optimizer(
+ params,
+ lr,
+ resume_optimizer_class = None,
+):
+ return make_lora_optimizer(params, lr, resume_optimizer_class)
def _free_text_encoders(pipe) -> None:
diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.py
index 10cd7e82a7a..d233428fe62 100644
--- a/studio/backend/core/training/diffusion_h3_trainer.py
+++ b/studio/backend/core/training/diffusion_h3_trainer.py
@@ -81,6 +81,9 @@ from core.training.diffusion_train_common import (
_latent_cache_forced,
_latent_cache_over_budget,
_restore_perf_flags,
+ enable_diffusion_gradient_checkpointing,
+ reset_training_peak_memory,
+ training_peak_memory_gb,
h3_train_unsupported_reason,
flow_bf16_trainable,
native_bf16_supported_xpu,
@@ -531,7 +534,11 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
from peft import LoraConfig
from peft.utils import get_peft_model_state_dict
- from core.training.diffusion_dit_trainer import _int8_quantize_base, _make_optimizer
+ from core.training.diffusion_dit_trainer import (
+ _int8_quantize_base,
+ _make_optimizer,
+ _maybe_compile_transformer,
+ )
out_dir = Path(cfg.output_dir).expanduser()
clip_paths = [p for p, _ in pairs]
@@ -638,11 +645,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
)
)
if cfg.gradient_checkpointing:
- import functools
- import torch.utils.checkpoint as _ckpt
- transformer.enable_gradient_checkpointing(
- gradient_checkpointing_func = functools.partial(_ckpt.checkpoint, use_reentrant = False)
- )
+ enable_diffusion_gradient_checkpointing(transformer)
cast_training_params(transformer, dtype = torch.float32)
lora_params = [p for p in transformer.parameters() if p.requires_grad]
if not lora_params:
@@ -658,6 +661,10 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
_int8_quantize_base(transformer, cfg.resolved_family)
ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None
+ # dynamic=True keeps one graph per block across caption lengths; int8 stays eager like the DiT families.
+ compiled = _maybe_compile_transformer(
+ transformer, cfg, base_precision == "nf4", device, on_event, base_precision
+ )
optimizer = _make_optimizer(lora_params, cfg.learning_rate)
lr_sched = get_scheduler(
cfg.lr_scheduler,
@@ -671,7 +678,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
_emit(
on_event,
"model_load_completed",
- compiled = False,
+ compiled = compiled,
base_precision = base_precision,
sequence_length = h3_packed_sequence_length(
max(e.shape[1] for e in caption_embeds.values()), num_frames, height, width
@@ -692,6 +699,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
t_start = time.time()
t_steady = None
done = 0
+ reset_training_peak_memory(device)
for opt_step in range(cfg.train_steps):
optimizer.zero_grad(set_to_none = True)
step_loss = 0.0
@@ -770,8 +778,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav
if done == 1:
t_steady = now
if done % cfg.log_every == 0 or done == cfg.train_steps:
- if device == "cuda":
- peak_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)
+ peak_gb = training_peak_memory_gb(device)
per_step = cfg.gradient_accumulation_steps
if t_steady is not None and done > 1:
sps = round((done - 1) * per_step / max(now - t_steady, 1e-6), 3)
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.py
index 65a4b098c6b..3ba83459fdb 100644
--- a/studio/backend/core/training/diffusion_lora_trainer.py
+++ b/studio/backend/core/training/diffusion_lora_trainer.py
@@ -46,7 +46,10 @@ from core.training.diffusion_train_common import ( # noqa: F401
StopCb,
DiffusionLoraConfig,
LATENT_CACHE_OVER_BUDGET,
- bitsandbytes_optimizer_supported,
+ enable_diffusion_gradient_checkpointing,
+ TrainingPeakMemory,
+ make_lora_optimizer,
+ recorded_optimizer_class,
native_bf16_supported,
_apply_perf_flags,
_assert_trusted_base_model,
@@ -454,7 +457,7 @@ def run_diffusion_lora_training(
)
)
if cfg.gradient_checkpointing:
- unet.enable_gradient_checkpointing()
+ enable_diffusion_gradient_checkpointing(unet)
# LoRA params must be fp32 for a stable optimizer under mixed precision.
if weight_dtype != torch.float32:
cast_training_params(unet, dtype = torch.float32)
@@ -468,7 +471,9 @@ def run_diffusion_lora_training(
)
lora_params = [p for p in unet.parameters() if p.requires_grad]
- optimizer = _make_lora_optimizer(lora_params, cfg.learning_rate)
+ optimizer = _make_lora_optimizer(
+ lora_params, cfg.learning_rate, recorded_optimizer_class(cfg, identity)
+ )
# A frozen fp16 UNet (pre-Ampere / RDNA2 fallback) underflows the LoRA gradients without loss scaling.
scaler = torch.amp.GradScaler("cuda", enabled = weight_dtype is torch.float16)
# The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the
@@ -735,6 +740,7 @@ def run_diffusion_lora_training(
if resumed < cfg.train_steps:
_sample(resumed)
+ train_peak = TrainingPeakMemory(device)
for opt_step in range(resumed, cfg.train_steps):
if stop_latched:
stopped = True
@@ -841,8 +847,7 @@ def run_diffusion_lora_training(
t_steady = now
if done % cfg.log_every == 0 or done == cfg.train_steps:
# ``learning_rate`` (not ``lr``) is the field the Unsloth training pump reads.
- if device == "cuda":
- peak_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)
+ peak_gb = train_peak.gb()
per_step = cfg.train_batch_size * cfg.gradient_accumulation_steps
if t_steady is not None and ran_here > 1:
samples_per_second = round(
@@ -865,7 +870,9 @@ def run_diffusion_lora_training(
stop_now = _check_stop()
if not stop_now and sample_plan is not None and sample_plan.due(done, cfg.train_steps):
+ train_peak.pause()
_sample(done)
+ train_peak.resume()
stop_now = stop_latched
# Skipped on the final step and when stopping, since the stop path writes one at the exact step.
if (
@@ -937,29 +944,12 @@ def run_diffusion_lora_training(
_restore_perf_flags(snap)
-def _make_lora_optimizer(params: list, lr: float) -> Any:
- """8-bit AdamW (bitsandbytes) by default -- half the optimizer state, no meaningful
- quality cost for LoRA -- falling back to torch AdamW (fused on CUDA) when unavailable.
- UNSLOTH_DIFFUSION_FP32_OPTIM forces plain (non-fused) AdamW: the accuracy guard wants the
- reference optimizer, so it must not take the fused path."""
- import torch
-
- if os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM", "") in ("1", "true"):
- return torch.optim.AdamW(params, lr = lr)
- # Checked before construction, not around it: on XPU the 8-bit optimizer builds fine and
- # only dies at the first step(), which the except below would never see.
- if bitsandbytes_optimizer_supported():
- try:
- import bitsandbytes as bnb
- return bnb.optim.AdamW8bit(params, lr = lr)
- except Exception: # noqa: BLE001 -- bnb missing / no CUDA: fall back to torch AdamW
- pass
- if torch.cuda.is_available():
- try:
- return torch.optim.AdamW(params, lr = lr, fused = True)
- except Exception: # noqa: BLE001 -- fused unsupported on this build/device
- pass
- return torch.optim.AdamW(params, lr = lr)
+def _make_lora_optimizer(
+ params: list,
+ lr: float,
+ resume_optimizer_class = None,
+) -> Any:
+ return make_lora_optimizer(params, lr, resume_optimizer_class)
def run_diffusion_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> None:
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index 3c67a344067..fe9bc913afa 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -824,8 +824,8 @@ def bitsandbytes_optimizer_supported() -> bool:
Keyed on get_device(), NOT torch.xpu.is_available(): a hybrid host with an Intel iGPU beside
an NVIDIA card reports both and detection prefers CUDA. Answering the presence question there
- would drop 8-bit on a CUDA run, and worse, change optimizer_key() so restore_resume_state
- refuses every existing AdamW8bit checkpoint with a ResumeError.
+ would make make_lora_optimizer skip AdamW8bit, so restore_resume_state refuses every existing
+ AdamW8bit checkpoint with a ResumeError.
"""
try:
from utils.hardware import DeviceType, get_device
@@ -834,6 +834,139 @@ def bitsandbytes_optimizer_supported() -> bool:
return True
+def _is_bnb_8bit_optimizer_key(name: Optional[str]) -> bool:
+ return isinstance(name, str) and name.startswith("bitsandbytes.") and "8bit" in name
+
+
+def recorded_optimizer_class(cfg: Any, identity: Any) -> Optional[str]:
+ """Optimizer class of the bundle being resumed, else None; never raises (restore_resume_state reports errors)."""
+ if not getattr(cfg, "resume_from_checkpoint", None):
+ return None
+ try:
+ from core.training.diffusion_checkpoint import preflight_resume, read_checkpoint
+ path, _step = preflight_resume(
+ cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps
+ )
+ value = (read_checkpoint(path) or {}).get("optimizer_class")
+ except Exception: # noqa: BLE001
+ return None
+ return value if isinstance(value, str) else None
+
+
+def resume_steps_left(cfg: Any, identity: Any) -> int:
+ """Optimizer steps this process will run: the target minus the resumed bundle's step; the target when unknown."""
+ if not getattr(cfg, "resume_from_checkpoint", None):
+ return cfg.train_steps
+ try:
+ from core.training.diffusion_checkpoint import preflight_resume
+ _path, step = preflight_resume(
+ cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps
+ )
+ except Exception: # noqa: BLE001 -- restore_resume_state reports the real problem
+ return cfg.train_steps
+ return max(0, cfg.train_steps - int(step))
+
+
+class TrainingPeakMemory:
+ """Training-only peak: ``pause`` banks it before a preview round, ``resume`` resets after, so VAE decode is excluded."""
+
+ def __init__(self, device: str):
+ self.device = device
+ self.banked = 0.0
+ reset_training_peak_memory(device)
+
+ def pause(self) -> None:
+ self.banked = max(self.banked, training_peak_memory_gb(self.device))
+
+ def resume(self) -> None:
+ reset_training_peak_memory(self.device)
+
+ def gb(self) -> float:
+ return max(self.banked, training_peak_memory_gb(self.device))
+
+
+def _accelerator_memory_module(device: str) -> Any:
+ import torch
+
+ module = getattr(torch, device, None) if device in ("cuda", "xpu") else None
+ if module is None or not hasattr(module, "max_memory_allocated"):
+ return None
+ return module
+
+
+def reset_training_peak_memory(device: str) -> None:
+ """Start the reported peak at the loop: the load / encode peak is higher and not the training cost."""
+ module = _accelerator_memory_module(device)
+ if module is not None:
+ try:
+ module.reset_peak_memory_stats()
+ except Exception: # noqa: BLE001 -- reporting only
+ pass
+
+
+def training_peak_memory_gb(device: str) -> float:
+ """Peak allocated GB since reset_training_peak_memory on CUDA / XPU; 0.0 where unmeasured (reported as None)."""
+ module = _accelerator_memory_module(device)
+ if module is None:
+ return 0.0
+ try:
+ return round(module.max_memory_allocated() / 1e9, 2)
+ except Exception: # noqa: BLE001 -- reporting only
+ return 0.0
+
+
+GC_MODE_ENV = "UNSLOTH_DIFFUSION_GC_MODE"
+GC_EVERY_N_ENV = "UNSLOTH_DIFFUSION_GC_EVERY_N"
+
+
+def enable_diffusion_gradient_checkpointing(model: Any, mode: Optional[str] = None) -> str:
+ """ "plain" checkpoints every block, "partial" every Nth; selective / CPU-offloaded were slower or diverged under regional compile."""
+ mode = (mode or os.environ.get(GC_MODE_ENV, "") or "plain").strip().lower()
+ every_n = 1
+ if mode == "partial":
+ try:
+ every_n = max(2, int(os.environ.get(GC_EVERY_N_ENV, "") or 2))
+ except ValueError:
+ every_n = 2
+ try:
+ from unsloth_zoo.block_checkpointing import enable_block_gradient_checkpointing
+ except ImportError:
+ import functools
+
+ import torch.utils.checkpoint as _ckpt
+
+ model.enable_gradient_checkpointing(
+ gradient_checkpointing_func = functools.partial(_ckpt.checkpoint, use_reentrant = False)
+ )
+ return "plain"
+ enable_block_gradient_checkpointing(model, every_n_blocks = every_n)
+ return "partial" if every_n > 1 else "plain"
+
+
+def make_lora_optimizer(
+ params: list,
+ lr: float,
+ resume_optimizer_class: Optional[str] = None,
+) -> Any:
+ """torch AdamW (AdamW8bit step() syncs per parameter); AdamW8bit only to resume its own bundle, never on XPU (dies at step())."""
+ import torch
+
+ if os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM", "") in ("1", "true"):
+ return torch.optim.AdamW(params, lr = lr)
+ if _is_bnb_8bit_optimizer_key(resume_optimizer_class) and bitsandbytes_optimizer_supported():
+ try:
+ import bitsandbytes as bnb
+ return bnb.optim.AdamW8bit(params, lr = lr)
+ except Exception: # noqa: BLE001 -- restore_resume_state then names the mismatch
+ pass
+ if torch.cuda.is_available():
+ try:
+ return torch.optim.AdamW(params, lr = lr, fused = True)
+ except Exception: # noqa: BLE001 -- fused unsupported on this build/device
+ pass
+ return torch.optim.AdamW(params, lr = lr)
+
+
def training_precision_preflight_error(resolved_family: str, base_precision: str) -> Optional[str]:
"""Reason the requested DiT precision cannot run on this host, else None -- checked by the start
route BEFORE evicting resident GPU workloads (the trainer's own checks fire only in the
@@ -929,8 +1062,7 @@ def family_train_infos() -> list[dict[str, Any]]:
if is_dit
else None
)
- # H3 refuses compile_transformer="on" (its packed sequence changes length with every caption, so each step
- # would re-trace), so advertising the control would offer a selection that always 400s.
+ # H3 compiles under the auto rule; the panel control stays DiT / SDXL only.
supports_compile = bool(not dit_block) and name in _DIT_TRAIN_FAMILIES
if not is_dit or dit_block:
fam_modes: list[str] = []
@@ -1603,14 +1735,6 @@ def h3_train_unsupported_reason(cfg: Any) -> Optional[str]:
"row layout is set by the clip's own geometry and its caption's length. Use "
"gradient_accumulation_steps to raise the effective batch."
)
- # torch.compile is never invoked here (the packed layout changes shape with every caption length), so an explicit
- # "on" would be accepted and then ignored.
- if str(getattr(cfg, "compile_transformer", "auto") or "auto").strip().lower() == "on":
- return (
- "MiniMax-H3 does not compile: its packed sequence changes length with every clip's "
- "caption, so torch.compile would re-trace each step. Use compile_transformer "
- "'off' or 'auto'."
- )
# No conditioning cache exists on this path, so accepting the directory would promise a saving that never happens.
if str(getattr(cfg, "cond_cache_dir", "") or "").strip():
return (
@@ -2083,6 +2207,9 @@ def _reapply_lr_schedule(optimizer: Any, lr_scheduler: Any) -> None:
pass
+_OPTIMIZER_KERNEL_KEYS = ("fused", "foreach")
+
+
def restore_resume_state(
cfg: DiffusionLoraConfig,
*,
@@ -2118,9 +2245,7 @@ def restore_resume_state(
load_trainable_state_dict(model, ckpt.tensors("adapter"))
optimizer_state = ckpt.torch_state("optimizer")
if optimizer_state is not None:
- # The trainers pick their optimizer from the HOST (bnb present, fused kernel available,
- # UNSLOTH_DIFFUSION_FP32_OPTIM), so foreign moments arrive legitimately (state1/state2 versus
- # exp_avg/exp_avg_sq): shapes match, load_state_dict accepts them, and the first step dies on a bare KeyError.
+ # Foreign moments (AdamW8bit vs torch AdamW) load cleanly, then the first step dies on a bare KeyError.
saved_optimizer = ckpt.optimizer_class
live_optimizer = optimizer_key(optimizer)
if not saved_optimizer:
@@ -2147,9 +2272,22 @@ def restore_resume_state(
"than this build produces, so its moments cannot be matched to this run's "
"tensors. Start a new run, or resume on the version that wrote it."
)
- # load_state_dict replaces the param groups too, so the checkpoint's learning rate wins over a changed cfg,
- # the same semantics as HF Trainer's resume.
+ # Checkpoint lr wins (HF Trainer semantics); the fused/foreach kernel choice stays this host's.
+ kernels = [
+ {k: g[k] for k in _OPTIMIZER_KERNEL_KEYS if k in g} for g in optimizer.param_groups
+ ]
+ saved_groups = (
+ optimizer_state.get("param_groups") if isinstance(optimizer_state, dict) else None
+ )
+ if isinstance(saved_groups, list) and len(saved_groups) == len(kernels):
+ # torch places "step" by the SAVED fused flag: non-fused bundles keep it on CPU, which fused rejects.
+ optimizer_state = {
+ **optimizer_state,
+ "param_groups": [{**g, **kept} for g, kept in zip(saved_groups, kernels)],
+ }
optimizer.load_state_dict(optimizer_state)
+ for group, kept in zip(optimizer.param_groups, kernels):
+ group.update(kept)
elif optimizer is not None:
# Every bundle this writer produces has an optimizer, so continuing without one restarts Adam's moments from
# zero at step N while reporting a clean resume.
diff --git a/studio/backend/tests/test_diffusion_checkpoint_resume.py b/studio/backend/tests/test_diffusion_checkpoint_resume.py
index de06fe3100b..81e1187e463 100644
--- a/studio/backend/tests/test_diffusion_checkpoint_resume.py
+++ b/studio/backend/tests/test_diffusion_checkpoint_resume.py
@@ -2121,22 +2121,21 @@ def test_a_first_periodic_save_does_not_spend_the_previous_runs_bundles(run_dir)
assert "discard_existing = False," in source, name
-def test_both_trainers_honour_the_fp32_optimizer_override(run_dir):
+def test_both_trainers_honour_the_fp32_optimizer_override(run_dir, monkeypatch):
"""The preflight refuses 8-bit moments when the override is set, which is only sound if
every trainer actually obeys it -- otherwise DiT checkpoints written on this host become
- unresumable on the same host."""
- trainers = Path(dc.__file__).parent
- for name in ("diffusion_lora_trainer.py", "diffusion_dit_trainer.py"):
- source = (trainers / name).read_text(encoding = "utf-8")
- marker = source.find("def _make_optimizer")
- if marker < 0:
- marker = source.find("def _make_lora_optimizer")
- assert marker > 0, name
- body = source[marker : marker + 1200]
- # The env READ, not a mention of it in prose.
- read = 'os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM"'
- assert read in body, name
- assert body.index(read) < body.index("AdamW8bit"), name
+ unresumable on the same host. Asked of the factories themselves, including for a resume of
+ an 8-bit bundle, which is the one case that otherwise builds AdamW8bit."""
+ import core.training.diffusion_dit_trainer as dit
+ import core.training.diffusion_lora_trainer as sdxl
+
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")
+ for factory in (sdxl._make_lora_optimizer, dit._make_optimizer):
+ for resume in (None, "bitsandbytes.optim.adamw.AdamW8bit"):
+ param = torch.nn.Parameter(torch.zeros(2, 2))
+ optimizer = factory([param], 1e-4, resume)
+ assert type(optimizer) is torch.optim.AdamW, factory
+ assert not optimizer.defaults.get("fused"), factory
def test_a_completed_run_does_not_leave_its_periodic_bundles_behind(run_dir):
diff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.py
index f8e1c729641..09d8aa28aaf 100644
--- a/studio/backend/tests/test_diffusion_h3_trainer.py
+++ b/studio/backend/tests/test_diffusion_h3_trainer.py
@@ -1090,7 +1090,6 @@ def test_the_knobs_h3_cannot_honour_are_refused_or_normalised():
refused = {
"mixed_precision": ("no", "requires bf16"),
"train_batch_size": (2, "batch size 1"),
- "compile_transformer": ("on", "does not compile"),
"cond_cache_dir": ("/tmp/cond", "conditioning cache"),
"cfg_dropout": (0.1, "guidance-distilled"),
"weighting_scheme": ("bell", "timestep-weighted"),
@@ -1171,7 +1170,6 @@ def test_h3_is_advertised_as_trainable_with_the_precisions_it_has():
# The case the bug was: a host that CAN train the DiT families must be able to train H3.
if reference["precision_modes"]:
assert h3["precision_modes"], "an empty list disables Start in the Train panel"
- # compile is the one DiT lever that must NOT follow: "on" is refused by the trainer.
assert h3["supports_compile"] is False
assert infos["sdxl"]["supports_compile"] is True
if reference["precision_modes"]:
@@ -1885,3 +1883,104 @@ def test_the_advertised_h3_vram_floor_covers_the_measured_run_peak():
the chip is what a user sizes a host from."""
from core.training.diffusion_train_common import _FAMILY_TRAIN_SPECS
assert _FAMILY_TRAIN_SPECS["minimax-h3"]["qlora_vram_gb"] >= 78
+
+
+def test_h3_accepts_every_compile_mode():
+ """dynamic=True compiles one graph per block across caption lengths, so "on" is no longer refused."""
+ from dataclasses import replace as _replace
+
+ from core.training.diffusion_train_common import h3_train_unsupported_reason
+
+ base = _h3_cfg().normalized()
+ for mode in ("off", "on", "auto"):
+ assert h3_train_unsupported_reason(_replace(base, compile_transformer = mode)) is None, mode
+
+
[email protected]("precision, expect_bnb", [("bf16", False), ("nf4", True), ("int8", False)])
+def test_h3_routes_its_transformer_through_the_shared_regional_compile(
+ precision, expect_bnb, monkeypatch, tmp_path
+):
+ """H3 hands its transformer to the DiT compile wrapper after int8 conversion, before the optimizer."""
+ import torch
+
+ import core.training.diffusion_dit_trainer as dit
+ import core.training.diffusion_h3_trainer as diffusion_h3_trainer
+ from core.training.diffusion_h3_trainer import _train_h3
+
+ order: list[str] = []
+
+ class _Tiny(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(4, 4)
+
+ def add_adapter(self, _config):
+ self.lora = torch.nn.Linear(4, 4)
+
+ class _Pipe:
+ text_encoder = processor = vae = audio_vae = None
+
+ seen = {}
+
+ class _Stop(Exception):
+ pass
+
+ def _compile(transformer, cfg, base_is_bnb, device, on_event, base_precision):
+ order.append("compile")
+ seen.update(base_is_bnb = base_is_bnb, base_precision = base_precision)
+ return False
+
+ def _optimizer(*_a, **_k):
+ order.append("optimizer")
+ raise _Stop
+
+ monkeypatch.setattr(diffusion_h3_trainer, "_load_conditioners", lambda cfg, device: _Pipe())
+ monkeypatch.setattr(
+ diffusion_h3_trainer, "_encode_prompt", lambda pipe, cap, device: torch.zeros(1, 3, 8)
+ )
+ monkeypatch.setattr(diffusion_h3_trainer, "_dataset_canvas", lambda path, short: (64, 64))
+ monkeypatch.setattr(diffusion_h3_trainer, "decode_clip", lambda path, **k: (None, None))
+ lf = h3_video_latent_frames(H3_TRAIN_NUM_FRAMES)
+ na = h3_audio_latent_count(H3_TRAIN_NUM_FRAMES)
+ monkeypatch.setattr(
+ diffusion_h3_trainer,
+ "_encode_video_stats",
+ lambda vae, frames, device: (torch.zeros(1, 24, lf, 4, 4), torch.zeros(1, 24, lf, 4, 4)),
+ )
+ monkeypatch.setattr(
+ diffusion_h3_trainer,
+ "_encode_audio_latents",
+ lambda avae, wav, device: torch.zeros(2, 32, na),
+ )
+ monkeypatch.setattr(
+ diffusion_h3_trainer, "_load_transformer", lambda cfg, device, precision: _Tiny()
+ )
+ monkeypatch.setattr(
+ dit, "_int8_quantize_base", lambda transformer, family: order.append("int8")
+ )
+ monkeypatch.setattr(dit, "_maybe_compile_transformer", _compile)
+ monkeypatch.setattr(dit, "_make_optimizer", _optimizer)
+
+ cfg = _replace_cfg(_h3_cfg(base_precision = precision, gradient_checkpointing = False))
+ with pytest.raises(_Stop):
+ _train_h3(
+ cfg,
+ [("clip.mp4", "a caption")],
+ random.Random(0),
+ "cpu",
+ torch.float32,
+ lambda *_a, **_k: None,
+ lambda: False,
+ lambda: True,
+ )
+ assert seen == {"base_is_bnb": expect_bnb, "base_precision": precision}
+ assert order == (["int8"] if precision == "int8" else []) + ["compile", "optimizer"]
+
+
+def _replace_cfg(cfg):
+ from dataclasses import replace as _replace
+
+ from core.training.diffusion_train_common import train_recipe_overrides
+
+ cfg = cfg.normalized()
+ return _replace(cfg, **train_recipe_overrides(cfg))
diff --git a/studio/backend/tests/test_diffusion_train_perf.py b/studio/backend/tests/test_diffusion_train_perf.py
index ff72ff64ec0..a03add4322b 100644
--- a/studio/backend/tests/test_diffusion_train_perf.py
+++ b/studio/backend/tests/test_diffusion_train_perf.py
@@ -15,6 +15,7 @@ inject in-thread fakes exactly like ``test_diffusion_training.py``.
from __future__ import annotations
import itertools
+from unittest.mock import patch
from pathlib import Path
@@ -225,8 +226,12 @@ def test_should_compile_policy():
assert _should_compile(_cfg(compile_transformer = "off"), False, "cuda") is False
# on always compiles on cuda.
assert _should_compile(_cfg(compile_transformer = "on"), False, "cuda") is True
- # auto stays off over a bitsandbytes base (graph breaks in the dequant path).
- assert _should_compile(_cfg(compile_transformer = "auto"), True, "cuda") is False
+ # auto over an nf4 base follows the torch / bitsandbytes version gate.
+ import core.training.diffusion_dit_trainer as dit
+
+ for supported in (True, False):
+ with patch.object(dit, "_nf4_compile_supported", lambda s = supported: s):
+ assert _should_compile(_cfg(compile_transformer = "auto"), True, "cuda") is supported
# auto turns on for the dense bf16 base precision on cuda.
assert (
_should_compile(_cfg(compile_transformer = "auto"), False, "cuda", base_precision = "bf16")
diff --git a/studio/backend/tests/test_diffusion_train_speed.py b/studio/backend/tests/test_diffusion_train_speed.py
new file mode 100644
index 00000000000..183af280e34
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_train_speed.py
@@ -0,0 +1,512 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Diffusion trainer speed defaults: optimizer factory, nf4 compile gate, checkpointing modes."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+import random
+import sys
+from pathlib import Path
+from types import ModuleType
+
+import pytest
+import torch
+
+import core.training.diffusion_dit_trainer as dit
+import core.training.diffusion_lora_trainer as sdxl
+import core.training.diffusion_train_common as dtc
+from core.training import diffusion_checkpoint as dc
+from core.training.diffusion_train_common import (
+ DiffusionLoraConfig,
+ PermutationBatchSampler,
+ make_lora_optimizer,
+ recorded_optimizer_class,
+ restore_resume_state,
+ write_resume_checkpoint,
+)
+
+BNB_KEY = "bitsandbytes.optim.adamw.AdamW8bit"
+
+
+class _Fake8bit(torch.optim.AdamW):
+ """Stands in for bnb.optim.AdamW8bit."""
+
+
+def fake_bnb(monkeypatch):
+ mod = ModuleType("bitsandbytes")
+ optim = ModuleType("bitsandbytes.optim")
+ optim.AdamW8bit = _Fake8bit
+ mod.optim = optim
+ monkeypatch.setitem(sys.modules, "bitsandbytes", mod)
+ monkeypatch.setitem(sys.modules, "bitsandbytes.optim", optim)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
+ monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: True)
+ return mod
+
+
+def _params():
+ p = torch.nn.Parameter(torch.zeros(2, 2))
+ p.grad = torch.ones(2, 2)
+ return [p]
+
+
[email protected]("factory", [dit._make_optimizer, sdxl._make_lora_optimizer])
+def test_fresh_run_builds_torch_adamw_even_with_bitsandbytes(fake_bnb, factory):
+ opt = factory(_params(), 1e-4)
+ assert type(opt) is torch.optim.AdamW
+ opt.step()
+
+
[email protected]("factory", [dit._make_optimizer, sdxl._make_lora_optimizer])
+def test_resuming_an_8bit_bundle_rebuilds_adamw8bit(fake_bnb, factory):
+ opt = factory(_params(), 1e-4, BNB_KEY)
+ assert isinstance(opt, _Fake8bit)
+ assert type(factory(_params(), 1e-4, "torch.optim.adamw.AdamW")) is torch.optim.AdamW
+ assert type(factory(_params(), 1e-4, None)) is torch.optim.AdamW
+
+
+def test_8bit_resume_on_xpu_falls_back_to_torch_adamw(fake_bnb, monkeypatch):
+ monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: False)
+ opt = make_lora_optimizer(_params(), 1e-4, BNB_KEY)
+ assert type(opt) is torch.optim.AdamW
+ opt.step()
+
+
+def test_fp32_override_still_forces_plain_adamw(fake_bnb, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")
+ for resume in (None, BNB_KEY):
+ opt = make_lora_optimizer(_params(), 1e-4, resume)
+ assert type(opt) is torch.optim.AdamW
+ assert not opt.defaults.get("fused")
+
+
[email protected](not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")
+def test_cuda_default_is_fused(monkeypatch):
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
+ p = torch.nn.Parameter(torch.zeros(4, 4, device = "cuda"))
+ opt = make_lora_optimizer([p], 1e-4)
+ assert type(opt) is torch.optim.AdamW and opt.defaults.get("fused") is True
+
+
+def _identity() -> dc.CheckpointIdentity:
+ return dc.CheckpointIdentity(
+ family = "sdxl",
+ base_model = "stabilityai/sdxl-turbo",
+ lora_target_modules = ("to_k", "to_q", "to_v", "to_out.0"),
+ lora_rank = 16,
+ lora_alpha = 16,
+ precision = "bf16",
+ base_precision = "nf4",
+ resolution = 1024,
+ base_revision = "rev-deadbeef",
+ dataset_fingerprint = "ds-3-cafe",
+ )
+
+
+def _cfg(out: Path, **kw) -> DiffusionLoraConfig:
+ return DiffusionLoraConfig(
+ base_model = "stabilityai/sdxl-turbo",
+ data_dir = "unused",
+ output_dir = str(out),
+ train_steps = 500,
+ **kw,
+ )
+
+
+def run_dir():
+ from utils.paths import outputs_root
+
+ d = outputs_root() / "speed-run"
+ d.mkdir(parents = True, exist_ok = True)
+ return d
+
+
+def _write(
+ out: Path,
+ model,
+ optimizer,
+ step: int = 3,
+):
+ sched = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda s: 1.0)
+ written, error = write_resume_checkpoint(
+ _cfg(out),
+ step = step,
+ model = model,
+ optimizer = optimizer,
+ lr_scheduler = sched,
+ identity = _identity(),
+ sampler = PermutationBatchSampler(7, random.Random(0)),
+ rng_streams = {"loop": random.Random(0), "variant": random.Random(1)},
+ )
+ assert error is None and written
+ return sched
+
+
+def test_recorded_optimizer_class_reads_the_bundle_being_resumed(run_dir, monkeypatch):
+ # The resume preflight refuses an 8-bit bundle on a host without bitsandbytes, so pretend it is installed.
+ real_find_spec = importlib.util.find_spec
+ monkeypatch.setattr(
+ importlib.util,
+ "find_spec",
+ lambda name, *a, **k: object() if name == "bitsandbytes" else real_find_spec(name, *a, **k),
+ )
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
+ tmp_path = run_dir
+ model = torch.nn.Linear(4, 4, bias = False)
+ _write(tmp_path, model, torch.optim.AdamW(model.parameters(), lr = 1e-3))
+ cfg = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "checkpoint-3"))
+ assert recorded_optimizer_class(cfg, _identity()) == "torch.optim.adamw.AdamW"
+ manifest_path = tmp_path / "checkpoint-3" / dc.TRAINER_STATE_FILENAME
+ manifest = json.loads(manifest_path.read_text(encoding = "utf-8"))
+ manifest["optimizer_class"] = BNB_KEY
+ manifest_path.write_text(json.dumps(manifest), encoding = "utf-8")
+ assert recorded_optimizer_class(cfg, _identity()) == BNB_KEY
+ assert recorded_optimizer_class(_cfg(tmp_path), _identity()) is None
+ bogus = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "nope"))
+ assert recorded_optimizer_class(bogus, _identity()) is None
+
+
[email protected](not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")
+def test_a_fused_bundle_resumed_under_the_fp32_override_stays_non_fused(run_dir, monkeypatch):
+ # load_state_dict restores the saved fused flag; the reference-optimizer override must survive it.
+ model = torch.nn.Linear(4, 4, bias = False).cuda()
+ fused = torch.optim.AdamW(model.parameters(), lr = 1e-3, fused = True)
+ model.weight.grad = torch.ones_like(model.weight)
+ fused.step()
+ _write(run_dir, model, fused)
+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")
+ live = torch.nn.Linear(4, 4, bias = False).cuda()
+ opt = make_lora_optimizer(list(live.parameters()), 1e-3)
+ assert not opt.param_groups[0].get("fused")
+ restore_resume_state(
+ _cfg(run_dir, resume_from_checkpoint = str(run_dir / "checkpoint-3")),
+ model = live,
+ optimizer = opt,
+ lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),
+ identity = _identity(),
+ )
+ assert not opt.param_groups[0].get("fused")
+ assert opt.state[live.weight]["exp_avg"].abs().sum() > 0
+
+
[email protected](not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")
+def test_a_non_fused_bundle_resumes_into_fused_adamw(run_dir, monkeypatch):
+ # A plain AdamW bundle keeps its step on the CPU; the fused kernel needs it on the param's device.
+ torch.manual_seed(0)
+ model = torch.nn.Linear(4, 4, bias = False).cuda()
+ plain = torch.optim.AdamW(model.parameters(), lr = 1e-3)
+ model.weight.grad = torch.ones_like(model.weight)
+ plain.step()
+ _write(run_dir, model, plain)
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
+ live = torch.nn.Linear(4, 4, bias = False).cuda()
+ opt = make_lora_optimizer(list(live.parameters()), 1e-3)
+ assert opt.param_groups[0].get("fused")
+ restore_resume_state(
+ _cfg(run_dir, resume_from_checkpoint = str(run_dir / "checkpoint-3")),
+ model = live,
+ optimizer = opt,
+ lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),
+ identity = _identity(),
+ )
+ assert opt.param_groups[0].get("fused")
+ live.weight.grad = torch.ones_like(live.weight)
+ opt.step()
+ assert opt.state[live.weight]["step"].item() == 2
+
+
[email protected](not torch.cuda.is_available(), reason = "bitsandbytes AdamW8bit needs CUDA")
+def test_old_8bit_bundle_resumes_and_continues_identically(run_dir, monkeypatch):
+ tmp_path = run_dir
+ bnb = pytest.importorskip("bitsandbytes")
+ if not hasattr(getattr(bnb, "optim", None), "AdamW8bit"):
+ pytest.skip("bitsandbytes is stubbed out in this environment")
+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)
+ monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: True)
+
+ def _model():
+ torch.manual_seed(0)
+ return torch.nn.Linear(128, 64, bias = False).cuda()
+
+ def _step(model, opt, i):
+ opt.zero_grad(set_to_none = True)
+ model.weight.grad = torch.full_like(model.weight, 0.01 * (i + 1))
+ opt.step()
+
+ ref = _model()
+ ref_opt = bnb.optim.AdamW8bit(ref.parameters(), lr = 1e-3)
+ for i in range(3):
+ _step(ref, ref_opt, i)
+ _write(tmp_path, ref, ref_opt)
+
+ cfg = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "checkpoint-3"))
+ live = _model()
+ opt = make_lora_optimizer(
+ list(live.parameters()), 1e-3, recorded_optimizer_class(cfg, _identity())
+ )
+ assert dc.optimizer_key(opt) == BNB_KEY
+ restored = restore_resume_state(
+ cfg,
+ model = live,
+ optimizer = opt,
+ lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),
+ identity = _identity(),
+ )
+ assert restored.step == 3
+ for i in range(3, 6):
+ _step(ref, ref_opt, i)
+ _step(live, opt, i)
+ assert torch.equal(live.weight, ref.weight)
+
+
+def _ccfg(mode = "auto"):
+ return DiffusionLoraConfig(
+ base_model = "Tongyi-MAI/Z-Image",
+ data_dir = "unused",
+ output_dir = "unused",
+ compile_transformer = mode,
+ )
+
+
+def test_auto_compiles_nf4_only_when_the_versions_are_supported(monkeypatch):
+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
+ assert dit._should_compile(_ccfg(), True, "cuda", "nf4") is True
+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: False)
+ assert dit._should_compile(_ccfg(), True, "cuda", "nf4") is False
+ assert dit._should_compile(_ccfg("on"), True, "cuda", "nf4") is True
+ assert dit._should_compile(_ccfg("off"), True, "cuda", "nf4") is False
+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
+ assert dit._should_compile(_ccfg(), True, "xpu", "nf4") is False
+ assert dit._should_compile(_ccfg(), True, "cpu", "nf4") is False
+
+
+def test_auto_keeps_a_short_nf4_run_eager(monkeypatch):
+ from dataclasses import replace
+
+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
+ short = replace(_ccfg(), train_steps = dit.NF4_COMPILE_MIN_STEPS - 1)
+ assert dit._should_compile(short, True, "cuda", "nf4") is False
+ assert (
+ dit._should_compile(replace(short, compile_transformer = "on"), True, "cuda", "nf4") is True
+ )
+ assert dit._should_compile(short, False, "cuda", "bf16") is True
+ at_floor = replace(_ccfg(), train_steps = dit.NF4_COMPILE_MIN_STEPS)
+ assert dit._should_compile(at_floor, True, "cuda", "nf4") is True
+
+
+ "torch_version, bnb_version, rocm, expected",
+ [
+ ("2.13.0+cu130", "0.50.2", False, True),
+ ("2.10.0+cu128", "0.46.1", False, True),
+ ("2.9.1+cu128", "0.50.2", False, False),
+ ("2.13.0", "0.45.5", False, False),
+ ("2.13.0", "0.46.0", False, False),
+ ("2.13.0+rocm7.0", "0.50.2", True, False),
+ ("2.11.0.dev20260101+cu130", "0.49.0.dev0", False, True),
+ ("2.10.0rc1", "0.50.2", False, False),
+ ("2.13.0", "0.46.1rc1", False, False),
+ ("2.10.0.dev20250901+cu128", "0.50.2", False, False),
+ ],
+)
+def test_nf4_compile_version_floor(monkeypatch, torch_version, bnb_version, rocm, expected):
+ import importlib.metadata
+
+ monkeypatch.setattr(torch, "__version__", torch_version)
+ monkeypatch.setattr(dit, "torch_is_rocm", lambda: rocm)
+ real = importlib.metadata.version
+ monkeypatch.setattr(
+ importlib.metadata,
+ "version",
+ lambda name: bnb_version if name == "bitsandbytes" else real(name),
+ )
+ assert dit._nf4_compile_supported() is expected
+
+
+def test_nf4_compile_gate_fails_closed_without_bitsandbytes(monkeypatch):
+ import importlib.metadata
+
+ def _missing(name):
+ raise importlib.metadata.PackageNotFoundError(name)
+
+ monkeypatch.setattr(importlib.metadata, "version", _missing)
+ assert dit._nf4_compile_supported() is False
+
+
+def _tiny_zimage():
+ diffusers = pytest.importorskip("diffusers")
+ peft = pytest.importorskip("peft")
+ torch.manual_seed(0)
+ model = diffusers.ZImageTransformer2DModel(
+ dim = 64,
+ n_layers = 4,
+ n_refiner_layers = 1,
+ n_heads = 2,
+ n_kv_heads = 2,
+ cap_feat_dim = 32,
+ axes_dims = [8, 12, 12],
+ axes_lens = [64, 32, 32],
+ )
+ model.requires_grad_(False)
+ model.add_adapter(
+ peft.LoraConfig(r = 4, lora_alpha = 4, init_lora_weights = False, target_modules = ["to_q", "to_v"])
+ )
+ return model
+
+
+def _lora_grads(model):
+ torch.manual_seed(1)
+ x, cap, t = [torch.randn(16, 1, 8, 8)], [torch.randn(6, 32)], torch.tensor([0.3])
+ model.train()
+ out = model(x, t, cap, return_dict = False)[0]
+ sum((o**2).mean() for o in out).backward()
+ return {n: p.grad.clone() for n, p in model.named_parameters() if p.requires_grad}
+
+
[email protected]("mode", ["plain", "partial"])
+def test_checkpoint_modes_match_uncheckpointed_grads(mode, monkeypatch):
+ monkeypatch.delenv(dtc.GC_MODE_ENV, raising = False)
+ reference = _lora_grads(_tiny_zimage())
+ model = _tiny_zimage()
+ calls = []
+ real_checkpoint = torch.utils.checkpoint.checkpoint
+ monkeypatch.setattr(
+ torch.utils.checkpoint,
+ "checkpoint",
+ lambda fn, *a, **k: calls.append(k.get("use_reentrant")) or real_checkpoint(fn, *a, **k),
+ )
+ assert dtc.enable_diffusion_gradient_checkpointing(model, mode) == mode
+ grads = _lora_grads(model)
+ assert grads.keys() == reference.keys()
+ for name in reference:
+ torch.testing.assert_close(grads[name], reference[name], rtol = 1e-5, atol = 1e-6)
+ # 6 ZImageTransformerBlocks (4 layers + 1 noise + 1 context refiner): partial skips every 2nd.
+ assert calls and set(calls) == {False}
+ assert len(calls) == (6 if mode == "plain" else 3)
+
+
+def _fake_zoo_helper(monkeypatch):
+ import sys
+ import types
+
+ called = []
+ module = types.ModuleType("unsloth_zoo.block_checkpointing")
+ module.enable_block_gradient_checkpointing = lambda model, every_n_blocks: called.append(
+ every_n_blocks
+ )
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.block_checkpointing", module)
+ return called
+
+
+def test_checkpoint_mode_comes_from_the_env_and_defaults_to_plain(monkeypatch):
+ called = _fake_zoo_helper(monkeypatch)
+ monkeypatch.delenv(dtc.GC_MODE_ENV, raising = False)
+ monkeypatch.delenv(dtc.GC_EVERY_N_ENV, raising = False)
+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == "plain"
+ for value, expected in (("partial", "partial"), ("PARTIAL ", "partial"), ("bogus", "plain")):
+ monkeypatch.setenv(dtc.GC_MODE_ENV, value)
+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == expected
+ monkeypatch.setenv(dtc.GC_MODE_ENV, "partial")
+ monkeypatch.setenv(dtc.GC_EVERY_N_ENV, "x")
+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == "partial"
+ monkeypatch.setenv(dtc.GC_EVERY_N_ENV, "4")
+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == "partial"
+ assert called == [1, 2, 2, 1, 2, 4]
+
+
+def test_checkpointing_falls_back_to_every_block_without_the_zoo_helper(monkeypatch):
+ import sys
+
+ seen = {}
+
+ class _Model(torch.nn.Module):
+ def enable_gradient_checkpointing(self, gradient_checkpointing_func = None):
+ seen["func"] = gradient_checkpointing_func
+
+ monkeypatch.setenv(dtc.GC_MODE_ENV, "partial")
+ monkeypatch.setitem(sys.modules, "unsloth_zoo.block_checkpointing", None)
+ assert dtc.enable_diffusion_gradient_checkpointing(_Model()) == "plain"
+ assert seen["func"].keywords == {"use_reentrant": False}
+
+
+class _FakeAccel:
+ def __init__(self, peak_bytes):
+ self.peak_bytes, self.resets = peak_bytes, 0
+
+ def reset_peak_memory_stats(self):
+ self.resets += 1
+
+ def max_memory_allocated(self):
+ return self.peak_bytes
+
+
[email protected]("device", ["cuda", "xpu"])
+def test_training_peak_is_reported_on_cuda_and_xpu(device, monkeypatch):
+ accel = _FakeAccel(3_456_000_000)
+ monkeypatch.setattr(torch, device, accel, raising = False)
+ dtc.reset_training_peak_memory(device)
+ assert accel.resets == 1
+ assert dtc.training_peak_memory_gb(device) == 3.46
+
+
[email protected]("device", ["cpu", "mps"])
+def test_training_peak_is_unmeasured_elsewhere(device):
+ dtc.reset_training_peak_memory(device)
+ assert dtc.training_peak_memory_gb(device) == 0.0
+
+
+def test_a_failing_peak_probe_reports_unmeasured(monkeypatch):
+ class _Broken(_FakeAccel):
+ def max_memory_allocated(self):
+ raise RuntimeError("no device")
+
+ def reset_peak_memory_stats(self):
+ raise RuntimeError("no device")
+
+ monkeypatch.setattr(torch, "xpu", _Broken(0), raising = False)
+ dtc.reset_training_peak_memory("xpu")
+ assert dtc.training_peak_memory_gb("xpu") == 0.0
+
+
+def test_a_preview_round_never_reads_as_training_peak(monkeypatch):
+ class _Accel(_FakeAccel):
+ def reset_peak_memory_stats(self):
+ self.resets += 1
+ self.peak_bytes = 0
+
+ accel = _Accel(0)
+ monkeypatch.setattr(torch, "xpu", accel, raising = False)
+ peak = dtc.TrainingPeakMemory("xpu")
+ accel.peak_bytes = 5_000_000_000 # training steps
+ peak.pause()
+ accel.peak_bytes = 9_000_000_000 # the preview's VAE decode
+ peak.resume()
+ accel.peak_bytes = 4_000_000_000 # training again
+ assert peak.gb() == 5.0
+ accel.peak_bytes = 6_000_000_000
+ assert peak.gb() == 6.0
+
+
+def test_a_short_resume_keeps_nf4_eager(monkeypatch):
+ from dataclasses import replace
+
+ import core.training.diffusion_checkpoint as ckpt
+
+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)
+ cfg = replace(_ccfg(), train_steps = 500, resume_from_checkpoint = "/runs/a/checkpoint-450")
+ monkeypatch.setattr(ckpt, "preflight_resume", lambda *a, **k: ("/runs/a/checkpoint-450", 450))
+ left = dtc.resume_steps_left(cfg, identity = None)
+ assert left == 50
+ assert dit._should_compile(cfg, True, "cuda", "nf4", left) is False
+ assert dit._should_compile(cfg, True, "cuda", "nf4") is True # a fresh 500-step run compiles
+ # An unreadable bundle keeps the target (restore_resume_state reports it).
+ monkeypatch.setattr(
+ ckpt, "preflight_resume", lambda *a, **k: (_ for _ in ()).throw(OSError("gone"))
+ )
+ assert dtc.resume_steps_left(cfg, identity = None) == 500
+ assert dtc.resume_steps_left(replace(cfg, resume_from_checkpoint = None), None) == 500
diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.py
index 2cf52062712..be44e90c03e 100644
--- a/studio/backend/tests/test_gpu_selection.py
+++ b/studio/backend/tests/test_gpu_selection.py
@@ -2690,10 +2690,12 @@ class TestXpuBitsandbytesOptimizerGate(unittest.TestCase):
self.assertTrue(self._probe()())
def test_diffusion_factories_skip_bnb_on_xpu_and_keep_it_elsewhere(self):
+ """torch AdamW everywhere; AdamW8bit only to resume its own bundle, never on XPU."""
import torch
import core.training.diffusion_dit_trainer as dit_mod
import core.training.diffusion_lora_trainer as lora_mod
+ import core.training.diffusion_train_common as common_mod
class _Bnb8bitMarker(torch.optim.AdamW):
"""Stands in for bnb.optim.AdamW8bit: constructs fine, dies at the first step
@@ -2711,32 +2713,32 @@ class TestXpuBitsandbytesOptimizerGate(unittest.TestCase):
("sdxl_lora", lora_mod, "_make_lora_optimizer"),
("dit", dit_mod, "_make_optimizer"),
)
+ resume_key = "bitsandbytes.optim.adamw.AdamW8bit"
for label, module, factory_name in cases:
factory = getattr(module, factory_name)
for on_xpu in (True, False):
- with self.subTest(trainer = label, xpu = on_xpu):
- param = torch.nn.Parameter(torch.zeros(2, 2))
- param.grad = torch.ones(2, 2)
- with (
- patch.dict(
- sys.modules,
- {"bitsandbytes": fake_bnb, "bitsandbytes.optim": fake_optim},
- ),
- patch.object(
- module,
- "bitsandbytes_optimizer_supported",
- lambda supported = not on_xpu: supported,
- ),
- ):
- optimizer = factory([param], 1e-4)
-
- if on_xpu:
- # Must not be the bnb optimizer, and must survive an actual step.
- self.assertNotIsInstance(optimizer, _Bnb8bitMarker)
- optimizer.step()
- else:
- # Unchanged off XPU: still the 8-bit optimizer, not a blanket disable.
- self.assertIsInstance(optimizer, _Bnb8bitMarker)
+ for resume in (None, resume_key):
+ with self.subTest(trainer = label, xpu = on_xpu, resume = resume):
+ param = torch.nn.Parameter(torch.zeros(2, 2))
+ param.grad = torch.ones(2, 2)
+ with (
+ patch.dict(
+ sys.modules,
+ {"bitsandbytes": fake_bnb, "bitsandbytes.optim": fake_optim},
+ ),
+ patch.object(
+ common_mod,
+ "bitsandbytes_optimizer_supported",
+ lambda supported = not on_xpu: supported,
+ ),
+ ):
+ optimizer = factory([param], 1e-4, resume)
+
+ if on_xpu or resume is None:
+ self.assertNotIsInstance(optimizer, _Bnb8bitMarker)
+ optimizer.step()
+ else:
+ self.assertIsInstance(optimizer, _Bnb8bitMarker)
class TestCliDefaultOptimizerFollowsTheDevicePolicy(unittest.TestCase):