Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: faster image LoRA training (fused AdamW, compile nf4 bases) (#13236)
Daniel Han9ef7ef32e379
diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.pyindex db77af2ce3e..4ca7af78ca3 100644--- a/studio/backend/core/training/diffusion_checkpoint.py+++ b/studio/backend/core/training/diffusion_checkpoint.py@@ -1817,10 +1817,10 @@ def _join_clauses(items: list[str]) -> str:def _assert_optimizer_buildable(path: Path, manifest: dict[str, Any]) -> None:"""Refuse moments this host provably cannot build an optimizer for, BEFORE teardown.- The trainers choose bitsandbytes AdamW8bit or torch AdamW from the HOST, so a bundle can arrive- with foreign moments, and the child's own check then fires after the route has evicted the- resident inference models and loaded a multi-GB base, for a run that is guaranteed to terminate- without training.+ Bundles from older builds carry bitsandbytes AdamW8bit moments, which the trainers can only+ restore into AdamW8bit. Without bitsandbytes the child's own check fires after the route has+ evicted the resident inference models and loaded a multi-GB base, for a run that is guaranteed+ to terminate without training.Deliberately one-directional, and deliberately import-free. ``find_spec`` answers "isbitsandbytes installed" without creating a CUDA context in the Unsloth process, but it cannotdiff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex 833aa2ed0c1..9cc9d418628 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -21,7 +21,7 @@ the VAE freed: the cache stores the posterior's affine parameters (mean/std foldfamily's latent normalisation), so every step still draws a fresh VAE sample --distribution-identical to encoding in the loop, without keeping the VAE resident or paying aper-step encode. The transformer trains as a QLoRA (nf4) adapter by default with gradient-checkpointing and 8-bit AdamW, so only the (small) LoRA params + optimizer state and the frozen+checkpointing and fused AdamW, so only the (small) LoRA params + optimizer state and the frozen4-bit base sit in VRAM during the loop."""@@ -46,7 +46,11 @@ from core.training.diffusion_train_common import (EventCb,LATENT_CACHE_OVER_BUDGET,StopCb,- bitsandbytes_optimizer_supported,+ enable_diffusion_gradient_checkpointing,+ TrainingPeakMemory,+ resume_steps_left,+ make_lora_optimizer,+ recorded_optimizer_class,_apply_perf_flags,_assert_trusted_base_model,_emit,@@ -1883,6 +1887,7 @@ def _should_compile(base_is_bnb,device,base_precision = "nf4",+ steps_left = None,) -> bool:mode = (cfg.compile_transformer or "auto").strip().lower()if device != "cuda" or mode == "off":@@ -1893,11 +1898,35 @@ def _should_compile(return Falseif mode == "on":return True- # Regional compile is the whole point of the dense modes (2.6x measured on Z-Image bf16) but is fragile over- # bitsandbytes 4-bit modules, so it stays off for QLoRA.+ if base_precision == "nf4":+ steps = cfg.train_steps if steps_left is None else steps_left+ return steps >= NF4_COMPILE_MIN_STEPS and _nf4_compile_supported()return base_precision in ("bf16", "fp8", "mxfp8")+# Lowest stack where bnb 4-bit ops are torch custom ops (0 graph breaks, loss matches eager); ROCm unmeasured, stays eager.+NF4_COMPILE_MIN_TORCH = "2.10"+NF4_COMPILE_MIN_BNB = "0.46.1"+# The nf4 compile only pays back after a few hundred steps, so a short run stays eager.+NF4_COMPILE_MIN_STEPS = 200+++def _nf4_compile_supported() -> bool:+ try:+ import importlib.metadata++ import torch+ from packaging.version import Version++ if torch_is_rocm():+ return False+ if Version(torch.__version__) < Version(NF4_COMPILE_MIN_TORCH):+ return False+ return Version(importlib.metadata.version("bitsandbytes")) >= Version(NF4_COMPILE_MIN_BNB)+ except Exception: # noqa: BLE001 -- unknown versions keep the historical eager path+ return False++def _maybe_compile_transformer(transformer,cfg,@@ -1905,12 +1934,13 @@ def _maybe_compile_transformer(device,on_event,base_precision = "nf4",+ steps_left = None,) -> bool:"""Regionally compile the transformer blocks (diffusers compile_repeated_blocks) after the LoRAis attached. Never fatal: a wrap failure falls back to eager with a warning event, anddynamo's suppress_errors keeps a frame that fails to COMPILE at the first step running eagerinstead of raising mid-run."""- if not _should_compile(cfg, base_is_bnb, device, base_precision):+ if not _should_compile(cfg, base_is_bnb, device, base_precision, steps_left):if base_precision in ("fp8", "mxfp8"):_emit(on_event,@@ -2242,13 +2272,7 @@ def _train_dit())if cfg.gradient_checkpointing:- # Non-reentrant checkpointing: reentrant recompute of a bnb 4-bit LoRA linear can trip an illegal memory- # access on the larger FLUX transformer, and it is the recommended mode anyway.- import functools- import torch.utils.checkpoint as _ckpt- transformer.enable_gradient_checkpointing(- gradient_checkpointing_func = functools.partial(_ckpt.checkpoint, use_reentrant = False)- )+ enable_diffusion_gradient_checkpointing(transformer)cast_training_params(transformer, dtype = torch.float32)lora_params = [p for p in transformer.parameters() if p.requires_grad]@@ -2265,8 +2289,15 @@ def _train_dit(# conversions so they track the final fp32 objects.ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None+ # A resume runs only the remaining steps, so the nf4 compile break-even is judged on those.compiled = _maybe_compile_transformer(- transformer, cfg, base_is_bnb, device, on_event, base_precision+ transformer,+ cfg,+ base_is_bnb,+ device,+ on_event,+ base_precision,+ resume_steps_left(cfg, identity),)# Compiled Qwen graphs need one fixed text length across steps: pin the pad bucket to the dataset's longest# caption.@@ -2274,7 +2305,9 @@ def _train_dit(if compiled and spec.family == "qwen-image":qwen_pad_to = max(e[0].shape[1] for e in caption_embeds.values())- optimizer = _make_optimizer(lora_params, cfg.learning_rate)+ optimizer = _make_optimizer(+ lora_params, cfg.learning_rate, recorded_optimizer_class(cfg, identity)+ )scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token)@@ -2474,6 +2507,7 @@ def _train_dit(if sample_plan is not None and resumed < cfg.train_steps:_sample(resumed)+ train_peak = TrainingPeakMemory(device)for opt_step in range(resumed, cfg.train_steps):if stop_latched:stopped = True@@ -2562,8 +2596,7 @@ def _train_dit(# Step 1 pays the one-time costs (cudnn autotune, compile warmup), so the reported rate starts after it.t_steady = nowif done % cfg.log_every == 0 or done == cfg.train_steps:- if device == "cuda":- peak_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)+ peak_gb = train_peak.gb()per_step = batch_size * cfg.gradient_accumulation_stepsif t_steady is not None and ran_here > 1:sps = round((ran_here - 1) * per_step / max(now - t_steady, 1e-6), 3)@@ -2583,7 +2616,9 @@ def _train_dit()stop_now = _check_stop()if not stop_now and sample_plan is not None and sample_plan.due(done, cfg.train_steps):+ train_peak.pause()_sample(done)+ train_peak.resume()stop_now = stop_latched# Skipped on the final step and when stopping, since the stop path writes one at the exact step.if (@@ -2657,31 +2692,12 @@ def _train_dit(return str(out_dir)-def _make_optimizer(params, lr):- """8-bit AdamW (bitsandbytes) when available -- half the optimizer state, no accuracy regression- for LoRA -- else torch AdamW, fused on CUDA (with a fallback when this build/device lacks the- fused kernel). UNSLOTH_DIFFUSION_FP32_OPTIM forces plain (non-fused) AdamW, as it does for- SDXL: the accuracy guard wants the reference optimizer, and a host where the override means- one thing for one trainer and nothing for the other cannot answer "can this checkpoint be- resumed here" before the run starts."""- import torch-- if os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM", "") in ("1", "true"):- return torch.optim.AdamW(params, lr = lr)- # Checked before construction, not around it: on XPU the 8-bit optimizer builds fine and- # only dies at the first step(), which the except below would never see.- if bitsandbytes_optimizer_supported():- try:- import bitsandbytes as bnb- return bnb.optim.AdamW8bit(params, lr = lr)- except Exception: # noqa: BLE001 -- bnb missing / no CUDA: fall back to torch AdamW- pass- if torch.cuda.is_available():- try:- return torch.optim.AdamW(params, lr = lr, fused = True)- except Exception: # noqa: BLE001 -- fused unsupported on this build/device- pass- return torch.optim.AdamW(params, lr = lr)+def _make_optimizer(+ params,+ lr,+ resume_optimizer_class = None,+):+ return make_lora_optimizer(params, lr, resume_optimizer_class)def _free_text_encoders(pipe) -> None:diff --git a/studio/backend/core/training/diffusion_h3_trainer.py b/studio/backend/core/training/diffusion_h3_trainer.pyindex 10cd7e82a7a..d233428fe62 100644--- a/studio/backend/core/training/diffusion_h3_trainer.py+++ b/studio/backend/core/training/diffusion_h3_trainer.py@@ -81,6 +81,9 @@ from core.training.diffusion_train_common import (_latent_cache_forced,_latent_cache_over_budget,_restore_perf_flags,+ enable_diffusion_gradient_checkpointing,+ reset_training_peak_memory,+ training_peak_memory_gb,h3_train_unsupported_reason,flow_bf16_trainable,native_bf16_supported_xpu,@@ -531,7 +534,11 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savfrom peft import LoraConfigfrom peft.utils import get_peft_model_state_dict- from core.training.diffusion_dit_trainer import _int8_quantize_base, _make_optimizer+ from core.training.diffusion_dit_trainer import (+ _int8_quantize_base,+ _make_optimizer,+ _maybe_compile_transformer,+ )out_dir = Path(cfg.output_dir).expanduser()clip_paths = [p for p, _ in pairs]@@ -638,11 +645,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav))if cfg.gradient_checkpointing:- import functools- import torch.utils.checkpoint as _ckpt- transformer.enable_gradient_checkpointing(- gradient_checkpointing_func = functools.partial(_ckpt.checkpoint, use_reentrant = False)- )+ enable_diffusion_gradient_checkpointing(transformer)cast_training_params(transformer, dtype = torch.float32)lora_params = [p for p in transformer.parameters() if p.requires_grad]if not lora_params:@@ -658,6 +661,10 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav_int8_quantize_base(transformer, cfg.resolved_family)ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None+ # dynamic=True keeps one graph per block across caption lengths; int8 stays eager like the DiT families.+ compiled = _maybe_compile_transformer(+ transformer, cfg, base_precision == "nf4", device, on_event, base_precision+ )optimizer = _make_optimizer(lora_params, cfg.learning_rate)lr_sched = get_scheduler(cfg.lr_scheduler,@@ -671,7 +678,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _sav_emit(on_event,"model_load_completed",- compiled = False,+ compiled = compiled,base_precision = base_precision,sequence_length = h3_packed_sequence_length(max(e.shape[1] for e in caption_embeds.values()), num_frames, height, width@@ -692,6 +699,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savt_start = time.time()t_steady = Nonedone = 0+ reset_training_peak_memory(device)for opt_step in range(cfg.train_steps):optimizer.zero_grad(set_to_none = True)step_loss = 0.0@@ -770,8 +778,7 @@ def _train_h3(cfg, pairs, rng, device, weight_dtype, on_event, _check_stop, _savif done == 1:t_steady = nowif done % cfg.log_every == 0 or done == cfg.train_steps:- if device == "cuda":- peak_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)+ peak_gb = training_peak_memory_gb(device)per_step = cfg.gradient_accumulation_stepsif t_steady is not None and done > 1:sps = round((done - 1) * per_step / max(now - t_steady, 1e-6), 3)diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.pyindex 65a4b098c6b..3ba83459fdb 100644--- a/studio/backend/core/training/diffusion_lora_trainer.py+++ b/studio/backend/core/training/diffusion_lora_trainer.py@@ -46,7 +46,10 @@ from core.training.diffusion_train_common import ( # noqa: F401StopCb,DiffusionLoraConfig,LATENT_CACHE_OVER_BUDGET,- bitsandbytes_optimizer_supported,+ enable_diffusion_gradient_checkpointing,+ TrainingPeakMemory,+ make_lora_optimizer,+ recorded_optimizer_class,native_bf16_supported,_apply_perf_flags,_assert_trusted_base_model,@@ -454,7 +457,7 @@ def run_diffusion_lora_training())if cfg.gradient_checkpointing:- unet.enable_gradient_checkpointing()+ enable_diffusion_gradient_checkpointing(unet)# LoRA params must be fp32 for a stable optimizer under mixed precision.if weight_dtype != torch.float32:cast_training_params(unet, dtype = torch.float32)@@ -468,7 +471,9 @@ def run_diffusion_lora_training()lora_params = [p for p in unet.parameters() if p.requires_grad]- optimizer = _make_lora_optimizer(lora_params, cfg.learning_rate)+ optimizer = _make_lora_optimizer(+ lora_params, cfg.learning_rate, recorded_optimizer_class(cfg, identity)+ )# A frozen fp16 UNet (pre-Ampere / RDNA2 fallback) underflows the LoRA gradients without loss scaling.scaler = torch.amp.GradScaler("cuda", enabled = weight_dtype is torch.float16)# The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the@@ -735,6 +740,7 @@ def run_diffusion_lora_training(if resumed < cfg.train_steps:_sample(resumed)+ train_peak = TrainingPeakMemory(device)for opt_step in range(resumed, cfg.train_steps):if stop_latched:stopped = True@@ -841,8 +847,7 @@ def run_diffusion_lora_training(t_steady = nowif done % cfg.log_every == 0 or done == cfg.train_steps:# ``learning_rate`` (not ``lr``) is the field the Unsloth training pump reads.- if device == "cuda":- peak_gb = round(torch.cuda.max_memory_allocated() / 1e9, 2)+ peak_gb = train_peak.gb()per_step = cfg.train_batch_size * cfg.gradient_accumulation_stepsif t_steady is not None and ran_here > 1:samples_per_second = round(@@ -865,7 +870,9 @@ def run_diffusion_lora_training(stop_now = _check_stop()if not stop_now and sample_plan is not None and sample_plan.due(done, cfg.train_steps):+ train_peak.pause()_sample(done)+ train_peak.resume()stop_now = stop_latched# Skipped on the final step and when stopping, since the stop path writes one at the exact step.if (@@ -937,29 +944,12 @@ def run_diffusion_lora_training(_restore_perf_flags(snap)-def _make_lora_optimizer(params: list, lr: float) -> Any:- """8-bit AdamW (bitsandbytes) by default -- half the optimizer state, no meaningful- quality cost for LoRA -- falling back to torch AdamW (fused on CUDA) when unavailable.- UNSLOTH_DIFFUSION_FP32_OPTIM forces plain (non-fused) AdamW: the accuracy guard wants the- reference optimizer, so it must not take the fused path."""- import torch-- if os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM", "") in ("1", "true"):- return torch.optim.AdamW(params, lr = lr)- # Checked before construction, not around it: on XPU the 8-bit optimizer builds fine and- # only dies at the first step(), which the except below would never see.- if bitsandbytes_optimizer_supported():- try:- import bitsandbytes as bnb- return bnb.optim.AdamW8bit(params, lr = lr)- except Exception: # noqa: BLE001 -- bnb missing / no CUDA: fall back to torch AdamW- pass- if torch.cuda.is_available():- try:- return torch.optim.AdamW(params, lr = lr, fused = True)- except Exception: # noqa: BLE001 -- fused unsupported on this build/device- pass- return torch.optim.AdamW(params, lr = lr)+def _make_lora_optimizer(+ params: list,+ lr: float,+ resume_optimizer_class = None,+) -> Any:+ return make_lora_optimizer(params, lr, resume_optimizer_class)def run_diffusion_training_process(*, event_queue: Any, stop_queue: Any, config: dict) -> None:diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex 3c67a344067..fe9bc913afa 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -824,8 +824,8 @@ def bitsandbytes_optimizer_supported() -> bool:Keyed on get_device(), NOT torch.xpu.is_available(): a hybrid host with an Intel iGPU besidean NVIDIA card reports both and detection prefers CUDA. Answering the presence question there- would drop 8-bit on a CUDA run, and worse, change optimizer_key() so restore_resume_state- refuses every existing AdamW8bit checkpoint with a ResumeError.+ would make make_lora_optimizer skip AdamW8bit, so restore_resume_state refuses every existing+ AdamW8bit checkpoint with a ResumeError."""try:from utils.hardware import DeviceType, get_device@@ -834,6 +834,139 @@ def bitsandbytes_optimizer_supported() -> bool:return True+def _is_bnb_8bit_optimizer_key(name: Optional[str]) -> bool:+ return isinstance(name, str) and name.startswith("bitsandbytes.") and "8bit" in name+++def recorded_optimizer_class(cfg: Any, identity: Any) -> Optional[str]:+ """Optimizer class of the bundle being resumed, else None; never raises (restore_resume_state reports errors)."""+ if not getattr(cfg, "resume_from_checkpoint", None):+ return None+ try:+ from core.training.diffusion_checkpoint import preflight_resume, read_checkpoint+ path, _step = preflight_resume(+ cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps+ )+ value = (read_checkpoint(path) or {}).get("optimizer_class")+ except Exception: # noqa: BLE001+ return None+ return value if isinstance(value, str) else None+++def resume_steps_left(cfg: Any, identity: Any) -> int:+ """Optimizer steps this process will run: the target minus the resumed bundle's step; the target when unknown."""+ if not getattr(cfg, "resume_from_checkpoint", None):+ return cfg.train_steps+ try:+ from core.training.diffusion_checkpoint import preflight_resume+ _path, step = preflight_resume(+ cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps+ )+ except Exception: # noqa: BLE001 -- restore_resume_state reports the real problem+ return cfg.train_steps+ return max(0, cfg.train_steps - int(step))+++class TrainingPeakMemory:+ """Training-only peak: ``pause`` banks it before a preview round, ``resume`` resets after, so VAE decode is excluded."""++ def __init__(self, device: str):+ self.device = device+ self.banked = 0.0+ reset_training_peak_memory(device)++ def pause(self) -> None:+ self.banked = max(self.banked, training_peak_memory_gb(self.device))++ def resume(self) -> None:+ reset_training_peak_memory(self.device)++ def gb(self) -> float:+ return max(self.banked, training_peak_memory_gb(self.device))+++def _accelerator_memory_module(device: str) -> Any:+ import torch++ module = getattr(torch, device, None) if device in ("cuda", "xpu") else None+ if module is None or not hasattr(module, "max_memory_allocated"):+ return None+ return module+++def reset_training_peak_memory(device: str) -> None:+ """Start the reported peak at the loop: the load / encode peak is higher and not the training cost."""+ module = _accelerator_memory_module(device)+ if module is not None:+ try:+ module.reset_peak_memory_stats()+ except Exception: # noqa: BLE001 -- reporting only+ pass+++def training_peak_memory_gb(device: str) -> float:+ """Peak allocated GB since reset_training_peak_memory on CUDA / XPU; 0.0 where unmeasured (reported as None)."""+ module = _accelerator_memory_module(device)+ if module is None:+ return 0.0+ try:+ return round(module.max_memory_allocated() / 1e9, 2)+ except Exception: # noqa: BLE001 -- reporting only+ return 0.0+++GC_MODE_ENV = "UNSLOTH_DIFFUSION_GC_MODE"+GC_EVERY_N_ENV = "UNSLOTH_DIFFUSION_GC_EVERY_N"+++def enable_diffusion_gradient_checkpointing(model: Any, mode: Optional[str] = None) -> str:+ """ "plain" checkpoints every block, "partial" every Nth; selective / CPU-offloaded were slower or diverged under regional compile."""+ mode = (mode or os.environ.get(GC_MODE_ENV, "") or "plain").strip().lower()+ every_n = 1+ if mode == "partial":+ try:+ every_n = max(2, int(os.environ.get(GC_EVERY_N_ENV, "") or 2))+ except ValueError:+ every_n = 2+ try:+ from unsloth_zoo.block_checkpointing import enable_block_gradient_checkpointing+ except ImportError:+ import functools++ import torch.utils.checkpoint as _ckpt++ model.enable_gradient_checkpointing(+ gradient_checkpointing_func = functools.partial(_ckpt.checkpoint, use_reentrant = False)+ )+ return "plain"+ enable_block_gradient_checkpointing(model, every_n_blocks = every_n)+ return "partial" if every_n > 1 else "plain"+++def make_lora_optimizer(+ params: list,+ lr: float,+ resume_optimizer_class: Optional[str] = None,+) -> Any:+ """torch AdamW (AdamW8bit step() syncs per parameter); AdamW8bit only to resume its own bundle, never on XPU (dies at step())."""+ import torch++ if os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM", "") in ("1", "true"):+ return torch.optim.AdamW(params, lr = lr)+ if _is_bnb_8bit_optimizer_key(resume_optimizer_class) and bitsandbytes_optimizer_supported():+ try:+ import bitsandbytes as bnb+ return bnb.optim.AdamW8bit(params, lr = lr)+ except Exception: # noqa: BLE001 -- restore_resume_state then names the mismatch+ pass+ if torch.cuda.is_available():+ try:+ return torch.optim.AdamW(params, lr = lr, fused = True)+ except Exception: # noqa: BLE001 -- fused unsupported on this build/device+ pass+ return torch.optim.AdamW(params, lr = lr)++def training_precision_preflight_error(resolved_family: str, base_precision: str) -> Optional[str]:"""Reason the requested DiT precision cannot run on this host, else None -- checked by the startroute BEFORE evicting resident GPU workloads (the trainer's own checks fire only in the@@ -929,8 +1062,7 @@ def family_train_infos() -> list[dict[str, Any]]:if is_ditelse None)- # H3 refuses compile_transformer="on" (its packed sequence changes length with every caption, so each step- # would re-trace), so advertising the control would offer a selection that always 400s.+ # H3 compiles under the auto rule; the panel control stays DiT / SDXL only.supports_compile = bool(not dit_block) and name in _DIT_TRAIN_FAMILIESif not is_dit or dit_block:fam_modes: list[str] = []@@ -1603,14 +1735,6 @@ def h3_train_unsupported_reason(cfg: Any) -> Optional[str]:"row layout is set by the clip's own geometry and its caption's length. Use ""gradient_accumulation_steps to raise the effective batch.")- # torch.compile is never invoked here (the packed layout changes shape with every caption length), so an explicit- # "on" would be accepted and then ignored.- if str(getattr(cfg, "compile_transformer", "auto") or "auto").strip().lower() == "on":- return (- "MiniMax-H3 does not compile: its packed sequence changes length with every clip's "- "caption, so torch.compile would re-trace each step. Use compile_transformer "- "'off' or 'auto'."- )# No conditioning cache exists on this path, so accepting the directory would promise a saving that never happens.if str(getattr(cfg, "cond_cache_dir", "") or "").strip():return (@@ -2083,6 +2207,9 @@ def _reapply_lr_schedule(optimizer: Any, lr_scheduler: Any) -> None:pass+_OPTIMIZER_KERNEL_KEYS = ("fused", "foreach")++def restore_resume_state(cfg: DiffusionLoraConfig,*,@@ -2118,9 +2245,7 @@ def restore_resume_state(load_trainable_state_dict(model, ckpt.tensors("adapter"))optimizer_state = ckpt.torch_state("optimizer")if optimizer_state is not None:- # The trainers pick their optimizer from the HOST (bnb present, fused kernel available,- # UNSLOTH_DIFFUSION_FP32_OPTIM), so foreign moments arrive legitimately (state1/state2 versus- # exp_avg/exp_avg_sq): shapes match, load_state_dict accepts them, and the first step dies on a bare KeyError.+ # Foreign moments (AdamW8bit vs torch AdamW) load cleanly, then the first step dies on a bare KeyError.saved_optimizer = ckpt.optimizer_classlive_optimizer = optimizer_key(optimizer)if not saved_optimizer:@@ -2147,9 +2272,22 @@ def restore_resume_state("than this build produces, so its moments cannot be matched to this run's ""tensors. Start a new run, or resume on the version that wrote it.")- # load_state_dict replaces the param groups too, so the checkpoint's learning rate wins over a changed cfg,- # the same semantics as HF Trainer's resume.+ # Checkpoint lr wins (HF Trainer semantics); the fused/foreach kernel choice stays this host's.+ kernels = [+ {k: g[k] for k in _OPTIMIZER_KERNEL_KEYS if k in g} for g in optimizer.param_groups+ ]+ saved_groups = (+ optimizer_state.get("param_groups") if isinstance(optimizer_state, dict) else None+ )+ if isinstance(saved_groups, list) and len(saved_groups) == len(kernels):+ # torch places "step" by the SAVED fused flag: non-fused bundles keep it on CPU, which fused rejects.+ optimizer_state = {+ **optimizer_state,+ "param_groups": [{**g, **kept} for g, kept in zip(saved_groups, kernels)],+ }optimizer.load_state_dict(optimizer_state)+ for group, kept in zip(optimizer.param_groups, kernels):+ group.update(kept)elif optimizer is not None:# Every bundle this writer produces has an optimizer, so continuing without one restarts Adam's moments from# zero at step N while reporting a clean resume.diff --git a/studio/backend/tests/test_diffusion_checkpoint_resume.py b/studio/backend/tests/test_diffusion_checkpoint_resume.pyindex de06fe3100b..81e1187e463 100644--- a/studio/backend/tests/test_diffusion_checkpoint_resume.py+++ b/studio/backend/tests/test_diffusion_checkpoint_resume.py@@ -2121,22 +2121,21 @@ def test_a_first_periodic_save_does_not_spend_the_previous_runs_bundles(run_dir)assert "discard_existing = False," in source, name-def test_both_trainers_honour_the_fp32_optimizer_override(run_dir):+def test_both_trainers_honour_the_fp32_optimizer_override(run_dir, monkeypatch):"""The preflight refuses 8-bit moments when the override is set, which is only sound ifevery trainer actually obeys it -- otherwise DiT checkpoints written on this host become- unresumable on the same host."""- trainers = Path(dc.__file__).parent- for name in ("diffusion_lora_trainer.py", "diffusion_dit_trainer.py"):- source = (trainers / name).read_text(encoding = "utf-8")- marker = source.find("def _make_optimizer")- if marker < 0:- marker = source.find("def _make_lora_optimizer")- assert marker > 0, name- body = source[marker : marker + 1200]- # The env READ, not a mention of it in prose.- read = 'os.environ.get("UNSLOTH_DIFFUSION_FP32_OPTIM"'- assert read in body, name- assert body.index(read) < body.index("AdamW8bit"), name+ unresumable on the same host. Asked of the factories themselves, including for a resume of+ an 8-bit bundle, which is the one case that otherwise builds AdamW8bit."""+ import core.training.diffusion_dit_trainer as dit+ import core.training.diffusion_lora_trainer as sdxl++ monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")+ for factory in (sdxl._make_lora_optimizer, dit._make_optimizer):+ for resume in (None, "bitsandbytes.optim.adamw.AdamW8bit"):+ param = torch.nn.Parameter(torch.zeros(2, 2))+ optimizer = factory([param], 1e-4, resume)+ assert type(optimizer) is torch.optim.AdamW, factory+ assert not optimizer.defaults.get("fused"), factorydef test_a_completed_run_does_not_leave_its_periodic_bundles_behind(run_dir):diff --git a/studio/backend/tests/test_diffusion_h3_trainer.py b/studio/backend/tests/test_diffusion_h3_trainer.pyindex f8e1c729641..09d8aa28aaf 100644--- a/studio/backend/tests/test_diffusion_h3_trainer.py+++ b/studio/backend/tests/test_diffusion_h3_trainer.py@@ -1090,7 +1090,6 @@ def test_the_knobs_h3_cannot_honour_are_refused_or_normalised():refused = {"mixed_precision": ("no", "requires bf16"),"train_batch_size": (2, "batch size 1"),- "compile_transformer": ("on", "does not compile"),"cond_cache_dir": ("/tmp/cond", "conditioning cache"),"cfg_dropout": (0.1, "guidance-distilled"),"weighting_scheme": ("bell", "timestep-weighted"),@@ -1171,7 +1170,6 @@ def test_h3_is_advertised_as_trainable_with_the_precisions_it_has():# The case the bug was: a host that CAN train the DiT families must be able to train H3.if reference["precision_modes"]:assert h3["precision_modes"], "an empty list disables Start in the Train panel"- # compile is the one DiT lever that must NOT follow: "on" is refused by the trainer.assert h3["supports_compile"] is Falseassert infos["sdxl"]["supports_compile"] is Trueif reference["precision_modes"]:@@ -1885,3 +1883,104 @@ def test_the_advertised_h3_vram_floor_covers_the_measured_run_peak():the chip is what a user sizes a host from."""from core.training.diffusion_train_common import _FAMILY_TRAIN_SPECSassert _FAMILY_TRAIN_SPECS["minimax-h3"]["qlora_vram_gb"] >= 78+++def test_h3_accepts_every_compile_mode():+ """dynamic=True compiles one graph per block across caption lengths, so "on" is no longer refused."""+ from dataclasses import replace as _replace++ from core.training.diffusion_train_common import h3_train_unsupported_reason++ base = _h3_cfg().normalized()+ for mode in ("off", "on", "auto"):+ assert h3_train_unsupported_reason(_replace(base, compile_transformer = mode)) is None, mode++[email protected]("precision, expect_bnb", [("bf16", False), ("nf4", True), ("int8", False)])+def test_h3_routes_its_transformer_through_the_shared_regional_compile(+ precision, expect_bnb, monkeypatch, tmp_path+):+ """H3 hands its transformer to the DiT compile wrapper after int8 conversion, before the optimizer."""+ import torch++ import core.training.diffusion_dit_trainer as dit+ import core.training.diffusion_h3_trainer as diffusion_h3_trainer+ from core.training.diffusion_h3_trainer import _train_h3++ order: list[str] = []++ class _Tiny(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(4, 4)++ def add_adapter(self, _config):+ self.lora = torch.nn.Linear(4, 4)++ class _Pipe:+ text_encoder = processor = vae = audio_vae = None++ seen = {}++ class _Stop(Exception):+ pass++ def _compile(transformer, cfg, base_is_bnb, device, on_event, base_precision):+ order.append("compile")+ seen.update(base_is_bnb = base_is_bnb, base_precision = base_precision)+ return False++ def _optimizer(*_a, **_k):+ order.append("optimizer")+ raise _Stop++ monkeypatch.setattr(diffusion_h3_trainer, "_load_conditioners", lambda cfg, device: _Pipe())+ monkeypatch.setattr(+ diffusion_h3_trainer, "_encode_prompt", lambda pipe, cap, device: torch.zeros(1, 3, 8)+ )+ monkeypatch.setattr(diffusion_h3_trainer, "_dataset_canvas", lambda path, short: (64, 64))+ monkeypatch.setattr(diffusion_h3_trainer, "decode_clip", lambda path, **k: (None, None))+ lf = h3_video_latent_frames(H3_TRAIN_NUM_FRAMES)+ na = h3_audio_latent_count(H3_TRAIN_NUM_FRAMES)+ monkeypatch.setattr(+ diffusion_h3_trainer,+ "_encode_video_stats",+ lambda vae, frames, device: (torch.zeros(1, 24, lf, 4, 4), torch.zeros(1, 24, lf, 4, 4)),+ )+ monkeypatch.setattr(+ diffusion_h3_trainer,+ "_encode_audio_latents",+ lambda avae, wav, device: torch.zeros(2, 32, na),+ )+ monkeypatch.setattr(+ diffusion_h3_trainer, "_load_transformer", lambda cfg, device, precision: _Tiny()+ )+ monkeypatch.setattr(+ dit, "_int8_quantize_base", lambda transformer, family: order.append("int8")+ )+ monkeypatch.setattr(dit, "_maybe_compile_transformer", _compile)+ monkeypatch.setattr(dit, "_make_optimizer", _optimizer)++ cfg = _replace_cfg(_h3_cfg(base_precision = precision, gradient_checkpointing = False))+ with pytest.raises(_Stop):+ _train_h3(+ cfg,+ [("clip.mp4", "a caption")],+ random.Random(0),+ "cpu",+ torch.float32,+ lambda *_a, **_k: None,+ lambda: False,+ lambda: True,+ )+ assert seen == {"base_is_bnb": expect_bnb, "base_precision": precision}+ assert order == (["int8"] if precision == "int8" else []) + ["compile", "optimizer"]+++def _replace_cfg(cfg):+ from dataclasses import replace as _replace++ from core.training.diffusion_train_common import train_recipe_overrides++ cfg = cfg.normalized()+ return _replace(cfg, **train_recipe_overrides(cfg))diff --git a/studio/backend/tests/test_diffusion_train_perf.py b/studio/backend/tests/test_diffusion_train_perf.pyindex ff72ff64ec0..a03add4322b 100644--- a/studio/backend/tests/test_diffusion_train_perf.py+++ b/studio/backend/tests/test_diffusion_train_perf.py@@ -15,6 +15,7 @@ inject in-thread fakes exactly like ``test_diffusion_training.py``.from __future__ import annotationsimport itertools+from unittest.mock import patchfrom pathlib import Path@@ -225,8 +226,12 @@ def test_should_compile_policy():assert _should_compile(_cfg(compile_transformer = "off"), False, "cuda") is False# on always compiles on cuda.assert _should_compile(_cfg(compile_transformer = "on"), False, "cuda") is True- # auto stays off over a bitsandbytes base (graph breaks in the dequant path).- assert _should_compile(_cfg(compile_transformer = "auto"), True, "cuda") is False+ # auto over an nf4 base follows the torch / bitsandbytes version gate.+ import core.training.diffusion_dit_trainer as dit++ for supported in (True, False):+ with patch.object(dit, "_nf4_compile_supported", lambda s = supported: s):+ assert _should_compile(_cfg(compile_transformer = "auto"), True, "cuda") is supported# auto turns on for the dense bf16 base precision on cuda.assert (_should_compile(_cfg(compile_transformer = "auto"), False, "cuda", base_precision = "bf16")diff --git a/studio/backend/tests/test_diffusion_train_speed.py b/studio/backend/tests/test_diffusion_train_speed.pynew file mode 100644index 00000000000..183af280e34--- /dev/null+++ b/studio/backend/tests/test_diffusion_train_speed.py@@ -0,0 +1,512 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Diffusion trainer speed defaults: optimizer factory, nf4 compile gate, checkpointing modes."""++from __future__ import annotations++import importlib.util+import json+import random+import sys+from pathlib import Path+from types import ModuleType++import pytest+import torch++import core.training.diffusion_dit_trainer as dit+import core.training.diffusion_lora_trainer as sdxl+import core.training.diffusion_train_common as dtc+from core.training import diffusion_checkpoint as dc+from core.training.diffusion_train_common import (+ DiffusionLoraConfig,+ PermutationBatchSampler,+ make_lora_optimizer,+ recorded_optimizer_class,+ restore_resume_state,+ write_resume_checkpoint,+)++BNB_KEY = "bitsandbytes.optim.adamw.AdamW8bit"+++class _Fake8bit(torch.optim.AdamW):+ """Stands in for bnb.optim.AdamW8bit."""+++def fake_bnb(monkeypatch):+ mod = ModuleType("bitsandbytes")+ optim = ModuleType("bitsandbytes.optim")+ optim.AdamW8bit = _Fake8bit+ mod.optim = optim+ monkeypatch.setitem(sys.modules, "bitsandbytes", mod)+ monkeypatch.setitem(sys.modules, "bitsandbytes.optim", optim)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)+ monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: True)+ return mod+++def _params():+ p = torch.nn.Parameter(torch.zeros(2, 2))+ p.grad = torch.ones(2, 2)+ return [p]++[email protected]("factory", [dit._make_optimizer, sdxl._make_lora_optimizer])+def test_fresh_run_builds_torch_adamw_even_with_bitsandbytes(fake_bnb, factory):+ opt = factory(_params(), 1e-4)+ assert type(opt) is torch.optim.AdamW+ opt.step()++[email protected]("factory", [dit._make_optimizer, sdxl._make_lora_optimizer])+def test_resuming_an_8bit_bundle_rebuilds_adamw8bit(fake_bnb, factory):+ opt = factory(_params(), 1e-4, BNB_KEY)+ assert isinstance(opt, _Fake8bit)+ assert type(factory(_params(), 1e-4, "torch.optim.adamw.AdamW")) is torch.optim.AdamW+ assert type(factory(_params(), 1e-4, None)) is torch.optim.AdamW+++def test_8bit_resume_on_xpu_falls_back_to_torch_adamw(fake_bnb, monkeypatch):+ monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: False)+ opt = make_lora_optimizer(_params(), 1e-4, BNB_KEY)+ assert type(opt) is torch.optim.AdamW+ opt.step()+++def test_fp32_override_still_forces_plain_adamw(fake_bnb, monkeypatch):+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")+ for resume in (None, BNB_KEY):+ opt = make_lora_optimizer(_params(), 1e-4, resume)+ assert type(opt) is torch.optim.AdamW+ assert not opt.defaults.get("fused")++[email protected](not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")+def test_cuda_default_is_fused(monkeypatch):+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)+ p = torch.nn.Parameter(torch.zeros(4, 4, device = "cuda"))+ opt = make_lora_optimizer([p], 1e-4)+ assert type(opt) is torch.optim.AdamW and opt.defaults.get("fused") is True+++def _identity() -> dc.CheckpointIdentity:+ return dc.CheckpointIdentity(+ family = "sdxl",+ base_model = "stabilityai/sdxl-turbo",+ lora_target_modules = ("to_k", "to_q", "to_v", "to_out.0"),+ lora_rank = 16,+ lora_alpha = 16,+ precision = "bf16",+ base_precision = "nf4",+ resolution = 1024,+ base_revision = "rev-deadbeef",+ dataset_fingerprint = "ds-3-cafe",+ )+++def _cfg(out: Path, **kw) -> DiffusionLoraConfig:+ return DiffusionLoraConfig(+ base_model = "stabilityai/sdxl-turbo",+ data_dir = "unused",+ output_dir = str(out),+ train_steps = 500,+ **kw,+ )+++def run_dir():+ from utils.paths import outputs_root++ d = outputs_root() / "speed-run"+ d.mkdir(parents = True, exist_ok = True)+ return d+++def _write(+ out: Path,+ model,+ optimizer,+ step: int = 3,+):+ sched = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda s: 1.0)+ written, error = write_resume_checkpoint(+ _cfg(out),+ step = step,+ model = model,+ optimizer = optimizer,+ lr_scheduler = sched,+ identity = _identity(),+ sampler = PermutationBatchSampler(7, random.Random(0)),+ rng_streams = {"loop": random.Random(0), "variant": random.Random(1)},+ )+ assert error is None and written+ return sched+++def test_recorded_optimizer_class_reads_the_bundle_being_resumed(run_dir, monkeypatch):+ # The resume preflight refuses an 8-bit bundle on a host without bitsandbytes, so pretend it is installed.+ real_find_spec = importlib.util.find_spec+ monkeypatch.setattr(+ importlib.util,+ "find_spec",+ lambda name, *a, **k: object() if name == "bitsandbytes" else real_find_spec(name, *a, **k),+ )+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)+ tmp_path = run_dir+ model = torch.nn.Linear(4, 4, bias = False)+ _write(tmp_path, model, torch.optim.AdamW(model.parameters(), lr = 1e-3))+ cfg = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "checkpoint-3"))+ assert recorded_optimizer_class(cfg, _identity()) == "torch.optim.adamw.AdamW"+ manifest_path = tmp_path / "checkpoint-3" / dc.TRAINER_STATE_FILENAME+ manifest = json.loads(manifest_path.read_text(encoding = "utf-8"))+ manifest["optimizer_class"] = BNB_KEY+ manifest_path.write_text(json.dumps(manifest), encoding = "utf-8")+ assert recorded_optimizer_class(cfg, _identity()) == BNB_KEY+ assert recorded_optimizer_class(_cfg(tmp_path), _identity()) is None+ bogus = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "nope"))+ assert recorded_optimizer_class(bogus, _identity()) is None++[email protected](not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")+def test_a_fused_bundle_resumed_under_the_fp32_override_stays_non_fused(run_dir, monkeypatch):+ # load_state_dict restores the saved fused flag; the reference-optimizer override must survive it.+ model = torch.nn.Linear(4, 4, bias = False).cuda()+ fused = torch.optim.AdamW(model.parameters(), lr = 1e-3, fused = True)+ model.weight.grad = torch.ones_like(model.weight)+ fused.step()+ _write(run_dir, model, fused)+ monkeypatch.setenv("UNSLOTH_DIFFUSION_FP32_OPTIM", "1")+ live = torch.nn.Linear(4, 4, bias = False).cuda()+ opt = make_lora_optimizer(list(live.parameters()), 1e-3)+ assert not opt.param_groups[0].get("fused")+ restore_resume_state(+ _cfg(run_dir, resume_from_checkpoint = str(run_dir / "checkpoint-3")),+ model = live,+ optimizer = opt,+ lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),+ identity = _identity(),+ )+ assert not opt.param_groups[0].get("fused")+ assert opt.state[live.weight]["exp_avg"].abs().sum() > 0++[email protected](not torch.cuda.is_available(), reason = "fused AdamW needs CUDA")+def test_a_non_fused_bundle_resumes_into_fused_adamw(run_dir, monkeypatch):+ # A plain AdamW bundle keeps its step on the CPU; the fused kernel needs it on the param's device.+ torch.manual_seed(0)+ model = torch.nn.Linear(4, 4, bias = False).cuda()+ plain = torch.optim.AdamW(model.parameters(), lr = 1e-3)+ model.weight.grad = torch.ones_like(model.weight)+ plain.step()+ _write(run_dir, model, plain)+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)+ live = torch.nn.Linear(4, 4, bias = False).cuda()+ opt = make_lora_optimizer(list(live.parameters()), 1e-3)+ assert opt.param_groups[0].get("fused")+ restore_resume_state(+ _cfg(run_dir, resume_from_checkpoint = str(run_dir / "checkpoint-3")),+ model = live,+ optimizer = opt,+ lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),+ identity = _identity(),+ )+ assert opt.param_groups[0].get("fused")+ live.weight.grad = torch.ones_like(live.weight)+ opt.step()+ assert opt.state[live.weight]["step"].item() == 2++[email protected](not torch.cuda.is_available(), reason = "bitsandbytes AdamW8bit needs CUDA")+def test_old_8bit_bundle_resumes_and_continues_identically(run_dir, monkeypatch):+ tmp_path = run_dir+ bnb = pytest.importorskip("bitsandbytes")+ if not hasattr(getattr(bnb, "optim", None), "AdamW8bit"):+ pytest.skip("bitsandbytes is stubbed out in this environment")+ monkeypatch.delenv("UNSLOTH_DIFFUSION_FP32_OPTIM", raising = False)+ monkeypatch.setattr(dtc, "bitsandbytes_optimizer_supported", lambda: True)++ def _model():+ torch.manual_seed(0)+ return torch.nn.Linear(128, 64, bias = False).cuda()++ def _step(model, opt, i):+ opt.zero_grad(set_to_none = True)+ model.weight.grad = torch.full_like(model.weight, 0.01 * (i + 1))+ opt.step()++ ref = _model()+ ref_opt = bnb.optim.AdamW8bit(ref.parameters(), lr = 1e-3)+ for i in range(3):+ _step(ref, ref_opt, i)+ _write(tmp_path, ref, ref_opt)++ cfg = _cfg(tmp_path, resume_from_checkpoint = str(tmp_path / "checkpoint-3"))+ live = _model()+ opt = make_lora_optimizer(+ list(live.parameters()), 1e-3, recorded_optimizer_class(cfg, _identity())+ )+ assert dc.optimizer_key(opt) == BNB_KEY+ restored = restore_resume_state(+ cfg,+ model = live,+ optimizer = opt,+ lr_scheduler = torch.optim.lr_scheduler.LambdaLR(opt, lambda s: 1.0),+ identity = _identity(),+ )+ assert restored.step == 3+ for i in range(3, 6):+ _step(ref, ref_opt, i)+ _step(live, opt, i)+ assert torch.equal(live.weight, ref.weight)+++def _ccfg(mode = "auto"):+ return DiffusionLoraConfig(+ base_model = "Tongyi-MAI/Z-Image",+ data_dir = "unused",+ output_dir = "unused",+ compile_transformer = mode,+ )+++def test_auto_compiles_nf4_only_when_the_versions_are_supported(monkeypatch):+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)+ assert dit._should_compile(_ccfg(), True, "cuda", "nf4") is True+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: False)+ assert dit._should_compile(_ccfg(), True, "cuda", "nf4") is False+ assert dit._should_compile(_ccfg("on"), True, "cuda", "nf4") is True+ assert dit._should_compile(_ccfg("off"), True, "cuda", "nf4") is False+ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)+ assert dit._should_compile(_ccfg(), True, "xpu", "nf4") is False+ assert dit._should_compile(_ccfg(), True, "cpu", "nf4") is False+++def test_auto_keeps_a_short_nf4_run_eager(monkeypatch):+ from dataclasses import replace++ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)+ short = replace(_ccfg(), train_steps = dit.NF4_COMPILE_MIN_STEPS - 1)+ assert dit._should_compile(short, True, "cuda", "nf4") is False+ assert (+ dit._should_compile(replace(short, compile_transformer = "on"), True, "cuda", "nf4") is True+ )+ assert dit._should_compile(short, False, "cuda", "bf16") is True+ at_floor = replace(_ccfg(), train_steps = dit.NF4_COMPILE_MIN_STEPS)+ assert dit._should_compile(at_floor, True, "cuda", "nf4") is True+++ "torch_version, bnb_version, rocm, expected",+ [+ ("2.13.0+cu130", "0.50.2", False, True),+ ("2.10.0+cu128", "0.46.1", False, True),+ ("2.9.1+cu128", "0.50.2", False, False),+ ("2.13.0", "0.45.5", False, False),+ ("2.13.0", "0.46.0", False, False),+ ("2.13.0+rocm7.0", "0.50.2", True, False),+ ("2.11.0.dev20260101+cu130", "0.49.0.dev0", False, True),+ ("2.10.0rc1", "0.50.2", False, False),+ ("2.13.0", "0.46.1rc1", False, False),+ ("2.10.0.dev20250901+cu128", "0.50.2", False, False),+ ],+)+def test_nf4_compile_version_floor(monkeypatch, torch_version, bnb_version, rocm, expected):+ import importlib.metadata++ monkeypatch.setattr(torch, "__version__", torch_version)+ monkeypatch.setattr(dit, "torch_is_rocm", lambda: rocm)+ real = importlib.metadata.version+ monkeypatch.setattr(+ importlib.metadata,+ "version",+ lambda name: bnb_version if name == "bitsandbytes" else real(name),+ )+ assert dit._nf4_compile_supported() is expected+++def test_nf4_compile_gate_fails_closed_without_bitsandbytes(monkeypatch):+ import importlib.metadata++ def _missing(name):+ raise importlib.metadata.PackageNotFoundError(name)++ monkeypatch.setattr(importlib.metadata, "version", _missing)+ assert dit._nf4_compile_supported() is False+++def _tiny_zimage():+ diffusers = pytest.importorskip("diffusers")+ peft = pytest.importorskip("peft")+ torch.manual_seed(0)+ model = diffusers.ZImageTransformer2DModel(+ dim = 64,+ n_layers = 4,+ n_refiner_layers = 1,+ n_heads = 2,+ n_kv_heads = 2,+ cap_feat_dim = 32,+ axes_dims = [8, 12, 12],+ axes_lens = [64, 32, 32],+ )+ model.requires_grad_(False)+ model.add_adapter(+ peft.LoraConfig(r = 4, lora_alpha = 4, init_lora_weights = False, target_modules = ["to_q", "to_v"])+ )+ return model+++def _lora_grads(model):+ torch.manual_seed(1)+ x, cap, t = [torch.randn(16, 1, 8, 8)], [torch.randn(6, 32)], torch.tensor([0.3])+ model.train()+ out = model(x, t, cap, return_dict = False)[0]+ sum((o**2).mean() for o in out).backward()+ return {n: p.grad.clone() for n, p in model.named_parameters() if p.requires_grad}++[email protected]("mode", ["plain", "partial"])+def test_checkpoint_modes_match_uncheckpointed_grads(mode, monkeypatch):+ monkeypatch.delenv(dtc.GC_MODE_ENV, raising = False)+ reference = _lora_grads(_tiny_zimage())+ model = _tiny_zimage()+ calls = []+ real_checkpoint = torch.utils.checkpoint.checkpoint+ monkeypatch.setattr(+ torch.utils.checkpoint,+ "checkpoint",+ lambda fn, *a, **k: calls.append(k.get("use_reentrant")) or real_checkpoint(fn, *a, **k),+ )+ assert dtc.enable_diffusion_gradient_checkpointing(model, mode) == mode+ grads = _lora_grads(model)+ assert grads.keys() == reference.keys()+ for name in reference:+ torch.testing.assert_close(grads[name], reference[name], rtol = 1e-5, atol = 1e-6)+ # 6 ZImageTransformerBlocks (4 layers + 1 noise + 1 context refiner): partial skips every 2nd.+ assert calls and set(calls) == {False}+ assert len(calls) == (6 if mode == "plain" else 3)+++def _fake_zoo_helper(monkeypatch):+ import sys+ import types++ called = []+ module = types.ModuleType("unsloth_zoo.block_checkpointing")+ module.enable_block_gradient_checkpointing = lambda model, every_n_blocks: called.append(+ every_n_blocks+ )+ monkeypatch.setitem(sys.modules, "unsloth_zoo.block_checkpointing", module)+ return called+++def test_checkpoint_mode_comes_from_the_env_and_defaults_to_plain(monkeypatch):+ called = _fake_zoo_helper(monkeypatch)+ monkeypatch.delenv(dtc.GC_MODE_ENV, raising = False)+ monkeypatch.delenv(dtc.GC_EVERY_N_ENV, raising = False)+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == "plain"+ for value, expected in (("partial", "partial"), ("PARTIAL ", "partial"), ("bogus", "plain")):+ monkeypatch.setenv(dtc.GC_MODE_ENV, value)+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == expected+ monkeypatch.setenv(dtc.GC_MODE_ENV, "partial")+ monkeypatch.setenv(dtc.GC_EVERY_N_ENV, "x")+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == "partial"+ monkeypatch.setenv(dtc.GC_EVERY_N_ENV, "4")+ assert dtc.enable_diffusion_gradient_checkpointing(object()) == "partial"+ assert called == [1, 2, 2, 1, 2, 4]+++def test_checkpointing_falls_back_to_every_block_without_the_zoo_helper(monkeypatch):+ import sys++ seen = {}++ class _Model(torch.nn.Module):+ def enable_gradient_checkpointing(self, gradient_checkpointing_func = None):+ seen["func"] = gradient_checkpointing_func++ monkeypatch.setenv(dtc.GC_MODE_ENV, "partial")+ monkeypatch.setitem(sys.modules, "unsloth_zoo.block_checkpointing", None)+ assert dtc.enable_diffusion_gradient_checkpointing(_Model()) == "plain"+ assert seen["func"].keywords == {"use_reentrant": False}+++class _FakeAccel:+ def __init__(self, peak_bytes):+ self.peak_bytes, self.resets = peak_bytes, 0++ def reset_peak_memory_stats(self):+ self.resets += 1++ def max_memory_allocated(self):+ return self.peak_bytes++[email protected]("device", ["cuda", "xpu"])+def test_training_peak_is_reported_on_cuda_and_xpu(device, monkeypatch):+ accel = _FakeAccel(3_456_000_000)+ monkeypatch.setattr(torch, device, accel, raising = False)+ dtc.reset_training_peak_memory(device)+ assert accel.resets == 1+ assert dtc.training_peak_memory_gb(device) == 3.46++[email protected]("device", ["cpu", "mps"])+def test_training_peak_is_unmeasured_elsewhere(device):+ dtc.reset_training_peak_memory(device)+ assert dtc.training_peak_memory_gb(device) == 0.0+++def test_a_failing_peak_probe_reports_unmeasured(monkeypatch):+ class _Broken(_FakeAccel):+ def max_memory_allocated(self):+ raise RuntimeError("no device")++ def reset_peak_memory_stats(self):+ raise RuntimeError("no device")++ monkeypatch.setattr(torch, "xpu", _Broken(0), raising = False)+ dtc.reset_training_peak_memory("xpu")+ assert dtc.training_peak_memory_gb("xpu") == 0.0+++def test_a_preview_round_never_reads_as_training_peak(monkeypatch):+ class _Accel(_FakeAccel):+ def reset_peak_memory_stats(self):+ self.resets += 1+ self.peak_bytes = 0++ accel = _Accel(0)+ monkeypatch.setattr(torch, "xpu", accel, raising = False)+ peak = dtc.TrainingPeakMemory("xpu")+ accel.peak_bytes = 5_000_000_000 # training steps+ peak.pause()+ accel.peak_bytes = 9_000_000_000 # the preview's VAE decode+ peak.resume()+ accel.peak_bytes = 4_000_000_000 # training again+ assert peak.gb() == 5.0+ accel.peak_bytes = 6_000_000_000+ assert peak.gb() == 6.0+++def test_a_short_resume_keeps_nf4_eager(monkeypatch):+ from dataclasses import replace++ import core.training.diffusion_checkpoint as ckpt++ monkeypatch.setattr(dit, "_nf4_compile_supported", lambda: True)+ cfg = replace(_ccfg(), train_steps = 500, resume_from_checkpoint = "/runs/a/checkpoint-450")+ monkeypatch.setattr(ckpt, "preflight_resume", lambda *a, **k: ("/runs/a/checkpoint-450", 450))+ left = dtc.resume_steps_left(cfg, identity = None)+ assert left == 50+ assert dit._should_compile(cfg, True, "cuda", "nf4", left) is False+ assert dit._should_compile(cfg, True, "cuda", "nf4") is True # a fresh 500-step run compiles+ # An unreadable bundle keeps the target (restore_resume_state reports it).+ monkeypatch.setattr(+ ckpt, "preflight_resume", lambda *a, **k: (_ for _ in ()).throw(OSError("gone"))+ )+ assert dtc.resume_steps_left(cfg, identity = None) == 500+ assert dtc.resume_steps_left(replace(cfg, resume_from_checkpoint = None), None) == 500diff --git a/studio/backend/tests/test_gpu_selection.py b/studio/backend/tests/test_gpu_selection.pyindex 2cf52062712..be44e90c03e 100644--- a/studio/backend/tests/test_gpu_selection.py+++ b/studio/backend/tests/test_gpu_selection.py@@ -2690,10 +2690,12 @@ class TestXpuBitsandbytesOptimizerGate(unittest.TestCase):self.assertTrue(self._probe()())def test_diffusion_factories_skip_bnb_on_xpu_and_keep_it_elsewhere(self):+ """torch AdamW everywhere; AdamW8bit only to resume its own bundle, never on XPU."""import torchimport core.training.diffusion_dit_trainer as dit_modimport core.training.diffusion_lora_trainer as lora_mod+ import core.training.diffusion_train_common as common_modclass _Bnb8bitMarker(torch.optim.AdamW):"""Stands in for bnb.optim.AdamW8bit: constructs fine, dies at the first step@@ -2711,32 +2713,32 @@ class TestXpuBitsandbytesOptimizerGate(unittest.TestCase):("sdxl_lora", lora_mod, "_make_lora_optimizer"),("dit", dit_mod, "_make_optimizer"),)+ resume_key = "bitsandbytes.optim.adamw.AdamW8bit"for label, module, factory_name in cases:factory = getattr(module, factory_name)for on_xpu in (True, False):- with self.subTest(trainer = label, xpu = on_xpu):- param = torch.nn.Parameter(torch.zeros(2, 2))- param.grad = torch.ones(2, 2)- with (- patch.dict(- sys.modules,- {"bitsandbytes": fake_bnb, "bitsandbytes.optim": fake_optim},- ),- patch.object(- module,- "bitsandbytes_optimizer_supported",- lambda supported = not on_xpu: supported,- ),- ):- optimizer = factory([param], 1e-4)-- if on_xpu:- # Must not be the bnb optimizer, and must survive an actual step.- self.assertNotIsInstance(optimizer, _Bnb8bitMarker)- optimizer.step()- else:- # Unchanged off XPU: still the 8-bit optimizer, not a blanket disable.- self.assertIsInstance(optimizer, _Bnb8bitMarker)+ for resume in (None, resume_key):+ with self.subTest(trainer = label, xpu = on_xpu, resume = resume):+ param = torch.nn.Parameter(torch.zeros(2, 2))+ param.grad = torch.ones(2, 2)+ with (+ patch.dict(+ sys.modules,+ {"bitsandbytes": fake_bnb, "bitsandbytes.optim": fake_optim},+ ),+ patch.object(+ common_mod,+ "bitsandbytes_optimizer_supported",+ lambda supported = not on_xpu: supported,+ ),+ ):+ optimizer = factory([param], 1e-4, resume)++ if on_xpu or resume is None:+ self.assertNotIsInstance(optimizer, _Bnb8bitMarker)+ optimizer.step()+ else:+ self.assertIsInstance(optimizer, _Bnb8bitMarker)class TestCliDefaultOptimizerFollowsTheDevicePolicy(unittest.TestCase):