Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: scale the SDXL loss in fp16 and keep image LoRA training off devices it cannot use (#13278)
Daniel Han34d8315bee44
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.pyindex bea2ba363f8..5f4f3a37d16 100644--- a/studio/backend/core/training/diffusion_lora_trainer.py+++ b/studio/backend/core/training/diffusion_lora_trainer.py@@ -29,6 +29,7 @@ from __future__ import annotationsimport argparseimport gc+import mathimport osimport randomimport time@@ -53,6 +54,7 @@ from core.training.diffusion_train_common import ( # noqa: F401_config_from_dict,_CONFIG_ALIASES,_emit,+ resolve_train_device,_latent_cache_forced,_latent_cache_over_budget,_plan_cache_variants,@@ -273,6 +275,14 @@ def _sample_sdxl_cached_latents(cache, idxs, variant_rng, device, weight_dtype):return latents, batch_time_ids+def _mps_available() -> bool:+ import torch+ try:+ return bool(torch.backends.mps.is_available())+ except Exception: # noqa: BLE001 -- an unprobeable backend is not one we train on+ return False++def run_diffusion_lora_training(config: DiffusionLoraConfig,*,@@ -355,6 +365,15 @@ def run_diffusion_lora_training(identity = identity,target_steps = cfg.train_steps,)+ if device == "cpu" and (resolve_train_device() == "xpu" or _mps_available()):+ _emit(+ on_event,+ "warning",+ message = (+ "SDXL training runs on the CPU in fp32 on this host: the SDXL trainer uses CUDA / ROCm "+ "GPUs only, so expect slow steps and previews."+ ),+ )_emit(on_event, "model_load_started", num_images = len(pairs))# Honour a stop requested before the (slow) base model load.@@ -413,6 +432,8 @@ def run_diffusion_lora_training(lora_params = [p for p in unet.parameters() if p.requires_grad]optimizer = _make_lora_optimizer(lora_params, cfg.learning_rate)+ # A frozen fp16 UNet (pre-Ampere / RDNA2 fallback) underflows the LoRA gradients without loss scaling.+ scaler = torch.amp.GradScaler("cuda", enabled = weight_dtype is torch.float16)# The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the# accumulation factor would stretch warmup.lr_sched = get_scheduler(@@ -519,6 +540,10 @@ def run_diffusion_lora_training(rng_streams = rng_streams,)resumed = restored.step if restored is not None else 0+ scaler_state = restored.progress.get("grad_scaler") if restored is not None else None+ if scaler.is_enabled() and isinstance(scaler_state, dict) and scaler_state:+ # Continue at the scale the run had backed off to, not 2**16, which would overflow again.+ scaler.load_state_dict(scaler_state)# Bound HERE: the checkpoint scan and the periodic saves both need it, and the two earlier# assignments live inside early-return branches.out_dir = Path(cfg.output_dir).expanduser()@@ -553,7 +578,7 @@ def run_diffusion_lora_training(on_event = on_event,sampler = index_sampler,rng_streams = rng_streams,- progress = {"running_loss": running_loss},+ progress = {"running_loss": running_loss, "grad_scaler": scaler.state_dict()},# NOT discard_existing: deleting the previous run's bundles at the FIRST periodic save spends them# before this run produced anything, so cancelling a retrain destroyed the thing being retrained.discard_existing = False,@@ -627,18 +652,25 @@ def run_diffusion_lora_training(else:loss = F.mse_loss(model_pred.float(), target.float(), reduction = "mean")- (loss / cfg.gradient_accumulation_steps).backward()+ scaler.scale(loss / cfg.gradient_accumulation_steps).backward()step_loss += float(loss.detach()) / cfg.gradient_accumulation_stepsmicro += 1# max_grad_norm at or below 0 disables clipping (Unsloth sends 0.0); passing 0.0 to clip_grad_norm_# would zero every gradient.grad_norm = None+ scaler.unscale_(optimizer)if cfg.max_grad_norm and cfg.max_grad_norm > 0:# Returned value is the total PRE-clip norm, reported to the UI chart.grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))- optimizer.step()- lr_sched.step()+ if not math.isfinite(grad_norm):+ grad_norm = None+ scale_before = scaler.get_scale() if scaler.is_enabled() else None+ scaler.step(optimizer)+ scaler.update()+ # An overflow step lowers the scale and skips the update; the schedule must not run ahead of the weights.+ if scale_before is None or scaler.get_scale() >= scale_before:+ lr_sched.step()running_loss += step_lossdone = opt_step + 1diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex 38041f2a2b0..98fe14cce23 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -770,12 +770,12 @@ def bf16_unsupported_reason(resolved_family: str) -> Optional[str]:def dit_accelerator_missing_reason(resolved_family: str) -> Optional[str]:"""Reason a DiT family cannot train on this host at all, else None. Never raises. nf4 is not a- CPU fallback: the 4-bit base load goes through diffusers' bitsandbytes quantizer, whose- validate_environment raises "No GPU found. A GPU is needed for quantization." unless CUDA,- XPU or MPS is present. Without this gate a GPU-less host accepts the default nf4 start,- evicts the resident Images pipeline, downloads the text encoders, and only then dies in the- child. SDXL keeps its own fp32-on-CPU path; MiniMax-H3 is covered too, since it loads its- denoiser through the same quantizer."""+ CPU fallback: the 4-bit base load goes through diffusers' bitsandbytes quantizer, which needs+ the model on CUDA or XPU. Without this gate such a host accepts the default nf4 start, evicts+ the resident Images pipeline, downloads the text encoders, and only then dies in the child.+ Apple MPS is not enough: resolve_train_device trains there on the CPU, and the quantizer+ refuses a CPU device_map. SDXL keeps its own fp32-on-CPU path; MiniMax-H3 is covered too,+ since it loads its denoiser through the same quantizer."""if (resolved_family or "").strip().lower() not in _FLOW_TRAIN_FAMILIES:return Nonetry:@@ -789,18 +789,14 @@ def dit_accelerator_missing_reason(resolved_family: str) -> Optional[str]:except Exception: # noqa: BLE001return False- if (- probe(torch.cuda)- # torch.backends.mps has carried is_available() since torch 1.12.- or probe(getattr(torch, "xpu", None))- or probe(getattr(torch.backends, "mps", None))- ):+ if probe(torch.cuda) or probe(getattr(torch, "xpu", None)):return Noneexcept Exception: # noqa: BLE001 -- probe failure must not block a startreturn Nonereturn (- "Training the DiT families needs a GPU: even the 4-bit (nf4) base load requires "- "CUDA, XPU or MPS, and this host has none. Train SDXL here, or use a GPU machine."+ "Training the DiT families needs an NVIDIA, AMD or Intel GPU: even the 4-bit (nf4) base "+ "load requires CUDA, ROCm or XPU, and this host has none (Apple Silicon included). Train "+ "SDXL here, or use a GPU machine.")diff --git a/studio/backend/tests/test_diffusion_base_precision.py b/studio/backend/tests/test_diffusion_base_precision.pyindex d544ac470bb..0f86f06a1f9 100644--- a/studio/backend/tests/test_diffusion_base_precision.py+++ b/studio/backend/tests/test_diffusion_base_precision.py@@ -782,8 +782,10 @@ def test_dit_accelerator_missing_reason_and_info_hide_train_without_a_gpu(monkeyelse:assert info["precision_modes"] != [] or name not in _DIT_TRAIN_FAMILIES- # Any accelerator clears it (MPS here, which bitsandbytes accepts).+ # MPS alone does not clear it: the trainer would run on the CPU, where the 4-bit load refuses.monkeypatch.setattr(torch.backends.mps, "is_available", lambda: True)+ assert "GPU" in (dit_accelerator_missing_reason("flux.1") or "")+ monkeypatch.setattr(torch.xpu, "is_available", lambda: True)assert dit_accelerator_missing_reason("flux.1") is Nonediff --git a/studio/backend/tests/test_diffusion_checkpoint_resume.py b/studio/backend/tests/test_diffusion_checkpoint_resume.pyindex 05978bb9abf..de06fe3100b 100644--- a/studio/backend/tests/test_diffusion_checkpoint_resume.py+++ b/studio/backend/tests/test_diffusion_checkpoint_resume.py@@ -402,6 +402,19 @@ def test_non_finite_progress_is_dropped_from_the_manifest(run_dir):assert dc.load_checkpoint(run_dir / "checkpoint-2").running_loss == 0.0+def test_the_fp16_loss_scaler_state_round_trips_through_progress(run_dir):+ import torch++ scaler = torch.amp.GradScaler("cuda", init_scale = 1024.0, enabled = True)+ state = scaler.state_dict()+ _Run(run_dir).save(2, progress = {"running_loss": 1.0, "grad_scaler": state})+ restored = dc.load_checkpoint(run_dir / "checkpoint-2").progress["grad_scaler"]+ assert restored == state+ fresh = torch.amp.GradScaler("cuda", enabled = True)+ fresh.load_state_dict(restored)+ assert fresh.state_dict() == state++def test_save_total_limit_keeps_only_the_newest_bundles(run_dir):run = _Run(run_dir, save_total_limit = 2)for step in (1, 2, 3, 4):diff --git a/studio/backend/tests/test_diffusion_lora_trainer.py b/studio/backend/tests/test_diffusion_lora_trainer.pyindex 651e7824f77..428d56d7396 100644--- a/studio/backend/tests/test_diffusion_lora_trainer.py+++ b/studio/backend/tests/test_diffusion_lora_trainer.py@@ -670,3 +670,34 @@ def test_gguf_substring_does_not_reject_local_diffusers_dir(tmp_path):# A *-GGUF repo id (not a local dir) still rejects.with pytest.raises(ValueError, match = "GGUF"):resolve_trainable_family("unsloth/FLUX.1-dev-GGUF")+++def test_sdxl_warns_when_an_unused_accelerator_leaves_it_on_the_cpu(monkeypatch, tmp_path):+ pytest.importorskip("diffusers")+ import torch+ from PIL import Image++ import core.training.diffusion_lora_trainer as sdxl++ data = tmp_path / "data"+ data.mkdir()+ Image.new("RGB", (64, 64)).save(data / "a.png")+ monkeypatch.setattr(sdxl, "_assert_trusted_base_model", lambda *_a, **_k: None)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setattr(torch.xpu, "is_available", lambda: False)+ cfg = DiffusionLoraConfig(+ base_model = "stabilityai/stable-diffusion-xl-base-1.0",+ data_dir = str(data),+ output_dir = str(tmp_path / "out"),+ instance_prompt = "a photo",+ )++ def warnings_for(mps):+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: mps)+ events = []+ sdxl.run_diffusion_lora_training(cfg, on_event = events.append, should_stop = lambda: True)+ return [e for e in events if e.get("type") == "warning" and "CPU" in e.get("message", "")]++ # A Mac (MPS present, SDXL trains on the CPU) is told why the run is slow; a CPU-only host is not.+ assert len(warnings_for(True)) == 1+ assert warnings_for(False) == []