Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: scale the SDXL loss in fp16 and keep image LoRA training off devices it cannot use (#13278)

Daniel Han34d8315bee44
diff --git a/studio/backend/core/training/diffusion_lora_trainer.py b/studio/backend/core/training/diffusion_lora_trainer.py
index bea2ba363f8..5f4f3a37d16 100644
--- a/studio/backend/core/training/diffusion_lora_trainer.py
+++ b/studio/backend/core/training/diffusion_lora_trainer.py
@@ -29,6 +29,7 @@ from __future__ import annotations
import argparse
import gc
+import math
import os
import random
import time
@@ -53,6 +54,7 @@ from core.training.diffusion_train_common import ( # noqa: F401
_config_from_dict,
_CONFIG_ALIASES,
_emit,
+ resolve_train_device,
_latent_cache_forced,
_latent_cache_over_budget,
_plan_cache_variants,
@@ -273,6 +275,14 @@ def _sample_sdxl_cached_latents(cache, idxs, variant_rng, device, weight_dtype):
return latents, batch_time_ids
+def _mps_available() -> bool:
+ import torch
+ try:
+ return bool(torch.backends.mps.is_available())
+ except Exception: # noqa: BLE001 -- an unprobeable backend is not one we train on
+ return False
+
+
def run_diffusion_lora_training(
config: DiffusionLoraConfig,
*,
@@ -355,6 +365,15 @@ def run_diffusion_lora_training(
identity = identity,
target_steps = cfg.train_steps,
)
+ if device == "cpu" and (resolve_train_device() == "xpu" or _mps_available()):
+ _emit(
+ on_event,
+ "warning",
+ message = (
+ "SDXL training runs on the CPU in fp32 on this host: the SDXL trainer uses CUDA / ROCm "
+ "GPUs only, so expect slow steps and previews."
+ ),
+ )
_emit(on_event, "model_load_started", num_images = len(pairs))
# Honour a stop requested before the (slow) base model load.
@@ -413,6 +432,8 @@ def run_diffusion_lora_training(
lora_params = [p for p in unet.parameters() if p.requires_grad]
optimizer = _make_lora_optimizer(lora_params, cfg.learning_rate)
+ # A frozen fp16 UNet (pre-Ampere / RDNA2 fallback) underflows the LoRA gradients without loss scaling.
+ scaler = torch.amp.GradScaler("cuda", enabled = weight_dtype is torch.float16)
# The scheduler advances once per optimizer update, so count warmup/decay in optimizer steps; the
# accumulation factor would stretch warmup.
lr_sched = get_scheduler(
@@ -519,6 +540,10 @@ def run_diffusion_lora_training(
rng_streams = rng_streams,
)
resumed = restored.step if restored is not None else 0
+ scaler_state = restored.progress.get("grad_scaler") if restored is not None else None
+ if scaler.is_enabled() and isinstance(scaler_state, dict) and scaler_state:
+ # Continue at the scale the run had backed off to, not 2**16, which would overflow again.
+ scaler.load_state_dict(scaler_state)
# Bound HERE: the checkpoint scan and the periodic saves both need it, and the two earlier
# assignments live inside early-return branches.
out_dir = Path(cfg.output_dir).expanduser()
@@ -553,7 +578,7 @@ def run_diffusion_lora_training(
on_event = on_event,
sampler = index_sampler,
rng_streams = rng_streams,
- progress = {"running_loss": running_loss},
+ progress = {"running_loss": running_loss, "grad_scaler": scaler.state_dict()},
# NOT discard_existing: deleting the previous run's bundles at the FIRST periodic save spends them
# before this run produced anything, so cancelling a retrain destroyed the thing being retrained.
discard_existing = False,
@@ -627,18 +652,25 @@ def run_diffusion_lora_training(
else:
loss = F.mse_loss(model_pred.float(), target.float(), reduction = "mean")
- (loss / cfg.gradient_accumulation_steps).backward()
+ scaler.scale(loss / cfg.gradient_accumulation_steps).backward()
step_loss += float(loss.detach()) / cfg.gradient_accumulation_steps
micro += 1
# max_grad_norm at or below 0 disables clipping (Unsloth sends 0.0); passing 0.0 to clip_grad_norm_
# would zero every gradient.
grad_norm = None
+ scaler.unscale_(optimizer)
if cfg.max_grad_norm and cfg.max_grad_norm > 0:
# Returned value is the total PRE-clip norm, reported to the UI chart.
grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))
- optimizer.step()
- lr_sched.step()
+ if not math.isfinite(grad_norm):
+ grad_norm = None
+ scale_before = scaler.get_scale() if scaler.is_enabled() else None
+ scaler.step(optimizer)
+ scaler.update()
+ # An overflow step lowers the scale and skips the update; the schedule must not run ahead of the weights.
+ if scale_before is None or scaler.get_scale() >= scale_before:
+ lr_sched.step()
running_loss += step_loss
done = opt_step + 1
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index 38041f2a2b0..98fe14cce23 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -770,12 +770,12 @@ def bf16_unsupported_reason(resolved_family: str) -> Optional[str]:
def dit_accelerator_missing_reason(resolved_family: str) -> Optional[str]:
"""Reason a DiT family cannot train on this host at all, else None. Never raises. nf4 is not a
- CPU fallback: the 4-bit base load goes through diffusers' bitsandbytes quantizer, whose
- validate_environment raises "No GPU found. A GPU is needed for quantization." unless CUDA,
- XPU or MPS is present. Without this gate a GPU-less host accepts the default nf4 start,
- evicts the resident Images pipeline, downloads the text encoders, and only then dies in the
- child. SDXL keeps its own fp32-on-CPU path; MiniMax-H3 is covered too, since it loads its
- denoiser through the same quantizer."""
+ CPU fallback: the 4-bit base load goes through diffusers' bitsandbytes quantizer, which needs
+ the model on CUDA or XPU. Without this gate such a host accepts the default nf4 start, evicts
+ the resident Images pipeline, downloads the text encoders, and only then dies in the child.
+ Apple MPS is not enough: resolve_train_device trains there on the CPU, and the quantizer
+ refuses a CPU device_map. SDXL keeps its own fp32-on-CPU path; MiniMax-H3 is covered too,
+ since it loads its denoiser through the same quantizer."""
if (resolved_family or "").strip().lower() not in _FLOW_TRAIN_FAMILIES:
return None
try:
@@ -789,18 +789,14 @@ def dit_accelerator_missing_reason(resolved_family: str) -> Optional[str]:
except Exception: # noqa: BLE001
return False
- if (
- probe(torch.cuda)
- # torch.backends.mps has carried is_available() since torch 1.12.
- or probe(getattr(torch, "xpu", None))
- or probe(getattr(torch.backends, "mps", None))
- ):
+ if probe(torch.cuda) or probe(getattr(torch, "xpu", None)):
return None
except Exception: # noqa: BLE001 -- probe failure must not block a start
return None
return (
- "Training the DiT families needs a GPU: even the 4-bit (nf4) base load requires "
- "CUDA, XPU or MPS, and this host has none. Train SDXL here, or use a GPU machine."
+ "Training the DiT families needs an NVIDIA, AMD or Intel GPU: even the 4-bit (nf4) base "
+ "load requires CUDA, ROCm or XPU, and this host has none (Apple Silicon included). Train "
+ "SDXL here, or use a GPU machine."
)
diff --git a/studio/backend/tests/test_diffusion_base_precision.py b/studio/backend/tests/test_diffusion_base_precision.py
index d544ac470bb..0f86f06a1f9 100644
--- a/studio/backend/tests/test_diffusion_base_precision.py
+++ b/studio/backend/tests/test_diffusion_base_precision.py
@@ -782,8 +782,10 @@ def test_dit_accelerator_missing_reason_and_info_hide_train_without_a_gpu(monkey
else:
assert info["precision_modes"] != [] or name not in _DIT_TRAIN_FAMILIES
- # Any accelerator clears it (MPS here, which bitsandbytes accepts).
+ # MPS alone does not clear it: the trainer would run on the CPU, where the 4-bit load refuses.
monkeypatch.setattr(torch.backends.mps, "is_available", lambda: True)
+ assert "GPU" in (dit_accelerator_missing_reason("flux.1") or "")
+ monkeypatch.setattr(torch.xpu, "is_available", lambda: True)
assert dit_accelerator_missing_reason("flux.1") is None
diff --git a/studio/backend/tests/test_diffusion_checkpoint_resume.py b/studio/backend/tests/test_diffusion_checkpoint_resume.py
index 05978bb9abf..de06fe3100b 100644
--- a/studio/backend/tests/test_diffusion_checkpoint_resume.py
+++ b/studio/backend/tests/test_diffusion_checkpoint_resume.py
@@ -402,6 +402,19 @@ def test_non_finite_progress_is_dropped_from_the_manifest(run_dir):
assert dc.load_checkpoint(run_dir / "checkpoint-2").running_loss == 0.0
+def test_the_fp16_loss_scaler_state_round_trips_through_progress(run_dir):
+ import torch
+
+ scaler = torch.amp.GradScaler("cuda", init_scale = 1024.0, enabled = True)
+ state = scaler.state_dict()
+ _Run(run_dir).save(2, progress = {"running_loss": 1.0, "grad_scaler": state})
+ restored = dc.load_checkpoint(run_dir / "checkpoint-2").progress["grad_scaler"]
+ assert restored == state
+ fresh = torch.amp.GradScaler("cuda", enabled = True)
+ fresh.load_state_dict(restored)
+ assert fresh.state_dict() == state
+
+
def test_save_total_limit_keeps_only_the_newest_bundles(run_dir):
run = _Run(run_dir, save_total_limit = 2)
for step in (1, 2, 3, 4):
diff --git a/studio/backend/tests/test_diffusion_lora_trainer.py b/studio/backend/tests/test_diffusion_lora_trainer.py
index 651e7824f77..428d56d7396 100644
--- a/studio/backend/tests/test_diffusion_lora_trainer.py
+++ b/studio/backend/tests/test_diffusion_lora_trainer.py
@@ -670,3 +670,34 @@ def test_gguf_substring_does_not_reject_local_diffusers_dir(tmp_path):
# A *-GGUF repo id (not a local dir) still rejects.
with pytest.raises(ValueError, match = "GGUF"):
resolve_trainable_family("unsloth/FLUX.1-dev-GGUF")
+
+
+def test_sdxl_warns_when_an_unused_accelerator_leaves_it_on_the_cpu(monkeypatch, tmp_path):
+ pytest.importorskip("diffusers")
+ import torch
+ from PIL import Image
+
+ import core.training.diffusion_lora_trainer as sdxl
+
+ data = tmp_path / "data"
+ data.mkdir()
+ Image.new("RGB", (64, 64)).save(data / "a.png")
+ monkeypatch.setattr(sdxl, "_assert_trusted_base_model", lambda *_a, **_k: None)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(torch.xpu, "is_available", lambda: False)
+ cfg = DiffusionLoraConfig(
+ base_model = "stabilityai/stable-diffusion-xl-base-1.0",
+ data_dir = str(data),
+ output_dir = str(tmp_path / "out"),
+ instance_prompt = "a photo",
+ )
+
+ def warnings_for(mps):
+ monkeypatch.setattr(torch.backends.mps, "is_available", lambda: mps)
+ events = []
+ sdxl.run_diffusion_lora_training(cfg, on_event = events.append, should_stop = lambda: True)
+ return [e for e in events if e.get("type") == "warning" and "CPU" in e.get("message", "")]
+
+ # A Mac (MPS present, SDXL trains on the CPU) is told why the run is slow; a CPU-only host is not.
+ assert len(warnings_for(True)) == 1
+ assert warnings_for(False) == []