Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: train Z-Image LoRAs on attention and feed-forward (#13233)

Daniel Han070a0a59d41a
diff --git a/studio/backend/core/training/diffusion_checkpoint.py b/studio/backend/core/training/diffusion_checkpoint.py
index 56aad4e4a21..db77af2ce3e 100644
--- a/studio/backend/core/training/diffusion_checkpoint.py
+++ b/studio/backend/core/training/diffusion_checkpoint.py
@@ -1887,6 +1887,21 @@ def _assert_loadable(path: Path, manifest: dict[str, Any]) -> None:
) from error
+def recorded_resume_targets(path_value: Any) -> Optional[tuple[str, ...]]:
+ """LoRA targets recorded by the bundle a resume would continue, else None. Never raises."""
+ try:
+ root = resolve_resume_dir(str(path_value))
+ explicit = read_checkpoint(root) if checkpoint_step(root) >= 0 else None
+ candidates = [(root, explicit)] if explicit is not None else iter_valid_checkpoints(root)
+ for _candidate, manifest in candidates:
+ saved = CheckpointIdentity.from_dict((manifest or {}).get("identity"))
+ if saved is not None and saved.lora_target_modules:
+ return tuple(saved.lora_target_modules)
+ except Exception: # noqa: BLE001 -- best effort; the preflight reports the real problem
+ pass
+ return None
+
+
def recorded_bucketing(path_value: str) -> bool:
"""Whether the resumed bundle trained bucketed (pre-field or unreadable: False; preflight_resume refuses)."""
try:
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 38047ed9764..833aa2ed0c1 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -96,8 +96,7 @@ from core.training.diffusion_train_extras import (
save_ema_adapter,
)
-# Per-family LoRA target modules (attention projections). FLUX / Qwen double-stream blocks also carry added-kv
-# projections; Z-Image is single-stream.
+# FLUX / Qwen double-stream blocks also carry added-kv projections; Z-Image is single-stream.
_FLUX_TARGETS = (
"to_q",
"to_k",
@@ -109,7 +108,8 @@ _FLUX_TARGETS = (
"to_add_out",
)
_QWEN_TARGETS = _FLUX_TARGETS
-_ZIMAGE_TARGETS = ("to_q", "to_k", "to_v", "to_out.0")
+# Attention + SwiGLU (DiffSynth-Studio's Z-Image recipe); old attention-only runs keep theirs on resume.
+_ZIMAGE_TARGETS = ("to_q", "to_k", "to_v", "to_out.0", "w1", "w2", "w3")
# The Krea 2 authors' recommended defaults (their DreamBooth script): attention + SwiGLU + text-fusion projector +
# conditioning embedders. For long runs they suggest narrowing to attention.
_KREA2_TARGETS = (
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index cb920f858ac..6ebba8c4849 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -1260,6 +1260,10 @@ class DiffusionLoraConfig:
raise ValueError("learning_rate must be > 0")
alpha = self.lora_alpha if self.lora_alpha is not None else self.lora_rank
targets = tuple(self.lora_target_modules) or DEFAULT_LORA_TARGETS
+ # Resume with unset targets keeps the recorded ones, so changing a family default never strands old runs.
+ if resume_from_checkpoint and resolved_family != "sdxl" and targets == DEFAULT_LORA_TARGETS:
+ from core.training.diffusion_checkpoint import recorded_resume_targets
+ targets = recorded_resume_targets(resume_from_checkpoint) or targets
# A blank Hub token (the Unsloth default when none is configured) must load anonymously, not as an explicit
# empty credential.
token = self.hf_token.strip() if isinstance(self.hf_token, str) else self.hf_token
diff --git a/studio/backend/routes/training.py b/studio/backend/routes/training.py
index 275b57d2c6f..805f4e3880a 100644
--- a/studio/backend/routes/training.py
+++ b/studio/backend/routes/training.py
@@ -3523,7 +3523,8 @@ async def start_diffusion_training(
from core.training.diffusion_lora_trainer import _config_from_dict
try:
- normalized_cfg = _config_from_dict(config).normalized()
+ # Off the event loop: resume may read checkpoint bundles.
+ normalized_cfg = await asyncio.to_thread(_config_from_dict(config).normalized)
except ValueError as e:
raise HTTPException(status_code = 400, detail = str(e))
diff --git a/studio/backend/tests/test_diffusion_accuracy_defaults.py b/studio/backend/tests/test_diffusion_accuracy_defaults.py
new file mode 100644
index 00000000000..589dd4c9765
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_accuracy_defaults.py
@@ -0,0 +1,213 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Per-family accuracy defaults of the DiT LoRA trainer, and that runs started under the previous
+defaults still resume with what they were trained with. CPU only."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+import torch
+
+from core.training import diffusion_checkpoint as dc
+from core.training.diffusion_dit_trainer import _SPECS, _select_lora_targets
+from core.training.diffusion_train_common import DEFAULT_LORA_TARGETS, DiffusionLoraConfig
+
+_OLD_ZIMAGE_TARGETS = ("to_q", "to_k", "to_v", "to_out.0")
+
+
+def _zimage_cfg(**kw) -> DiffusionLoraConfig:
+ return DiffusionLoraConfig(
+ base_model = "Tongyi-MAI/Z-Image-Turbo", data_dir = "d", output_dir = "o", **kw
+ ).normalized()
+
+
+def test_zimage_default_targets_cover_attention_and_feed_forward():
+ targets = _SPECS["z-image"].lora_targets
+ assert set(_OLD_ZIMAGE_TARGETS) <= set(targets)
+ assert {"w1", "w2", "w3"} <= set(targets)
+ cfg = _zimage_cfg()
+ assert _select_lora_targets(cfg.lora_target_modules, targets) == targets
+ assert dc.identity_for_config(cfg).lora_target_modules == targets
+
+
+def test_zimage_feed_forward_names_match_the_diffusers_model():
+ diffusers = pytest.importorskip("diffusers")
+ model_cls = getattr(diffusers, "ZImageTransformer2DModel", None)
+ if model_cls is None:
+ pytest.skip("diffusers has no Z-Image")
+ model = model_cls(
+ all_patch_size = (2,),
+ all_f_patch_size = (1,),
+ in_channels = 4,
+ dim = 64,
+ n_layers = 1,
+ n_refiner_layers = 1,
+ n_heads = 2,
+ n_kv_heads = 2,
+ cap_feat_dim = 32,
+ axes_dims = [8, 12, 12],
+ axes_lens = [64, 32, 32],
+ )
+ leaves = {
+ n.rsplit(".", 1)[-1] for n, m in model.named_modules() if isinstance(m, torch.nn.Linear)
+ }
+ assert {"w1", "w2", "w3"} <= leaves
+
+
+def run_dir():
+ from utils.paths import outputs_root
+
+ d = outputs_root() / "zimage-old-run"
+ d.mkdir(parents = True, exist_ok = True)
+ return d
+
+
+def _write_bundle(run_dir: Path, identity: dc.CheckpointIdentity) -> None:
+ param = torch.nn.Parameter(torch.zeros(2, 2))
+ optimizer = torch.optim.AdamW([param], lr = 1e-3)
+ dc.save_checkpoint(
+ output_dir = str(run_dir),
+ step = 11,
+ adapter_state = {"w": torch.zeros(2, 2)},
+ identity = identity,
+ target_steps = 500,
+ optimizer = optimizer,
+ lr_scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda _: 1.0),
+ rng = dc.capture_rng_state({}),
+ sampler_state = {"n": 1, "order": [0], "pos": 0},
+ )
+
+
+def _old_zimage_identity(**overrides) -> dc.CheckpointIdentity:
+ fresh = dc.identity_for_config(_zimage_cfg())
+ fields = {**fresh.as_dict(), "lora_target_modules": _OLD_ZIMAGE_TARGETS, "flow_shift": "1.0"}
+ fields.update(overrides)
+ return dc.CheckpointIdentity.from_dict(fields)
+
+
+def test_resuming_an_old_zimage_run_keeps_its_recorded_targets(run_dir):
+ saved = _old_zimage_identity()
+ _write_bundle(run_dir, saved)
+ cfg = _zimage_cfg(resume_from_checkpoint = str(run_dir))
+ assert cfg.lora_target_modules == _OLD_ZIMAGE_TARGETS
+ # The model's own schedule: 1.0 before the numeric flow_shift became the effective shift, None after.
+ assert cfg.flow_shift in (None, 1.0)
+ assert _select_lora_targets(cfg.lora_target_modules, _SPECS["z-image"].lora_targets) == (
+ _OLD_ZIMAGE_TARGETS
+ )
+ incoming = dc.identity_for_config(cfg)
+ assert incoming.lora_target_modules == saved.lora_target_modules
+ assert incoming.flow_shift == saved.flow_shift
+ cfg = _zimage_cfg(resume_from_checkpoint = str(run_dir / "checkpoint-11"))
+ assert cfg.lora_target_modules == _OLD_ZIMAGE_TARGETS
+
+
+def test_a_save_killed_mid_promotion_still_resumes_on_its_targets(run_dir):
+ # A crash mid-promotion leaves only the aged staging bundle, which the preflight restores.
+ import os
+ import time
+
+ _write_bundle(run_dir, _old_zimage_identity())
+ aside = run_dir / f"{dc._STAGING_PREFIX}replaced-11-cafebabe"
+ os.replace(run_dir / "checkpoint-11", aside)
+ old = time.time() - (dc._LIVE_REPLACEMENT_GRACE_SECONDS + 60)
+ os.utime(aside, (old, old))
+ cfg = _zimage_cfg(resume_from_checkpoint = str(run_dir))
+ assert cfg.lora_target_modules == _OLD_ZIMAGE_TARGETS
+
+
+def test_a_run_folder_named_like_a_bundle_still_resumes_on_its_targets():
+ from utils.paths import outputs_root
+
+ run_dir = outputs_root() / "checkpoint-2026"
+ run_dir.mkdir(parents = True, exist_ok = True)
+ _write_bundle(run_dir, _old_zimage_identity())
+ cfg = _zimage_cfg(resume_from_checkpoint = str(run_dir))
+ assert cfg.lora_target_modules == _OLD_ZIMAGE_TARGETS
+
+
+def test_start_route_reads_the_recorded_targets_off_the_event_loop(run_dir, monkeypatch):
+ import asyncio
+ import threading
+
+ import routes.training as tr
+ from core.training import diffusion_train_common as dtc
+ from models.training import DiffusionTrainingStartRequest
+
+ _write_bundle(run_dir, _old_zimage_identity())
+ seen = {}
+ real = dc.recorded_resume_targets
+
+ def _recorded(path_value):
+ seen["thread"] = threading.current_thread()
+ return real(path_value)
+
+ class _Stop(Exception):
+ pass
+
+ def _stop(cfg):
+ seen["targets"] = cfg.lora_target_modules
+ raise _Stop
+
+ monkeypatch.setattr(dc, "recorded_resume_targets", _recorded)
+ monkeypatch.setattr(dtc, "h3_train_unsupported_reason", _stop)
+ monkeypatch.setattr(tr, "_resolve_diffusion_data_dir", lambda raw: Path(raw))
+ monkeypatch.setattr(tr, "validate_job_paths", lambda _config: None)
+ monkeypatch.setattr(
+ tr,
+ "get_training_backend",
+ lambda: type("B", (), {"is_training_active": lambda self: False})(),
+ )
+ body = DiffusionTrainingStartRequest(
+ base_model = "Tongyi-MAI/Z-Image-Turbo",
+ data_dir = "d",
+ output_dir = "zimage-resumed",
+ resume_from_checkpoint = str(run_dir),
+ )
+
+ async def _run():
+ loop_thread = threading.current_thread()
+ with pytest.raises(_Stop):
+ await tr.start_diffusion_training(body = body, current_subject = "u", via_api_key = False)
+ return loop_thread
+
+ loop_thread = asyncio.run(_run())
+ assert seen["targets"] == _OLD_ZIMAGE_TARGETS
+ assert seen["thread"] is not loop_thread
+
+
+def test_explicit_request_targets_win_over_the_recorded_ones(run_dir):
+ _write_bundle(run_dir, _old_zimage_identity())
+ cfg = _zimage_cfg(
+ resume_from_checkpoint = str(run_dir),
+ lora_target_modules = ("to_q", "to_v"),
+ )
+ assert cfg.lora_target_modules == ("to_q", "to_v")
+
+
+def test_unreadable_resume_path_falls_back_to_the_family_defaults(tmp_path):
+ cfg = _zimage_cfg(resume_from_checkpoint = str(tmp_path / "nowhere"))
+ assert cfg.lora_target_modules == DEFAULT_LORA_TARGETS
+ assert dc.recorded_resume_targets(str(tmp_path / "nowhere")) is None
+
+
+def test_zimage_default_keeps_the_checkpoints_built_in_shift():
+ # Shift 2-3 (DiffSynth / musubi) hurt likeness, so keep the scheduler's static shift (3 Turbo, 6 base).
+ diffusers = pytest.importorskip("diffusers")
+ from core.training.diffusion_dit_trainer import _training_sigma_table
+ for shift, base_model in ((3.0, "Tongyi-MAI/Z-Image-Turbo"), (6.0, "Tongyi-MAI/Z-Image")):
+ cfg = DiffusionLoraConfig(base_model = base_model, data_dir = "d", output_dir = "o").normalized()
+ assert cfg.resolved_family == "z-image" and cfg.flow_shift in (None, 1.0)
+ sched = diffusers.FlowMatchEulerDiscreteScheduler(num_train_timesteps = 1000, shift = shift)
+ table = _training_sigma_table(sched, cfg.flow_shift)
+ assert table is sched.sigmas
+ t = torch.linspace(1.0, 1.0 / 1000, 1000)
+ assert torch.allclose(table.float(), shift * t / (1 + (shift - 1) * t), atol = 1e-5)
+ u = torch.sigmoid(torch.randn(200_000, generator = torch.Generator().manual_seed(0)))
+ sig = table[(u * 1000).long().clamp(0, 999)].float()
+ expected = float((shift * u / (1 + (shift - 1) * u) > 0.8).float().mean())
+ assert abs(float((sig > 0.8).float().mean()) - expected) < 0.01
diff --git a/studio/backend/tests/test_diffusion_dit_trainer.py b/studio/backend/tests/test_diffusion_dit_trainer.py
index d01a3d01815..c66fd27ccbb 100644
--- a/studio/backend/tests/test_diffusion_dit_trainer.py
+++ b/studio/backend/tests/test_diffusion_dit_trainer.py
@@ -194,8 +194,7 @@ def test_zimage_offers_the_undistilled_base_the_upstream_recipe_trains_on():
_assert_trusted_base_model("Tongyi-MAI/Z-Image")
with pytest.raises(ValueError, match = "untrusted"):
_assert_trusted_base_model("someone/random-z-image-finetune")
- # The upstream script's target list; the family spec must already match it.
- assert _SPECS["z-image"].lora_targets == ("to_q", "to_k", "to_v", "to_out.0")
+ assert _SPECS["z-image"].lora_targets == ("to_q", "to_k", "to_v", "to_out.0", "w1", "w2", "w3")
# No deploy pairing: an adapter previews on whichever checkpoint it trained on. A family-wide
# one would also rewrite the nf4 Turbo base, sending a QLoRA run's preview to a dense fp32 load.
assert fam.deploy_base_repo is None