Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: Wan2.2-TI2V-5B LoRA training (#13654)

Daniel Hanbc11dbef4556
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.py
index fb64b73a8bb..aee56a5f8fd 100644
--- a/studio/backend/core/inference/video_families.py
+++ b/studio/backend/core/inference/video_families.py
@@ -665,6 +665,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (
# temporal compression 4 gives valid frame counts 4k+1. Defaults 20 steps / CFG 5 (ComfyUI's template).
VideoFamily(
name = "wan2.2-ti2v-5b",
+ trainable = True,
+ label = "Wan2.2 TI2V 5B",
profile = FamilyProfile(
untiled_vae = UntiledVae(bytes_per_latent_pixel = 2.5 * 2**20, output_scale = (16, 4)),
preview_factors = "wan22_5b",
diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.py
index 927395219e4..6538f5dddd8 100644
--- a/studio/backend/core/training/diffusion_dit_trainer.py
+++ b/studio/backend/core/training/diffusion_dit_trainer.py
@@ -3,8 +3,8 @@
"""Flow-matching LoRA training for the rectified-flow DiT families.
-Image: FLUX.1-dev, FLUX.2 (dev + Klein), Qwen-Image, Z-Image, Krea 2. Video: LTX-2 (from a
-still-image dataset; see the ``_LTX2_TARGETS`` block for what a video family adds).
+Image: FLUX.1-dev, FLUX.2 (dev + Klein), Qwen-Image, Z-Image, Krea 2. Video: LTX-2 and Wan2.2-TI2V-5B (from a
+still-image dataset; see the ``_LTX2_TARGETS`` and ``_WAN_TARGETS`` blocks for what a video family adds).
These are rectified-flow transformers, not the SDXL U-Net, so they share only the plumbing in
``diffusion_train_common`` (config, dataset discovery, events, stop, publishing). The training math
@@ -183,6 +183,10 @@ class _FamilySpec:
latent_shape: Optional[Callable[..., tuple]] = None
decode_latents: Optional[Callable[..., Any]] = None
sample_forward_kwargs: Optional[dict] = None
+ # cfg -> FlowMatchEuler scheduler. None = load scheduler/ (only valid if it IS FlowMatchEuler; Wan ships UniPC).
+ training_scheduler: Optional[Callable[..., Any]] = None
+ # Saver takes ``adapter_metadata``; without it diffusers' loader assumes lora_alpha == rank.
+ save_takes_metadata: bool = False
def _gather_sigmas(sigma_table, indices, device, dtype, n_dim):
@@ -1644,6 +1648,194 @@ def _ltx2_save(pipe_cls, out_dir, transformer_lora_layers):
)
+# Wan2.2-TI2V-5B, trained from stills (a 1-frame clip is a valid T2V sample). Targets match DiffSynth-Studio /
+# musubi-tuner; bare "to_q" is safe (single stream), add_k/v_proj exist only on I2V-14B.
+_WAN_TARGETS = (
+ "to_q",
+ "to_k",
+ "to_v",
+ "to_out.0",
+ "ffn.net.0.proj",
+ "ffn.net.2",
+)
+_WAN_MAX_SEQUENCE_LENGTH = 512
+
+
+def _wan_pipeline_cls():
+ from diffusers import WanPipeline
+ return WanPipeline
+
+
+def _wan_load_conditioners(cfg, device, weight_dtype):
+ """UMT5 + tokenizer + VAE. VAE loads fp32 directly; the shared bf16-then-fp32 cast would round it through bf16."""
+ import torch
+ from diffusers import AutoencoderKLWan
+
+ vae = AutoencoderKLWan.from_pretrained(
+ cfg.base_model, subfolder = "vae", torch_dtype = torch.float32, token = cfg.hf_token
+ )
+ pipe = _wan_pipeline_cls().from_pretrained(
+ cfg.base_model,
+ transformer = None,
+ vae = vae,
+ torch_dtype = torch.bfloat16,
+ token = cfg.hf_token,
+ )
+ # VAE stays on host; the shared loop moves it only after UMT5-XXL (11.4 GB) is freed.
+ return pipe, pipe.vae
+
+
+def _wan_load_transformer(cfg, device, weight_dtype, base_precision):
+ from diffusers import WanTransformer3DModel
+ return _load_dit_transformer(WanTransformer3DModel, cfg, device, base_precision)
+
+
+def _wan_trim_padding(embeds):
+ """Drop trailing zero-pad rows; lossless since ``_wan_collate`` re-pads."""
+ nonzero = embeds[0].abs().amax(dim = -1) > 0
+ idx = nonzero.nonzero()
+ keep = int(idx.max()) + 1 if idx.numel() else 1
+ return embeds[:, :keep].contiguous()
+
+
+def _wan_encode_prompts(pipe, captions, device):
+ import torch
+
+ _encoders_to_device(pipe, device)
+ out = []
+ with torch.no_grad():
+ for cap in captions:
+ pe, _neg = pipe.encode_prompt(
+ prompt = cap,
+ do_classifier_free_guidance = False,
+ num_videos_per_prompt = 1,
+ max_sequence_length = _WAN_MAX_SEQUENCE_LENGTH,
+ device = device,
+ )
+ out.append((_wan_trim_padding(pe).cpu(),))
+ return out
+
+
+def _wan_latent_affine(vae, ref):
+ """Per-channel (mean, std); training latent is ``(z - mean) / std``."""
+ import torch
+
+ z = int(vae.config.z_dim)
+ mean = torch.tensor(vae.config.latents_mean, device = ref.device, dtype = ref.dtype).view(
+ 1, z, 1, 1, 1
+ )
+ std = torch.tensor(vae.config.latents_std, device = ref.device, dtype = ref.dtype).view(
+ 1, z, 1, 1, 1
+ )
+ return mean, std
+
+
+def _wan_pixels(vae, pixel_values):
+ return pixel_values.to(dtype = next(vae.parameters()).dtype).unsqueeze(2)
+
+
+def _wan_encode_latents(vae, pixel_values):
+ import torch
+
+ with torch.no_grad():
+ lat = vae.encode(_wan_pixels(vae, pixel_values)).latent_dist.sample()
+ mean, std = _wan_latent_affine(vae, lat)
+ return (lat - mean) / std
+
+
+def _wan_encode_latent_stats(vae, pixel_values):
+ import torch
+
+ with torch.no_grad():
+ dist = vae.encode(_wan_pixels(vae, pixel_values)).latent_dist
+ mean, std = _wan_latent_affine(vae, dist.mean)
+ return (dist.mean - mean) / std, dist.std / std
+
+
+def _wan_collate(
+ entries,
+ device,
+ weight_dtype,
+ pad_to = None,
+):
+ import torch
+ import torch.nn.functional as F
+
+ # Zero-pad as _get_t5_prompt_embeds does, on the DEVICE: host-side pad/cat stalled steps ~100 ms.
+ length = max(_WAN_MAX_SEQUENCE_LENGTH, *(e[0].shape[1] for e in entries))
+ padded = [
+ F.pad(
+ e[0].to(device = device, dtype = weight_dtype, non_blocking = True),
+ (0, 0, 0, length - e[0].shape[1]),
+ )
+ for e in entries
+ ]
+ return (padded[0] if len(padded) == 1 else torch.cat(padded),)
+
+
+def _wan_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype):
+ (prompt_embeds,) = embeds_batch
+ # [B] timestep == the pipeline's per-token expand_timesteps form for T2V.
+ return transformer(
+ hidden_states = noisy,
+ timestep = timesteps.to(device = noisy.device).reshape(-1),
+ encoder_hidden_states = prompt_embeds,
+ return_dict = False,
+ )[0]
+
+
+def _wan_save(
+ pipe_cls,
+ out_dir,
+ transformer_lora_layers,
+ adapter_metadata = None,
+):
+ # Older diffusers savers lack the metadata argument; they still get the weights.
+ import inspect
+
+ save = _wan_pipeline_cls().save_lora_weights
+ kwargs = {}
+ if (
+ adapter_metadata
+ and "transformer_lora_adapter_metadata" in inspect.signature(save).parameters
+ ):
+ kwargs["transformer_lora_adapter_metadata"] = dict(adapter_metadata)
+ save(
+ save_directory = out_dir,
+ transformer_lora_layers = transformer_lora_layers,
+ weight_name = DEFAULT_LORA_FILENAME,
+ **kwargs,
+ )
+
+
+def _lora_adapter_metadata(transformer) -> Optional[dict]:
+ try:
+ return transformer.peft_config["default"].to_dict()
+ except Exception: # noqa: BLE001 -- metadata is best-effort; the weights still save
+ return None
+
+
+def _adapter_saver(spec, transformer):
+ if not spec.save_takes_metadata:
+ return spec.save
+ import functools
+ return functools.partial(spec.save, adapter_metadata = _lora_adapter_metadata(transformer))
+
+
+def _wan_training_scheduler(cfg):
+ """FlowMatchEuler with Wan's UniPC ``flow_shift`` baked in; loading it from the UniPC config silently drops the
+ shift. UniPC flow sigmas equal FlowMatchEuler's static shift."""
+ from diffusers import FlowMatchEulerDiscreteScheduler
+
+ conf = FlowMatchEulerDiscreteScheduler.load_config(
+ cfg.base_model, subfolder = "scheduler", token = cfg.hf_token
+ )
+ return FlowMatchEulerDiscreteScheduler(
+ num_train_timesteps = int(conf.get("num_train_timesteps", 1000)),
+ shift = float(conf.get("flow_shift") or 1.0),
+ )
+
+
def _vae8_latent_shape(vae, height, width):
return (1, vae.config.latent_channels, height // 8, width // 8), (height // 16) * (width // 16)
@@ -1797,6 +1989,22 @@ _SPECS: dict[str, _FamilySpec] = {
forward = _ltx2_forward,
save = _ltx2_save,
),
+ "wan2.2-ti2v-5b": _FamilySpec(
+ family = "wan2.2-ti2v-5b",
+ lora_targets = _WAN_TARGETS,
+ force_bf16 = True,
+ dense_bf16_gb = 10.0,
+ load_conditioners = _wan_load_conditioners,
+ load_transformer = _wan_load_transformer,
+ encode_prompts = _wan_encode_prompts,
+ encode_latents = _wan_encode_latents,
+ encode_latent_stats = _wan_encode_latent_stats,
+ collate = _wan_collate,
+ forward = _wan_forward,
+ save = _wan_save,
+ training_scheduler = _wan_training_scheduler,
+ save_takes_metadata = True,
+ ),
}
@@ -2081,13 +2289,21 @@ def _sample_cached_latents(cache, idxs, variant_rng, device, weight_dtype):
a, b = variants[variant_rng.randrange(len(variants))] if len(variants) > 1 else variants[0]
parts_a.append(a)
parts_b.append(b)
- lat_a = torch.cat(parts_a).to(device, non_blocking = True)
+ lat_a = _to_device_cat(parts_a, device)
if parts_b[0] is None:
return lat_a.to(dtype = weight_dtype)
- lat_b = torch.cat(parts_b).to(device, non_blocking = True)
+ lat_b = _to_device_cat(parts_b, device)
return (lat_a + lat_b * torch.randn_like(lat_a)).to(dtype = weight_dtype)
+def _to_device_cat(parts, device):
+ """Copy pinned parts, then cat on ``device``: a host cat yields pageable memory and a sync copy."""
+ import torch
+
+ moved = [p.to(device, non_blocking = True) for p in parts]
+ return moved[0] if len(moved) == 1 else torch.cat(moved)
+
+
def _should_compile(
cfg,
base_is_bnb,
@@ -2225,7 +2441,7 @@ def run_dit_lora_training(
on_event: Optional[EventCb] = None,
should_stop: Optional[StopCb] = None,
) -> str:
- """Train a flow-matching DiT LoRA (FLUX.1 / FLUX.2 / Qwen-Image / Z-Image / Krea 2 / LTX-2) and
+ """Train a flow-matching DiT LoRA (FLUX.1 / FLUX.2 / Qwen-Image / Z-Image / Krea 2 / LTX-2 / Wan2.2-TI2V-5B) and
export it. Resumable: ``cfg.resume_from_checkpoint`` restores the adapter, optimizer moments,
LR position, EMA shadow, sampler cycle and RNG streams from a ``checkpoint-<N>`` bundle, and
the loop runs steps N+1..train_steps (the TARGET TOTAL). A stop-and-save and every
@@ -2445,6 +2661,9 @@ def _train_dit(
torch.cuda.empty_cache()
elif device == "xpu":
torch.xpu.empty_cache()
+ # Some families keep the VAE on host while text encoders are resident.
+ if vae is not None:
+ vae.to(device)
# The cache keeps the posterior affine parameters, so per-step sampling noise is preserved.
if use_cache:
@@ -2553,9 +2772,12 @@ def _train_dit(
optimizer = _make_optimizer(
lora_params, cfg.learning_rate, recorded_optimizer_class(cfg, identity)
)
- scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
- fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token
- )
+ if spec.training_scheduler is not None:
+ scheduler = spec.training_scheduler(fetch_cfg)
+ else:
+ scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
+ fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token
+ )
# getattr defaults keep an un-normalized config on the historical behaviour.
flow_shift = getattr(cfg, "flow_shift", None)
if flow_shift is None and spec.family in AUTO_FLOW_SHIFT_FAMILIES:
@@ -2892,14 +3114,15 @@ def _train_dit(
if stopped and done > 0:
_save_checkpoint(done)
layers = get_peft_model_state_dict(transformer)
- spec.save(pipe, str(out_dir), layers)
+ save_adapter = _adapter_saver(spec, transformer)
+ save_adapter(pipe, str(out_dir), layers)
lora_path = str(out_dir / DEFAULT_LORA_FILENAME)
# ``done`` (the step reached), not cfg.train_steps: a stop at 11/500 must not advertise 500.
catalog_path = _publish_to_lora_catalog(lora_path, cfg, done)
if ema is not None and ema.updates > 0:
try:
# Report the adapter FILE, like lora_path, so a caller can load it directly.
- ema_dir = save_ema_adapter(ema, transformer, spec.save, str(out_dir))
+ ema_dir = save_ema_adapter(ema, transformer, save_adapter, str(out_dir))
ema_path = str(Path(ema_dir) / DEFAULT_LORA_FILENAME)
except Exception as exc: # noqa: BLE001 -- the primary adapter is already saved
_emit(on_event, "warning", message = f"EMA adapter save failed: {exc}")
diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.py
index 5c2cd346817..f0bc5e709b9 100644
--- a/studio/backend/core/training/diffusion_train_common.py
+++ b/studio/backend/core/training/diffusion_train_common.py
@@ -71,7 +71,16 @@ _LR_SCHEDULERS: frozenset[str] = frozenset(
# DiT families whose fp32 RoPE/embedder overflow fp16, so they train in bf16 only. Keep in sync with the DiT trainer's
# own specs.
_FORCE_BF16_FAMILIES: frozenset[str] = frozenset(
- {"qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2", "minimax-h3"}
+ {
+ "qwen-image",
+ "z-image",
+ "krea-2",
+ "flux.2-klein",
+ "flux.2-dev",
+ "ltx-2",
+ "wan2.2-ti2v-5b",
+ "minimax-h3",
+ }
)
# ``VideoFamily.trainable``; every name in it MUST resolve through get_trainer, and a video base outside it is refused
@@ -87,7 +96,7 @@ TRAINABLE_VIDEO_FAMILIES: frozenset[str] = frozenset(f.name for f in _VIDEO_REGI
AUTO_FLOW_SHIFT_FAMILIES: frozenset[str] = frozenset({"qwen-image", "ltx-2", "minimax-h3"})
# Video latents are allocated on the family's spatial compression grid, so a training resolution off that grid
-# silently changes the latent geometry. LTX-2's VAE compresses 32x spatially, matching its resolution_multiple.
+# silently changes the latent geometry. LTX-2: 32x VAE; Wan2.2-TI2V-5B: 16x VAE x 2x2 patch.
_VIDEO_RESOLUTION_MULTIPLE = 32
_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}
@@ -465,7 +474,7 @@ def get_trainer(family: str) -> Callable[..., str]:
if key == "sdxl":
from core.training.diffusion_lora_trainer import run_diffusion_lora_training
return run_diffusion_lora_training
- if key in ("flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"):
+ if key in _DIT_TRAIN_FAMILIES:
from core.training.diffusion_dit_trainer import run_dit_lora_training
return run_dit_lora_training
# MiniMax-H3 denoises video and audio jointly over one packed sequence on two coupled schedules, which is outside
@@ -523,6 +532,14 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {
"lr_scheduler": "constant_with_warmup",
"lr_warmup_steps": 20,
},
+ # DiffSynth-Studio's Wan recipe; 704 is TI2V-5B's native short edge.
+ "wan2.2-ti2v-5b": {
+ "lora_rank": 32,
+ "learning_rate": 1e-4,
+ "resolution": 704,
+ "lr_scheduler": "constant_with_warmup",
+ "lr_warmup_steps": 20,
+ },
# resolution is the canvas SHORT EDGE and 768 is what the released checkpoint generates on; rank 16 not 32 because
# the adapter also serves the audio rows through one shared stack.
"minimax-h3": {
@@ -571,6 +588,13 @@ _FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {
"gated": False,
"note": "Video: trains a style LoRA on still images.",
},
+ # Measured on a B200 (nf4, 704px, rank 32): run peaks at 11.6 GB during the one-time UMT5-XXL encode.
+ "wan2.2-ti2v-5b": {
+ "params": "5B",
+ "qlora_vram_gb": 12,
+ "gated": False,
+ "note": "Video: trains a style LoRA on still images.",
+ },
# Measured on a B200: the loop peaks near 44 GB, but a 20-step run peaked at 77.76 GB with the 63 GiB Qwen3-VL
# conditioner resident, so 72 sizes users onto a card that OOMs later.
"minimax-h3": {
@@ -607,7 +631,16 @@ def _family_vram_note(name: str) -> str:
# The flow-matching DiT families (run by diffusion_dit_trainer): they expose base_precision / compile and need bf16 on
# CUDA. SDXL is absent (own mixed_precision path).
_DIT_TRAIN_FAMILIES = frozenset(
- {"flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"}
+ {
+ "flux.1",
+ "qwen-image",
+ "z-image",
+ "krea-2",
+ "flux.2-klein",
+ "flux.2-dev",
+ "ltx-2",
+ "wan2.2-ti2v-5b",
+ }
)
# Kept separate so the DiT-specific levers (compile, the shared sigma table) do not follow.
_FLOW_TRAIN_FAMILIES = _DIT_TRAIN_FAMILIES | {"minimax-h3"}
@@ -2016,6 +2049,7 @@ _TRAIN_EXTRA_TRUSTED_REPOS = frozenset(
"lightricks/ltx-2",
# MiniMax-H3's official base, for the same reason: safetensors-only, no remote code.
"minimaxai/minimax-h3",
+ "wan-ai/wan2.2-ti2v-5b-diffusers",
}
)
diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
index 3831ec8cf19..1644164d485 100644
--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json
@@ -3412,11 +3412,13 @@
"minimax-h3": "MiniMax-H3",
"qwen-image": "Qwen-Image",
"sdxl": "SDXL",
+ "wan2.2-ti2v-5b": "Wan2.2 TI2V 5B",
"z-image": "Z-Image"
},
"trainable_video_families": [
"ltx-2",
- "minimax-h3"
+ "minimax-h3",
+ "wan2.2-ti2v-5b"
],
"trusted_image_repos": [
"alpha-vllm/lumina-image-2.0",
diff --git a/studio/backend/tests/test_diffusion_dit_trainer.py b/studio/backend/tests/test_diffusion_dit_trainer.py
index ef71751a493..20b6e81bb7a 100644
--- a/studio/backend/tests/test_diffusion_dit_trainer.py
+++ b/studio/backend/tests/test_diffusion_dit_trainer.py
@@ -63,6 +63,7 @@ def test_specs_cover_the_dit_families():
"flux.2-dev",
# The first VIDEO family; its own assertions live in test_diffusion_dit_trainer_ltx2.
"ltx-2",
+ "wan2.2-ti2v-5b",
}
# FLUX / Qwen share the added-kv attention target set; Z-Image and Krea 2 are single-stream.
assert "add_q_proj" in _SPECS["flux.1"].lora_targets
diff --git a/studio/backend/tests/test_diffusion_dit_trainer_ltx2.py b/studio/backend/tests/test_diffusion_dit_trainer_ltx2.py
index 6c80ae84755..ea63c304723 100644
--- a/studio/backend/tests/test_diffusion_dit_trainer_ltx2.py
+++ b/studio/backend/tests/test_diffusion_dit_trainer_ltx2.py
@@ -462,8 +462,8 @@ def test_ltx2_bases_route_to_the_ltx2_trainer(base):
@pytest.mark.parametrize(
"base, family",
[
- ("Wan-AI/Wan2.2-TI2V-5B-Diffusers", "wan2.2-ti2v-5b"),
("Wan-AI/Wan2.2-T2V-A14B-Diffusers", "wan2.2-t2v-a14b"),
+ ("Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "wan2.1-t2v-1.3b"),
("hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", "hunyuanvideo-1.5"),
],
)
@@ -483,8 +483,8 @@ def test_explicit_video_model_family_override_is_honoured_and_gated():
# An opaque local path plus an explicit family is the documented way to train from a
# local checkout, so the override path needs the same video routing as detection.
assert resolve_trainable_family("/tmp/some-local-checkout", "ltx-2") == "ltx-2"
- with pytest.raises(ValueError, match = "wan2.2-ti2v-5b"):
- resolve_trainable_family("/tmp/some-local-checkout", "wan2.2-ti2v-5b")
+ with pytest.raises(ValueError, match = "wan2.2-t2v-a14b"):
+ resolve_trainable_family("/tmp/some-local-checkout", "wan2.2-t2v-a14b")
def test_unknown_model_family_lists_the_video_families_too():
diff --git a/studio/backend/tests/test_diffusion_dit_trainer_wan22.py b/studio/backend/tests/test_diffusion_dit_trainer_wan22.py
new file mode 100644
index 00000000000..a1f95be81c3
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_dit_trainer_wan22.py
@@ -0,0 +1,544 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU-only tests for the Wan2.2-TI2V-5B DiT LoRA trainer family (fakes plus a tiny random WanTransformer3DModel)."""
+
+from __future__ import annotations
+
+import json
+import math
+import types
+
+import pytest
+import torch
+
+from core.training import diffusion_dit_trainer as dit
+from core.training.diffusion_dit_trainer import (
+ _SPECS,
+ _WAN_MAX_SEQUENCE_LENGTH,
+ _WAN_TARGETS,
+ _select_lora_targets,
+ _training_sigma_table,
+)
+from core.training.diffusion_train_common import (
+ DEFAULT_LORA_FILENAME,
+ DEFAULT_LORA_TARGETS,
+ DiffusionLoraConfig,
+ TRAINABLE_VIDEO_FAMILIES,
+ _DIT_TRAIN_FAMILIES,
+ _FORCE_BF16_FAMILIES,
+ _assert_trusted_base_model,
+ _family_vram_note,
+ _publish_to_lora_catalog,
+ get_trainer,
+ resolve_trainable_family,
+ train_defaults,
+)
+
+FAMILY = "wan2.2-ti2v-5b"
+BASE = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"
+
+# Wan-AI/Wan2.2-TI2V-5B-Diffusers vae/config.json: 48 channels (latents_mean / latents_std are 48 long).
+Z_DIM = 48
+# Wan-AI/Wan2.2-TI2V-5B-Diffusers scheduler/scheduler_config.json, the keys the trainer reads.
+UNIPC_CONFIG = {
+ "_class_name": "UniPCMultistepScheduler",
+ "_diffusers_version": "0.35.0.dev0",
+ "flow_shift": 5.0,
+ "num_train_timesteps": 1000,
+ "prediction_type": "flow_prediction",
+ "use_flow_sigmas": True,
+ "solver_order": 2,
+}
+
+
+def _wan_classes():
+ try:
+ from diffusers import WanPipeline, WanTransformer3DModel
+ except Exception as exc: # noqa: BLE001 -- diffusers too old, or a broken optional dep
+ pytest.skip(f"diffusers Wan classes unavailable: {exc}")
+ return WanPipeline, WanTransformer3DModel
+
+
+def _tiny_transformer():
+ """A random 2-block WanTransformer3DModel with TI2V-5B's layout."""
+ _, WanTransformer3DModel = _wan_classes()
+ torch.manual_seed(0)
+ return WanTransformer3DModel(
+ patch_size = (1, 2, 2),
+ num_attention_heads = 2,
+ attention_head_dim = 12,
+ in_channels = Z_DIM,
+ out_channels = Z_DIM,
+ text_dim = 32,
+ freq_dim = 256,
+ ffn_dim = 32,
+ num_layers = 2,
+ cross_attn_norm = True,
+ qk_norm = "rms_norm_across_heads",
+ rope_max_seq_len = 32,
+ ).eval()
+
+
+def test_wan_spec_is_registered_and_bf16_only():
+ spec = _SPECS[FAMILY]
+ assert spec.family == FAMILY
+ assert spec.force_bf16 is True
+ assert FAMILY in _FORCE_BF16_FAMILIES
+ assert 8.0 < spec.dense_bf16_gb < 12.0
+ assert spec.lora_targets == _WAN_TARGETS
+ # Wan ships UniPC, so the spec must build its own FlowMatchEuler table.
+ assert spec.training_scheduler is dit._wan_training_scheduler
+ assert spec.decode_latents is None and spec.latent_shape is None
+
+
+def test_wan_is_a_trainable_video_family_with_a_trainer():
+ assert FAMILY in TRAINABLE_VIDEO_FAMILIES
+ assert FAMILY in _DIT_TRAIN_FAMILIES
+ assert get_trainer(FAMILY) is dit.run_dit_lora_training
+ for name in TRAINABLE_VIDEO_FAMILIES:
+ assert callable(get_trainer(name)), name
+
+
+ "base", [BASE, "wan-ai/wan2.2-ti2v-5b-diffusers", "/data/models/Wan2.2-TI2V-5B-Diffusers"]
+)
+def test_wan_bases_route_to_the_dit_trainer(base):
+ assert resolve_trainable_family(base) == FAMILY
+
+
+def test_explicit_model_family_override_routes_a_local_checkout():
+ assert resolve_trainable_family("/tmp/some-local-checkout", FAMILY) == FAMILY
+
+
+def test_the_a14b_moe_is_still_refused_and_names_the_5b():
+ # Step 2: two experts need two adapters, which this spec does not train.
+ with pytest.raises(ValueError) as exc:
+ resolve_trainable_family("Wan-AI/Wan2.2-T2V-A14B-Diffusers")
+ assert "wan2.2-t2v-a14b" in str(exc.value)
+ assert FAMILY in str(exc.value)
+
+
+def test_wan_official_base_passes_the_trusted_base_gate():
+ _assert_trusted_base_model(BASE)
+ with pytest.raises(ValueError, match = "untrusted"):
+ _assert_trusted_base_model("random-user/wan2.2-ti2v-5b-finetune")
+
+
+def test_wan_defaults_sit_on_the_32_grid():
+ d = train_defaults(FAMILY)
+ assert d["resolution"] % 32 == 0 # 16x VAE x 2x patch
+ assert d["lora_rank"] >= 16 and 0 < d["learning_rate"] <= 2e-4
+ assert d["lr_scheduler"] == "constant_with_warmup" and d["lr_warmup_steps"] > 0
+
+
+def test_wan_has_a_vram_row():
+ note = _family_vram_note(FAMILY)
+ assert "5B" in note and "QLoRA" in note
+
+
+def _cfg(tmp_path, **kw):
+ data = tmp_path / "data"
+ data.mkdir(exist_ok = True)
+ return DiffusionLoraConfig(
+ base_model = BASE, data_dir = str(data), output_dir = str(tmp_path / "run"), **kw
+ )
+
+
[email protected]("resolution, ok", [(704, True), (512, True), (520, False), (696, False)])
+def test_wan_resolution_must_sit_on_the_vae_grid(tmp_path, resolution, ok):
+ if ok:
+ assert _cfg(tmp_path, resolution = resolution).normalized().resolved_family == FAMILY
+ else:
+ with pytest.raises(ValueError, match = "multiple of 32"):
+ _cfg(tmp_path, resolution = resolution).normalized()
+
+
+def test_wan_rejects_fp16_before_loading(tmp_path):
+ with pytest.raises(ValueError, match = "bf16"):
+ _cfg(tmp_path, resolution = 704, mixed_precision = "fp16").normalized()
+
+
+def test_wan_default_flow_shift_stays_the_models_own_schedule(tmp_path):
+ cfg = _cfg(tmp_path, resolution = 704).normalized()
+ assert cfg.flow_shift is None
+
+
+def test_a_video_run_publishes_no_adapter_into_the_image_lora_catalog(tmp_path, monkeypatch):
+ catalog = tmp_path / "loras" / "diffusion"
+ catalog.mkdir(parents = True)
+ monkeypatch.setattr("core.inference.diffusion_lora.loras_dir", lambda: catalog)
+ adapter = tmp_path / "run" / DEFAULT_LORA_FILENAME
+ adapter.parent.mkdir(parents = True, exist_ok = True)
+ adapter.write_bytes(b"adapter-bytes")
+ cfg = _cfg(tmp_path, resolution = 704).normalized()
+ assert _publish_to_lora_catalog(str(adapter), cfg) is None
+ assert list(catalog.iterdir()) == []
+
+
+def test_family_train_infos_offers_wan(dit_train_host):
+ from core.training.diffusion_train_common import family_train_infos
+
+ infos = {i["name"]: i for i in family_train_infos()}
+ assert FAMILY in infos, "a trainable video family must be offered by /diffusion/info"
+ info = infos[FAMILY]
+ assert info["default_base"] == BASE
+ assert info["base_repos"] == [BASE]
+ assert info["label"] == "Wan2.2 TI2V 5B"
+ assert info["defaults"]["resolution"] % 32 == 0
+ assert info["deploy_base"] is None
+ assert "nf4" in info["precision_modes"]
+ assert info["supports_compile"] is True
+ assert "ltx-2" in infos and "sdxl" in infos
+
+
+def test_generic_default_targets_resolve_to_the_wan_set():
+ assert _select_lora_targets(DEFAULT_LORA_TARGETS, _WAN_TARGETS) == _WAN_TARGETS
+ explicit = ("to_q", "to_v")
+ assert _select_lora_targets(explicit, _WAN_TARGETS) == explicit
+
+
+def unipc_base(tmp_path):
+ root = tmp_path / "wan_base"
+ (root / "scheduler").mkdir(parents = True)
+ (root / "scheduler" / "scheduler_config.json").write_text(json.dumps(UNIPC_CONFIG))
+ return types.SimpleNamespace(base_model = str(root), hf_token = None)
+
+
+def test_training_scheduler_bakes_in_the_inference_flow_shift(unipc_base):
+ pytest.importorskip("diffusers")
+ sched = dit._wan_training_scheduler(unipc_base)
+ n = 1000
+ u = torch.linspace(1, n, n).flip(0) / n
+ expected = 5.0 * u / (1.0 + 4.0 * u)
+ assert sched.config.shift == 5.0
+ assert torch.allclose(sched.sigmas[:n].float(), expected, atol = 1e-6)
+ assert torch.allclose(sched.timesteps.float(), expected * n, atol = 1e-3)
+ # None (the default) trains on exactly this table; a number re-shifts from the unshifted u.
+ assert _training_sigma_table(sched, None) is sched.sigmas
+ assert torch.allclose(_training_sigma_table(sched, 1.0).float(), u, atol = 1e-6)
+
+
+def test_the_generic_scheduler_load_would_drop_the_shift(unipc_base):
+ """The trap the spec hook exists for: FlowMatchEuler.from_pretrained on Wan's UniPC config ignores flow_shift and
+ yields the identity (shift 1) table, so a run trains mostly at low noise."""
+ FlowMatchEulerDiscreteScheduler = pytest.importorskip(
+ "diffusers"
+ ).FlowMatchEulerDiscreteScheduler
+
+ generic = FlowMatchEulerDiscreteScheduler.from_pretrained(
+ unipc_base.base_model, subfolder = "scheduler"
+ )
+ ours = dit._wan_training_scheduler(unipc_base)
+ assert generic.config.shift == 1.0
+ mid = 500
+ assert ours.sigmas[mid] > generic.sigmas[mid] + 0.2
+
+
+class _FakeWanPipe:
+ """encode_prompt returns what WanPipeline._get_t5_prompt_embeds does: real rows then zeros to max_sequence_length."""
+
+ def __init__(self):
+ self.text_encoder = None
+ self.calls = []
+
+ def encode_prompt(
+ self,
+ prompt,
+ do_classifier_free_guidance,
+ num_videos_per_prompt,
+ max_sequence_length,
+ device,
+ ):
+ self.calls.append((prompt, max_sequence_length, do_classifier_free_guidance))
+ n = len(prompt.split()) + 1
+ g = torch.Generator().manual_seed(n)
+ emb = torch.zeros(1, max_sequence_length, 8)
+ emb[:, :n] = torch.randn(1, n, 8, generator = g) + 0.1
+ return emb.to(torch.bfloat16), None
+
+
+def test_encode_prompts_caches_trimmed_embeds_and_collate_restores_the_pipeline_tensor():
+ pipe = _FakeWanPipe()
+ caps = ["a cat", "a much longer caption about a yarn dog"]
+ cached = dit._wan_encode_prompts(pipe, caps, "cpu")
+ assert [c[1] for c in pipe.calls] == [_WAN_MAX_SEQUENCE_LENGTH] * 2
+ assert all(c[2] is False for c in pipe.calls)
+ assert [e[0].shape[1] for e in cached] == [3, 9] # trailing zero rows dropped
+ (batch,) = dit._wan_collate(cached, "cpu", torch.bfloat16)
+ assert batch.shape == (2, _WAN_MAX_SEQUENCE_LENGTH, 8)
+ for i, cap in enumerate(caps):
+ full, _ = _FakeWanPipe().encode_prompt(cap, False, 1, _WAN_MAX_SEQUENCE_LENGTH, "cpu")
+ assert torch.equal(
+ batch[i : i + 1], full
+ ), "collate must rebuild the exact zero-padded tensor"
+
+
+def test_trim_keeps_one_row_for_an_all_zero_embed():
+ out = dit._wan_trim_padding(torch.zeros(1, 16, 4))
+ assert out.shape == (1, 1, 4)
+
+
+class _FakeDist:
+ def __init__(self, mean, std):
+ self.mean, self.std = mean, std
+
+ def sample(self):
+ return self.mean
+
+
+class _FakeWanVae(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.w = torch.nn.Parameter(torch.zeros(1))
+ self.config = types.SimpleNamespace(
+ z_dim = Z_DIM,
+ latents_mean = [0.1 * i for i in range(Z_DIM)],
+ latents_std = [1.0 + 0.05 * i for i in range(Z_DIM)],
+ )
+ self.seen = None
+
+ def encode(self, px):
+ self.seen = px
+ b, _c, f, h, w = px.shape
+ mean = torch.full((b, Z_DIM, f, h // 16, w // 16), 2.0)
+ std = torch.full_like(mean, 0.5)
+ return types.SimpleNamespace(latent_dist = _FakeDist(mean, std))
+
+
+def test_encode_latents_adds_a_frame_axis_and_normalises_like_the_pipeline():
+ vae = _FakeWanVae()
+ px = torch.zeros(2, 3, 64, 96)
+ lat = dit._wan_encode_latents(vae, px)
+ assert vae.seen.shape == (2, 3, 1, 64, 96)
+ assert lat.shape == (2, Z_DIM, 1, 4, 6)
+ # WanPipeline decodes with latents / (1 / std) + mean; that must invert the training normalisation.
+ mean = torch.tensor(vae.config.latents_mean).view(1, Z_DIM, 1, 1, 1)
+ inv_std = 1.0 / torch.tensor(vae.config.latents_std).view(1, Z_DIM, 1, 1, 1)
+ assert torch.allclose(lat / inv_std + mean, torch.full_like(lat, 2.0), atol = 1e-5)
+
+
+def test_encode_latent_stats_returns_the_posterior_affine_pair():
+ vae = _FakeWanVae()
+ a, b = dit._wan_encode_latent_stats(vae, torch.zeros(1, 3, 32, 32))
+ std = torch.tensor(vae.config.latents_std).view(1, Z_DIM, 1, 1, 1)
+ assert torch.allclose(b, 0.5 / std.expand_as(b))
+ assert torch.allclose(a, dit._wan_encode_latents(vae, torch.zeros(1, 3, 32, 32)))
+
+
+def test_forward_matches_the_pipelines_expand_timesteps_call():
+ transformer = _tiny_transformer()
+ noisy = torch.randn(2, Z_DIM, 1, 8, 12)
+ t = torch.tensor([937.5, 120.0])
+ emb = torch.randn(2, 16, 32)
+ with torch.no_grad():
+ ours = dit._wan_forward(transformer, noisy, t, None, (emb,), None, "cpu", torch.float32)
+ mask = torch.ones(1, 1, 8, 12)
+ per_sample = []
+ for i in range(2):
+ ts = (mask[0][:, ::2, ::2] * t[i]).flatten().unsqueeze(0)
+ per_sample.append(
+ transformer(
+ hidden_states = noisy[i : i + 1],
+ timestep = ts,
+ encoder_hidden_states = emb[i : i + 1],
+ return_dict = False,
+ )[0]
+ )
+ assert ours.shape == noisy.shape # aligned with target = noise - latents
+ assert torch.allclose(ours, torch.cat(per_sample), atol = 1e-6)
+
+
+def _adapted_tiny():
+ from peft import LoraConfig
+
+ transformer = _tiny_transformer()
+ transformer.add_adapter(
+ LoraConfig(
+ r = 4, lora_alpha = 4, init_lora_weights = "gaussian", target_modules = list(_WAN_TARGETS)
+ )
+ )
+ return transformer
+
+
+def test_targets_adapt_every_block_linear_and_nothing_else():
+ transformer = _adapted_tiny()
+ adapted = {n for n, m in transformer.named_modules() if hasattr(m, "lora_A")}
+ block_linears = {
+ f"blocks.{i}.{n}"
+ for i in range(2)
+ for n in (
+ "attn1.to_q",
+ "attn1.to_k",
+ "attn1.to_v",
+ "attn1.to_out.0",
+ "attn2.to_q",
+ "attn2.to_k",
+ "attn2.to_v",
+ "attn2.to_out.0",
+ "ffn.net.0.proj",
+ "ffn.net.2",
+ )
+ }
+ assert adapted == block_linears
+ assert not any(
+ n.startswith(("condition_embedder", "proj_out", "patch_embedding")) for n in adapted
+ )
+
+
+def test_saved_lora_has_diffusers_keys_and_loads_through_wanpipeline(tmp_path):
+ from peft.utils import get_peft_model_state_dict
+ from safetensors.torch import load_file
+
+ WanPipeline, _ = _wan_classes()
+ trained = _adapted_tiny()
+ with torch.no_grad():
+ for n, p in trained.named_parameters():
+ if "lora_B" in n:
+ p.normal_(0.0, 0.5)
+ dit._wan_save(None, str(tmp_path), get_peft_model_state_dict(trained))
+ path = tmp_path / DEFAULT_LORA_FILENAME
+ keys = set(load_file(str(path)))
+ assert keys and all(k.startswith("transformer.blocks.") for k in keys)
+ assert "transformer.blocks.0.attn1.to_q.lora_A.weight" in keys
+ assert "transformer.blocks.1.ffn.net.2.lora_B.weight" in keys
+
+ fresh = _tiny_transformer()
+ pipe = WanPipeline(
+ tokenizer = None,
+ text_encoder = None,
+ vae = None,
+ scheduler = None,
+ transformer = fresh,
+ expand_timesteps = True,
+ )
+ noisy = torch.randn(1, Z_DIM, 1, 8, 8)
+ emb = torch.randn(1, 16, 32)
+ t = torch.tensor([500.0])
+ with torch.no_grad():
+ base = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)
+ pipe.load_lora_weights(
+ str(tmp_path), weight_name = DEFAULT_LORA_FILENAME, adapter_name = "style"
+ )
+ loaded = {
+ n for n, m in fresh.named_modules() if hasattr(m, "lora_A") and "style" in m.lora_A
+ }
+ assert len(loaded) == 20
+ with_lora = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)
+ trained_out = dit._wan_forward(trained, noisy, t, None, (emb,), None, "cpu", torch.float32)
+ pipe.disable_lora()
+ off = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)
+ assert not torch.allclose(with_lora, base)
+ assert torch.allclose(
+ with_lora, trained_out, atol = 1e-5
+ ), "the loaded adapter must reproduce the trained one"
+ assert torch.equal(off, base)
+ assert math.isfinite(float(with_lora.abs().max()))
+
+
+def test_collate_copies_only_the_trimmed_rows_and_pads_on_the_device(monkeypatch):
+ """The pad to 512 must happen AFTER the move: padding on the host is a 4 MB pageable copy plus a multi-threaded
+ host fill + cat every step."""
+ seen = []
+ real_to = torch.Tensor.to
+
+ def spy(self, *a, **k):
+ seen.append(tuple(self.shape))
+ return real_to(self, *a, **k)
+
+ monkeypatch.setattr(torch.Tensor, "to", spy)
+ e = (torch.randn(1, 7, 8, dtype = torch.bfloat16),)
+ (out,) = dit._wan_collate([e], "cpu", torch.bfloat16)
+ assert seen == [(1, 7, 8)], "only the trimmed embed may be moved"
+ assert out.shape == (1, _WAN_MAX_SEQUENCE_LENGTH, 8)
+ assert torch.equal(out[:, :7], e[0]) and not out[:, 7:].any()
+
+
+def test_cached_latent_sampling_moves_before_joining_and_keeps_the_values():
+ cache = [[(torch.randn(1, Z_DIM, 1, 4, 4), torch.rand(1, Z_DIM, 1, 4, 4))] for _ in range(3)]
+ torch.manual_seed(7)
+ got = dit._sample_cached_latents(cache, [2, 0], None, "cpu", torch.float32)
+ torch.manual_seed(7)
+ a = torch.cat([cache[2][0][0], cache[0][0][0]])
+ b = torch.cat([cache[2][0][1], cache[0][0][1]])
+ assert torch.equal(got, a + b * torch.randn_like(a))
+ one = dit._to_device_cat([cache[1][0][0]], "cpu")
+ assert one is cache[1][0][0] or torch.equal(one, cache[1][0][0])
+
+
+def test_a_non_default_alpha_reloads_at_the_trained_strength(tmp_path):
+ """diffusers' loader assumes lora_alpha == rank unless the safetensors header carries the adapter config, so a
+ rank-4 / alpha-16 adapter (scale 4) used to come back at scale 1."""
+ from peft import LoraConfig
+ from peft.utils import get_peft_model_state_dict
+ from safetensors import safe_open
+
+ WanPipeline, _ = _wan_classes()
+ trained = _tiny_transformer()
+ trained.add_adapter(
+ LoraConfig(
+ r = 4, lora_alpha = 16, init_lora_weights = "gaussian", target_modules = list(_WAN_TARGETS)
+ )
+ )
+ with torch.no_grad():
+ for n, p in trained.named_parameters():
+ if "lora_B" in n:
+ p.normal_(0.0, 0.2)
+ spec = _SPECS[FAMILY]
+ assert spec.save_takes_metadata
+ dit._adapter_saver(spec, trained)(None, str(tmp_path), get_peft_model_state_dict(trained))
+ with safe_open(str(tmp_path / DEFAULT_LORA_FILENAME), "pt") as f:
+ meta = json.loads(f.metadata()["lora_adapter_metadata"])
+ assert meta["transformer.r"] == 4 and meta["transformer.lora_alpha"] == 16
+
+ fresh = _tiny_transformer()
+ pipe = WanPipeline(
+ tokenizer = None,
+ text_encoder = None,
+ vae = None,
+ scheduler = None,
+ transformer = fresh,
+ expand_timesteps = True,
+ )
+ pipe.load_lora_weights(str(tmp_path), weight_name = DEFAULT_LORA_FILENAME)
+ noisy, emb, t = torch.randn(1, Z_DIM, 1, 8, 8), torch.randn(1, 16, 32), torch.tensor([400.0])
+ with torch.no_grad():
+ got = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)
+ want = dit._wan_forward(trained, noisy, t, None, (emb,), None, "cpu", torch.float32)
+ assert torch.allclose(got, want, atol = 1e-5)
+
+
+def test_families_without_the_flag_keep_their_three_argument_saver():
+ for name, spec in _SPECS.items():
+ if not spec.save_takes_metadata:
+ assert dit._adapter_saver(spec, object()) is spec.save, name
+
+
+def test_load_conditioners_leaves_the_vae_on_the_host(monkeypatch):
+ moved = []
+
+ class _Vae:
+ def to(self, *a, **k):
+ moved.append((a, k))
+ return self
+
+ class _Pipe:
+ def __init__(self, vae):
+ self.vae = vae
+
+ _wan_classes() # skip where this diffusers cannot import the Wan stack
+ import diffusers
+
+ monkeypatch.setattr(
+ diffusers.AutoencoderKLWan, "from_pretrained", classmethod(lambda cls, *a, **k: _Vae())
+ )
+ monkeypatch.setattr(
+ dit,
+ "_wan_pipeline_cls",
+ lambda: types.SimpleNamespace(from_pretrained = lambda *a, **k: _Pipe(k["vae"])),
+ )
+ pipe, vae = dit._wan_load_conditioners(
+ types.SimpleNamespace(base_model = BASE, hf_token = None), "cuda", None
+ )
+ assert vae is pipe.vae and moved == [], "the VAE must wait for the encoders to be freed"