Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: Wan2.2-TI2V-5B LoRA training (#13654)
Daniel Hanbc11dbef4556
diff --git a/studio/backend/core/inference/video_families.py b/studio/backend/core/inference/video_families.pyindex fb64b73a8bb..aee56a5f8fd 100644--- a/studio/backend/core/inference/video_families.py+++ b/studio/backend/core/inference/video_families.py@@ -665,6 +665,8 @@ _FAMILIES: tuple[VideoFamily, ...] = (# temporal compression 4 gives valid frame counts 4k+1. Defaults 20 steps / CFG 5 (ComfyUI's template).VideoFamily(name = "wan2.2-ti2v-5b",+ trainable = True,+ label = "Wan2.2 TI2V 5B",profile = FamilyProfile(untiled_vae = UntiledVae(bytes_per_latent_pixel = 2.5 * 2**20, output_scale = (16, 4)),preview_factors = "wan22_5b",diff --git a/studio/backend/core/training/diffusion_dit_trainer.py b/studio/backend/core/training/diffusion_dit_trainer.pyindex 927395219e4..6538f5dddd8 100644--- a/studio/backend/core/training/diffusion_dit_trainer.py+++ b/studio/backend/core/training/diffusion_dit_trainer.py@@ -3,8 +3,8 @@"""Flow-matching LoRA training for the rectified-flow DiT families.-Image: FLUX.1-dev, FLUX.2 (dev + Klein), Qwen-Image, Z-Image, Krea 2. Video: LTX-2 (from a-still-image dataset; see the ``_LTX2_TARGETS`` block for what a video family adds).+Image: FLUX.1-dev, FLUX.2 (dev + Klein), Qwen-Image, Z-Image, Krea 2. Video: LTX-2 and Wan2.2-TI2V-5B (from a+still-image dataset; see the ``_LTX2_TARGETS`` and ``_WAN_TARGETS`` blocks for what a video family adds).These are rectified-flow transformers, not the SDXL U-Net, so they share only the plumbing in``diffusion_train_common`` (config, dataset discovery, events, stop, publishing). The training math@@ -183,6 +183,10 @@ class _FamilySpec:latent_shape: Optional[Callable[..., tuple]] = Nonedecode_latents: Optional[Callable[..., Any]] = Nonesample_forward_kwargs: Optional[dict] = None+ # cfg -> FlowMatchEuler scheduler. None = load scheduler/ (only valid if it IS FlowMatchEuler; Wan ships UniPC).+ training_scheduler: Optional[Callable[..., Any]] = None+ # Saver takes ``adapter_metadata``; without it diffusers' loader assumes lora_alpha == rank.+ save_takes_metadata: bool = Falsedef _gather_sigmas(sigma_table, indices, device, dtype, n_dim):@@ -1644,6 +1648,194 @@ def _ltx2_save(pipe_cls, out_dir, transformer_lora_layers):)+# Wan2.2-TI2V-5B, trained from stills (a 1-frame clip is a valid T2V sample). Targets match DiffSynth-Studio /+# musubi-tuner; bare "to_q" is safe (single stream), add_k/v_proj exist only on I2V-14B.+_WAN_TARGETS = (+ "to_q",+ "to_k",+ "to_v",+ "to_out.0",+ "ffn.net.0.proj",+ "ffn.net.2",+)+_WAN_MAX_SEQUENCE_LENGTH = 512+++def _wan_pipeline_cls():+ from diffusers import WanPipeline+ return WanPipeline+++def _wan_load_conditioners(cfg, device, weight_dtype):+ """UMT5 + tokenizer + VAE. VAE loads fp32 directly; the shared bf16-then-fp32 cast would round it through bf16."""+ import torch+ from diffusers import AutoencoderKLWan++ vae = AutoencoderKLWan.from_pretrained(+ cfg.base_model, subfolder = "vae", torch_dtype = torch.float32, token = cfg.hf_token+ )+ pipe = _wan_pipeline_cls().from_pretrained(+ cfg.base_model,+ transformer = None,+ vae = vae,+ torch_dtype = torch.bfloat16,+ token = cfg.hf_token,+ )+ # VAE stays on host; the shared loop moves it only after UMT5-XXL (11.4 GB) is freed.+ return pipe, pipe.vae+++def _wan_load_transformer(cfg, device, weight_dtype, base_precision):+ from diffusers import WanTransformer3DModel+ return _load_dit_transformer(WanTransformer3DModel, cfg, device, base_precision)+++def _wan_trim_padding(embeds):+ """Drop trailing zero-pad rows; lossless since ``_wan_collate`` re-pads."""+ nonzero = embeds[0].abs().amax(dim = -1) > 0+ idx = nonzero.nonzero()+ keep = int(idx.max()) + 1 if idx.numel() else 1+ return embeds[:, :keep].contiguous()+++def _wan_encode_prompts(pipe, captions, device):+ import torch++ _encoders_to_device(pipe, device)+ out = []+ with torch.no_grad():+ for cap in captions:+ pe, _neg = pipe.encode_prompt(+ prompt = cap,+ do_classifier_free_guidance = False,+ num_videos_per_prompt = 1,+ max_sequence_length = _WAN_MAX_SEQUENCE_LENGTH,+ device = device,+ )+ out.append((_wan_trim_padding(pe).cpu(),))+ return out+++def _wan_latent_affine(vae, ref):+ """Per-channel (mean, std); training latent is ``(z - mean) / std``."""+ import torch++ z = int(vae.config.z_dim)+ mean = torch.tensor(vae.config.latents_mean, device = ref.device, dtype = ref.dtype).view(+ 1, z, 1, 1, 1+ )+ std = torch.tensor(vae.config.latents_std, device = ref.device, dtype = ref.dtype).view(+ 1, z, 1, 1, 1+ )+ return mean, std+++def _wan_pixels(vae, pixel_values):+ return pixel_values.to(dtype = next(vae.parameters()).dtype).unsqueeze(2)+++def _wan_encode_latents(vae, pixel_values):+ import torch++ with torch.no_grad():+ lat = vae.encode(_wan_pixels(vae, pixel_values)).latent_dist.sample()+ mean, std = _wan_latent_affine(vae, lat)+ return (lat - mean) / std+++def _wan_encode_latent_stats(vae, pixel_values):+ import torch++ with torch.no_grad():+ dist = vae.encode(_wan_pixels(vae, pixel_values)).latent_dist+ mean, std = _wan_latent_affine(vae, dist.mean)+ return (dist.mean - mean) / std, dist.std / std+++def _wan_collate(+ entries,+ device,+ weight_dtype,+ pad_to = None,+):+ import torch+ import torch.nn.functional as F++ # Zero-pad as _get_t5_prompt_embeds does, on the DEVICE: host-side pad/cat stalled steps ~100 ms.+ length = max(_WAN_MAX_SEQUENCE_LENGTH, *(e[0].shape[1] for e in entries))+ padded = [+ F.pad(+ e[0].to(device = device, dtype = weight_dtype, non_blocking = True),+ (0, 0, 0, length - e[0].shape[1]),+ )+ for e in entries+ ]+ return (padded[0] if len(padded) == 1 else torch.cat(padded),)+++def _wan_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype):+ (prompt_embeds,) = embeds_batch+ # [B] timestep == the pipeline's per-token expand_timesteps form for T2V.+ return transformer(+ hidden_states = noisy,+ timestep = timesteps.to(device = noisy.device).reshape(-1),+ encoder_hidden_states = prompt_embeds,+ return_dict = False,+ )[0]+++def _wan_save(+ pipe_cls,+ out_dir,+ transformer_lora_layers,+ adapter_metadata = None,+):+ # Older diffusers savers lack the metadata argument; they still get the weights.+ import inspect++ save = _wan_pipeline_cls().save_lora_weights+ kwargs = {}+ if (+ adapter_metadata+ and "transformer_lora_adapter_metadata" in inspect.signature(save).parameters+ ):+ kwargs["transformer_lora_adapter_metadata"] = dict(adapter_metadata)+ save(+ save_directory = out_dir,+ transformer_lora_layers = transformer_lora_layers,+ weight_name = DEFAULT_LORA_FILENAME,+ **kwargs,+ )+++def _lora_adapter_metadata(transformer) -> Optional[dict]:+ try:+ return transformer.peft_config["default"].to_dict()+ except Exception: # noqa: BLE001 -- metadata is best-effort; the weights still save+ return None+++def _adapter_saver(spec, transformer):+ if not spec.save_takes_metadata:+ return spec.save+ import functools+ return functools.partial(spec.save, adapter_metadata = _lora_adapter_metadata(transformer))+++def _wan_training_scheduler(cfg):+ """FlowMatchEuler with Wan's UniPC ``flow_shift`` baked in; loading it from the UniPC config silently drops the+ shift. UniPC flow sigmas equal FlowMatchEuler's static shift."""+ from diffusers import FlowMatchEulerDiscreteScheduler++ conf = FlowMatchEulerDiscreteScheduler.load_config(+ cfg.base_model, subfolder = "scheduler", token = cfg.hf_token+ )+ return FlowMatchEulerDiscreteScheduler(+ num_train_timesteps = int(conf.get("num_train_timesteps", 1000)),+ shift = float(conf.get("flow_shift") or 1.0),+ )++def _vae8_latent_shape(vae, height, width):return (1, vae.config.latent_channels, height // 8, width // 8), (height // 16) * (width // 16)@@ -1797,6 +1989,22 @@ _SPECS: dict[str, _FamilySpec] = {forward = _ltx2_forward,save = _ltx2_save,),+ "wan2.2-ti2v-5b": _FamilySpec(+ family = "wan2.2-ti2v-5b",+ lora_targets = _WAN_TARGETS,+ force_bf16 = True,+ dense_bf16_gb = 10.0,+ load_conditioners = _wan_load_conditioners,+ load_transformer = _wan_load_transformer,+ encode_prompts = _wan_encode_prompts,+ encode_latents = _wan_encode_latents,+ encode_latent_stats = _wan_encode_latent_stats,+ collate = _wan_collate,+ forward = _wan_forward,+ save = _wan_save,+ training_scheduler = _wan_training_scheduler,+ save_takes_metadata = True,+ ),}@@ -2081,13 +2289,21 @@ def _sample_cached_latents(cache, idxs, variant_rng, device, weight_dtype):a, b = variants[variant_rng.randrange(len(variants))] if len(variants) > 1 else variants[0]parts_a.append(a)parts_b.append(b)- lat_a = torch.cat(parts_a).to(device, non_blocking = True)+ lat_a = _to_device_cat(parts_a, device)if parts_b[0] is None:return lat_a.to(dtype = weight_dtype)- lat_b = torch.cat(parts_b).to(device, non_blocking = True)+ lat_b = _to_device_cat(parts_b, device)return (lat_a + lat_b * torch.randn_like(lat_a)).to(dtype = weight_dtype)+def _to_device_cat(parts, device):+ """Copy pinned parts, then cat on ``device``: a host cat yields pageable memory and a sync copy."""+ import torch++ moved = [p.to(device, non_blocking = True) for p in parts]+ return moved[0] if len(moved) == 1 else torch.cat(moved)++def _should_compile(cfg,base_is_bnb,@@ -2225,7 +2441,7 @@ def run_dit_lora_training(on_event: Optional[EventCb] = None,should_stop: Optional[StopCb] = None,) -> str:- """Train a flow-matching DiT LoRA (FLUX.1 / FLUX.2 / Qwen-Image / Z-Image / Krea 2 / LTX-2) and+ """Train a flow-matching DiT LoRA (FLUX.1 / FLUX.2 / Qwen-Image / Z-Image / Krea 2 / LTX-2 / Wan2.2-TI2V-5B) andexport it. Resumable: ``cfg.resume_from_checkpoint`` restores the adapter, optimizer moments,LR position, EMA shadow, sampler cycle and RNG streams from a ``checkpoint-<N>`` bundle, andthe loop runs steps N+1..train_steps (the TARGET TOTAL). A stop-and-save and every@@ -2445,6 +2661,9 @@ def _train_dit(torch.cuda.empty_cache()elif device == "xpu":torch.xpu.empty_cache()+ # Some families keep the VAE on host while text encoders are resident.+ if vae is not None:+ vae.to(device)# The cache keeps the posterior affine parameters, so per-step sampling noise is preserved.if use_cache:@@ -2553,9 +2772,12 @@ def _train_dit(optimizer = _make_optimizer(lora_params, cfg.learning_rate, recorded_optimizer_class(cfg, identity))- scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(- fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token- )+ if spec.training_scheduler is not None:+ scheduler = spec.training_scheduler(fetch_cfg)+ else:+ scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(+ fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token+ )# getattr defaults keep an un-normalized config on the historical behaviour.flow_shift = getattr(cfg, "flow_shift", None)if flow_shift is None and spec.family in AUTO_FLOW_SHIFT_FAMILIES:@@ -2892,14 +3114,15 @@ def _train_dit(if stopped and done > 0:_save_checkpoint(done)layers = get_peft_model_state_dict(transformer)- spec.save(pipe, str(out_dir), layers)+ save_adapter = _adapter_saver(spec, transformer)+ save_adapter(pipe, str(out_dir), layers)lora_path = str(out_dir / DEFAULT_LORA_FILENAME)# ``done`` (the step reached), not cfg.train_steps: a stop at 11/500 must not advertise 500.catalog_path = _publish_to_lora_catalog(lora_path, cfg, done)if ema is not None and ema.updates > 0:try:# Report the adapter FILE, like lora_path, so a caller can load it directly.- ema_dir = save_ema_adapter(ema, transformer, spec.save, str(out_dir))+ ema_dir = save_ema_adapter(ema, transformer, save_adapter, str(out_dir))ema_path = str(Path(ema_dir) / DEFAULT_LORA_FILENAME)except Exception as exc: # noqa: BLE001 -- the primary adapter is already saved_emit(on_event, "warning", message = f"EMA adapter save failed: {exc}")diff --git a/studio/backend/core/training/diffusion_train_common.py b/studio/backend/core/training/diffusion_train_common.pyindex 5c2cd346817..f0bc5e709b9 100644--- a/studio/backend/core/training/diffusion_train_common.py+++ b/studio/backend/core/training/diffusion_train_common.py@@ -71,7 +71,16 @@ _LR_SCHEDULERS: frozenset[str] = frozenset(# DiT families whose fp32 RoPE/embedder overflow fp16, so they train in bf16 only. Keep in sync with the DiT trainer's# own specs._FORCE_BF16_FAMILIES: frozenset[str] = frozenset(- {"qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2", "minimax-h3"}+ {+ "qwen-image",+ "z-image",+ "krea-2",+ "flux.2-klein",+ "flux.2-dev",+ "ltx-2",+ "wan2.2-ti2v-5b",+ "minimax-h3",+ })# ``VideoFamily.trainable``; every name in it MUST resolve through get_trainer, and a video base outside it is refused@@ -87,7 +96,7 @@ TRAINABLE_VIDEO_FAMILIES: frozenset[str] = frozenset(f.name for f in _VIDEO_REGIAUTO_FLOW_SHIFT_FAMILIES: frozenset[str] = frozenset({"qwen-image", "ltx-2", "minimax-h3"})# Video latents are allocated on the family's spatial compression grid, so a training resolution off that grid-# silently changes the latent geometry. LTX-2's VAE compresses 32x spatially, matching its resolution_multiple.+# silently changes the latent geometry. LTX-2: 32x VAE; Wan2.2-TI2V-5B: 16x VAE x 2x2 patch._VIDEO_RESOLUTION_MULTIPLE = 32_IMAGE_EXTS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}@@ -465,7 +474,7 @@ def get_trainer(family: str) -> Callable[..., str]:if key == "sdxl":from core.training.diffusion_lora_trainer import run_diffusion_lora_trainingreturn run_diffusion_lora_training- if key in ("flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"):+ if key in _DIT_TRAIN_FAMILIES:from core.training.diffusion_dit_trainer import run_dit_lora_trainingreturn run_dit_lora_training# MiniMax-H3 denoises video and audio jointly over one packed sequence on two coupled schedules, which is outside@@ -523,6 +532,14 @@ FAMILY_TRAIN_DEFAULTS: dict[str, dict[str, Any]] = {"lr_scheduler": "constant_with_warmup","lr_warmup_steps": 20,},+ # DiffSynth-Studio's Wan recipe; 704 is TI2V-5B's native short edge.+ "wan2.2-ti2v-5b": {+ "lora_rank": 32,+ "learning_rate": 1e-4,+ "resolution": 704,+ "lr_scheduler": "constant_with_warmup",+ "lr_warmup_steps": 20,+ },# resolution is the canvas SHORT EDGE and 768 is what the released checkpoint generates on; rank 16 not 32 because# the adapter also serves the audio rows through one shared stack."minimax-h3": {@@ -571,6 +588,13 @@ _FAMILY_TRAIN_SPECS: dict[str, dict[str, Any]] = {"gated": False,"note": "Video: trains a style LoRA on still images.",},+ # Measured on a B200 (nf4, 704px, rank 32): run peaks at 11.6 GB during the one-time UMT5-XXL encode.+ "wan2.2-ti2v-5b": {+ "params": "5B",+ "qlora_vram_gb": 12,+ "gated": False,+ "note": "Video: trains a style LoRA on still images.",+ },# Measured on a B200: the loop peaks near 44 GB, but a 20-step run peaked at 77.76 GB with the 63 GiB Qwen3-VL# conditioner resident, so 72 sizes users onto a card that OOMs later."minimax-h3": {@@ -607,7 +631,16 @@ def _family_vram_note(name: str) -> str:# The flow-matching DiT families (run by diffusion_dit_trainer): they expose base_precision / compile and need bf16 on# CUDA. SDXL is absent (own mixed_precision path)._DIT_TRAIN_FAMILIES = frozenset(- {"flux.1", "qwen-image", "z-image", "krea-2", "flux.2-klein", "flux.2-dev", "ltx-2"}+ {+ "flux.1",+ "qwen-image",+ "z-image",+ "krea-2",+ "flux.2-klein",+ "flux.2-dev",+ "ltx-2",+ "wan2.2-ti2v-5b",+ })# Kept separate so the DiT-specific levers (compile, the shared sigma table) do not follow._FLOW_TRAIN_FAMILIES = _DIT_TRAIN_FAMILIES | {"minimax-h3"}@@ -2016,6 +2049,7 @@ _TRAIN_EXTRA_TRUSTED_REPOS = frozenset("lightricks/ltx-2",# MiniMax-H3's official base, for the same reason: safetensors-only, no remote code."minimaxai/minimax-h3",+ "wan-ai/wan2.2-ti2v-5b-diffusers",})diff --git a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json b/studio/backend/tests/fixtures/diffusion_family_data_freeze.jsonindex 3831ec8cf19..1644164d485 100644--- a/studio/backend/tests/fixtures/diffusion_family_data_freeze.json+++ b/studio/backend/tests/fixtures/diffusion_family_data_freeze.json@@ -3412,11 +3412,13 @@"minimax-h3": "MiniMax-H3","qwen-image": "Qwen-Image","sdxl": "SDXL",+ "wan2.2-ti2v-5b": "Wan2.2 TI2V 5B","z-image": "Z-Image"},"trainable_video_families": ["ltx-2",- "minimax-h3"+ "minimax-h3",+ "wan2.2-ti2v-5b"],"trusted_image_repos": ["alpha-vllm/lumina-image-2.0",diff --git a/studio/backend/tests/test_diffusion_dit_trainer.py b/studio/backend/tests/test_diffusion_dit_trainer.pyindex ef71751a493..20b6e81bb7a 100644--- a/studio/backend/tests/test_diffusion_dit_trainer.py+++ b/studio/backend/tests/test_diffusion_dit_trainer.py@@ -63,6 +63,7 @@ def test_specs_cover_the_dit_families():"flux.2-dev",# The first VIDEO family; its own assertions live in test_diffusion_dit_trainer_ltx2."ltx-2",+ "wan2.2-ti2v-5b",}# FLUX / Qwen share the added-kv attention target set; Z-Image and Krea 2 are single-stream.assert "add_q_proj" in _SPECS["flux.1"].lora_targetsdiff --git a/studio/backend/tests/test_diffusion_dit_trainer_ltx2.py b/studio/backend/tests/test_diffusion_dit_trainer_ltx2.pyindex 6c80ae84755..ea63c304723 100644--- a/studio/backend/tests/test_diffusion_dit_trainer_ltx2.py+++ b/studio/backend/tests/test_diffusion_dit_trainer_ltx2.py@@ -462,8 +462,8 @@ def test_ltx2_bases_route_to_the_ltx2_trainer(base):@pytest.mark.parametrize("base, family",[- ("Wan-AI/Wan2.2-TI2V-5B-Diffusers", "wan2.2-ti2v-5b"),("Wan-AI/Wan2.2-T2V-A14B-Diffusers", "wan2.2-t2v-a14b"),+ ("Wan-AI/Wan2.1-T2V-1.3B-Diffusers", "wan2.1-t2v-1.3b"),("hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v", "hunyuanvideo-1.5"),],)@@ -483,8 +483,8 @@ def test_explicit_video_model_family_override_is_honoured_and_gated():# An opaque local path plus an explicit family is the documented way to train from a# local checkout, so the override path needs the same video routing as detection.assert resolve_trainable_family("/tmp/some-local-checkout", "ltx-2") == "ltx-2"- with pytest.raises(ValueError, match = "wan2.2-ti2v-5b"):- resolve_trainable_family("/tmp/some-local-checkout", "wan2.2-ti2v-5b")+ with pytest.raises(ValueError, match = "wan2.2-t2v-a14b"):+ resolve_trainable_family("/tmp/some-local-checkout", "wan2.2-t2v-a14b")def test_unknown_model_family_lists_the_video_families_too():diff --git a/studio/backend/tests/test_diffusion_dit_trainer_wan22.py b/studio/backend/tests/test_diffusion_dit_trainer_wan22.pynew file mode 100644index 00000000000..a1f95be81c3--- /dev/null+++ b/studio/backend/tests/test_diffusion_dit_trainer_wan22.py@@ -0,0 +1,544 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU-only tests for the Wan2.2-TI2V-5B DiT LoRA trainer family (fakes plus a tiny random WanTransformer3DModel)."""++from __future__ import annotations++import json+import math+import types++import pytest+import torch++from core.training import diffusion_dit_trainer as dit+from core.training.diffusion_dit_trainer import (+ _SPECS,+ _WAN_MAX_SEQUENCE_LENGTH,+ _WAN_TARGETS,+ _select_lora_targets,+ _training_sigma_table,+)+from core.training.diffusion_train_common import (+ DEFAULT_LORA_FILENAME,+ DEFAULT_LORA_TARGETS,+ DiffusionLoraConfig,+ TRAINABLE_VIDEO_FAMILIES,+ _DIT_TRAIN_FAMILIES,+ _FORCE_BF16_FAMILIES,+ _assert_trusted_base_model,+ _family_vram_note,+ _publish_to_lora_catalog,+ get_trainer,+ resolve_trainable_family,+ train_defaults,+)++FAMILY = "wan2.2-ti2v-5b"+BASE = "Wan-AI/Wan2.2-TI2V-5B-Diffusers"++# Wan-AI/Wan2.2-TI2V-5B-Diffusers vae/config.json: 48 channels (latents_mean / latents_std are 48 long).+Z_DIM = 48+# Wan-AI/Wan2.2-TI2V-5B-Diffusers scheduler/scheduler_config.json, the keys the trainer reads.+UNIPC_CONFIG = {+ "_class_name": "UniPCMultistepScheduler",+ "_diffusers_version": "0.35.0.dev0",+ "flow_shift": 5.0,+ "num_train_timesteps": 1000,+ "prediction_type": "flow_prediction",+ "use_flow_sigmas": True,+ "solver_order": 2,+}+++def _wan_classes():+ try:+ from diffusers import WanPipeline, WanTransformer3DModel+ except Exception as exc: # noqa: BLE001 -- diffusers too old, or a broken optional dep+ pytest.skip(f"diffusers Wan classes unavailable: {exc}")+ return WanPipeline, WanTransformer3DModel+++def _tiny_transformer():+ """A random 2-block WanTransformer3DModel with TI2V-5B's layout."""+ _, WanTransformer3DModel = _wan_classes()+ torch.manual_seed(0)+ return WanTransformer3DModel(+ patch_size = (1, 2, 2),+ num_attention_heads = 2,+ attention_head_dim = 12,+ in_channels = Z_DIM,+ out_channels = Z_DIM,+ text_dim = 32,+ freq_dim = 256,+ ffn_dim = 32,+ num_layers = 2,+ cross_attn_norm = True,+ qk_norm = "rms_norm_across_heads",+ rope_max_seq_len = 32,+ ).eval()+++def test_wan_spec_is_registered_and_bf16_only():+ spec = _SPECS[FAMILY]+ assert spec.family == FAMILY+ assert spec.force_bf16 is True+ assert FAMILY in _FORCE_BF16_FAMILIES+ assert 8.0 < spec.dense_bf16_gb < 12.0+ assert spec.lora_targets == _WAN_TARGETS+ # Wan ships UniPC, so the spec must build its own FlowMatchEuler table.+ assert spec.training_scheduler is dit._wan_training_scheduler+ assert spec.decode_latents is None and spec.latent_shape is None+++def test_wan_is_a_trainable_video_family_with_a_trainer():+ assert FAMILY in TRAINABLE_VIDEO_FAMILIES+ assert FAMILY in _DIT_TRAIN_FAMILIES+ assert get_trainer(FAMILY) is dit.run_dit_lora_training+ for name in TRAINABLE_VIDEO_FAMILIES:+ assert callable(get_trainer(name)), name+++ "base", [BASE, "wan-ai/wan2.2-ti2v-5b-diffusers", "/data/models/Wan2.2-TI2V-5B-Diffusers"]+)+def test_wan_bases_route_to_the_dit_trainer(base):+ assert resolve_trainable_family(base) == FAMILY+++def test_explicit_model_family_override_routes_a_local_checkout():+ assert resolve_trainable_family("/tmp/some-local-checkout", FAMILY) == FAMILY+++def test_the_a14b_moe_is_still_refused_and_names_the_5b():+ # Step 2: two experts need two adapters, which this spec does not train.+ with pytest.raises(ValueError) as exc:+ resolve_trainable_family("Wan-AI/Wan2.2-T2V-A14B-Diffusers")+ assert "wan2.2-t2v-a14b" in str(exc.value)+ assert FAMILY in str(exc.value)+++def test_wan_official_base_passes_the_trusted_base_gate():+ _assert_trusted_base_model(BASE)+ with pytest.raises(ValueError, match = "untrusted"):+ _assert_trusted_base_model("random-user/wan2.2-ti2v-5b-finetune")+++def test_wan_defaults_sit_on_the_32_grid():+ d = train_defaults(FAMILY)+ assert d["resolution"] % 32 == 0 # 16x VAE x 2x patch+ assert d["lora_rank"] >= 16 and 0 < d["learning_rate"] <= 2e-4+ assert d["lr_scheduler"] == "constant_with_warmup" and d["lr_warmup_steps"] > 0+++def test_wan_has_a_vram_row():+ note = _family_vram_note(FAMILY)+ assert "5B" in note and "QLoRA" in note+++def _cfg(tmp_path, **kw):+ data = tmp_path / "data"+ data.mkdir(exist_ok = True)+ return DiffusionLoraConfig(+ base_model = BASE, data_dir = str(data), output_dir = str(tmp_path / "run"), **kw+ )++[email protected]("resolution, ok", [(704, True), (512, True), (520, False), (696, False)])+def test_wan_resolution_must_sit_on_the_vae_grid(tmp_path, resolution, ok):+ if ok:+ assert _cfg(tmp_path, resolution = resolution).normalized().resolved_family == FAMILY+ else:+ with pytest.raises(ValueError, match = "multiple of 32"):+ _cfg(tmp_path, resolution = resolution).normalized()+++def test_wan_rejects_fp16_before_loading(tmp_path):+ with pytest.raises(ValueError, match = "bf16"):+ _cfg(tmp_path, resolution = 704, mixed_precision = "fp16").normalized()+++def test_wan_default_flow_shift_stays_the_models_own_schedule(tmp_path):+ cfg = _cfg(tmp_path, resolution = 704).normalized()+ assert cfg.flow_shift is None+++def test_a_video_run_publishes_no_adapter_into_the_image_lora_catalog(tmp_path, monkeypatch):+ catalog = tmp_path / "loras" / "diffusion"+ catalog.mkdir(parents = True)+ monkeypatch.setattr("core.inference.diffusion_lora.loras_dir", lambda: catalog)+ adapter = tmp_path / "run" / DEFAULT_LORA_FILENAME+ adapter.parent.mkdir(parents = True, exist_ok = True)+ adapter.write_bytes(b"adapter-bytes")+ cfg = _cfg(tmp_path, resolution = 704).normalized()+ assert _publish_to_lora_catalog(str(adapter), cfg) is None+ assert list(catalog.iterdir()) == []+++def test_family_train_infos_offers_wan(dit_train_host):+ from core.training.diffusion_train_common import family_train_infos++ infos = {i["name"]: i for i in family_train_infos()}+ assert FAMILY in infos, "a trainable video family must be offered by /diffusion/info"+ info = infos[FAMILY]+ assert info["default_base"] == BASE+ assert info["base_repos"] == [BASE]+ assert info["label"] == "Wan2.2 TI2V 5B"+ assert info["defaults"]["resolution"] % 32 == 0+ assert info["deploy_base"] is None+ assert "nf4" in info["precision_modes"]+ assert info["supports_compile"] is True+ assert "ltx-2" in infos and "sdxl" in infos+++def test_generic_default_targets_resolve_to_the_wan_set():+ assert _select_lora_targets(DEFAULT_LORA_TARGETS, _WAN_TARGETS) == _WAN_TARGETS+ explicit = ("to_q", "to_v")+ assert _select_lora_targets(explicit, _WAN_TARGETS) == explicit+++def unipc_base(tmp_path):+ root = tmp_path / "wan_base"+ (root / "scheduler").mkdir(parents = True)+ (root / "scheduler" / "scheduler_config.json").write_text(json.dumps(UNIPC_CONFIG))+ return types.SimpleNamespace(base_model = str(root), hf_token = None)+++def test_training_scheduler_bakes_in_the_inference_flow_shift(unipc_base):+ pytest.importorskip("diffusers")+ sched = dit._wan_training_scheduler(unipc_base)+ n = 1000+ u = torch.linspace(1, n, n).flip(0) / n+ expected = 5.0 * u / (1.0 + 4.0 * u)+ assert sched.config.shift == 5.0+ assert torch.allclose(sched.sigmas[:n].float(), expected, atol = 1e-6)+ assert torch.allclose(sched.timesteps.float(), expected * n, atol = 1e-3)+ # None (the default) trains on exactly this table; a number re-shifts from the unshifted u.+ assert _training_sigma_table(sched, None) is sched.sigmas+ assert torch.allclose(_training_sigma_table(sched, 1.0).float(), u, atol = 1e-6)+++def test_the_generic_scheduler_load_would_drop_the_shift(unipc_base):+ """The trap the spec hook exists for: FlowMatchEuler.from_pretrained on Wan's UniPC config ignores flow_shift and+ yields the identity (shift 1) table, so a run trains mostly at low noise."""+ FlowMatchEulerDiscreteScheduler = pytest.importorskip(+ "diffusers"+ ).FlowMatchEulerDiscreteScheduler++ generic = FlowMatchEulerDiscreteScheduler.from_pretrained(+ unipc_base.base_model, subfolder = "scheduler"+ )+ ours = dit._wan_training_scheduler(unipc_base)+ assert generic.config.shift == 1.0+ mid = 500+ assert ours.sigmas[mid] > generic.sigmas[mid] + 0.2+++class _FakeWanPipe:+ """encode_prompt returns what WanPipeline._get_t5_prompt_embeds does: real rows then zeros to max_sequence_length."""++ def __init__(self):+ self.text_encoder = None+ self.calls = []++ def encode_prompt(+ self,+ prompt,+ do_classifier_free_guidance,+ num_videos_per_prompt,+ max_sequence_length,+ device,+ ):+ self.calls.append((prompt, max_sequence_length, do_classifier_free_guidance))+ n = len(prompt.split()) + 1+ g = torch.Generator().manual_seed(n)+ emb = torch.zeros(1, max_sequence_length, 8)+ emb[:, :n] = torch.randn(1, n, 8, generator = g) + 0.1+ return emb.to(torch.bfloat16), None+++def test_encode_prompts_caches_trimmed_embeds_and_collate_restores_the_pipeline_tensor():+ pipe = _FakeWanPipe()+ caps = ["a cat", "a much longer caption about a yarn dog"]+ cached = dit._wan_encode_prompts(pipe, caps, "cpu")+ assert [c[1] for c in pipe.calls] == [_WAN_MAX_SEQUENCE_LENGTH] * 2+ assert all(c[2] is False for c in pipe.calls)+ assert [e[0].shape[1] for e in cached] == [3, 9] # trailing zero rows dropped+ (batch,) = dit._wan_collate(cached, "cpu", torch.bfloat16)+ assert batch.shape == (2, _WAN_MAX_SEQUENCE_LENGTH, 8)+ for i, cap in enumerate(caps):+ full, _ = _FakeWanPipe().encode_prompt(cap, False, 1, _WAN_MAX_SEQUENCE_LENGTH, "cpu")+ assert torch.equal(+ batch[i : i + 1], full+ ), "collate must rebuild the exact zero-padded tensor"+++def test_trim_keeps_one_row_for_an_all_zero_embed():+ out = dit._wan_trim_padding(torch.zeros(1, 16, 4))+ assert out.shape == (1, 1, 4)+++class _FakeDist:+ def __init__(self, mean, std):+ self.mean, self.std = mean, std++ def sample(self):+ return self.mean+++class _FakeWanVae(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.w = torch.nn.Parameter(torch.zeros(1))+ self.config = types.SimpleNamespace(+ z_dim = Z_DIM,+ latents_mean = [0.1 * i for i in range(Z_DIM)],+ latents_std = [1.0 + 0.05 * i for i in range(Z_DIM)],+ )+ self.seen = None++ def encode(self, px):+ self.seen = px+ b, _c, f, h, w = px.shape+ mean = torch.full((b, Z_DIM, f, h // 16, w // 16), 2.0)+ std = torch.full_like(mean, 0.5)+ return types.SimpleNamespace(latent_dist = _FakeDist(mean, std))+++def test_encode_latents_adds_a_frame_axis_and_normalises_like_the_pipeline():+ vae = _FakeWanVae()+ px = torch.zeros(2, 3, 64, 96)+ lat = dit._wan_encode_latents(vae, px)+ assert vae.seen.shape == (2, 3, 1, 64, 96)+ assert lat.shape == (2, Z_DIM, 1, 4, 6)+ # WanPipeline decodes with latents / (1 / std) + mean; that must invert the training normalisation.+ mean = torch.tensor(vae.config.latents_mean).view(1, Z_DIM, 1, 1, 1)+ inv_std = 1.0 / torch.tensor(vae.config.latents_std).view(1, Z_DIM, 1, 1, 1)+ assert torch.allclose(lat / inv_std + mean, torch.full_like(lat, 2.0), atol = 1e-5)+++def test_encode_latent_stats_returns_the_posterior_affine_pair():+ vae = _FakeWanVae()+ a, b = dit._wan_encode_latent_stats(vae, torch.zeros(1, 3, 32, 32))+ std = torch.tensor(vae.config.latents_std).view(1, Z_DIM, 1, 1, 1)+ assert torch.allclose(b, 0.5 / std.expand_as(b))+ assert torch.allclose(a, dit._wan_encode_latents(vae, torch.zeros(1, 3, 32, 32)))+++def test_forward_matches_the_pipelines_expand_timesteps_call():+ transformer = _tiny_transformer()+ noisy = torch.randn(2, Z_DIM, 1, 8, 12)+ t = torch.tensor([937.5, 120.0])+ emb = torch.randn(2, 16, 32)+ with torch.no_grad():+ ours = dit._wan_forward(transformer, noisy, t, None, (emb,), None, "cpu", torch.float32)+ mask = torch.ones(1, 1, 8, 12)+ per_sample = []+ for i in range(2):+ ts = (mask[0][:, ::2, ::2] * t[i]).flatten().unsqueeze(0)+ per_sample.append(+ transformer(+ hidden_states = noisy[i : i + 1],+ timestep = ts,+ encoder_hidden_states = emb[i : i + 1],+ return_dict = False,+ )[0]+ )+ assert ours.shape == noisy.shape # aligned with target = noise - latents+ assert torch.allclose(ours, torch.cat(per_sample), atol = 1e-6)+++def _adapted_tiny():+ from peft import LoraConfig++ transformer = _tiny_transformer()+ transformer.add_adapter(+ LoraConfig(+ r = 4, lora_alpha = 4, init_lora_weights = "gaussian", target_modules = list(_WAN_TARGETS)+ )+ )+ return transformer+++def test_targets_adapt_every_block_linear_and_nothing_else():+ transformer = _adapted_tiny()+ adapted = {n for n, m in transformer.named_modules() if hasattr(m, "lora_A")}+ block_linears = {+ f"blocks.{i}.{n}"+ for i in range(2)+ for n in (+ "attn1.to_q",+ "attn1.to_k",+ "attn1.to_v",+ "attn1.to_out.0",+ "attn2.to_q",+ "attn2.to_k",+ "attn2.to_v",+ "attn2.to_out.0",+ "ffn.net.0.proj",+ "ffn.net.2",+ )+ }+ assert adapted == block_linears+ assert not any(+ n.startswith(("condition_embedder", "proj_out", "patch_embedding")) for n in adapted+ )+++def test_saved_lora_has_diffusers_keys_and_loads_through_wanpipeline(tmp_path):+ from peft.utils import get_peft_model_state_dict+ from safetensors.torch import load_file++ WanPipeline, _ = _wan_classes()+ trained = _adapted_tiny()+ with torch.no_grad():+ for n, p in trained.named_parameters():+ if "lora_B" in n:+ p.normal_(0.0, 0.5)+ dit._wan_save(None, str(tmp_path), get_peft_model_state_dict(trained))+ path = tmp_path / DEFAULT_LORA_FILENAME+ keys = set(load_file(str(path)))+ assert keys and all(k.startswith("transformer.blocks.") for k in keys)+ assert "transformer.blocks.0.attn1.to_q.lora_A.weight" in keys+ assert "transformer.blocks.1.ffn.net.2.lora_B.weight" in keys++ fresh = _tiny_transformer()+ pipe = WanPipeline(+ tokenizer = None,+ text_encoder = None,+ vae = None,+ scheduler = None,+ transformer = fresh,+ expand_timesteps = True,+ )+ noisy = torch.randn(1, Z_DIM, 1, 8, 8)+ emb = torch.randn(1, 16, 32)+ t = torch.tensor([500.0])+ with torch.no_grad():+ base = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)+ pipe.load_lora_weights(+ str(tmp_path), weight_name = DEFAULT_LORA_FILENAME, adapter_name = "style"+ )+ loaded = {+ n for n, m in fresh.named_modules() if hasattr(m, "lora_A") and "style" in m.lora_A+ }+ assert len(loaded) == 20+ with_lora = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)+ trained_out = dit._wan_forward(trained, noisy, t, None, (emb,), None, "cpu", torch.float32)+ pipe.disable_lora()+ off = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)+ assert not torch.allclose(with_lora, base)+ assert torch.allclose(+ with_lora, trained_out, atol = 1e-5+ ), "the loaded adapter must reproduce the trained one"+ assert torch.equal(off, base)+ assert math.isfinite(float(with_lora.abs().max()))+++def test_collate_copies_only_the_trimmed_rows_and_pads_on_the_device(monkeypatch):+ """The pad to 512 must happen AFTER the move: padding on the host is a 4 MB pageable copy plus a multi-threaded+ host fill + cat every step."""+ seen = []+ real_to = torch.Tensor.to++ def spy(self, *a, **k):+ seen.append(tuple(self.shape))+ return real_to(self, *a, **k)++ monkeypatch.setattr(torch.Tensor, "to", spy)+ e = (torch.randn(1, 7, 8, dtype = torch.bfloat16),)+ (out,) = dit._wan_collate([e], "cpu", torch.bfloat16)+ assert seen == [(1, 7, 8)], "only the trimmed embed may be moved"+ assert out.shape == (1, _WAN_MAX_SEQUENCE_LENGTH, 8)+ assert torch.equal(out[:, :7], e[0]) and not out[:, 7:].any()+++def test_cached_latent_sampling_moves_before_joining_and_keeps_the_values():+ cache = [[(torch.randn(1, Z_DIM, 1, 4, 4), torch.rand(1, Z_DIM, 1, 4, 4))] for _ in range(3)]+ torch.manual_seed(7)+ got = dit._sample_cached_latents(cache, [2, 0], None, "cpu", torch.float32)+ torch.manual_seed(7)+ a = torch.cat([cache[2][0][0], cache[0][0][0]])+ b = torch.cat([cache[2][0][1], cache[0][0][1]])+ assert torch.equal(got, a + b * torch.randn_like(a))+ one = dit._to_device_cat([cache[1][0][0]], "cpu")+ assert one is cache[1][0][0] or torch.equal(one, cache[1][0][0])+++def test_a_non_default_alpha_reloads_at_the_trained_strength(tmp_path):+ """diffusers' loader assumes lora_alpha == rank unless the safetensors header carries the adapter config, so a+ rank-4 / alpha-16 adapter (scale 4) used to come back at scale 1."""+ from peft import LoraConfig+ from peft.utils import get_peft_model_state_dict+ from safetensors import safe_open++ WanPipeline, _ = _wan_classes()+ trained = _tiny_transformer()+ trained.add_adapter(+ LoraConfig(+ r = 4, lora_alpha = 16, init_lora_weights = "gaussian", target_modules = list(_WAN_TARGETS)+ )+ )+ with torch.no_grad():+ for n, p in trained.named_parameters():+ if "lora_B" in n:+ p.normal_(0.0, 0.2)+ spec = _SPECS[FAMILY]+ assert spec.save_takes_metadata+ dit._adapter_saver(spec, trained)(None, str(tmp_path), get_peft_model_state_dict(trained))+ with safe_open(str(tmp_path / DEFAULT_LORA_FILENAME), "pt") as f:+ meta = json.loads(f.metadata()["lora_adapter_metadata"])+ assert meta["transformer.r"] == 4 and meta["transformer.lora_alpha"] == 16++ fresh = _tiny_transformer()+ pipe = WanPipeline(+ tokenizer = None,+ text_encoder = None,+ vae = None,+ scheduler = None,+ transformer = fresh,+ expand_timesteps = True,+ )+ pipe.load_lora_weights(str(tmp_path), weight_name = DEFAULT_LORA_FILENAME)+ noisy, emb, t = torch.randn(1, Z_DIM, 1, 8, 8), torch.randn(1, 16, 32), torch.tensor([400.0])+ with torch.no_grad():+ got = dit._wan_forward(fresh, noisy, t, None, (emb,), None, "cpu", torch.float32)+ want = dit._wan_forward(trained, noisy, t, None, (emb,), None, "cpu", torch.float32)+ assert torch.allclose(got, want, atol = 1e-5)+++def test_families_without_the_flag_keep_their_three_argument_saver():+ for name, spec in _SPECS.items():+ if not spec.save_takes_metadata:+ assert dit._adapter_saver(spec, object()) is spec.save, name+++def test_load_conditioners_leaves_the_vae_on_the_host(monkeypatch):+ moved = []++ class _Vae:+ def to(self, *a, **k):+ moved.append((a, k))+ return self++ class _Pipe:+ def __init__(self, vae):+ self.vae = vae++ _wan_classes() # skip where this diffusers cannot import the Wan stack+ import diffusers++ monkeypatch.setattr(+ diffusers.AutoencoderKLWan, "from_pretrained", classmethod(lambda cls, *a, **k: _Vae())+ )+ monkeypatch.setattr(+ dit,+ "_wan_pipeline_cls",+ lambda: types.SimpleNamespace(from_pretrained = lambda *a, **k: _Pipe(k["vae"])),+ )+ pipe, vae = dit._wan_load_conditioners(+ types.SimpleNamespace(base_model = BASE, hf_token = None), "cuda", None+ )+ assert vae is pipe.vae and moved == [], "the VAE must wait for the encoders to be freed"