Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

117 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Flow-matching LoRA training for the rectified-flow DiT families.

Image: FLUX.1-dev, FLUX.2 (dev + Klein), Qwen-Image, Z-Image, Krea 2. Video: LTX-2 and Wan2.2-TI2V-5B (from a
still-image dataset; see the ``_LTX2_TARGETS`` and ``_WAN_TARGETS`` blocks for what a video family adds).

These are rectified-flow transformers, not the SDXL U-Net, so they share only the plumbing in
``diffusion_train_common`` (config, dataset discovery, events, stop, publishing). The training math
here is flow matching: sample a sigma with the logit-normal density used by the diffusers dreambooth
scripts, form ``noisy = (1 - sigma) * latents + sigma * noise``, predict the velocity, and regress
it onto ``target = noise - latents``. The per-family differences (latent normalisation + packing,
the transformer forward signature, embedding collation, and the LoRA save entrypoint) live in small
``_FamilySpec`` objects; the loop itself is family-agnostic. Verified against diffusers 0.38.0.

Memory: the text encoder(s) are the largest module (T5-XXL ~9 GB for FLUX, Qwen2.5-VL ~7 GB for
Qwen-Image, Qwen3 for Z-Image), so captions are encoded ONCE up front and the encoders are freed
before the loop. VAE latents are likewise precomputed into a small CPU cache (``cache_latents``) and
the VAE freed: the cache stores the posterior's affine parameters (mean/std folded through the
family's latent normalisation), so every step still draws a fresh VAE sample --
distribution-identical to encoding in the loop, without keeping the VAE resident or paying a
per-step encode. The transformer trains as a QLoRA (nf4) adapter by default with gradient
checkpointing and fused AdamW, so only the (small) LoRA params + optimizer state and the frozen
4-bit base sit in VRAM during the loop.
"""

from __future__ import annotations

import gc
import math
import os
import random
import time
from contextlib import nullcontext
from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any, Callable, Optional

from core.training.diffusion_load_progress import report_model_load_progress
from core._torchao_stub import is_stubbed, torch_is_rocm
from core.training.diffusion_train_common import (
    AUTO_FLOW_SHIFT_FAMILIES,
    DEFAULT_LORA_FILENAME,
    DEFAULT_LORA_TARGETS,
    DiffusionLoraConfig,
    EventCb,
    LATENT_CACHE_OVER_BUDGET,
    StopCb,
    enable_diffusion_gradient_checkpointing,
    TrainingPeakMemory,
    resume_steps_left,
    make_lora_optimizer,
    recorded_optimizer_class,
    _apply_perf_flags,
    _assert_trusted_base_model,
    _emit,
    training_fetch_repo,
    _latent_cache_forced,
    _latent_cache_over_budget,
    _plan_cache_variants,
    _publish_to_lora_catalog,
    _restore_perf_flags,
    bucket_groups,
    discover_image_caption_pairs,
    has_functional_torchao,
    flow_bf16_trainable,
    native_bf16_supported_xpu,
    PermutationBatchSampler,
    plan_image_canvases,
    repo_is_prequantized,
    resolve_bucketing,
    resolve_train_device,
    resolve_train_steps,
    restore_resume_state,
    write_resume_checkpoint,
)
from core.training.diffusion_checkpoint import (
    clear_own_checkpoints,
    discard_preexisting_checkpoints,
    retire_own_checkpoints,
    resumed_into_this_dir,
    snapshot_checkpoints,
    identity_for_config,
    with_cache_mode,
    with_resolved_base_precision,
    with_resolved_revision,
    preflight_resume,
)
from core.training.diffusion_samples import (
    discard_samples,
    euler_flow_sample,
    isolated_sampling,
    plan_samples,
    run_sample_round,
)
from core.training.diffusion_train_extras import (
    BucketBatchSampler,
    LoRAEMA,
    accumulate_loss,
    cover_resize_dims,
    loss_value,
    PersistentConditioningCache,
    save_ema_adapter,
)

# FLUX / Qwen double-stream blocks also carry added-kv projections; Z-Image is single-stream.
_FLUX_TARGETS = (
    "to_q",
    "to_k",
    "to_v",
    "to_out.0",
    "add_q_proj",
    "add_k_proj",
    "add_v_proj",
    "to_add_out",
)
_QWEN_TARGETS = _FLUX_TARGETS
# Attention + SwiGLU (DiffSynth-Studio's Z-Image recipe); old attention-only runs keep theirs on resume.
_ZIMAGE_TARGETS = ("to_q", "to_k", "to_v", "to_out.0", "w1", "w2", "w3")
# The Krea 2 authors' recommended defaults (their DreamBooth script): attention + SwiGLU + text-fusion projector +
# conditioning embedders. For long runs they suggest narrowing to attention.
_KREA2_TARGETS = (
    "img_in",
    "final_layer.linear",
    "to_q",
    "to_k",
    "to_v",
    "to_out.0",
    "to_gate",
    "ff.up",
    "ff.down",
    "text_fusion.projector",
    "txt_in.linear_1",
    "txt_in.linear_2",
    "time_embed.linear_1",
    "time_embed.linear_2",
    "time_mod_proj",
)


def _select_lora_targets(
    cfg_targets: tuple[str, ...], spec_targets: tuple[str, ...]
) -> tuple[str, ...]:
    """The generic DEFAULT_LORA_TARGETS means unset, so the family's targets apply; other tuples win."""
    if tuple(cfg_targets) == DEFAULT_LORA_TARGETS:
        return tuple(spec_targets)
    return tuple(cfg_targets)


@dataclass
class _FamilySpec:
    """Everything the shared loop needs that differs by family."""

    family: str
    lora_targets: tuple[str, ...]
    # bf16 only (Z-Image overflows fp16 and its RoPE/embedder run in fp32).
    force_bf16: bool
    # A family covering several sizes (flux.2-klein 4B/9B) must use _dense_bf16_gb.
    dense_bf16_gb: float
    # Phased load: the transformer never coexists with the text encoders + VAE.
    load_conditioners: Callable[..., tuple[Any, Any]]
    load_transformer: Callable[..., Any]
    # Encode a list of captions -> a per-caption tuple of CPU tensors (the family's embeds).
    encode_prompts: Callable[..., list[tuple]]
    # Encode a pixel tensor [B,3,H,W] in [-1,1] -> latents (family-normalised, on device).
    encode_latents: Callable[..., Any]
    # (A, B) posterior params so a per-step sample is A + B * randn; B is None if deterministic.
    encode_latent_stats: Callable[..., tuple]
    # pad_to pins a fixed text length for variable-length embeds (compile).
    collate: Callable[..., tuple]
    # Returns model_pred aligned with target = noise - latents.
    forward: Callable[..., Any]
    # Save the LoRA in diffusers format via the family pipeline's save_lora_weights.
    save: Callable[..., None]
    # (vae, h, w) -> (latent shape, packed seq len for the shift), and latents -> pixels. None = no sampling path.
    latent_shape: Optional[Callable[..., tuple]] = None
    decode_latents: Optional[Callable[..., Any]] = None
    sample_forward_kwargs: Optional[dict] = None
    # cfg -> FlowMatchEuler scheduler. None = load scheduler/ (only valid if it IS FlowMatchEuler; Wan ships UniPC).
    training_scheduler: Optional[Callable[..., Any]] = None
    # Saver takes ``adapter_metadata``; without it diffusers' loader assumes lora_alpha == rank.
    save_takes_metadata: bool = False


def _gather_sigmas(sigma_table, indices, device, dtype, n_dim):
    """Gathers by index, so sigma_table must stay aligned with the timesteps the indices were drawn from."""
    sigma = sigma_table[indices].to(device = device, non_blocking = True).to(dtype = dtype).flatten()
    while sigma.ndim < n_dim:
        sigma = sigma.unsqueeze(-1)
    return sigma


def _unshifted_sigmas(scheduler):
    """Uniform u before any init-time static shift, as ``FlowMatchEulerDiscreteScheduler.__init__`` builds it."""
    import numpy as np
    import torch

    sc = scheduler.config
    if getattr(sc, "use_dynamic_shifting", False) or float(getattr(sc, "shift", 1.0) or 1.0) == 1.0:
        return scheduler.sigmas
    n = int(sc.num_train_timesteps)
    u = torch.from_numpy(np.linspace(1, n, n, dtype = np.float32)[::-1].copy()) / n
    return u.to(scheduler.sigmas.dtype)


def _training_sigma_table(scheduler, flow_shift):
    """None: the scheduler's own sigmas. "auto": the inference distribution, which dynamic-shifting
    schedulers never bake into ``sigmas`` (Qwen pins mu = max_shift = log 3). A number: the effective
    shift s*u/(1+(s-1)*u) on unshifted u (musubi/kohya); applying it on top of a baked-in shift is WRONG."""
    sigmas = scheduler.sigmas
    if flow_shift is None:
        return sigmas
    if flow_shift == "auto":
        sc = scheduler.config
        if not getattr(sc, "use_dynamic_shifting", False):
            return sigmas  # static-shift family: its init already baked the shift in
        mu = float(getattr(sc, "max_shift", None) or math.log(3.0))
        shifted = scheduler.time_shift(mu, 1.0, sigmas)
        if getattr(sc, "shift_terminal", None):
            # Stretch must see the full descending table, never a batch.
            shifted = scheduler.stretch_shift_to_terminal(shifted)
        return shifted
    s = float(flow_shift)
    u = _unshifted_sigmas(scheduler)
    if s == 1.0:
        return u
    return s * u / (1.0 + (s - 1.0) * u)


def _bell_loss_weights(num_train_timesteps):
    """bsmntw-style bell loss-weight table over the training schedule: a Gaussian bell centered
    mid-schedule, floored at 0 and normalized to mean 1 (so the expected loss scale is
    unchanged). Indexed by round(sigma * num_train_timesteps)."""
    import torch

    steps = num_train_timesteps
    t = torch.arange(steps, dtype = torch.float32)
    w = torch.exp(-2.0 * ((t - steps / 2) / steps) ** 2)
    w = w - w.min()
    return w * (steps / w.sum())


def _sample_timesteps(scheduler, batch_size, device):
    """Logit-normal density timestep sampling (weighting_scheme='logit_normal'), returning
    (timesteps, indices) into the scheduler's schedule."""
    from diffusers.training_utils import compute_density_for_timestep_sampling

    u = compute_density_for_timestep_sampling(
        weighting_scheme = "logit_normal",
        batch_size = batch_size,
        logit_mean = 0.0,
        logit_std = 1.0,
        mode_scale = 1.29,
    )
    num_train = scheduler.config.num_train_timesteps
    indices = (u * num_train).long().clamp(0, num_train - 1)
    # Gather on host then one async copy; device-table indexing with host indices blocks.
    timesteps = scheduler.timesteps[indices].to(device, non_blocking = True)
    return timesteps, indices


def _empty_device_cache(device) -> None:
    import torch
    if device == "cuda":
        torch.cuda.empty_cache()
    elif device == "xpu":
        torch.xpu.empty_cache()


def _encoders_to_device(pipe, device) -> None:
    """A QLoRA load leaves text encoders on CPU, so move them; a 4-bit encoder already placed is skipped."""
    for attr in ("text_encoder", "text_encoder_2", "text_encoder_3"):
        enc = getattr(pipe, attr, None)
        if enc is None:
            continue
        try:
            enc.to(device)
        except (ValueError, RuntimeError, NotImplementedError):
            pass


def _bnb_4bit_config():
    from diffusers import BitsAndBytesConfig as DiffusersBnb
    import torch
    return DiffusersBnb(
        load_in_4bit = True,
        bnb_4bit_quant_type = "nf4",
        bnb_4bit_compute_dtype = torch.bfloat16,
        # Double quant re-quantizes absmax scales to 8-bit, saving ~0.4 bits/param at no fidelity cost.
        bnb_4bit_use_double_quant = True,
    )


# Alias kept for callers; the heuristic moved to diffusion_train_common for config validation.
_repo_is_prequantized = repo_is_prequantized


def _load_quantized_transformer(transformer_cls, cfg, device):
    """Load ``cfg.base_model``'s transformer subfolder as a trainable nf4 QLoRA module."""
    import torch
    return transformer_cls.from_pretrained(
        cfg.base_model,
        subfolder = "transformer",
        quantization_config = _bnb_4bit_config(),
        device_map = {"": device},
        torch_dtype = torch.bfloat16,
        token = cfg.hf_token,
    )


def _load_pipe_without_transformer(pipe_cls, cfg, device):
    """Load a pipeline for conditioning only: ``transformer = None`` skips the multi-GB denoiser
    entirely (the documented diffusers pattern), leaving just the text encoders + tokenizers +
    VAE + scheduler. The transformer loads later, after these are freed."""
    import torch

    pipe = pipe_cls.from_pretrained(
        cfg.base_model,
        transformer = None,
        torch_dtype = torch.bfloat16,
        token = cfg.hf_token,
    )
    # Stays on host until the text encoders are freed (never resident beside them).
    pipe.vae.to(dtype = torch.float32)
    return pipe, pipe.vae


def _load_dit_transformer(transformer_cls, cfg, device, base_precision):
    """Load the transformer alone in the resolved ``base_precision``. nf4: a prequant (bnb-4bit)
    repo carries its quantization config and loads 4-bit as-is, while a dense base is quantized
    to nf4 on the fly; the memory floor. bf16 / fp8 / mxfp8: the dense transformer, converted AFTER the LoRA
    attaches (fp8 stores its frozen linears in e4m3, about half the bf16 weight memory; mxfp8 keeps bf16 storage).
    int8: the dense transformer quantized in place to torchao weight-only int8 (the
    PEFT-attachable scheme), roughly halving the bf16 weight footprint."""
    import torch

    if base_precision == "nf4":
        if not repo_is_prequantized(cfg.base_model):
            return _load_quantized_transformer(transformer_cls, cfg, device)
        transformer = transformer_cls.from_pretrained(
            cfg.base_model,
            subfolder = "transformer",
            torch_dtype = torch.bfloat16,
            token = cfg.hf_token,
        )
        # A prequant load is already device-placed by bitsandbytes.
        if not getattr(transformer, "is_loaded_in_4bit", False):
            transformer = transformer.to(device)
        return transformer

    if base_precision == "fp8":
        transformer = _load_fp8_prequant_transformer(transformer_cls, cfg, device)
        if transformer is not None:
            return transformer

    # int8 quantizes AFTER the LoRA attaches: quantizing first makes peft dispatch TorchaoLoraLinear, whose peft-0.18
    # constructor rejects the torchao-0.16 config API (peft 0.19+ accepts it; the order works for both).
    transformer = transformer_cls.from_pretrained(
        cfg.base_model,
        subfolder = "transformer",
        torch_dtype = torch.bfloat16,
        token = cfg.hf_token,
    )
    # int8 stays on host: _int8_quantize_base streams Linears to device so the bf16 base is never whole in VRAM.
    return transformer if base_precision == "int8" else transformer.to(device)


def _load_fp8_prequant_transformer(transformer_cls, cfg, device):
    """The family's hosted fp8 denoiser (the file Studio inference uses) as the fp8 base, skipping the bf16 transformer
    download; None (load bf16 and quantize, as before) when none is mapped for this exact base or it fails to load."""
    import logging

    from core.training.diffusion_fp8_prequant import load_fp8_transformer, resolve_fp8_checkpoint

    log = logging.getLogger(__name__)
    path = resolve_fp8_checkpoint(
        getattr(cfg, "resolved_family", None), cfg.base_model, cfg.hf_token, logger = log
    )
    if path is None:
        return None
    try:
        transformer = load_fp8_transformer(
            transformer_cls, cfg.base_model, path, device, hf_token = cfg.hf_token
        )
    except Exception as exc:  # noqa: BLE001 -- the dense path still works
        log.warning("fp8 prequant: %s did not load (%s); loading bf16", path, exc)
        gc.collect()
        return None
    transformer._studio_fp8_prequant_path = path
    log.info("fp8 base loaded from the pre-quantized checkpoint %s", path)
    return transformer


def _int8_quantize_base(
    transformer,
    family: Optional[str] = None,
    device = None,
) -> None:
    """Passing family keeps training and inference on one exclusion list, else the first forward raises. With device,
    a host-resident base moves to it one module at a time as it quantizes (peak = int8 base + one bf16 Linear)."""
    from core.inference.diffusion_transformer_quant import (
        _quiet_config,
        apply_small_m_padding,
        exclude_tokens_for_scheme,
        make_filter_fn,
    )
    from torchao.quantization import Int8WeightOnlyConfig, quantize_

    # Int8WeightOnlyConfig defaults to set_inductor_config=True, process-wide, and the trainer compiles next.
    quantize_(
        transformer,
        _quiet_config(Int8WeightOnlyConfig),
        filter_fn = make_filter_fn(
            512, exclude_name_tokens = exclude_tokens_for_scheme("int8", family, baked = True)
        ),
        **({} if device is None else {"device": device}),
    )
    # After quantize_: the helper reparents the Linears it wraps. Not best-effort: a raise leaves the base quantized but not safely runnable.
    apply_small_m_padding(transformer, "int8", family)
    if device is not None:
        # quantize_ moves every module it visits; this catches anything outside its walk (root params / buffers).
        transformer.to(device)
        gc.collect()
        _empty_device_cache(device)


# Small-M linears stay bf16 (casts outweigh GEMM); final_layer is Krea 2's output projection.
_LOWP_EXTRA_EXCLUDE_TOKENS = ("final_layer",)


def _lowp_exclude_tokens(family: Optional[str] = None) -> tuple[str, ...]:
    from core.inference.diffusion_transformer_quant import exclude_tokens_for_scheme
    return tuple(exclude_tokens_for_scheme("int8", family, baked = True)) + _LOWP_EXTRA_EXCLUDE_TOKENS


def _lowp_module_filter(
    mod,
    fqn: str,
    align: int,
    exclude_tokens: tuple[str, ...] = (),
) -> bool:
    import torch.nn as nn

    if not isinstance(mod, nn.Linear):
        return False
    if "lora_" in fqn:
        return False
    if fqn.endswith("proj_out") or ".proj_out." in fqn:
        return False
    name = fqn.lower()
    if any(tok in name for tok in exclude_tokens):
        return False
    return mod.in_features % align == 0 and mod.out_features % align == 0


def _fp8_module_filter(
    mod,
    fqn: str,
    exclude_tokens: tuple[str, ...] = (),
) -> bool:
    """Dims divisible by 16 (the diffusers FLUX2 reference filter)."""
    return _lowp_module_filter(mod, fqn, 16, exclude_tokens)


def _in_repeated_blocks(transformer, module_filter):
    """Restrict the filter to the regionally compiled ``_repeated_blocks`` (if any)."""
    names = set(getattr(transformer, "_repeated_blocks", None) or ())
    prefixes = tuple(
        f"{fqn}." for fqn, mod in transformer.named_modules() if mod.__class__.__name__ in names
    )
    if not prefixes:
        return module_filter
    return lambda mod, fqn: fqn.startswith(prefixes) and module_filter(mod, fqn)


def _lowp_targets(transformer, base_filter, family: Optional[str]) -> list:
    tokens = _lowp_exclude_tokens(family)
    keep = _in_repeated_blocks(transformer, lambda mod, fqn: base_filter(mod, fqn, tokens))
    return [
        (fqn, mod)
        for fqn, mod in transformer.named_modules()
        if not getattr(mod, "_studio_lowp", None) and keep(mod, fqn)
    ]


def _require_lowp_modules(transformer, mode: str) -> bool:
    """Raises when nothing converted (FLUX.1 / LTX-2 on torchao 0.17, all biased), so bf16 is recorded."""
    if not any(getattr(mod, "_studio_lowp", None) == mode for mod in transformer.modules()):
        raise RuntimeError("no eligible linear layers")
    return True


def _convert_fp8(transformer, family: Optional[str] = None) -> bool:
    """Frozen base linears to weight-cached fp8 (``diffusion_frozen_fp8``): rowwise float scales, as DiT activation
    outliers (Z-Image ~2.7e6) crush a tensorwise one."""
    from core.training.diffusion_frozen_fp8 import convert_frozen_fp8

    _, failed = convert_frozen_fp8(
        transformer, {fqn for fqn, _ in _lowp_targets(transformer, _fp8_module_filter, family)}
    )
    transformer._studio_fp8_failed = failed
    return _require_lowp_modules(transformer, "fp8")


def _probe_lowp(convert, transformer) -> None:
    """Tiny fwd+bwd probe on 48 unaligned rows: an unverified arch falls back to bf16 at load."""
    import torch
    import torch.nn as nn

    try:
        device = next(transformer.parameters()).device
    except StopIteration:
        return
    if device.type != "cuda":
        return

    class _Probe(nn.Module):
        def __init__(self):
            super().__init__()
            self.ff = nn.Linear(64, 64, bias = False)

        def forward(self, x):
            return self.ff(x)

    # Forked RNG: the probe's init + randn run after the seed and would shift the noise / timestep draws vs bf16.
    with torch.random.fork_rng(devices = [device]):
        probe = _Probe().to(device = device, dtype = torch.bfloat16).requires_grad_(False)
        convert(probe)
        x = torch.randn(1, 48, 64, device = device, requires_grad = True)
        with torch.autocast("cuda", dtype = torch.bfloat16):
            y = probe(x)
        y.float().sum().backward()
        torch.cuda.synchronize(device)
    if not (torch.isfinite(y).all() and torch.isfinite(x.grad).all()):
        raise RuntimeError("probe GEMM returned non-finite values")


def _apply_fp8_training(
    transformer,
    on_event,
    family: Optional[str] = None,
) -> bool:
    """fp8 compute + storage for the frozen base, after add_adapter. Never fatal: falls back to bf16."""
    from core.training.diffusion_fp8_prequant import dequantize_placeholders
    try:
        _probe_lowp(_convert_fp8, transformer)
        converted = _convert_fp8(transformer, family)
        # A prequant base's fp8 layers the conversion did not take stay bf16, as on the dense path.
        dequantize_placeholders(transformer)
        failed = getattr(transformer, "_studio_fp8_failed", 0)
        if failed:
            _emit(
                on_event,
                "warning",
                message = f"fp8: {failed} layer(s) could not be converted and stay bf16.",
            )
        import torch

        device = next(transformer.parameters()).device.type
        if device == "cuda":
            torch.cuda.empty_cache()
        elif device == "xpu":
            torch.xpu.empty_cache()
        return converted
    except Exception as exc:  # noqa: BLE001 -- fp8 is an optimisation, never fatal
        dequantize_placeholders(transformer)
        _emit(on_event, "warning", message = f"fp8 training unavailable, using bf16 compute: {exc}")
        return False


def _mx_module_filter(
    mod,
    fqn: str,
    exclude_tokens: tuple[str, ...] = (),
) -> bool:
    """32-wide MX blocks and a bf16 weight (the MX casts assert it)."""
    import torch
    return _lowp_module_filter(mod, fqn, 32, exclude_tokens) and mod.weight.dtype == torch.bfloat16


def _mxfp8_training_config():
    """RCEIL MX config: torchao 0.14-0.16 ``MXLinearConfig``, else 0.17+ with hp wgrad: torchao computes grad_weight
    even for a frozen weight, and its dim1 cast asserts tokens % 32 == 0 (the gradient is discarded anyway)."""
    try:
        from torchao.prototype.mx_formats import MXLinearConfig
        return MXLinearConfig.from_recipe_name("mxfp8_cublas_rceil")
    except ImportError:
        from torchao.prototype.moe_training.config import (
            MXFP8TrainingOpConfig,
            MXFP8TrainingRecipe,
        )
        return MXFP8TrainingOpConfig.from_recipe(MXFP8TrainingRecipe.MXFP8_RCEIL_WGRAD_WITH_HP)


def _mx_linear_class():
    try:
        from torchao.prototype.moe_training.mxfp8_linear import MXFP8Linear
        return MXFP8Linear
    except ImportError:
        return None


_ROW_PADDED_MX: dict = {}


def _row_padded_mx_linear(base_cls):
    """torchao <= 0.16 MXLinear has no hp wgrad and asserts tokens % 32 == 0: pad zero rows, slice off
    (MX scales are per block along the feature dim, so real rows are unchanged)."""
    cls = _ROW_PADDED_MX.get(base_cls)
    if cls is None:
        import torch.nn.functional as F
        def forward(self, x):
            shape = x.shape
            rows = x.reshape(-1, shape[-1])
            m = rows.shape[0]
            y = base_cls.forward(self, F.pad(rows, (0, 0, 0, (-m) % 32)))[:m]
            return y.reshape(*shape[:-1], y.shape[-1])

        cls = _ROW_PADDED_MX[base_cls] = type(
            f"RowPadded{base_cls.__name__}", (base_cls,), {"forward": forward}
        )
    return cls


def _autocast_mx_input(module, args):
    import torch

    x = args[0] if args else None
    if (
        isinstance(x, torch.Tensor)
        and x.dtype == torch.float32
        and x.is_cuda
        and torch.is_autocast_enabled("cuda")
    ):
        return (x.to(torch.get_autocast_dtype("cuda")), *args[1:])
    return None


def _convert_mxfp8(transformer, family: Optional[str] = None) -> bool:
    import torch.nn as nn

    config = _mxfp8_training_config()
    targets = [
        (fqn, mod)
        for fqn, mod in _lowp_targets(transformer, _mx_module_filter, family)
        if type(mod) is nn.Linear
    ]
    if not hasattr(config, "wgrad_with_hp"):
        from torchao.quantization import quantize_

        names = {fqn for fqn, _ in targets}
        quantize_(transformer, config, filter_fn = lambda mod, fqn: fqn in names)
        for _, mod in targets:
            mod.__class__ = _row_padded_mx_linear(mod.__class__)
    else:
        linear_cls = _mx_linear_class()
        if linear_cls is not None:
            for _, mod in targets:
                mod.__class__ = linear_cls
                mod.kernel_preference = config.kernel_preference
                mod.scale_calculation_mode = config.scale_calculation_mode
                mod.wgrad_with_hp = config.wgrad_with_hp
        else:
            from torchao.quantization import quantize_

            targets = [(fqn, mod) for fqn, mod in targets if mod.bias is None]
            names = {fqn for fqn, _ in targets}
            quantize_(transformer, config, filter_fn = lambda mod, fqn: fqn in names)
        # The MX linear asserts bf16 input above autocast (fp32 LayerNorm output into FLUX.2 ff.linear_in raised).
        for _, mod in targets:
            mod.register_forward_pre_hook(_autocast_mx_input)
    for _, mod in targets:
        mod._studio_lowp = "mxfp8"
    return _require_lowp_modules(transformer, "mxfp8")


def _apply_mxfp8_training(
    transformer,
    on_event,
    family: Optional[str] = None,
) -> bool:
    """Best-effort, never fatal; runs after add_adapter; torchao 0.18+ uses MXFP8Linear (bias
    dropped, pytorch/ao issue 4846); 0.17 keeps biased linears bf16."""
    try:
        _probe_lowp(_convert_mxfp8, transformer)
        return _convert_mxfp8(transformer, family)
    except Exception as exc:  # noqa: BLE001 -- mxfp8 is an optimisation, never fatal
        _emit(on_event, "warning", message = f"mxfp8 training unavailable, using bf16 compute: {exc}")
        return False


def _pick_auto_precision(
    prequant,
    device,
    free_gb,
    dense_gb,
    capability,
    has_fp8,
    has_torchao = True,
) -> str:
    """Pure policy for base_precision="auto": nf4 for a prequant base or no CUDA; else the fastest
    dense mode whose weights + headroom (activations, optimizer, cache) fit the free VRAM at
    decision time. bf16 + regional compile is the measured speed winner (2.3-2.6x over nf4 on
    B200); fp8 stays an explicit opt-in: it trades e4m3 rounding (~0.5% loss vs bf16) for ~half the base weight
    memory and only ~7% speed at LoRA-training shapes. int8 must still
    materialise the full bf16 transformer before ``quantize_`` shrinks it module-by-module, so
    its band requires the dense-load transient (1.15x dense) to fit -- what int8 buys in that
    band is steady-state headroom for activations and the latent cache, not load-time memory.
    int8 also needs torchao at runtime (``_int8_quantize_base`` has no fallback, unlike fp8), so
    auto only picks it when torchao is importable and drops to nf4 otherwise.
    ``capability``/``has_fp8`` remain parameters so the policy can be revisited per GPU
    generation without changing callers."""
    _ = capability, has_fp8
    if prequant or device != "cuda" or not free_gb or not dense_gb:
        return "nf4"
    if free_gb > dense_gb * 1.5:
        return "bf16"
    if free_gb > dense_gb * 1.15:
        return "int8" if has_torchao else "nf4"
    return "nf4"


def _dense_bf16_gb(spec, base_model: str) -> float:
    """Per-base overrides only, never the family table: flux.2-klein 9B is 2.3x its 4B default. Never raises, so a sizing lookup cannot fail a run."""
    try:
        from core.inference.diffusion_auto_policy import base_repo_bf16_components_gb
        components = base_repo_bf16_components_gb(base_model)
        if components:
            return float(components[0])
    except Exception:  # noqa: BLE001 -- table miss / import failure -> the family number
        pass
    return float(spec.dense_bf16_gb)


def _resolve_base_precision(cfg, spec, device) -> str:
    """Explicit dense modes are re-checked against the live device, since they are CUDA-only."""
    mode = (cfg.base_precision or "nf4").strip().lower()
    if mode != "auto":
        if mode in ("bf16", "int8", "fp8", "mxfp8") and device != "cuda":
            raise ValueError(
                f"base_precision={mode!r} needs a CUDA GPU; this host has none. "
                f"Use base_precision='nf4' or 'auto'."
            )
        # Mirror the pre-eviction ROCm guard inside the trainer child.
        if mode in ("int8", "fp8", "mxfp8") and torch_is_rocm():
            raise ValueError(
                f"base_precision={mode!r} is a torchao NVIDIA tensor-core path (int8 sm_80+, fp8 "
                "sm_89+, mxfp8 sm_100+) and this is a ROCm/AMD GPU. Use base_precision='nf4', "
                "'bf16', or 'auto'."
            )
        # int8 has no runtime fallback, so gate on a FUNCTIONAL torchao, not find_spec, which the Windows-ROCm stub satisfies.
        if mode == "int8" and not has_functional_torchao():
            raise ValueError(
                "base_precision='int8' needs a functional torchao install; this host's "
                "torchao is missing or the non-functional Windows-ROCm stub. Use "
                "base_precision='nf4', 'bf16', or 'auto'."
            )
        # The stub fakes float8 and mx_formats success, so runs would silently train bf16; key on the stub, not has_functional_torchao(), which probes int8.
        if mode in ("fp8", "mxfp8") and is_stubbed("torchao"):
            raise ValueError(
                f"base_precision={mode!r} is not available on this host: torchao is the "
                "non-functional Windows-ROCm stub. Use base_precision='nf4', 'bf16', or 'auto'."
            )
        # mxfp8 needs sm100+ (fails after a full dense load); fp8 needs sm89+ (fails in the training
        # loop).
        if mode == "fp8" and device == "cuda":
            try:
                import torch
                fp8_ok = torch.cuda.get_device_capability() >= (8, 9)
            except Exception:  # noqa: BLE001 -- probe failure -> treat as unsupported, fail fast
                fp8_ok = False
            if not fp8_ok:
                raise ValueError(
                    "base_precision='fp8' needs an Ada, Hopper or Blackwell (sm89+) GPU; this GPU is "
                    "older. Use base_precision='bf16', 'int8', 'nf4', or 'auto'."
                )
        if mode == "mxfp8" and device == "cuda":
            try:
                import torch
                blackwell = torch.cuda.get_device_capability() >= (10, 0)
            except Exception:  # noqa: BLE001 -- probe failure -> treat as unsupported, fail fast
                blackwell = False
            if not blackwell:
                raise ValueError(
                    "base_precision='mxfp8' needs a Blackwell (sm100+) GPU; this GPU is older. "
                    "Use base_precision='bf16', 'int8', 'nf4', or 'auto'."
                )
        return mode
    # auto may resolve to dense modes only under bf16 compute; otherwise stay on nf4.
    if getattr(cfg, "mixed_precision", "bf16") != "bf16":
        return "nf4"
    prequant = repo_is_prequantized(cfg.base_model)
    free_gb = None
    capability = None
    has_fp8 = False
    # Auto must not select a torchao mode on ROCm.
    has_torchao = has_functional_torchao() and not torch_is_rocm()
    if device == "cuda":
        try:
            import torch

            # Windows ROCm over-reports free VRAM, which would pick a dense precision the card cannot hold.
            from utils.hardware import trusted_mem_get_info

            free_gb = trusted_mem_get_info()[0] / 1e9
            capability = torch.cuda.get_device_capability()
            has_fp8 = hasattr(torch, "float8_e4m3fn")
        except Exception:  # noqa: BLE001 -- probe failure -> the safe mode
            pass
    return _pick_auto_precision(
        prequant,
        device,
        free_gb,
        _dense_bf16_gb(spec, cfg.base_model),
        capability,
        has_fp8,
        has_torchao,
    )


def _flux_load_conditioners(cfg, device, weight_dtype):
    from diffusers import FluxPipeline
    return _load_pipe_without_transformer(FluxPipeline, cfg, device)


def _flux_load_transformer(cfg, device, weight_dtype, base_precision):
    from diffusers import FluxTransformer2DModel
    return _load_dit_transformer(FluxTransformer2DModel, cfg, device, base_precision)


def _flux_encode_prompts(pipe, captions, device):
    import torch

    _encoders_to_device(pipe, device)
    out = []
    with torch.no_grad():
        for cap in captions:
            pe, pooled, text_ids = pipe.encode_prompt(
                prompt = cap,
                prompt_2 = cap,
                device = device,
                num_images_per_prompt = 1,
                max_sequence_length = 512,
            )
            out.append((pe.cpu(), pooled.cpu(), text_ids.cpu()))
    return out


def _flux_encode_latents(vae, pixel_values):
    import torch

    with torch.no_grad():
        lat = vae.encode(pixel_values.to(torch.float32)).latent_dist.sample()
    lat = (lat - vae.config.shift_factor) * vae.config.scaling_factor
    return lat


def _flux_encode_latent_stats(vae, pixel_values):
    import torch

    with torch.no_grad():
        dist = vae.encode(pixel_values.to(torch.float32)).latent_dist
    scale = vae.config.scaling_factor
    return (dist.mean - vae.config.shift_factor) * scale, dist.std * scale


def _flux_collate(
    entries,
    device,
    weight_dtype,
    pad_to = None,
):
    import torch

    # FLUX embeds are fixed-length, so a plain cat batches them; text_ids are shared across prompts.
    pe = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
    pooled = _to_device_cat([e[1] for e in entries], device).to(dtype = weight_dtype)
    text_ids = entries[0][2].to(device = device, non_blocking = True).to(dtype = torch.float32)
    return (pe, pooled, text_ids)


# Per-run cache of step-invariant FLUX conditioning tensors; cleared at run start.
_FLUX_STATIC: dict[tuple, tuple] = {}


def _flux_static_inputs(
    bsz,
    h,
    w,
    device,
    guidance_value = 1.0,
):
    import torch
    from diffusers import FluxPipeline

    key = (bsz, h, w, str(device), float(guidance_value))
    hit = _FLUX_STATIC.get(key)
    if hit is None:
        # Position ids drive RoPE and are indices, so keep them float32.
        img_ids = FluxPipeline._prepare_latent_image_ids(bsz, h // 2, w // 2, device, torch.float32)
        guidance = torch.full((bsz,), float(guidance_value), device = device, dtype = torch.float32)
        hit = _FLUX_STATIC[key] = (img_ids, guidance)
    return hit


def _flux_forward(
    transformer,
    noisy,
    timesteps,
    sigmas,
    embeds_batch,
    cfg,
    device,
    weight_dtype,
    guidance = 1.0,
):
    from diffusers import FluxPipeline

    pe, pooled, text_ids = embeds_batch
    bsz, c, h, w = noisy.shape
    packed = FluxPipeline._pack_latents(noisy, bsz, c, h, w)
    # Training feeds guidance 1.0 (diffusers dreambooth convention); sampling feeds the inference 3.5.
    img_ids, guidance = _flux_static_inputs(bsz, h, w, device, guidance)
    model_pred = transformer(
        hidden_states = packed,
        timestep = timesteps / 1000,
        guidance = guidance,
        pooled_projections = pooled,
        encoder_hidden_states = pe,
        txt_ids = text_ids,
        img_ids = img_ids,
        return_dict = False,
    )[0]
    return FluxPipeline._unpack_latents(model_pred, h * 8, w * 8, 8)


def _flux_save(pipe_cls, out_dir, transformer_lora_layers):
    from diffusers import FluxPipeline
    FluxPipeline.save_lora_weights(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
    )


def _qwen_load_conditioners(cfg, device, weight_dtype):
    from diffusers import QwenImagePipeline
    return _load_pipe_without_transformer(QwenImagePipeline, cfg, device)


def _qwen_load_transformer(cfg, device, weight_dtype, base_precision):
    # Prequant default loads 4-bit nf4 as-is; dense modes need the 20B Qwen/Qwen-Image base.
    from diffusers import QwenImageTransformer2DModel
    return _load_dit_transformer(QwenImageTransformer2DModel, cfg, device, base_precision)


def _qwen_encode_prompts(pipe, captions, device):
    import torch

    _encoders_to_device(pipe, device)
    out = []
    with torch.no_grad():
        for cap in captions:
            pe, mask = pipe.encode_prompt(
                prompt = cap,
                device = device,
                num_images_per_prompt = 1,
                max_sequence_length = 1024,
            )
            out.append((pe.cpu(), mask.cpu() if mask is not None else None))
    return out


def _qwen_latent_affine(vae, ref):
    import torch

    z = vae.config.z_dim
    mean = torch.tensor(vae.config.latents_mean, device = ref.device, dtype = ref.dtype)
    std = torch.tensor(vae.config.latents_std, device = ref.device, dtype = ref.dtype)
    return mean.view(1, z, 1, 1, 1), std.view(1, z, 1, 1, 1)


def _qwen_encode_latents(vae, pixel_values):
    import torch

    # AutoencoderKLQwenImage is a 3D (video) VAE: add a temporal dim.
    px = pixel_values.to(torch.float32).unsqueeze(2)
    with torch.no_grad():
        lat = vae.encode(px).latent_dist.sample()
    mean, std = _qwen_latent_affine(vae, lat)
    return (lat - mean) / std


def _qwen_encode_latent_stats(vae, pixel_values):
    import torch

    px = pixel_values.to(torch.float32).unsqueeze(2)
    with torch.no_grad():
        dist = vae.encode(px).latent_dist
    mean, std = _qwen_latent_affine(vae, dist.mean)
    return (dist.mean - mean) / std, dist.std / std


def _qwen_collate(
    entries,
    device,
    weight_dtype,
    pad_to = None,
):
    import torch
    import torch.nn.functional as F

    # Qwen embeds vary in length: pad to batch max or a pinned pad_to bucket, with the mask padded alike.
    seqs = [e[0].shape[1] for e in entries]
    target = max(pad_to or 0, max(seqs))
    pes, masks = [], []
    # Pad/cat on device: a host pad/cat is pageable and its copy blocks.
    for pe, mask in entries:
        s = pe.shape[1]
        pe = pe.to(device, non_blocking = True)
        if mask is None:
            mask = torch.ones((1, s), dtype = torch.int64, device = pe.device)
        else:
            mask = mask.to(device, non_blocking = True)
        if s < target:
            pe = F.pad(pe, (0, 0, 0, target - s))
            mask = F.pad(mask, (0, target - s))
        pes.append(pe)
        masks.append(mask)
    pe_b = (pes[0] if len(pes) == 1 else torch.cat(pes)).to(dtype = weight_dtype)
    mask_b = masks[0] if len(masks) == 1 else torch.cat(masks)
    # A single unpadded sample keeps the legacy None mask, avoiding a behaviour change for existing single-image runs.
    if len(entries) == 1 and entries[0][1] is None and target == seqs[0]:
        mask_b = None
    return (pe_b, mask_b)


def _qwen_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype):
    from diffusers import QwenImagePipeline

    pe, mask = embeds_batch
    bsz, c, f, h, w = noisy.shape
    packed = QwenImagePipeline._pack_latents(noisy, bsz, c, h, w)
    # Each entry is a LIST of one tuple: the transformer indexes sample[0] / sample[1:].
    img_shapes = [[(1, h // 2, w // 2)]] * bsz
    pred = transformer(
        hidden_states = packed,
        encoder_hidden_states = pe,
        encoder_hidden_states_mask = mask,
        timestep = timesteps / 1000,
        img_shapes = img_shapes,
        return_dict = False,
    )[0]
    return QwenImagePipeline._unpack_latents(pred, h * 8, w * 8, 8)


def _qwen_save(pipe_cls, out_dir, transformer_lora_layers):
    from diffusers import QwenImagePipeline
    QwenImagePipeline.save_lora_weights(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
    )


def _zimage_load_conditioners(cfg, device, weight_dtype):
    from diffusers import ZImagePipeline
    return _load_pipe_without_transformer(ZImagePipeline, cfg, device)


def _zimage_load_transformer(cfg, device, weight_dtype, base_precision):
    # Prequant default loads 4-bit as-is under nf4; the dense modes use the bf16 Tongyi-MAI base. Z-Image is bf16 only
    # (its RoPE/embedder run fp32; fp16 overflows).
    from diffusers import ZImageTransformer2DModel
    return _load_dit_transformer(ZImageTransformer2DModel, cfg, device, base_precision)


def _zimage_encode_prompts(pipe, captions, device):
    import torch

    _encoders_to_device(pipe, device)
    out = []
    with torch.no_grad():
        for cap in captions:
            pe, _neg = pipe.encode_prompt(
                prompt = cap,
                device = device,
                do_classifier_free_guidance = False,
                max_sequence_length = 512,
            )
            # pe is a list of one variable-length [seq, 2560] tensor per prompt.
            emb = pe[0] if isinstance(pe, (list, tuple)) else pe
            out.append((emb.cpu(),))
    return out


def _zimage_encode_latents(vae, pixel_values):
    import torch
    with torch.no_grad():
        lat = vae.encode(pixel_values.to(torch.float32)).latent_dist.mode()
    return (lat - vae.config.shift_factor) * vae.config.scaling_factor


def _zimage_encode_latent_stats(vae, pixel_values):
    # Z-Image trains from the posterior mode, so B is None and the loop skips the per-step draw.
    return _zimage_encode_latents(vae, pixel_values), None


def _zimage_collate(
    entries,
    device,
    weight_dtype,
    pad_to = None,
):
    caps = [e[0].to(device = device, non_blocking = True).to(dtype = weight_dtype) for e in entries]
    return (caps,)


def _zimage_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype):
    import torch

    (caps,) = embeds_batch
    # Timestep convention is REVERSED ((1000 - t) / 1000) and the prediction is NEGATED.
    x_list = list(noisy.unsqueeze(2).unbind(dim = 0))
    t_norm = (1000 - timesteps) / 1000
    out = transformer(x_list, t_norm, list(caps), return_dict = False)[0]
    return -torch.stack(out, dim = 0).squeeze(2)


def _zimage_save(pipe_cls, out_dir, transformer_lora_layers):
    from diffusers import ZImagePipeline
    ZImagePipeline.save_lora_weights(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
    )


def _krea2_load_conditioners(cfg, device, weight_dtype):
    # The krea repo ships transformers-5.x configs the pinned 4.x cannot parse (diffusion_krea2.py).
    import torch
    from core.inference.diffusion_krea2 import load_krea2_pipeline

    pipe = load_krea2_pipeline(
        cfg.base_model, torch.bfloat16, hf_token = cfg.hf_token, with_transformer = False
    )
    # Host until the text encoders are freed, as _load_pipe_without_transformer.
    pipe.vae.to(dtype = torch.float32)
    return pipe, pipe.vae


def _krea2_load_transformer(cfg, device, weight_dtype, base_precision):
    # No prequant repo yet, so nf4 quantizes the 12B transformer on the fly.
    from diffusers import Krea2Transformer2DModel
    return _load_dit_transformer(Krea2Transformer2DModel, cfg, device, base_precision)


def _krea2_encode_prompts(pipe, captions, device):
    import torch

    _encoders_to_device(pipe, device)
    out = []
    with torch.no_grad():
        for cap in captions:
            # encode_prompt pads to max_sequence_length, so embeds keep static shapes and the mask is [1, 512].
            pe, mask = pipe.encode_prompt(
                prompt = cap,
                device = device,
                num_images_per_prompt = 1,
                max_sequence_length = 512,
            )
            out.append((pe.cpu(), mask.cpu()))
    return out


# Krea 2 uses the Qwen-Image VAE normalisation (latents_mean/latents_std), so encoding is shared.
_krea2_encode_latents = _qwen_encode_latents
_krea2_encode_latent_stats = _qwen_encode_latent_stats


def _krea2_collate(
    entries,
    device,
    weight_dtype,
    pad_to = None,
):
    # Fixed-length embeds, so collation is a plain concat with the mask riding along; ``pad_to`` is moot.
    pe_b = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
    mask_b = _to_device_cat([e[1] for e in entries], device)
    return (pe_b, mask_b)


def _krea2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype):
    from diffusers import Krea2Pipeline

    pe, mask = embeds_batch
    # Krea2Pipeline._pack_latents is an instance method, so packing is inlined.
    bsz, c, _f, h, w = noisy.shape
    packed = noisy.reshape(bsz, c, h // 2, 2, w // 2, 2)
    packed = packed.permute(0, 2, 4, 1, 3, 5).reshape(bsz, (h // 2) * (w // 2), c * 4)
    # Text tokens sit at the rotary origin, so one shared position grid serves the batch.
    position_ids = Krea2Pipeline.prepare_position_ids(pe.shape[1], h // 2, w // 2, device)
    pred = transformer(
        hidden_states = packed,
        encoder_hidden_states = pe,
        timestep = timesteps / 1000,
        position_ids = position_ids,
        encoder_attention_mask = mask,
        return_dict = False,
    )[0]
    pred = pred.view(bsz, h // 2, w // 2, c, 2, 2)
    pred = pred.permute(0, 3, 1, 4, 2, 5)
    return pred.reshape(bsz, c, 1, h, w)


def _krea2_save(pipe_cls, out_dir, transformer_lora_layers):
    from diffusers import Krea2Pipeline
    Krea2Pipeline.save_lora_weights(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
    )


# FLUX.2 latents train patchified and batch-norm-normalised, from the posterior MODE.
_FLUX2_COMMON_TARGETS = (
    # Double-stream blocks: separate q/k/v plus the ModuleList out proj.
    "to_k",
    "to_q",
    "to_v",
    "to_out.0",
    # Single-stream blocks use one fused qkv+mlp input projection.
    "to_qkv_mlp_proj",
)
# A bare "to_out" also matches the double-stream ModuleList, which PEFT cannot wrap.
_FLUX2_KLEIN_TARGETS = _FLUX2_COMMON_TARGETS + tuple(
    f"single_transformer_blocks.{i}.attn.to_out" for i in range(24)
)
_FLUX2_DEV_TARGETS = _FLUX2_COMMON_TARGETS + tuple(
    f"single_transformer_blocks.{i}.attn.to_out" for i in range(48)
)
# Train dev with guidance 3.5 as the references do; Klein applies it only if guidance_embeds is set.
_FLUX2_TRAIN_GUIDANCE = 3.5


def _flux2_load_conditioners(cfg, device, weight_dtype):
    from diffusers import Flux2Pipeline
    return _load_pipe_without_transformer(Flux2Pipeline, cfg, device)


def _flux2_klein_load_conditioners(cfg, device, weight_dtype):
    from diffusers import Flux2KleinPipeline
    return _load_pipe_without_transformer(Flux2KleinPipeline, cfg, device)


def _flux2_load_transformer(cfg, device, weight_dtype, base_precision):
    from diffusers import Flux2Transformer2DModel
    return _load_dit_transformer(Flux2Transformer2DModel, cfg, device, base_precision)


def _flux2_encode_prompts(pipe, captions, device):
    import torch

    _encoders_to_device(pipe, device)
    out = []
    with torch.no_grad():
        for cap in captions:
            # Embeds are fixed-length (max_sequence_length); text_ids are per-caption [1, txt_len, 4].
            pe, text_ids = pipe.encode_prompt(
                prompt = cap,
                device = device,
                num_images_per_prompt = 1,
                max_sequence_length = 512,
            )
            out.append((pe.cpu(), text_ids.cpu()))
    return out


def _flux2_encode_latents(vae, pixel_values):
    import torch
    from diffusers import Flux2Pipeline

    with torch.no_grad():
        lat = vae.encode(pixel_values.to(torch.float32)).latent_dist.mode()
    lat = Flux2Pipeline._patchify_latents(lat)
    # The FLUX.2 VAE normalises latents with its BatchNorm running stats, not shift/scaling_factor.
    mean = vae.bn.running_mean.view(1, -1, 1, 1).to(lat)
    std = torch.sqrt(vae.bn.running_var.view(1, -1, 1, 1) + vae.config.batch_norm_eps).to(lat)
    return (lat - mean) / std


def _flux2_encode_latent_stats(vae, pixel_values):
    # FLUX.2 caches the posterior mode, so B is None and the loop skips the per-step sampling draw.
    return _flux2_encode_latents(vae, pixel_values), None


def _flux2_collate(
    entries,
    device,
    weight_dtype,
    pad_to = None,
):
    import torch

    # text_ids are PER-SAMPLE (unlike FLUX.1's shared grid).
    pe = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
    text_ids = _to_device_cat([e[1] for e in entries], device).to(dtype = torch.float32)
    return (pe, text_ids)


# The ids derive only from the latent shape, so rebuilding them every step is allocator churn.
_FLUX2_STATIC: dict[tuple, Any] = {}


def _flux2_static_img_ids(latents, device):
    from diffusers import Flux2Pipeline

    key = (latents.shape[0], latents.shape[-2], latents.shape[-1], str(device))
    hit = _FLUX2_STATIC.get(key)
    if hit is None:
        hit = _FLUX2_STATIC[key] = Flux2Pipeline._prepare_latent_ids(latents).to(device = device)
    return hit


def _flux2_forward(
    transformer,
    noisy,
    timesteps,
    sigmas,
    embeds_batch,
    cfg,
    device,
    weight_dtype,
    guidance_value = None,
):
    import torch
    from diffusers import Flux2Pipeline

    pe, text_ids = embeds_batch
    # noisy is already patchified, so packing is just the pipeline's [B,C,H,W] to [B,H*W,C] flatten.
    packed = Flux2Pipeline._pack_latents(noisy)
    img_ids = _flux2_static_img_ids(noisy, device)
    guidance = None
    if getattr(transformer.config, "guidance_embeds", False):
        guidance = torch.full(
            (noisy.shape[0],),
            _FLUX2_TRAIN_GUIDANCE if guidance_value is None else guidance_value,
            device = device,
            dtype = torch.float32,
        )
    pred = transformer(
        hidden_states = packed,
        timestep = timesteps / 1000,
        guidance = guidance,
        encoder_hidden_states = pe,
        txt_ids = text_ids,
        img_ids = img_ids,
        return_dict = False,
    )[0]
    pred = pred[:, : packed.size(1)]
    # diffusers 0.39 stacks the unpacked samples; other builds return a list.
    unpacked = Flux2Pipeline._unpack_latents_with_ids(pred, img_ids)
    if isinstance(unpacked, (list, tuple)):
        unpacked = torch.stack(unpacked)
    return unpacked


def _flux2_save(pipe_cls, out_dir, transformer_lora_layers):
    from diffusers import Flux2Pipeline
    Flux2Pipeline.save_lora_weights(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
    )


def _flux2_klein_save(pipe_cls, out_dir, transformer_lora_layers):
    from diffusers import Flux2KleinPipeline
    Flux2KleinPipeline.save_lora_weights(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
    )


# Targets fully qualified: a bare to_q would also match audio and cross-modality attention, so keep audio out of the LoRA targets and the loss.
_LTX2_TARGETS = (
    "attn1.to_q",
    "attn1.to_k",
    "attn1.to_v",
    "attn1.to_out.0",
    "attn2.to_q",
    "attn2.to_k",
    "attn2.to_v",
    "attn2.to_out.0",
)

# Rotary time is in SECONDS: at 24 fps a still lands on a generated clip's first frame
# (Lightricks pin images to fps=1.0 instead).
_LTX2_TRAIN_FPS = 24.0


def _ltx2_pipeline_cls():
    """``diffusers.LTX2Pipeline``, importable on the pinned transformers (see ltx2_import_compat)."""
    from core.inference.ltx2_import_compat import ensure_ltx2_pipelines_importable

    ensure_ltx2_pipelines_importable()
    from diffusers import LTX2Pipeline

    return LTX2Pipeline


def _ltx2_load_conditioners(cfg, device, weight_dtype):
    LTX2Pipeline = _ltx2_pipeline_cls()

    pipe, vae = _load_pipe_without_transformer(LTX2Pipeline, cfg, device)
    # The connectors are not a text_encoder attribute, so _encoders_to_device never reaches them.
    if getattr(pipe, "connectors", None) is not None:
        pipe.connectors.to(device)
    return pipe, vae


def _ltx2_load_transformer(cfg, device, weight_dtype, base_precision):
    from diffusers import LTX2VideoTransformer3DModel
    return _load_dit_transformer(LTX2VideoTransformer3DModel, cfg, device, base_precision)


def _ltx2_encode_prompts(pipe, captions, device):
    import torch

    _encoders_to_device(pipe, device)
    # Gemma3 hidden states are stacked per layer; only the connector output reaches the transformer.
    out = []
    with torch.no_grad():
        for cap in captions:
            pe, mask, _neg, _neg_mask = pipe.encode_prompt(
                prompt = cap,
                do_classifier_free_guidance = False,
                num_videos_per_prompt = 1,
                max_sequence_length = 1024,
                device = device,
            )
            # Read AFTER encode_prompt, which sets padding_side="left"; the connector mask depends on it.
            padding_side = getattr(getattr(pipe, "tokenizer", None), "padding_side", "left")
            video_emb, audio_emb, conn_mask = pipe.connectors(pe, mask, padding_side = padding_side)
            out.append((video_emb.cpu(), audio_emb.cpu(), conn_mask.cpu()))
    return out


def _ltx2_latent_affine(vae, ref):
    mean = vae.latents_mean.to(device = ref.device, dtype = ref.dtype).view(1, -1, 1, 1, 1)
    std = vae.latents_std.to(device = ref.device, dtype = ref.dtype).view(1, -1, 1, 1, 1)
    # scaling_factor is 1.0 on the shipped checkpoint but is applied for fidelity to _normalize_latents.
    return mean, std / float(vae.config.scaling_factor or 1.0)


def _ltx2_encode_latents(vae, pixel_values):
    import torch

    # Stills are 1-frame clips; a 512px still encodes to [B,128,1,16,16] at 32x spatial, 8x temporal.
    px = pixel_values.to(torch.float32).unsqueeze(2)
    with torch.no_grad():
        lat = vae.encode(px).latent_dist.sample()
    mean, std = _ltx2_latent_affine(vae, lat)
    return (lat - mean) / std


def _ltx2_encode_latent_stats(vae, pixel_values):
    import torch

    px = pixel_values.to(torch.float32).unsqueeze(2)
    with torch.no_grad():
        dist = vae.encode(px).latent_dist
    mean, std = _ltx2_latent_affine(vae, dist.mean)
    return (dist.mean - mean) / std, dist.std / std


def _ltx2_collate(
    entries,
    device,
    weight_dtype,
    pad_to = None,
):
    # Connector embeds are fixed at [1, 1024, 3840] via max_sequence_length, so pad_to is moot.
    video = _to_device_cat([e[0] for e in entries], device).to(dtype = weight_dtype)
    audio = _to_device_cat([e[1] for e in entries], device).to(dtype = weight_dtype)
    mask = _to_device_cat([e[2] for e in entries], device)
    return (video, audio, mask)


def _ltx2_audio_token_count(config, num_pixel_frames: int, fps: float) -> int:
    """Audio token count is read from the transformer config, so no audio VAE is needed; floored at one token since the transformer always indexes audio."""
    per_second = (
        float(config.audio_sampling_rate)
        / float(config.audio_hop_length)
        / float(config.audio_scale_factor)
    )
    return max(1, round((num_pixel_frames / float(fps)) * per_second))


def _ltx2_pack(latents, conf):
    """[B,C,F,H,W] -> [B, F*H*W, C] via the pipeline's own patchifier."""
    LTX2Pipeline = _ltx2_pipeline_cls()
    return LTX2Pipeline._pack_latents(latents, conf.patch_size, conf.patch_size_t)


def _ltx2_unpack(pred, f, h, w, conf):
    """The inverse of ``_ltx2_pack``, back to the 5-D shape ``target = noise - latents`` has."""
    LTX2Pipeline = _ltx2_pipeline_cls()
    return LTX2Pipeline._unpack_latents(pred, f, h, w, conf.patch_size, conf.patch_size_t)


def _ltx2_audio_state(sigmas, bsz, audio_len, channels, device, dtype):
    """Still images have no audio: the placeholder follows a zero clean latent's flow state; it exists only because forward requires the argument."""
    import torch

    noise = torch.randn((bsz, audio_len, channels), device = device, dtype = dtype)
    # sigmas arrives broadcast to the 5-D video latent; the audio stream is 3-D.
    return sigmas.reshape(bsz, 1, 1) * noise


def _ltx2_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype):
    video_emb, audio_emb, mask = embeds_batch
    bsz, _c, f, h, w = noisy.shape
    conf = transformer.config
    packed = _ltx2_pack(noisy, conf)

    # vae_scale_factors is (temporal, height, width), so [0] is the 8x temporal compression.
    num_pixel_frames = (f - 1) * int(conf.vae_scale_factors[0]) + 1
    audio_len = _ltx2_audio_token_count(conf, num_pixel_frames, _LTX2_TRAIN_FPS)
    audio_noisy = _ltx2_audio_state(
        sigmas, bsz, audio_len, conf.audio_in_channels, packed.device, packed.dtype
    )

    pred, _audio_pred = transformer(
        hidden_states = packed,
        audio_hidden_states = audio_noisy,
        encoder_hidden_states = video_emb,
        audio_encoder_hidden_states = audio_emb,
        # LTX-2 conditions on the UNSCALED timestep, unlike FLUX / Qwen's timestep / 1000.
        timestep = timesteps,
        sigma = timesteps,
        encoder_attention_mask = mask,
        audio_encoder_attention_mask = mask,
        num_frames = f,
        height = h,
        width = w,
        fps = _LTX2_TRAIN_FPS,
        audio_num_frames = audio_len,
        # Keep the placeholder audio stream from perturbing the video prediction.
        isolate_modalities = True,
        return_dict = False,
    )
    return _ltx2_unpack(pred, f, h, w, conf)


def _ltx2_save(pipe_cls, out_dir, transformer_lora_layers):
    LTX2Pipeline = _ltx2_pipeline_cls()
    LTX2Pipeline.save_lora_weights(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
    )


# Wan2.2-TI2V-5B, trained from stills (a 1-frame clip is a valid T2V sample). Targets match DiffSynth-Studio /
# musubi-tuner; bare "to_q" is safe (single stream), add_k/v_proj exist only on I2V-14B.
_WAN_TARGETS = (
    "to_q",
    "to_k",
    "to_v",
    "to_out.0",
    "ffn.net.0.proj",
    "ffn.net.2",
)
_WAN_MAX_SEQUENCE_LENGTH = 512


def _wan_pipeline_cls():
    from diffusers import WanPipeline
    return WanPipeline


def _wan_load_conditioners(cfg, device, weight_dtype):
    """UMT5 + tokenizer + VAE. VAE loads fp32 directly; the shared bf16-then-fp32 cast would round it through bf16."""
    import torch
    from diffusers import AutoencoderKLWan

    vae = AutoencoderKLWan.from_pretrained(
        cfg.base_model, subfolder = "vae", torch_dtype = torch.float32, token = cfg.hf_token
    )
    pipe = _wan_pipeline_cls().from_pretrained(
        cfg.base_model,
        transformer = None,
        vae = vae,
        torch_dtype = torch.bfloat16,
        token = cfg.hf_token,
    )
    # VAE stays on host; the shared loop moves it only after UMT5-XXL (11.4 GB) is freed.
    return pipe, pipe.vae


def _wan_load_transformer(cfg, device, weight_dtype, base_precision):
    from diffusers import WanTransformer3DModel
    return _load_dit_transformer(WanTransformer3DModel, cfg, device, base_precision)


def _wan_trim_padding(embeds):
    """Drop trailing zero-pad rows; lossless since ``_wan_collate`` re-pads."""
    nonzero = embeds[0].abs().amax(dim = -1) > 0
    idx = nonzero.nonzero()
    keep = int(idx.max()) + 1 if idx.numel() else 1
    return embeds[:, :keep].contiguous()


def _wan_encode_prompts(pipe, captions, device):
    import torch

    _encoders_to_device(pipe, device)
    out = []
    with torch.no_grad():
        for cap in captions:
            pe, _neg = pipe.encode_prompt(
                prompt = cap,
                do_classifier_free_guidance = False,
                num_videos_per_prompt = 1,
                max_sequence_length = _WAN_MAX_SEQUENCE_LENGTH,
                device = device,
            )
            out.append((_wan_trim_padding(pe).cpu(),))
    return out


def _wan_latent_affine(vae, ref):
    """Per-channel (mean, std); training latent is ``(z - mean) / std``."""
    import torch

    z = int(vae.config.z_dim)
    mean = torch.tensor(vae.config.latents_mean, device = ref.device, dtype = ref.dtype).view(
        1, z, 1, 1, 1
    )
    std = torch.tensor(vae.config.latents_std, device = ref.device, dtype = ref.dtype).view(
        1, z, 1, 1, 1
    )
    return mean, std


def _wan_pixels(vae, pixel_values):
    return pixel_values.to(dtype = next(vae.parameters()).dtype).unsqueeze(2)


def _wan_encode_latents(vae, pixel_values):
    import torch

    with torch.no_grad():
        lat = vae.encode(_wan_pixels(vae, pixel_values)).latent_dist.sample()
    mean, std = _wan_latent_affine(vae, lat)
    return (lat - mean) / std


def _wan_encode_latent_stats(vae, pixel_values):
    import torch

    with torch.no_grad():
        dist = vae.encode(_wan_pixels(vae, pixel_values)).latent_dist
    mean, std = _wan_latent_affine(vae, dist.mean)
    return (dist.mean - mean) / std, dist.std / std


def _wan_collate(
    entries,
    device,
    weight_dtype,
    pad_to = None,
):
    import torch
    import torch.nn.functional as F

    # Zero-pad as _get_t5_prompt_embeds does, on the DEVICE: host-side pad/cat stalled steps ~100 ms.
    length = max(_WAN_MAX_SEQUENCE_LENGTH, *(e[0].shape[1] for e in entries))
    padded = [
        F.pad(
            e[0].to(device = device, dtype = weight_dtype, non_blocking = True),
            (0, 0, 0, length - e[0].shape[1]),
        )
        for e in entries
    ]
    return (padded[0] if len(padded) == 1 else torch.cat(padded),)


def _wan_forward(transformer, noisy, timesteps, sigmas, embeds_batch, cfg, device, weight_dtype):
    (prompt_embeds,) = embeds_batch
    # [B] timestep == the pipeline's per-token expand_timesteps form for T2V.
    return transformer(
        hidden_states = noisy,
        timestep = timesteps.to(device = noisy.device).reshape(-1),
        encoder_hidden_states = prompt_embeds,
        return_dict = False,
    )[0]


def _wan_save(
    pipe_cls,
    out_dir,
    transformer_lora_layers,
    adapter_metadata = None,
):
    # Older diffusers savers lack the metadata argument; they still get the weights.
    import inspect

    save = _wan_pipeline_cls().save_lora_weights
    kwargs = {}
    if (
        adapter_metadata
        and "transformer_lora_adapter_metadata" in inspect.signature(save).parameters
    ):
        kwargs["transformer_lora_adapter_metadata"] = dict(adapter_metadata)
    save(
        save_directory = out_dir,
        transformer_lora_layers = transformer_lora_layers,
        weight_name = DEFAULT_LORA_FILENAME,
        **kwargs,
    )


def _lora_adapter_metadata(transformer) -> Optional[dict]:
    try:
        return transformer.peft_config["default"].to_dict()
    except Exception:  # noqa: BLE001 -- metadata is best-effort; the weights still save
        return None


def _adapter_saver(spec, transformer):
    if not spec.save_takes_metadata:
        return spec.save
    import functools
    return functools.partial(spec.save, adapter_metadata = _lora_adapter_metadata(transformer))


def _wan_training_scheduler(cfg):
    """FlowMatchEuler with Wan's UniPC ``flow_shift`` baked in; loading it from the UniPC config silently drops the
    shift. UniPC flow sigmas equal FlowMatchEuler's static shift."""
    from diffusers import FlowMatchEulerDiscreteScheduler

    conf = FlowMatchEulerDiscreteScheduler.load_config(
        cfg.base_model, subfolder = "scheduler", token = cfg.hf_token
    )
    return FlowMatchEulerDiscreteScheduler(
        num_train_timesteps = int(conf.get("num_train_timesteps", 1000)),
        shift = float(conf.get("flow_shift") or 1.0),
    )


def _vae8_latent_shape(vae, height, width):
    return (1, vae.config.latent_channels, height // 8, width // 8), (height // 16) * (width // 16)


def _flux_decode_latents(vae, latents):
    lat = latents / vae.config.scaling_factor + vae.config.shift_factor
    return vae.decode(lat.to(vae.dtype)).sample


def _qwen_latent_shape(vae, height, width):
    return (1, vae.config.z_dim, 1, height // 8, width // 8), (height // 16) * (width // 16)


def _qwen_decode_latents(vae, latents):
    mean, std = _qwen_latent_affine(vae, latents)
    return vae.decode((latents * std + mean).to(vae.dtype)).sample[:, :, 0]


def _flux2_latent_shape(vae, height, width):
    return (1, vae.config.latent_channels * 4, height // 16, width // 16), (height // 16) * (
        width // 16
    )


def _flux2_decode_latents(vae, latents):
    import torch
    from diffusers import Flux2Pipeline

    mean = vae.bn.running_mean.view(1, -1, 1, 1).to(latents)
    std = torch.sqrt(vae.bn.running_var.view(1, -1, 1, 1) + vae.config.batch_norm_eps).to(latents)
    lat = Flux2Pipeline._unpatchify_latents(latents * std + mean)
    return vae.decode(lat.to(vae.dtype)).sample


_SPECS: dict[str, _FamilySpec] = {
    "flux.1": _FamilySpec(
        family = "flux.1",
        lora_targets = _FLUX_TARGETS,
        force_bf16 = False,
        dense_bf16_gb = 23.8,
        load_conditioners = _flux_load_conditioners,
        load_transformer = _flux_load_transformer,
        encode_prompts = _flux_encode_prompts,
        encode_latents = _flux_encode_latents,
        encode_latent_stats = _flux_encode_latent_stats,
        collate = _flux_collate,
        forward = _flux_forward,
        save = _flux_save,
        latent_shape = _vae8_latent_shape,
        decode_latents = _flux_decode_latents,
        sample_forward_kwargs = {"guidance": 3.5},
    ),
    "qwen-image": _FamilySpec(
        family = "qwen-image",
        lora_targets = _QWEN_TARGETS,
        force_bf16 = True,
        dense_bf16_gb = 41.0,
        load_conditioners = _qwen_load_conditioners,
        load_transformer = _qwen_load_transformer,
        encode_prompts = _qwen_encode_prompts,
        encode_latents = _qwen_encode_latents,
        encode_latent_stats = _qwen_encode_latent_stats,
        collate = _qwen_collate,
        forward = _qwen_forward,
        save = _qwen_save,
        latent_shape = _qwen_latent_shape,
        decode_latents = _qwen_decode_latents,
    ),
    "z-image": _FamilySpec(
        family = "z-image",
        lora_targets = _ZIMAGE_TARGETS,
        force_bf16 = True,
        dense_bf16_gb = 12.3,
        load_conditioners = _zimage_load_conditioners,
        load_transformer = _zimage_load_transformer,
        encode_prompts = _zimage_encode_prompts,
        encode_latents = _zimage_encode_latents,
        encode_latent_stats = _zimage_encode_latent_stats,
        collate = _zimage_collate,
        forward = _zimage_forward,
        save = _zimage_save,
        latent_shape = _vae8_latent_shape,
        decode_latents = _flux_decode_latents,
    ),
    "krea-2": _FamilySpec(
        family = "krea-2",
        lora_targets = _KREA2_TARGETS,
        force_bf16 = True,
        dense_bf16_gb = 26.3,
        load_conditioners = _krea2_load_conditioners,
        load_transformer = _krea2_load_transformer,
        encode_prompts = _krea2_encode_prompts,
        encode_latents = _krea2_encode_latents,
        encode_latent_stats = _krea2_encode_latent_stats,
        collate = _krea2_collate,
        forward = _krea2_forward,
        save = _krea2_save,
        latent_shape = _qwen_latent_shape,
        decode_latents = _qwen_decode_latents,
    ),
    "flux.2-klein": _FamilySpec(
        family = "flux.2-klein",
        lora_targets = _FLUX2_KLEIN_TARGETS,
        # The upstream references train in bf16; fp16 is unvalidated on the FLUX.2 stack.
        force_bf16 = True,
        dense_bf16_gb = 8.1,
        load_conditioners = _flux2_klein_load_conditioners,
        load_transformer = _flux2_load_transformer,
        encode_prompts = _flux2_encode_prompts,
        encode_latents = _flux2_encode_latents,
        encode_latent_stats = _flux2_encode_latent_stats,
        collate = _flux2_collate,
        forward = _flux2_forward,
        save = _flux2_klein_save,
        latent_shape = _flux2_latent_shape,
        decode_latents = _flux2_decode_latents,
    ),
    "flux.2-dev": _FamilySpec(
        family = "flux.2-dev",
        lora_targets = _FLUX2_DEV_TARGETS,
        force_bf16 = True,
        # Mistral conditioners (~46 GB bf16) load, encode and free before the 32B DiT lands on device.
        dense_bf16_gb = 64.5,
        load_conditioners = _flux2_load_conditioners,
        load_transformer = _flux2_load_transformer,
        encode_prompts = _flux2_encode_prompts,
        encode_latents = _flux2_encode_latents,
        encode_latent_stats = _flux2_encode_latent_stats,
        collate = _flux2_collate,
        forward = _flux2_forward,
        save = _flux2_save,
        latent_shape = _flux2_latent_shape,
        decode_latents = _flux2_decode_latents,
        # Flux2Pipeline's default guidance_scale, which Studio's FLUX.2-dev inference also uses.
        sample_forward_kwargs = {"guidance_value": 4.0},
    ),
    "ltx-2": _FamilySpec(
        family = "ltx-2",
        lora_targets = _LTX2_TARGETS,
        force_bf16 = True,
        # Gemma3-12B conditioners load, encode and free before the 19B DiT (37.76 GB bf16) lands on device.
        dense_bf16_gb = 37.8,
        load_conditioners = _ltx2_load_conditioners,
        load_transformer = _ltx2_load_transformer,
        encode_prompts = _ltx2_encode_prompts,
        encode_latents = _ltx2_encode_latents,
        encode_latent_stats = _ltx2_encode_latent_stats,
        collate = _ltx2_collate,
        forward = _ltx2_forward,
        save = _ltx2_save,
    ),
    "wan2.2-ti2v-5b": _FamilySpec(
        family = "wan2.2-ti2v-5b",
        lora_targets = _WAN_TARGETS,
        force_bf16 = True,
        dense_bf16_gb = 10.0,
        load_conditioners = _wan_load_conditioners,
        load_transformer = _wan_load_transformer,
        encode_prompts = _wan_encode_prompts,
        encode_latents = _wan_encode_latents,
        encode_latent_stats = _wan_encode_latent_stats,
        collate = _wan_collate,
        forward = _wan_forward,
        save = _wan_save,
        training_scheduler = _wan_training_scheduler,
        save_takes_metadata = True,
    ),
}


# HF repos gated by license acceptance; training needs a token whose account accepted it. Checked by name, no network, so a missing token fails fast.
_GATED_TRAIN_REPOS = frozenset({"black-forest-labs/flux.1-dev", "black-forest-labs/flux.2-dev"})


def _assert_gated_access(base_model: str, hf_token: Optional[str]) -> None:
    """Raise a clear error before loading a gated base without a token."""
    from core.inference.diffusion_families import _is_local_path

    name = str(base_model or "").strip().lower()
    # A local clone named like the vendor repo is weights on disk, not a Hub fetch.
    if _is_local_path(base_model):
        return
    if name in _GATED_TRAIN_REPOS and not (hf_token and str(hf_token).strip()):
        raise ValueError(
            f"'{base_model}' is a gated Hugging Face repo. Accept its license on the Hub "
            f"and add your HF token in Unsloth settings before training from it."
        )


def _canvas(resolution):
    """(w, h) of the training canvas: a bucket tuple, or the legacy square ``resolution``."""
    if isinstance(resolution, (tuple, list)):
        return int(resolution[0]), int(resolution[1])
    return int(resolution), int(resolution)


def _open_resized(path, resolution):
    """Open + EXIF-orient + cover-resize to the canvas (same geometry as the SDXL loader; a square
    canvas is the legacy short-side resize). Returns the resized PIL image and its (rw, rh)."""
    from PIL import Image, ImageOps

    from core.inference.mcp_images import flattened_rgb

    img = flattened_rgb(ImageOps.exif_transpose(Image.open(path)), background = (255, 255, 255))
    rw, rh = cover_resize_dims(*img.size, *_canvas(resolution))
    return img.resize((rw, rh), Image.LANCZOS), rw, rh


def _to_unit_tensor(img):
    import numpy as np
    import torch

    arr = np.asarray(img, dtype = np.float32) / 255.0
    return torch.from_numpy(arr).permute(2, 0, 1) * 2.0 - 1.0


def _load_pixel_tensor(path, resolution, center_crop, random_flip, rng):
    """Image to [3,H,W] in [-1,1], SDXL loader geometry; DiT families take no SDXL time-ids."""
    from PIL import Image

    cw, ch = _canvas(resolution)
    img, rw, rh = _open_resized(path, resolution)
    if center_crop:
        left, top = (rw - cw) // 2, (rh - ch) // 2
    else:
        left = rng.randint(0, max(0, rw - cw))
        top = rng.randint(0, max(0, rh - ch))
    img = img.crop((left, top, left + cw, top + ch))
    if random_flip and rng.random() < 0.5:
        img = img.transpose(Image.FLIP_LEFT_RIGHT)
    return _to_unit_tensor(img)


def _load_pixel_tensor_planned(
    path,
    resolution,
    center_crop,
    u_left,
    u_top,
    flip,
    resized = None,
):
    """Deterministic pixel-crop variant for the latent cache: crop offsets are unit fractions over the randint range; center_crop matches the uncached path.
    ``resized`` is a prior ``_open_resized`` result, reused across variants."""
    from PIL import Image

    cw, ch = _canvas(resolution)
    img, rw, rh = resized if resized is not None else _open_resized(path, resolution)
    if center_crop:
        left, top = (rw - cw) // 2, (rh - ch) // 2
    else:
        left = min(int(u_left * (rw - cw + 1)), max(0, rw - cw))
        top = min(int(u_top * (rh - ch + 1)), max(0, rh - ch))
    img = img.crop((left, top, left + cw, top + ch))
    if flip:
        img = img.transpose(Image.FLIP_LEFT_RIGHT)
    return _to_unit_tensor(img)


def _build_latent_cache(
    spec,
    vae,
    image_paths,
    cfg,
    device,
    weight_dtype,
    on_event,
    check_stop,
    pcache = None,
    plan = None,
    canvases = None,
):
    """Precompute the latent cache: stats stay fp32, only the sample is cast to weight_dtype; returns None if a stop request interrupts the build."""

    if plan is None:
        plan = _plan_cache_variants(
            len(image_paths), cfg.cache_variants, cfg.center_crop, cfg.random_flip, cfg.seed
        )

    from core.inference.host_pinning import host_pinning_allowed

    pin = device == "cuda" and host_pinning_allowed()

    def _hold(t):
        if t is None:
            return None
        import torch

        t = t.to(torch.float32).cpu()
        if pin:
            try:
                t = t.pin_memory()
            except RuntimeError:
                pass
        return t

    cache: list[list[tuple]] = []
    total = len(image_paths)
    total_variants = sum(len(v) for v in plan)
    forced = _latent_cache_forced()
    gated = False
    for i, path in enumerate(image_paths):
        variants = []
        resized = None
        for u_left, u_top, flip in plan[i]:
            canvas = cfg.resolution if canvases is None else canvases[i]
            key = (
                pcache.latent_key(path, (u_left, u_top, flip), None if canvases is None else canvas)
                if pcache is not None
                else None
            )
            entry = pcache.get(key) if key is not None else None
            if entry is not None:
                a, b = entry
            else:
                if resized is None:
                    resized = _open_resized(path, canvas)
                px = (
                    _load_pixel_tensor_planned(
                        path, canvas, cfg.center_crop, u_left, u_top, flip, resized = resized
                    )
                    .unsqueeze(0)
                    .to(device)
                )
                a, b = spec.encode_latent_stats(vae, px)
                if key is not None:
                    try:
                        pcache.put(key, (a, b))
                    except Exception:  # noqa: BLE001 -- disk-full etc.: the run still trains
                        pass
            a, b = _hold(a), _hold(b)
            if not forced and not gated:
                # Size-gate the cache off the first REAL variant: the cache can exhaust pinned RAM.
                per_variant = a.numel() * a.element_size()
                if b is not None:
                    per_variant += b.numel() * b.element_size()
                if _latent_cache_over_budget(per_variant, total_variants):
                    _emit(
                        on_event,
                        "warning",
                        message = (
                            "Latent cache disabled: estimated "
                            f"{per_variant * total_variants / 1024 ** 3:.1f} GiB over the "
                            "budget; encoding latents per step instead. Set "
                            "UNSLOTH_DIFFUSION_FORCE_LATENT_CACHE=1 to keep it."
                        ),
                    )
                    return LATENT_CACHE_OVER_BUDGET
                gated = True
            variants.append((a, b))
        cache.append(variants)
        if (i + 1) % 4 == 0 or i + 1 == total:
            _emit(on_event, "preparing", stage = "cache_latents", done = i + 1, total = total)
        if check_stop():
            return None
    return cache


def _encode_prompts_cached(spec, pipe, to_encode, device, pcache):
    """Encode captions, serving hits from the persistent cache and writing misses back; result aligned with to_encode, and no cache means plain encode_prompts."""
    if pcache is None:
        return spec.encode_prompts(pipe, to_encode, device)
    hits: dict = {}
    misses = []
    for cap in to_encode:
        entry = pcache.get(pcache.text_key(cap))
        if entry is None:
            misses.append(cap)
        else:
            hits[cap] = entry
    if misses:
        for cap, emb in zip(misses, spec.encode_prompts(pipe, misses, device)):
            hits[cap] = emb
            try:
                pcache.put(pcache.text_key(cap), emb)
            except Exception:  # noqa: BLE001 -- disk-full etc.: the run still trains
                pass
    return [hits[cap] for cap in to_encode]


def _load_warm_conditioning(
    pcache,
    image_paths,
    plan,
    to_encode,
    device,
    canvases = None,
):
    """Full cache hit returns (caption_embeds, latent_cache), else (None, None) for the cold path;
    on a hit the VAE and text encoders are never loaded."""
    embeds = []
    for cap in to_encode:
        entry = pcache.get(pcache.text_key(cap))
        if entry is None:
            return None, None
        embeds.append(entry)

    from core.inference.host_pinning import host_pinning_allowed

    pin = device == "cuda" and host_pinning_allowed()

    def _hold(t):
        if t is None:
            return None
        import torch  # noqa: F401 -- pin_memory needs torch initialised

        t = t.contiguous()
        if pin:
            try:
                t = t.pin_memory()
            except RuntimeError:
                pass
        return t

    forced = _latent_cache_forced()
    total_variants = sum(len(v) for v in plan)
    gated = False
    cache: list[list[tuple]] = []
    for i, path in enumerate(image_paths):
        variants = []
        for variant in plan[i]:
            entry = pcache.get(
                pcache.latent_key(path, variant, None if canvases is None else canvases[i])
            )
            if entry is None:
                return None, None
            a, b = entry
            if not forced and not gated:
                per_variant = a.numel() * a.element_size()
                if b is not None:
                    per_variant += b.numel() * b.element_size()
                if _latent_cache_over_budget(per_variant, total_variants):
                    return None, None
                gated = True
            variants.append((_hold(a), _hold(b)))
        cache.append(variants)
    return {cap: emb for cap, emb in zip(to_encode, embeds)}, cache


def _sample_cached_latents(cache, idxs, variant_rng, device, weight_dtype):
    """Draws fresh noise per step in fp32, as an in-loop latent_dist.sample() does; cast after."""
    import torch

    parts_a, parts_b = [], []
    for i in idxs:
        variants = cache[i]
        a, b = variants[variant_rng.randrange(len(variants))] if len(variants) > 1 else variants[0]
        parts_a.append(a)
        parts_b.append(b)
    lat_a = _to_device_cat(parts_a, device)
    if parts_b[0] is None:
        return lat_a.to(dtype = weight_dtype)
    lat_b = _to_device_cat(parts_b, device)
    return (lat_a + lat_b * torch.randn_like(lat_a)).to(dtype = weight_dtype)


def _to_device_cat(parts, device):
    """Copy pinned parts, then cat on ``device``: a host cat yields pageable memory and a sync copy."""
    import torch

    moved = [p.to(device, non_blocking = True) for p in parts]
    return moved[0] if len(moved) == 1 else torch.cat(moved)


# Pinned (page-locked) host memory is not swappable, so a huge caption set stays pageable; the copies stay async.
_CAPTION_PIN_BUDGET_BYTES = 1024**3


def _pin_caption_embeds(caption_embeds: dict, device) -> dict:
    import torch

    from core.inference.host_pinning import host_pinning_allowed

    if device != "cuda" or not caption_embeds or not host_pinning_allowed():
        return caption_embeds
    tensors = [t for entry in caption_embeds.values() for t in entry if isinstance(t, torch.Tensor)]
    if any(t.device.type != "cpu" for t in tensors):
        return caption_embeds
    if sum(t.numel() * t.element_size() for t in tensors) > _CAPTION_PIN_BUDGET_BYTES:
        return caption_embeds

    def _pin(t):
        if not isinstance(t, torch.Tensor):
            return t
        try:
            return t.contiguous().pin_memory()
        except RuntimeError:
            return t

    return {cap: type(entry)(_pin(t) for t in entry) for cap, entry in caption_embeds.items()}


def _should_compile(
    cfg,
    base_is_bnb,
    device,
    base_precision = "nf4",
    steps_left = None,
) -> bool:
    mode = (cfg.compile_transformer or "auto").strip().lower()
    if device != "cuda" or mode == "off":
        return False
    # torch.compile cannot trace the torchao int8 subclass in training, so int8 runs eager.
    if base_precision == "int8":
        return False
    if mode == "on":
        return True
    if base_precision == "nf4":
        steps = cfg.train_steps if steps_left is None else steps_left
        return steps >= NF4_COMPILE_MIN_STEPS and _nf4_compile_supported()
    return base_precision in ("bf16", "fp8", "mxfp8")


# Lowest stack where bnb 4-bit ops are torch custom ops (0 graph breaks, loss matches eager); ROCm unmeasured, stays eager.
NF4_COMPILE_MIN_TORCH = "2.10"
NF4_COMPILE_MIN_BNB = "0.46.1"
# The nf4 compile only pays back after a few hundred steps, so a short run stays eager.
NF4_COMPILE_MIN_STEPS = 200


def _nf4_compile_supported() -> bool:
    try:
        import importlib.metadata

        import torch
        from packaging.version import Version

        if torch_is_rocm():
            return False
        if Version(torch.__version__) < Version(NF4_COMPILE_MIN_TORCH):
            return False
        return Version(importlib.metadata.version("bitsandbytes")) >= Version(NF4_COMPILE_MIN_BNB)
    except Exception:  # noqa: BLE001 -- unknown versions keep the historical eager path
        return False


def _maybe_compile_transformer(
    transformer,
    cfg,
    base_is_bnb,
    device,
    on_event,
    base_precision = "nf4",
    steps_left = None,
) -> bool:
    """Regionally compile the transformer blocks (diffusers compile_repeated_blocks) after the LoRA
    is attached. Never fatal: a wrap failure falls back to eager with a warning event, and
    dynamo's suppress_errors keeps a frame that fails to COMPILE at the first step running eager
    instead of raising mid-run."""
    if not _should_compile(cfg, base_is_bnb, device, base_precision, steps_left):
        if base_precision in ("fp8", "mxfp8"):
            _emit(
                on_event,
                "warning",
                message = (
                    f"{base_precision} training without torch.compile is slow; "
                    f"enable compile for the speedup."
                ),
            )
        return False
    import torch

    fn = getattr(transformer, "compile_repeated_blocks", None)
    if not callable(fn):
        _emit(
            on_event, "warning", message = "torch.compile unavailable for this model; running eager."
        )
        return False
    try:
        dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
        if dynamo_cfg is not None:
            # Z-Image has ~11 distinct block shapes, over dynamo's default recompile limit of 8.
            for attr in ("recompile_limit", "cache_size_limit"):
                if hasattr(dynamo_cfg, attr):
                    setattr(dynamo_cfg, attr, max(getattr(dynamo_cfg, attr) or 0, 64))
            if hasattr(dynamo_cfg, "suppress_errors"):
                dynamo_cfg.suppress_errors = True
        # dynamic=True matches the inference speed layer: on torch 2.10 / B200 the dynamic=False specialisation failed
        # with CUBLAS_STATUS_EXECUTION_FAILED.
        try:
            # torch 2.12 / 2.13 inductor CantSplit in the dynamic joint-stream cat backward (FLUX.1 fp8); probe-gated.
            from core.inference.diffusion_inductor_backports import install as _install_backports
            _install_backports()
        except Exception:  # noqa: BLE001 -- the backport is best-effort
            pass
        fn(fullgraph = not base_is_bnb, dynamic = True)
        return True
    except Exception as exc:  # noqa: BLE001 -- optimisation only, never fatal
        _emit(on_event, "warning", message = f"torch.compile disabled (eager fallback): {exc}")
        return False


def _dynamo_compile_counters() -> Optional[tuple[int, int, int]]:
    """(graph breaks, failed frames, unique graphs), or None."""
    try:
        from torch._dynamo.utils import counters
        frames = counters["frames"]
        return (
            sum(counters["graph_break"].values()),
            frames["total"] - frames["ok"],
            counters["stats"]["unique_graphs"],
        )
    except Exception:  # noqa: BLE001 -- diagnostics only
        return None


def _compile_engagement_warning(before, after) -> Optional[str]:
    """Why compiled blocks run eager since ``before`` (suppress_errors hides failures), else None."""
    if before is None or after is None:
        return None
    breaks, failed, graphs = (a - b for a, b in zip(after, before))
    if breaks > 0 or failed > 0:
        return (
            f"torch.compile did not fully engage ({breaks} graph break(s), {failed} frame(s) "
            f"fell back to eager): training continues, slower than compiled."
        )
    if graphs <= 0:
        return "torch.compile produced no graphs: the transformer blocks are running eager."
    return None


def run_dit_lora_training(
    config: DiffusionLoraConfig,
    *,
    on_event: Optional[EventCb] = None,
    should_stop: Optional[StopCb] = None,
) -> str:
    """Train a flow-matching DiT LoRA (FLUX.1 / FLUX.2 / Qwen-Image / Z-Image / Krea 2 / LTX-2 / Wan2.2-TI2V-5B) and
    export it. Resumable: ``cfg.resume_from_checkpoint`` restores the adapter, optimizer moments,
    LR position, EMA shadow, sampler cycle and RNG streams from a ``checkpoint-<N>`` bundle, and
    the loop runs steps N+1..train_steps (the TARGET TOTAL). A stop-and-save and every
    ``cfg.save_steps`` interval write such a bundle."""
    cfg = resolve_bucketing(config.normalized())
    spec = _SPECS.get(cfg.resolved_family)
    if spec is None:
        raise ValueError(f"No DiT trainer for family {cfg.resolved_family!r}")

    # DiT families need bf16, so fp16 is refused, not upgraded; validated before heavy imports so a host without diffusers still sees the real error.
    if cfg.mixed_precision == "fp16" and spec.force_bf16:
        raise ValueError(
            f"{spec.family} LoRA training requires bf16: fp16 overflows its fp32 RoPE / "
            f"embedder internals. Set mixed precision to bf16."
        )

    import torch

    rng = random.Random(cfg.seed)
    torch.manual_seed(cfg.seed)
    _FLUX_STATIC.clear()

    save_on_stop = True

    def _check_stop() -> bool:
        nonlocal save_on_stop
        if should_stop is None:
            return False
        sig = should_stop()
        if not sig:
            return False
        if isinstance(sig, dict) and sig.get("save") is False:
            save_on_stop = False
        return True

    device = resolve_train_device()
    # is_bf16_supported() counts emulation on CUDA and XPU, so gate on NATIVE bf16.
    if device == "cuda" and not flow_bf16_trainable():
        raise ValueError(
            "This trainer requires a bfloat16-capable GPU (Ampere or newer); "
            "this CUDA device does not support bf16."
        )
    if device == "xpu" and not native_bf16_supported_xpu():
        raise ValueError(
            "This trainer requires a bfloat16-capable GPU; this XPU device does not "
            "support bf16 natively."
        )
    weight_dtype = torch.bfloat16 if device in ("cuda", "xpu") else torch.float32

    _assert_trusted_base_model(cfg.base_model)
    # Check the fetched repo: the canonical id would raise for a gated base redirected to its mirror.
    _assert_gated_access(cfg.fetch_base_model or cfg.base_model, cfg.hf_token)
    pairs = discover_image_caption_pairs(
        cfg.data_dir, instance_prompt = cfg.instance_prompt, caption_column = cfg.caption_column
    )
    # Resolve num_epochs to train_steps now the size is known; rebind cfg so all reads agree.
    cfg = replace(
        cfg,
        train_steps = resolve_train_steps(cfg, len(pairs), [p for p, _ in pairs]),
        num_epochs = 0,
    )
    # Validate a resume against this run's identity BEFORE the multi-GB phased load, using the RESOLVED LoRA targets
    # rather than the generic default the config carries.
    identity = identity_for_config(
        cfg,
        dataset_pairs = pairs,
        resolved_targets = _select_lora_targets(cfg.lora_target_modules, spec.lora_targets),
    )
    if cfg.resume_from_checkpoint:
        preflight_resume(
            cfg.resume_from_checkpoint, identity = identity, target_steps = cfg.train_steps
        )
    _emit(
        on_event,
        "model_load_started",
        stage = "conditioning",
        num_images = len(pairs),
        fetch_repo = training_fetch_repo(cfg),
    )
    if _check_stop():
        out_dir = Path(cfg.output_dir).expanduser()
        _emit(
            on_event,
            "complete",
            output_dir = str(out_dir),
            lora_path = None,
            stopped = True,
            steps_run = 0,
            # A stop with save=false is a DISCARD however early it lands; otherwise the resume fallback offers the source bundle back as if the attempt were still live.
            discarded = not save_on_stop,
        )
        return str(out_dir)

    # Restore TF32 / cudnn.benchmark on exit, so in-process callers see no lasting change.
    perf_snap = _apply_perf_flags(cfg, device)
    try:
        return _train_dit(
            cfg,
            spec,
            pairs,
            rng,
            device,
            weight_dtype,
            on_event,
            _check_stop,
            lambda: save_on_stop,
            identity,
        )
    finally:
        _restore_perf_flags(perf_snap)


def _train_dit(
    cfg, spec, pairs, rng, device, weight_dtype, on_event, _check_stop, _save_on_stop, identity
):
    """The body of ``run_dit_lora_training``, split out so the backend perf flags are
    snapshot/restored around it in exactly one place."""
    import torch
    import torch.nn.functional as F
    from diffusers import FlowMatchEulerDiscreteScheduler
    from diffusers.optimization import get_scheduler
    from diffusers.training_utils import cast_training_params
    from peft import LoraConfig
    from peft.utils import get_peft_model_state_dict

    # diffusers honours no env var for its bars and is only imported from here.
    try:
        from loggers.config import quiet_third_party_progress_bars
        quiet_third_party_progress_bars()
    except Exception:  # noqa: BLE001 - never let log tidying stop a training run
        pass

    use_lora_targets = _select_lora_targets(cfg.lora_target_modules, spec.lora_targets)
    out_dir = Path(cfg.output_dir).expanduser()
    # Fetch from the byte-identical public mirror chosen in normalization; cfg.base_model stays canonical for the sidecar, completion event and resume identity.
    fetch_cfg = replace(cfg, base_model = cfg.fetch_base_model or cfg.base_model)

    # When every latent variant AND caption embedding is cached, the encoders never load.
    image_paths = [p for p, _ in pairs]
    captions = [c for _, c in pairs]
    uniq = sorted(set(captions))
    # CFG dropout needs the empty-prompt embedding precomputed before the encoders are freed.
    cfg_dropout = float(getattr(cfg, "cfg_dropout", 0.0) or 0.0)
    to_encode = uniq + ([""] if cfg_dropout > 0 and "" not in uniq else [])
    # Own dict: caption_embeds sizes the qwen compile pad bucket, which a long preview prompt must not grow.
    sample_plan = plan_samples(cfg, spec.family, captions) if spec.decode_latents else None
    sample_texts = sample_plan.encode_texts if sample_plan is not None else []
    sample_embeds: dict = {}
    # Kept past a failed round (which drops sample_plan) so a discard still removes what was written.
    sample_owner = sample_plan
    use_cache = cfg.cache_latents and os.environ.get(
        "UNSLOTH_DIFFUSION_NO_LATENT_CACHE", ""
    ) not in ("1", "true")
    pcache = None
    if getattr(cfg, "cond_cache_dir", None):
        try:
            # Namespace on the CHECKPOINT: keys carry only content, so caches would cross models.
            from .diffusion_train_extras import source_revision  # noqa: PLC0415
            namespace = f"{spec.family}_{cfg.base_model}_{source_revision(fetch_cfg.base_model)}"
            pcache = PersistentConditioningCache(cfg.cond_cache_dir, namespace, cfg.resolution)
        except Exception as exc:  # noqa: BLE001 -- the cache is an optimisation, never fatal
            _emit(on_event, "warning", message = f"conditioning cache disabled: {exc}")
    # Seeded crop/flip plan: stable cache keys across runs, so the warm check precedes loads.
    canvases, crop_room = plan_image_canvases(cfg, image_paths)
    plan = _plan_cache_variants(
        len(image_paths),
        cfg.cache_variants,
        cfg.center_crop,
        cfg.random_flip,
        cfg.seed,
        crop_room = crop_room,
    )

    pipe = None
    vae = None
    caption_embeds = None
    # Over the host-memory budget: keep the VAE and encode in-loop.
    latent_cache = None
    if pcache is not None and use_cache:
        caption_embeds, latent_cache = _load_warm_conditioning(
            pcache, image_paths, plan, to_encode, device, canvases
        )
        if caption_embeds is not None:
            _emit(
                on_event,
                "preparing",
                stage = "cache_latents",
                done = len(image_paths),
                total = len(image_paths),
            )
    if caption_embeds is not None and sample_texts:
        # Warm conditioning skips the VAE and encoders, but previews need the VAE and their own embeddings. RNG-isolated:
        # a run without previews never makes this load, so it must not move the training streams.
        _emit(
            on_event,
            "model_load_started",
            stage = "conditioning",
            fetch_repo = training_fetch_repo(cfg),
        )
        with isolated_sampling(device, False):
            with report_model_load_progress(on_event):
                pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
            encoded = _encode_prompts_cached(spec, pipe, sample_texts, device, pcache)
        sample_embeds = dict(zip(sample_texts, encoded))
        _free_text_encoders(pipe)
    if caption_embeds is None:
        latent_cache = None
        with report_model_load_progress(on_event):
            pipe, vae = spec.load_conditioners(fetch_cfg, device, weight_dtype)
        extra = [t for t in sample_texts if t not in to_encode]
        encoded = _encode_prompts_cached(spec, pipe, to_encode + extra, device, pcache)
        caption_embeds = {cap: emb for cap, emb in zip(to_encode, encoded)}
        all_embeds = dict(zip(to_encode + extra, encoded))
        sample_embeds = {t: all_embeds[t] for t in sample_texts}
        _free_text_encoders(pipe)
        gc.collect()
        if device == "cuda":
            torch.cuda.empty_cache()
        elif device == "xpu":
            torch.xpu.empty_cache()
        if vae is not None:
            vae.to(device)

        # The cache keeps the posterior affine parameters, so per-step sampling noise is preserved.
        if use_cache:
            latent_cache = _build_latent_cache(
                spec,
                vae,
                image_paths,
                cfg,
                device,
                weight_dtype,
                on_event,
                _check_stop,
                pcache = pcache,
                plan = plan,
                canvases = canvases,
            )
            if latent_cache is LATENT_CACHE_OVER_BUDGET:
                # Over the host-memory budget: keep the VAE and encode in-loop.
                latent_cache = None
            elif latent_cache is None:
                _emit(
                    on_event,
                    "complete",
                    output_dir = str(out_dir),
                    lora_path = None,
                    stopped = True,
                    steps_run = 0,
                    # A discard is a discard however early the stop lands.
                    discarded = not _save_on_stop(),
                )
                return str(out_dir)
    # VAE parked on the host between rounds when the latent cache would have freed it: training VRAM unchanged.
    sample_vae = vae if sample_plan is not None else None
    if sample_vae is not None and latent_cache is not None:
        sample_vae.to("cpu")
    if latent_cache is not None and vae is not None:
        try:
            pipe.vae = None
        except Exception:  # noqa: BLE001 -- a pipeline without a settable vae keeps it
            pass
        del vae
        vae = None
        gc.collect()
        if device == "cuda":
            torch.cuda.empty_cache()
        elif device == "xpu":
            torch.xpu.empty_cache()
    # Variant picks use their own stream so the training loop index/noise draws stay seed-deterministic.
    variant_rng = random.Random(cfg.seed + 1)
    caption_embeds = _pin_caption_embeds(caption_embeds, device)

    # Transformer loads last, at resolved precision: nf4 by default, 'auto' picks from free VRAM.
    base_precision = _resolve_base_precision(cfg, spec, device)
    _emit(
        on_event,
        "model_load_started",
        stage = "quantizing"
        if base_precision == "nf4" and not repo_is_prequantized(fetch_cfg.base_model)
        else "transformer",
        fetch_repo = training_fetch_repo(cfg),
    )
    with report_model_load_progress(on_event):
        transformer = spec.load_transformer(fetch_cfg, device, weight_dtype, base_precision)
    base_is_bnb = base_precision == "nf4"

    transformer.requires_grad_(False)
    transformer.add_adapter(
        LoraConfig(
            r = cfg.lora_rank,
            lora_alpha = cfg.lora_alpha,
            lora_dropout = cfg.lora_dropout,
            init_lora_weights = "gaussian",
            target_modules = list(use_lora_targets),
        )
    )
    if cfg.gradient_checkpointing:
        enable_diffusion_gradient_checkpointing(transformer)
    cast_training_params(transformer, dtype = torch.float32)
    lora_params = [p for p in transformer.parameters() if p.requires_grad]

    # Convert AFTER the LoRA attaches so the adapter modules stay high precision.
    if base_precision == "int8":
        _int8_quantize_base(transformer, cfg.resolved_family, device = device)
        # Re-collect: the move to device must not leave the optimizer holding host tensors.
        lora_params = [p for p in transformer.parameters() if p.requires_grad]
    if base_precision == "fp8" and not _apply_fp8_training(
        transformer, on_event, cfg.resolved_family
    ):
        base_precision = "bf16"
    if base_precision == "mxfp8" and not _apply_mxfp8_training(
        transformer, on_event, cfg.resolved_family
    ):
        base_precision = "bf16"

    # LoRA EMA shadows ONLY the trainable adapter params; initialised after the precision conversions so it tracks the final fp32 objects.
    ema = LoRAEMA(transformer, decay = cfg.ema_decay) if getattr(cfg, "ema_decay", 0.0) else None

    # A resume runs only the remaining steps, so the nf4 compile break-even is judged on those.
    compiled = _maybe_compile_transformer(
        transformer,
        cfg,
        base_is_bnb,
        device,
        on_event,
        base_precision,
        resume_steps_left(cfg, identity),
    )
    # Compiled Qwen graphs need one fixed text length across steps.
    qwen_pad_to = None
    if compiled and spec.family == "qwen-image":
        qwen_pad_to = max(e[0].shape[1] for e in caption_embeds.values())

    optimizer = _make_optimizer(
        lora_params, cfg.learning_rate, recorded_optimizer_class(cfg, identity)
    )
    if spec.training_scheduler is not None:
        scheduler = spec.training_scheduler(fetch_cfg)
    else:
        scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
            fetch_cfg.base_model, subfolder = "scheduler", token = cfg.hf_token
        )
    # getattr defaults keep an un-normalized config on the historical behaviour.
    flow_shift = getattr(cfg, "flow_shift", None)
    if flow_shift is None and spec.family in AUTO_FLOW_SHIFT_FAMILIES:
        flow_shift = "auto"
    sigma_table = _training_sigma_table(scheduler, flow_shift)
    shift_active = sigma_table is not scheduler.sigmas
    num_train_ts = scheduler.config.num_train_timesteps
    bell_weights = (
        _bell_loss_weights(num_train_ts).to(device)
        if str(getattr(cfg, "weighting_scheme", "none") or "none") == "bell"
        else None
    )
    # Warmup/decay count optimizer steps; scaling by accumulation would stretch warmup.
    lr_sched = get_scheduler(
        cfg.lr_scheduler,
        optimizer = optimizer,
        num_warmup_steps = cfg.lr_warmup_steps,
        num_training_steps = cfg.train_steps,
    )

    _emit(on_event, "model_load_completed", compiled = compiled, base_precision = base_precision)
    # Read the commit after the base is on disk: an earlier identity says unresolved, so resume cannot detect a moved repo. Record the repo actually fetched.
    identity = with_resolved_revision(identity, fetch_cfg.base_model)
    # See the SDXL trainer: the cache path the loop actually took, not the one requested.
    identity = with_cache_mode(identity, latent_cache is not None)
    # base_precision is resolved now (auto, or fp8/mxfp8 fallback to bf16); without this the bundle records a base it never trained on.
    identity = with_resolved_base_precision(identity, base_precision)

    transformer.train()
    n_images = len(image_paths)
    batch_size = cfg.train_batch_size
    # Permutation-cycle index sampler (shared with the SDXL trainer): visits every image once per cycle, so a short
    # run covers the whole dataset. Bucketed runs draw each batch from one bucket so its latents stack into one shape.
    index_sampler = (
        PermutationBatchSampler(n_images, rng)
        if canvases is None
        else BucketBatchSampler(bucket_groups(canvases), rng)
    )

    # Restores all run state before the loop so training resumes at resumed + 1; None for fresh runs.
    rng_streams = {"loop": rng, "variant": variant_rng}
    restored = restore_resume_state(
        cfg,
        model = transformer,
        optimizer = optimizer,
        lr_scheduler = lr_sched,
        identity = identity,
        on_event = on_event,
        ema = ema,
        sampler = index_sampler,
        rng_streams = rng_streams,
    )
    resumed = restored.step if restored is not None else 0
    # Resuming from A into reused dir B: the bundles of B are foreign, and the first save must clear them or a higher-numbered one outranks this run.
    resumed_here = bool(resumed) and resumed_into_this_dir(cfg, out_dir)
    # The bundles already here when this run started, so a discard removes only what this run wrote.
    preexisting_checkpoints = snapshot_checkpoints(out_dir)
    if restored is not None:
        was = restored.progress.get("resolved_base_precision")
        if was and was != base_precision:
            # Not fatal, since only the numerics of the frozen base change; base_precision auto picks from free VRAM, so this can happen silently.
            _emit(
                on_event,
                "warning",
                message = (
                    f"Resuming a checkpoint trained with a {was} base in {base_precision}; "
                    f"set base_precision explicitly to keep them identical."
                ),
            )

    stopped = False
    # Carried over so avg_loss stays an average over the whole run, not just since the resume.
    running_loss = restored.running_loss if restored is not None else 0.0
    peak_gb = 0.0
    t_start = time.time()
    t_steady = None
    t_rate = t_start
    # Starts at the resumed step so a no-op resume still reports the real step.
    done = resumed

    def _save_checkpoint(step: int) -> None:
        # write_resume_checkpoint emits its own saved/failed events, so a crash after a save stays resumable.
        _written, _error = write_resume_checkpoint(
            cfg,
            step = step,
            model = transformer,
            optimizer = optimizer,
            lr_scheduler = lr_sched,
            identity = identity,
            on_event = on_event,
            ema = ema,
            sampler = index_sampler,
            rng_streams = rng_streams,
            # The identity records the REQUESTED mode, which cannot tell nf4 from bf16 under "auto".
            progress = {"running_loss": running_loss, "resolved_base_precision": base_precision},
            # NOT discard_existing: a cancelled retrain would destroy the run being retrained.
            discard_existing = False,
            # A branched resume must not prune the higher-numbered checkpoints it did not write.
            preexisting = preexisting_checkpoints,
        )

    # Reconciles the fp32 LoRA params with the bnb 4-bit base matmuls.
    autocast = (
        torch.autocast(device_type = device, dtype = torch.bfloat16)
        if device in ("cuda", "xpu")
        else nullcontext()
    )

    def _render_dit(index, prompt):
        shape, seq_len = spec.latent_shape(
            sample_vae, sample_plan.resolution, sample_plan.resolution
        )
        extra = spec.sample_forward_kwargs or {}

        def velocity(x, t, sig, embeds):
            with autocast:
                return spec.forward(
                    transformer, x, t, sig, embeds, cfg, device, weight_dtype, **extra
                )

        cond = spec.collate([sample_embeds[prompt]], device, weight_dtype)
        uncond = (
            spec.collate([sample_embeds[""]], device, weight_dtype)
            if sample_plan.uses_cfg
            else None
        )
        latents = euler_flow_sample(
            plan = sample_plan,
            index = index,
            latent_shape = shape,
            image_seq_len = seq_len,
            family = spec.family,
            scheduler_config = scheduler.config,
            velocity = velocity,
            cond = cond,
            uncond = uncond,
            device = device,
            weight_dtype = weight_dtype,
            stop_requested = _stop_during_sample,
        )
        return spec.decode_latents(sample_vae, latents.to(next(sample_vae.parameters()).device))

    # The stop poll drains the request, so one seen mid-round is latched here for the loop to act on.
    stop_latched = False

    def _stop_during_sample() -> bool:
        nonlocal stop_latched
        if not stop_latched and _check_stop():
            stop_latched = True
        return stop_latched

    def _sample(step: int) -> None:
        nonlocal sample_plan, t_rate, t_steady
        if sample_plan is None:
            return
        t_round = time.time()
        parked = next(sample_vae.parameters()).device.type == "cpu"
        try:
            with isolated_sampling(device, compiled):
                transformer.eval()
                if parked:
                    sample_vae.to(device)
                run_sample_round(
                    sample_plan, step, _render_dit, on_event, _emit, _stop_during_sample
                )
        except Exception as exc:  # noqa: BLE001 -- previews are diagnostics, never a reason to lose the run
            _emit(on_event, "warning", message = f"Sample images disabled after an error: {exc}")
            sample_plan = None
        finally:
            transformer.train()
            if parked:
                sample_vae.to("cpu")
                if device == "cuda":
                    torch.cuda.empty_cache()
                elif device == "xpu":
                    torch.xpu.empty_cache()
            spent = time.time() - t_round
            t_rate += spent
            if t_steady is not None:
                t_steady += spent

    if sample_plan is not None and resumed < cfg.train_steps:
        _sample(resumed)

    train_peak = TrainingPeakMemory(device)
    compile_watch = _dynamo_compile_counters() if compiled else None
    for opt_step in range(resumed, cfg.train_steps):
        if stop_latched:
            stopped = True
            break
        optimizer.zero_grad(set_to_none = True)
        step_acc = None
        for _ in range(cfg.gradient_accumulation_steps):
            idxs = index_sampler.next_batch(batch_size)
            if canvases is not None:
                idxs = idxs[1]
            if latent_cache is not None:
                latents = _sample_cached_latents(
                    latent_cache, idxs, variant_rng, device, weight_dtype
                )
            else:
                px = torch.stack(
                    [
                        _load_pixel_tensor(
                            image_paths[i],
                            cfg.resolution if canvases is None else canvases[i],
                            cfg.center_crop,
                            cfg.random_flip,
                            rng,
                        )
                        for i in idxs
                    ]
                ).to(device)
                latents = spec.encode_latents(vae, px).to(weight_dtype)

            noise = torch.randn_like(latents)
            timesteps, t_indices = _sample_timesteps(scheduler, latents.shape[0], device)
            sigmas = _gather_sigmas(sigma_table, t_indices, device, weight_dtype, latents.ndim)
            if shift_active:
                # timestep = sigma * num_train_timesteps, gathered in fp32 so bf16 rounding never skews it.
                timesteps = (
                    sigma_table[t_indices]
                    .to(device = device, non_blocking = True)
                    .to(dtype = torch.float32)
                    .flatten()
                    * num_train_ts
                )
            noisy = (1.0 - sigmas) * latents + sigmas * noise

            embeds = spec.collate(
                [
                    caption_embeds[""]
                    if cfg_dropout > 0 and rng.random() < cfg_dropout
                    else caption_embeds[captions[i]]
                    for i in idxs
                ],
                device,
                weight_dtype,
                pad_to = qwen_pad_to,
            )
            with autocast:
                model_pred = spec.forward(
                    transformer, noisy, timesteps, sigmas, embeds, cfg, device, weight_dtype
                )
                target = noise - latents
                if bell_weights is None:
                    loss = F.mse_loss(model_pred.float(), target.float(), reduction = "mean")
                else:
                    per = F.mse_loss(model_pred.float(), target.float(), reduction = "none")
                    w_idx = (
                        (sigmas.flatten().float() * num_train_ts).long().clamp(0, num_train_ts - 1)
                    )
                    w = bell_weights[w_idx].view(-1, *([1] * (per.ndim - 1)))
                    loss = (per * w).mean()
            (loss / cfg.gradient_accumulation_steps).backward()
            step_acc = accumulate_loss(step_acc, loss, cfg.gradient_accumulation_steps)

        grad_norm = None
        if cfg.max_grad_norm and cfg.max_grad_norm > 0:
            # Pre-clip norm: the UI's health signal.
            grad_norm = float(torch.nn.utils.clip_grad_norm_(lora_params, cfg.max_grad_norm))
        step_loss = loss_value(step_acc)
        optimizer.step()
        lr_sched.step()
        if ema is not None:
            ema.update(transformer)

        running_loss += step_loss
        done = opt_step + 1
        now = time.time()
        # Rates count only the steps THIS process ran, so a resumed run does not divide by steps it never executed.
        ran_here = done - resumed
        if ran_here == 1:
            # Step 1 pays the one-time costs (cudnn autotune, compile warmup), so the reported rate starts after it.
            t_steady = now
        if compile_watch is not None:
            reason = _compile_engagement_warning(compile_watch, _dynamo_compile_counters())
            if reason:
                _emit(on_event, "warning", message = reason)
                compile_watch = None
        if done % cfg.log_every == 0 or done == cfg.train_steps:
            peak_gb = train_peak.gb()
            per_step = batch_size * cfg.gradient_accumulation_steps
            if t_steady is not None and ran_here > 1:
                sps = round((ran_here - 1) * per_step / max(now - t_steady, 1e-6), 3)
            else:
                sps = round(ran_here * per_step / max(now - t_rate, 1e-6), 3)
            _emit(
                on_event,
                "progress",
                step = done,
                total_steps = cfg.train_steps,
                loss = round(step_loss, 5),
                avg_loss = round(running_loss / done, 5),
                learning_rate = lr_sched.get_last_lr()[0],
                grad_norm = round(grad_norm, 5) if grad_norm is not None else None,
                samples_per_second = sps,
                peak_memory_gb = peak_gb or None,
            )
        stop_now = _check_stop()
        if not stop_now and sample_plan is not None and sample_plan.due(done, cfg.train_steps):
            train_peak.pause()
            _sample(done)
            train_peak.resume()
            stop_now = stop_latched
        # Skipped on the final step and when stopping, since the stop path writes one at the exact step.
        if (
            not stop_now
            and cfg.save_steps
            and done % cfg.save_steps == 0
            and done < cfg.train_steps
        ):
            _save_checkpoint(done)
        if stop_now:
            stopped = True
            break

    lora_path: Optional[str] = None
    catalog_path: Optional[str] = None
    ema_path: Optional[str] = None
    if not (stopped and not _save_on_stop()):
        out_dir.mkdir(parents = True, exist_ok = True)
        # Write the bundle BEFORE the adapter export: if that export fails, the run still comes back.
        if stopped and done > 0:
            _save_checkpoint(done)
        layers = get_peft_model_state_dict(transformer)
        save_adapter = _adapter_saver(spec, transformer)
        save_adapter(pipe, str(out_dir), layers)
        lora_path = str(out_dir / DEFAULT_LORA_FILENAME)
        # ``done`` (the step reached), not cfg.train_steps: a stop at 11/500 must not advertise 500.
        catalog_path = _publish_to_lora_catalog(lora_path, cfg, done)
        if ema is not None and ema.updates > 0:
            try:
                # Report the adapter FILE, like lora_path, so a caller can load it directly.
                ema_dir = save_ema_adapter(ema, transformer, save_adapter, str(out_dir))
                ema_path = str(Path(ema_dir) / DEFAULT_LORA_FILENAME)
            except Exception as exc:  # noqa: BLE001 -- the primary adapter is already saved
                _emit(on_event, "warning", message = f"EMA adapter save failed: {exc}")
        if not stopped:
            # A completed run must not leave its own checkpoint for a later resume to roll back to.
            retire_own_checkpoints(out_dir, preexisting_checkpoints, resumed_here = resumed_here)
        elif not resumed_here:
            # A partial save on a fresh retrain is a LOWER step than leftovers, which would outrank it.
            discard_preexisting_checkpoints(out_dir, preexisting_checkpoints)
    else:
        # Otherwise a discard leaves unresumable optimizer bundles with no delete path in the UI.
        clear_own_checkpoints(out_dir, preexisting_checkpoints)
        discard_samples(sample_owner)
        try:
            out_dir.rmdir()
        except OSError:
            # Not ours to remove if anything else is in it (a previous run's adapter).
            pass
    _emit(
        on_event,
        "complete",
        output_dir = str(out_dir),
        lora_path = lora_path,
        ema_path = ema_path,
        catalog_path = catalog_path,
        family = cfg.resolved_family,
        base_model = cfg.base_model,
        stopped = stopped,
        steps_run = done if cfg.train_steps else 0,
        wall_seconds = round(time.time() - t_start, 1),
        resumed_from_step = resumed or None,
        # Stop without saving discards the run, so its periodic checkpoints must not keep offering a resume.
        discarded = bool(stopped and not _save_on_stop()),
    )
    return str(out_dir)


def _make_optimizer(
    params,
    lr,
    resume_optimizer_class = None,
):
    return make_lora_optimizer(params, lr, resume_optimizer_class)


def _free_text_encoders(pipe) -> None:
    """Frees conditioning modules once embeddings are precomputed, so they do not sit in VRAM."""
    for attr in (
        "text_encoder",
        "text_encoder_2",
        "text_encoder_3",
        "tokenizer",
        "tokenizer_2",
        "tokenizer_3",
        "connectors",
        "audio_vae",
        "vocoder",
    ):
        if getattr(pipe, attr, None) is not None:
            try:
                setattr(pipe, attr, None)
            except Exception:  # noqa: BLE001
                pass