Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

94.5 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0

"""Opt-in speed modes off/eager/default/max (off by default); TF32 and cudnn.benchmark are process-
wide, restore at unload; torch imported lazily"""

from __future__ import annotations

import os
import re
import sys
import threading
from contextlib import contextmanager, nullcontext
from functools import lru_cache
from typing import Any, Callable, Iterator, Optional

from . import diffusion_compile_config as compile_config
from .diffusion_bg_compile import eager_forced as _bg_eager_forced
from .diffusion_bg_compile import yield_point as _bg_yield_point
from . import diffusion_gguf_compile as gguf_compile
from .diffusion_autoopt import targets as _targets

SPEED_OFF = "off"
SPEED_EAGER = "eager"
SPEED_DEFAULT = "default"
SPEED_MAX = "max"
SPEED_MODES = (SPEED_OFF, SPEED_EAGER, SPEED_DEFAULT, SPEED_MAX)


# Retaining its full size means it never rotates: the write index resets past the buffer end and updates stop landing.
_INDUCTOR_FLAGS = (
    ("emulate_precision_casts", "inductor_emulate_precision_casts"),
    ("coordinate_descent_tuning", "inductor_coordinate_descent_tuning"),
    ("coordinate_descent_check_all_directions", "inductor_coordinate_descent_check_all_directions"),
    ("force_fuse_int_mm_with_mul", "inductor_force_fuse_int_mm_with_mul"),
    ("fx_graph_cache", "inductor_fx_graph_cache"),
    ("dynamic_scale_rblock", "inductor_dynamic_scale_rblock"),
    ("memory_planning", "inductor_memory_planning"),
)
_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)
_DYNAMO_FLAGS = (("suppress_errors", "dynamo_suppress_errors"),)
_DYNAMO_MODULE = "torch._dynamo.config"
REDUCTION_FILTER_OPTION = "test_configs.force_filter_reduction_configs"
_INDUCTOR_MODULE = "torch._inductor.config"
_INDUCTOR_TRITON_MODULE = "torch._inductor.config.triton"


# allow_fp16_accumulation has one owner: writes in an open scope go to the recorded value.
_FP16_ACCUM_LOCK = threading.RLock()
_fp16_accum_scopes: list = []
_fp16_accum_base: Optional[bool] = None


def _read_fp16_accum(matmul: Any) -> bool:
    with _FP16_ACCUM_LOCK:
        if _fp16_accum_scopes:
            return bool(_fp16_accum_base)
        return bool(matmul.allow_fp16_accumulation)


def _write_fp16_accum(matmul: Any, value: Any) -> None:
    global _fp16_accum_base
    with _FP16_ACCUM_LOCK:
        if _fp16_accum_scopes:
            _fp16_accum_base = bool(value)
        else:
            matmul.allow_fp16_accumulation = bool(value)


@contextmanager
def fp16_accumulation_scope(value: bool) -> Iterator[None]:
    """Hold ``allow_fp16_accumulation`` at ``value`` for the body, then restore the latest process value."""
    global _fp16_accum_base
    import torch

    matmul = torch.backends.cuda.matmul
    token = object()
    with _FP16_ACCUM_LOCK:
        if not _fp16_accum_scopes:
            _fp16_accum_base = bool(matmul.allow_fp16_accumulation)
        _fp16_accum_scopes.append((token, bool(value)))
        matmul.allow_fp16_accumulation = bool(value)
    try:
        yield
    finally:
        with _FP16_ACCUM_LOCK:
            _fp16_accum_scopes[:] = [e for e in _fp16_accum_scopes if e[0] is not token]
            matmul.allow_fp16_accumulation = (
                _fp16_accum_scopes[-1][1] if _fp16_accum_scopes else bool(_fp16_accum_base)
            )


_CUDNN_BENCH_LOCK = threading.RLock()
_cudnn_bench_scopes: list = []
_cudnn_bench_base: Optional[bool] = None


def _read_cudnn_benchmark(cudnn: Any) -> bool:
    with _CUDNN_BENCH_LOCK:
        if _cudnn_bench_scopes:
            return bool(_cudnn_bench_base)
        return bool(cudnn.benchmark)


def _write_cudnn_benchmark(cudnn: Any, value: Any) -> None:
    global _cudnn_bench_base
    with _CUDNN_BENCH_LOCK:
        if _cudnn_bench_scopes:
            _cudnn_bench_base = bool(value)
        else:
            cudnn.benchmark = bool(value)


@contextmanager
def cudnn_benchmark_scope(value: bool) -> Iterator[None]:
    """Hold ``cudnn.benchmark`` at ``value`` for the body, then restore the latest process value."""
    global _cudnn_bench_base
    import torch

    cudnn = torch.backends.cudnn
    token = object()
    with _CUDNN_BENCH_LOCK:
        if not _cudnn_bench_scopes:
            _cudnn_bench_base = bool(cudnn.benchmark)
        _cudnn_bench_scopes.append((token, bool(value)))
        cudnn.benchmark = bool(value)
    try:
        yield
    finally:
        with _CUDNN_BENCH_LOCK:
            _cudnn_bench_scopes[:] = [e for e in _cudnn_bench_scopes if e[0] is not token]
            cudnn.benchmark = (
                _cudnn_bench_scopes[-1][1] if _cudnn_bench_scopes else bool(_cudnn_bench_base)
            )


def family_render_cudnn_scope(family: Any, device: Any) -> Any:
    """Hold ``cudnn.benchmark`` off for a render of a family that opts out of it. The flag is process-wide, so a
    co-resident or earlier load can have left it on, leaving the VAE convs on a per-process pick."""
    if getattr(family, "cudnn_benchmark", True) or not str(device or "").startswith("cuda"):
        return nullcontext()
    try:
        import torch
        if getattr(getattr(torch, "backends", None), "cudnn", None) is None:
            return nullcontext()
    except Exception:  # noqa: BLE001 - no torch: nothing to hold
        return nullcontext()
    return cudnn_benchmark_scope(False)


def snapshot_backend_flags() -> Optional[dict]:
    """Capture the process-wide torch backend flags this layer may mutate, for restore on unload. None
    without torch. Each flag is read defensively: a build missing one still captures the rest."""
    try:
        import torch
    except Exception:  # noqa: BLE001 - no torch -> nothing to snapshot/restore
        return None
    state: dict[str, Any] = {}
    matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
    if matmul is not None and hasattr(matmul, "allow_tf32"):
        state["matmul_tf32"] = bool(matmul.allow_tf32)
    if matmul is not None and hasattr(matmul, "allow_fp16_accumulation"):
        state["matmul_fp16_accum"] = _read_fp16_accum(matmul)
    cudnn = getattr(torch.backends, "cudnn", None)
    if cudnn is not None:
        if hasattr(cudnn, "allow_tf32"):
            state["cudnn_tf32"] = bool(cudnn.allow_tf32)
        if hasattr(cudnn, "benchmark"):
            state["cudnn_benchmark"] = _read_cudnn_benchmark(cudnn)
    inductor_cfg = _inductor_config()
    if inductor_cfg is not None:
        # Process-wide value: on torch 2.12+ a knob set on another thread is invisible here.
        for attr, key in _INDUCTOR_FLAGS:
            if hasattr(inductor_cfg, attr):
                state[key] = bool(compile_config.get_knob(_INDUCTOR_MODULE, attr))
        triton_cfg = getattr(inductor_cfg, "triton", None)
        if triton_cfg is not None:
            for attr, key in _INDUCTOR_TRITON_FLAGS:
                if hasattr(triton_cfg, attr):
                    state[key] = bool(compile_config.get_knob(_INDUCTOR_TRITON_MODULE, attr))
    dynamo_cfg = _dynamo_config()
    if dynamo_cfg is not None:
        for attr, key in _DYNAMO_FLAGS:
            if hasattr(dynamo_cfg, attr):
                state[key] = bool(compile_config.get_knob(_DYNAMO_MODULE, attr))
    getter = getattr(torch, "get_float32_matmul_precision", None)
    if callable(getter):
        try:
            state["matmul_precision"] = str(getter())
        except Exception:  # noqa: BLE001 - unreadable on this build: restore the rest
            pass
    return state


def release_rocm_tunableop(pipe: Any) -> None:
    """Unload: drop this pipe's TunableOp claim (process-global, so not in snapshot_backend_flags)."""
    mod = sys.modules.get(f"{__package__}.diffusion_rocm_tunableop")
    if mod is None or pipe is None:
        return
    try:
        mod.release(pipe)
    except Exception:  # noqa: BLE001 - teardown is best effort
        pass


def restore_backend_flags(state: Optional[dict]) -> None:
    """Restore the flags captured by ``snapshot_backend_flags``. No-op on None. Each flag is
    restored independently so one failure can't leak the others."""
    if not state:
        return
    try:
        import torch
    except Exception:  # noqa: BLE001 - no torch -> nothing to restore
        return

    def _set(obj: Any, attr: str, key: str) -> None:
        if obj is not None and key in state and hasattr(obj, attr):
            try:
                setattr(obj, attr, state[key])
            except Exception:  # noqa: BLE001 - best-effort per-flag restore
                pass

    # FIRST: on some builds set_float32_matmul_precision also writes matmul.allow_tf32.
    setter = getattr(torch, "set_float32_matmul_precision", None)
    if state.get("matmul_precision") and callable(setter):
        try:
            setter(state["matmul_precision"])
        except Exception:  # noqa: BLE001 - best-effort per-flag restore
            pass
    matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
    _set(matmul, "allow_tf32", "matmul_tf32")
    if (
        matmul is not None
        and "matmul_fp16_accum" in state
        and hasattr(matmul, "allow_fp16_accumulation")
    ):
        try:
            _write_fp16_accum(matmul, state["matmul_fp16_accum"])
        except Exception:  # noqa: BLE001 - best-effort per-flag restore
            pass
    cudnn = getattr(torch.backends, "cudnn", None)
    _set(cudnn, "allow_tf32", "cudnn_tf32")
    if cudnn is not None and "cudnn_benchmark" in state and hasattr(cudnn, "benchmark"):
        try:
            _write_cudnn_benchmark(cudnn, state["cudnn_benchmark"])
        except Exception:  # noqa: BLE001 - best-effort per-flag restore
            pass

    def _set_knob(obj: Any, module_name: str, attr: str, key: str) -> None:
        if key in state and compile_config.is_recorded(module_name, attr):
            compile_config.set_knob(module_name, attr, state[key])
        else:
            _set(obj, attr, key)

    inductor_cfg = _inductor_config()
    for attr, key in _INDUCTOR_FLAGS:
        _set_knob(inductor_cfg, _INDUCTOR_MODULE, attr, key)
    triton_cfg = getattr(inductor_cfg, "triton", None) if inductor_cfg is not None else None
    for attr, key in _INDUCTOR_TRITON_FLAGS:
        _set_knob(triton_cfg, _INDUCTOR_TRITON_MODULE, attr, key)
    dynamo_cfg = _dynamo_config()
    for attr, key in _DYNAMO_FLAGS:
        _set_knob(dynamo_cfg, _DYNAMO_MODULE, attr, key)


def _gguf_dequant_compilable(
    is_gguf: bool,
    on_cuda: bool,
    family_allows_compile: bool,
    target: Any = None,
) -> bool:
    return (
        is_gguf
        and on_cuda
        and family_allows_compile
        and not rocm_compile_switched_off(target)
        and torch_compile_runtime_available()
    )


def _pin_compile_defaults() -> None:
    """torch's defaults for every diffusion compile, against Unsloth's training config, which sets both process-wide
    on torch < 2.12 (#9897). ``memory_planning``: its pooled buffers come back from ``inductor::_alloc_from_pool`` as
    views AOTAutograd's custom-op aliasing check rejects at the first call (every gfx1151 render under ``CI``, a
    deprecation warning elsewhere). ``suppress_errors``: a failed compile would silently run eager; Studio's own
    guards fall back with a warning instead, as on torch 2.12+ where the render thread never saw the training config."""
    inductor_cfg = _inductor_config()
    if inductor_cfg is not None and hasattr(inductor_cfg, "memory_planning"):
        compile_config.set_knob(_INDUCTOR_MODULE, "memory_planning", False)
    dynamo_cfg = _dynamo_config()
    if dynamo_cfg is not None and hasattr(dynamo_cfg, "suppress_errors"):
        compile_config.set_knob(_DYNAMO_MODULE, "suppress_errors", False)


def _dynamo_config() -> Any:
    try:
        import torch
        return getattr(getattr(torch, "_dynamo", None), "config", None)
    except Exception:  # noqa: BLE001 - no dynamo -> nothing to snapshot/set
        return None


def _inductor_config() -> Any:
    """``torch._inductor.config`` or None. Read as attributes off the imported torch (not a
    submodule import) so a stubbed/partial torch reports None instead of a stale sys.modules hit."""
    try:
        import torch
        return getattr(getattr(torch, "_inductor", None), "config", None)
    except Exception:  # noqa: BLE001 - no inductor -> nothing to snapshot/set
        return None


def normalize_speed_mode(value: Optional[str]) -> str:
    """Lower/strip a requested speed mode (dashes ok); None / "" -> off."""
    if value is None:
        return SPEED_OFF
    normalized = str(value).strip().lower().replace("-", "_")
    if not normalized:
        return SPEED_OFF
    if normalized not in SPEED_MODES:
        raise ValueError(
            f"Unsupported diffusion speed_mode '{value}'. Use one of: {', '.join(SPEED_MODES)}."
        )
    return normalized


def resolve_speed_mode(
    value: Optional[str],
    *,
    is_gguf: bool,
    dense_default: str = SPEED_OFF,
) -> str:
    """Unset resolves to default for GGUF and dense_default otherwise; an explicit value is honored."""
    if value is None:
        return SPEED_DEFAULT if is_gguf else dense_default
    return normalize_speed_mode(value)


@lru_cache(maxsize = 1)
def torch_compile_runtime_available() -> bool:
    """Triton has no Windows wheel; TORCHDYNAMO_DISABLE is honored on every platform."""
    if os.environ.get("TORCHDYNAMO_DISABLE", "").strip() not in ("", "0"):
        return False
    if sys.platform != "win32":
        return True
    try:
        import triton  # noqa: F401, PLC0415
    except Exception:  # noqa: BLE001 -- absent or broken Triton means eager, never a failed load
        return False
    try:
        from .._msvc_env import crt_headers_reachable  # noqa: PLC0415
        return crt_headers_reachable()
    except Exception:  # noqa: BLE001 -- this runs during load; never fail it over a probe
        return True


def compile_eligible(
    target: Any,
    *,
    is_gguf: bool,
    family: Any,
    probe: bool = True,
) -> bool:
    """Whether the repeated block is regionally compiled: CUDA or ROCm bf16, or fp16 on NVIDIA
    sm_75+, compile-friendly family with inductor; probe=False never runs the ROCm probe,
    unprobed counts eligible, a recorded failure does not."""
    del is_gguf
    if not torch_compile_runtime_available():
        return False
    if not bool(getattr(target, "supports_default_torch_compile", False)):
        return False
    if not bool(getattr(family, "supports_torch_compile", True)):
        return False
    dtype = getattr(target, "dtype", None)
    if _is_bfloat16(dtype):
        return rocm_compile_probe_passes(target, probe = probe)
    # fp16-incompatible families run in fp32 (unmeasured under compile) or in fp16 behind diffusion_fp16_guard, which
    # must stay eager: the int8 fused MLP installed for compile replaces the guard's FFN forward.
    if bool(getattr(family, "fp16_incompatible", False)):
        return False
    return (
        _is_float16(dtype)
        and _fp16_compile_capable(target)
        and rocm_compile_probe_passes(target, probe = probe)
    )


def rocm_compile_probe_passes(target: Any, *, probe: bool = True) -> bool:
    if getattr(target, "backend", None) != "rocm":
        return True
    try:
        from . import diffusion_rocm_compile as rc
        if not probe:
            return rc.rocm_block_compile_known(getattr(target, "ordinal", None)) is not False
        return rc.rocm_block_compile_works(getattr(target, "ordinal", None))
    except Exception:  # noqa: BLE001
        return False


def rocm_probe_load_scoped(fn: Callable[..., Any]) -> Callable[..., Any]:
    """Wrap a load so a ROCm probe OOM is remembered for that load only (no re-probe within it, a retry in the next)."""
    from .diffusion_rocm_compile import load_scoped
    return load_scoped(fn)


def rocm_compile_switched_off(target: Any) -> bool:
    """``UNSLOTH_DIFFUSION_ROCM_COMPILE=0`` on a ROCm target: no compile at all, the GGUF dequant one included."""
    if getattr(target, "backend", None) != "rocm":
        return False
    try:
        from .diffusion_rocm_compile import rocm_compile_allowed
        return not rocm_compile_allowed()
    except Exception:  # noqa: BLE001
        return True


def fp16_compile_explicit_only(target: Any) -> bool:
    """fp16 compiles only on an explicit tier, never the deferred profile (T4 SDXL-Turbo: ~245 s for ~0.1 s/image)."""
    return _is_float16(getattr(target, "dtype", None))


def family_compiles_regionally(family: Any) -> bool:
    """False only for an EMPTY ``_repeated_blocks`` Studio cannot supply; unknown reads True."""
    if getattr(family, "denoiser_attr", "transformer") != "transformer":
        return True
    name = getattr(family, "transformer_class", None)
    if not isinstance(name, str) or not name:
        return True
    # Called before load_pipeline's guard; `import diffusers` imports torch._dynamo, so wait for the torch warm.
    try:
        from loggers import get_logger
        from utils.torch_warmup import close_dynamo_import_window
        close_dynamo_import_window(get_logger(__name__))
    except Exception:  # noqa: BLE001, S110 - optimisation only
        pass
    try:
        import diffusers
        cls = getattr(diffusers, name, None)
    except Exception:  # noqa: BLE001 - an unanswerable probe keeps today's behaviour
        return True
    if cls is None or not hasattr(cls, "_repeated_blocks"):
        return True
    if cls._repeated_blocks:
        return True
    from .diffusion_regional_compile import verified_repeated_blocks

    return bool(verified_repeated_blocks(name))


def _is_bfloat16(dtype: Any) -> bool:
    try:
        import torch
        return dtype is torch.bfloat16
    except Exception:
        return str(dtype).endswith("bfloat16")


def _is_float16(dtype: Any) -> bool:
    try:
        import torch
        return dtype is torch.float16
    except Exception:
        return str(dtype).endswith("float16") and not str(dtype).endswith("bfloat16")


# Only sm_75 (T4) was measured; Volta and older stay eager.
_FP16_COMPILE_MIN_CAPABILITY = (7, 5)


def _fp16_compile_capable(target: Any) -> bool:
    if getattr(target, "backend", "cuda") != "cuda":
        return False
    try:
        import torch

        ordinal = getattr(target, "ordinal", None)
        cap = (
            torch.cuda.get_device_capability()
            if ordinal is None
            else torch.cuda.get_device_capability(ordinal)
        )
        return tuple(cap)[:2] >= _FP16_COMPILE_MIN_CAPABILITY
    except Exception:  # noqa: BLE001 - an unanswerable probe keeps today's eager path
        return False


def apply_speed_optims(
    pipe: Any,
    target: Any,
    *,
    is_gguf: bool,
    family: Any,
    speed_mode: str = SPEED_OFF,
    cache_active: bool = False,
    offload_active: bool = False,
    cuda_graph_default: bool = True,
    cache_engaged: Optional[bool] = None,
    denoiser_offloaded: Optional[bool] = None,
    stream_int8_gemm: bool = False,
    logger: Any = None,
) -> dict[str, bool]:
    """Apply speed optims BEFORE placement/offload; offload hooks need compile without fullgraph."""
    applied = {
        "channels_last": False,
        "vae_fp16_decode": False,
        "vae_single_frame": False,
        "vae_fused": False,
        "cudnn_benchmark": False,
        "tf32": False,
        "fp16_accum": False,
        "fused_qkv": False,
        "compiled": False,
        "compiled_dequant": False,
        "rocm_query_chunks": False,
        "compiled_vae_decode": False,
        "cuda_graph": False,
        "int8_gemm": False,
    }
    mode = normalize_speed_mode(speed_mode)
    if getattr(target, "backend", None) in ("rocm", "mps"):
        # Memory, not speed, so before the `off` return: ROCm has no fused SDPA for the VAE's head dim 384 / 512.
        try:
            from .diffusion_vae_attn_chunked import install as install_vae_attn_chunks
            install_vae_attn_chunks(getattr(pipe, "vae", None), target, logger)
        except Exception as exc:  # noqa: BLE001 - keep stock attention
            _warn(logger, "vae attention chunks", exc)
    # Stability, not speed, so also before the `off` return.
    _enable_miopen_immediate(logger)
    if getattr(target, "backend", None) == "rocm":
        try:
            from . import diffusion_rocm_tunableop as rocm_tunableop
            if mode == SPEED_OFF:
                rocm_tunableop.uninstall(pipe)
            else:
                rocm_tunableop.install_for_pipe(pipe, getattr(target, "device", None), logger)
        except Exception as exc:  # noqa: BLE001 - keep torch's default GEMM pick
            _warn(logger, "rocm tunableop", exc)
    # TF32 (max) and cudnn.benchmark (any non-off CUDA load) are process-global; the caller restores them so a later
    # `off` load never inherits them.
    if mode == SPEED_OFF:
        return applied
    if mode in (SPEED_DEFAULT, SPEED_MAX):
        _pin_compile_defaults()
        if is_gguf:
            _prepare_lowbit_iq_tables(pipe, target)

    if getattr(target, "backend", None) == "rocm":
        try:
            from .diffusion_qwenimage21_rocm import install as install_rocm_query_chunks
            applied["rocm_query_chunks"] = install_rocm_query_chunks(pipe, target, logger)
        except Exception as exc:  # noqa: BLE001 - keep stock attention on unsupported installs
            _warn(logger, "qwen-image-2.1 ROCm query chunks", exc)

    on_cuda = getattr(target, "device", None) == "cuda"
    family_allows_compile = bool(getattr(family, "supports_torch_compile", True))
    filter_reductions = compile_config.family_filters_reductions(family)

    applied["vae_single_frame"] = _vae_single_frame(pipe, logger)
    # Lossless: a channels-last VAE speeds up its convs with no numeric change.
    applied["channels_last"] = _vae_channels_last(
        pipe, logger, fused = on_cuda and _fused_vae_planned(pipe)
    )
    # Near-lossless, not bit-identical, so never on "off" (returned above).
    applied["vae_fp16_decode"] = _video_vae_half_decode(pipe, target, family, logger)
    if on_cuda:
        applied["vae_fused"] = _install_fused_vae(
            pipe, logger, encoder = getattr(family, "vae_fused_encoder", True)
        )

    if (
        on_cuda
        and getattr(family, "cudnn_benchmark", True)
        and not _cudnn_benchmark_pointless(pipe)
    ):
        applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)

    if on_cuda:
        applied["fp16_accum"] = _enable_fp16_accumulation(
            family, logger, dtype = getattr(target, "dtype", None), speed_mode = mode
        )

    # default: GGUF compiles only the dequant chain, dense uses regional compile; max: autotune.
    if mode == SPEED_DEFAULT:
        # Asked directly: this arm never reaches compile_eligible(), unlike the dense arm below.
        if _gguf_dequant_compilable(is_gguf, on_cuda, family_allows_compile, target):
            applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)
        elif compile_eligible(target, is_gguf = is_gguf, family = family) and not fp16_unet_offloaded(
            target, pipe, offload_active = offload_active
        ):
            # A U-Net fuses QKV before its whole-module compile; DiTs fuse on max only.
            if _denoiser_unet(pipe) is not None:
                applied["fused_qkv"] = _fuse_qkv(pipe, logger)
            applied["compiled"] = _compile_repeated_blocks(
                pipe,
                logger,
                max_autotune = False,
                cache_active = cache_active,
                offload_active = offload_active,
                denoiser_offloaded = denoiser_offloaded,
                onload_device = _onload_device(target) if stream_int8_gemm else None,
                filter_reductions = filter_reductions,
            )
    elif mode == SPEED_MAX:
        if compile_eligible(target, is_gguf = is_gguf, family = family) and not fp16_unet_offloaded(
            target, pipe, offload_active = offload_active
        ):
            applied["compiled"] = _compile_repeated_blocks(
                pipe,
                logger,
                max_autotune = True,
                cache_active = cache_active,
                offload_active = offload_active,
                denoiser_offloaded = denoiser_offloaded,
                onload_device = _onload_device(target) if stream_int8_gemm else None,
                filter_reductions = filter_reductions,
            )
        if not applied["compiled"] and _gguf_dequant_compilable(
            is_gguf, on_cuda, family_allows_compile, target
        ):
            # No block compile (ROCm, or its setup failed): keep default's dequant compile rather than run slower.
            applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)

    if applied["compiled"]:
        applied["int8_gemm"] = any(
            bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)
        )

    if applied["compiled"] and _vae_decode_compile_allowed(pipe, mode):
        # A U-Net keeps the decode recipe its whole-module compile was measured with.
        applied["compiled_vae_decode"] = _compile_vae_decode(
            pipe,
            logger,
            max_autotune = mode == SPEED_MAX and _denoiser_unet(pipe) is None,
            eager_when_tiled = _vae_eager_when_tiled(pipe),
        )

    # Fused passes feed cuDNN channels-last activations; contiguous weights would be relaid out per call.
    fused_cl = bool(getattr(getattr(pipe, "vae", None), "_unsloth_vae_fused_cl_weights", False))
    if (
        applied["channels_last"]
        and not fused_cl
        and not _channels_last_decode_wins(
            pipe, target, applied["compiled_vae_decode"], offload_active
        )
    ):
        applied["channels_last"] = not _vae_contiguous(pipe, logger)
    elif applied["compiled_vae_decode"] and not _channels_last_decode_wins(
        pipe, target, False, offload_active
    ):
        # A decode compile that fails at first call falls back eager: relayout then.
        try:
            pipe.vae._unsloth_eager_contiguous = True
        except Exception:  # noqa: BLE001
            pass

    if mode in (SPEED_DEFAULT, SPEED_MAX) and on_cuda and tf32_enabled():
        applied["tf32"] = _enable_tf32(logger)
    if mode == SPEED_MAX:
        applied["fused_qkv"] = _fuse_qkv(pipe, logger)

    # Deliberately NOT gated on applied["compiled"]: a launch-bound step survives the compile.
    if mode in (SPEED_DEFAULT, SPEED_MAX):
        cuda_graph = None
        ok, reason = False, "cuda graph layer unavailable"
        moved = bool(offload_active if denoiser_offloaded is None else denoiser_offloaded)
        try:
            from . import diffusion_cuda_graph as cuda_graph  # noqa: PLC0415 - import cycle

            after_placement = moved and cuda_graph.offload_graphs_enabled()
            ok, reason = cuda_graph.graph_eligible(
                target,
                family = family,
                pipe = pipe,
                offload_active = moved and not after_placement,
                cache_active = cache_active if cache_engaged is None else bool(cache_engaged),
                speed_mode = mode,
                family_default = cuda_graph_default,
                logger = logger,
                offloaded = after_placement,
            )
            if ok and after_placement:
                # The slot and the recorded copies depend on the hooks placement installs: arm_graphs_after_placement.
                ok, reason = False, "armed after placement"
                try:
                    pipe._unsloth_cuda_graph_after_placement = True
                    # A family that opts in only offloaded must not get a resident graph if the denoiser stays put.
                    pipe._unsloth_cuda_graph_offload_only = not bool(
                        getattr(family, "supports_cuda_graph", cuda_graph_default)
                    )
                except Exception:  # noqa: BLE001
                    pass
        except Exception as exc:  # noqa: BLE001 - an unimportable graph layer means eager, never a failed load
            _warn(logger, "cuda graph eligibility", exc)
        # Stashed either way: status reports WHY graphs are off, not just that they are.
        try:
            pipe._unsloth_cuda_graph_reason = reason
        except Exception:  # noqa: BLE001
            pass
        if ok and cuda_graph is not None:
            try:
                applied["cuda_graph"] = bool(cuda_graph.install_cuda_graphs(pipe, logger = logger))
            except Exception as exc:  # noqa: BLE001 - the load proceeds eager
                _warn(logger, "cuda graph capture", exc)

    return applied


def _prepare_lowbit_iq_tables(pipe: Any, target: Any) -> None:
    """Low-bit IQ lookup tables on the execution device before compile, so a compiled block reads a device constant."""
    device = getattr(target, "torch_device", None)
    if not isinstance(device, str) or not device:
        return
    try:
        from .diffusion_gguf_lowbit import prepare_lowbit_iq_tables

        modules = list(_denoiser_dits(pipe))
        unet = _denoiser_unet(pipe)
        if unet is not None:
            modules.append(unet)
        if modules:
            prepare_lowbit_iq_tables(device, *modules)
    except Exception:  # noqa: BLE001 - optimisation only: the lookup still builds traceably
        pass


def _onload_device(target: Any) -> Optional[str]:
    """Onload device of an offloaded denoiser, else None. NVIDIA only: ROCm runs int8 weight-only, never this GEMM."""
    if getattr(target, "device", None) != "cuda" or getattr(target, "backend", "cuda") != "cuda":
        return None
    device = getattr(target, "torch_device", None)
    return str(device) if isinstance(device, str) and device.startswith("cuda") else "cuda"


def arm_graphs_after_placement(
    pipe: Any,
    applied: dict,
    logger: Any = None,
) -> dict:
    """Arm the denoiser graphs for the placement that actually happened: replaces a resident-planned graph a hook now
    wraps, and arms the one apply_speed_optims deferred for an offloaded denoiser (the copies are recorded in it)."""
    pending = bool(getattr(pipe, "_unsloth_cuda_graph_after_placement", False))
    hooked = bool(applied.get("cuda_graph")) and _denoiser_moves(pipe)
    if not (pending or hooked):
        return applied
    try:
        from . import diffusion_cuda_graph as cuda_graph  # noqa: PLC0415 - import cycle

        if not cuda_graph.offload_graphs_enabled():
            cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
            pipe._unsloth_cuda_graphs = ()
            pipe._unsloth_cuda_graph_reason = "offload active"
            applied["cuda_graph"] = False
            return applied
        from .diffusion_cfg_batch import cfg_pair_decline  # noqa: PLC0415 - import cycle

        if cfg_pair_decline(pipe, graphs = False) is None:
            cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
            pipe._unsloth_cuda_graphs = ()
            pipe._unsloth_cuda_graph_after_placement = False
            pipe._unsloth_cuda_graph_reason = "guidance branches pair per streamed block"
            applied["cuda_graph"] = False
            if logger is not None:
                logger.info(
                    "diffusion.cuda_graph: off for this load (both guidance branches share each streamed "
                    "block's upload instead)"
                )
            return applied
        handles, reason = cuda_graph.arm_after_placement(pipe, logger = logger)
        pipe._unsloth_cuda_graph_after_placement = False
        pipe._unsloth_cuda_graph_reason = reason
        applied["cuda_graph"] = bool(handles)
    except Exception as exc:  # noqa: BLE001 - the load proceeds eager
        _warn(logger, "cuda graph arm after placement", exc)
        applied["cuda_graph"] = False
    return applied


def _denoiser_moves(pipe: Any) -> bool:
    for module in _denoiser_dits(pipe) or [m for m in [_denoiser_unet(pipe)] if m is not None]:
        if getattr(module, "_hf_hook", None) is not None:
            return True
        hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}
        if any("offload" in str(key) for key in hooks):
            return True
        try:
            if any(getattr(m, "_hf_hook", None) is not None for m in module.modules()):
                return True
        except Exception:  # noqa: BLE001
            return True
    return False


def engage_pinned_denoisers(
    pipe: Any,
    applied: dict,
    logger: Any = None,
) -> dict:
    """Install the int8 GEMM the plan refused once every denoiser group is pinned (before the first forward). Graphs
    stay off: an oversized request can still stream the groups, and the hooks' copy-stream wait breaks capture."""
    if getattr(pipe, "_unsloth_cuda_graph_reason", None) == "offload active":
        try:
            pipe._unsloth_cuda_graph_reason = "denoiser pinned resident under offload hooks"
        except Exception:  # noqa: BLE001
            pass
    if not applied.get("compiled") or applied.get("int8_gemm"):
        return applied
    try:
        from .diffusion_int8_gemm import install as install_int8_gemm
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "int8 fused-dequant gemm", exc)
        return applied
    for transformer in _denoiser_dits(pipe):
        try:
            transformer._unsloth_int8_gemm = install_int8_gemm(
                transformer, logger, offload_active = False
            )
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "int8 fused-dequant gemm", exc)
    applied["int8_gemm"] = any(
        bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)
    )
    return applied


def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool:
    """An offloaded fp16 U-Net stays eager: fused QKV costs more transfer than compile saves (L4: 5.43 vs 4.86 s)."""
    return (
        bool(offload_active)
        and _is_float16(getattr(target, "dtype", None))
        and _denoiser_unet(pipe) is not None
    )


def _vae_single_frame(pipe: Any, logger: Any) -> bool:
    try:
        from . import diffusion_vae_single_frame  # noqa: PLC0415
        return diffusion_vae_single_frame.install(getattr(pipe, "vae", None), logger)
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "vae single-frame path", exc)
        return False


# channels_last: slower on the stock path (72.7 vs 89 ms), faster once the fused norms install (104.7 vs 121.0 ms).
_VAE_CHANNELS_LAST_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})


# DiT-only VAEs: cudnn.benchmark saves ~3 ms per decode (fused, 1024) but re-tunes 0.7-2 s at every new resolution.
_CUDNN_BENCHMARK_DENY_VAES: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})


def _cudnn_benchmark_pointless(pipe: Any) -> bool:
    if _denoiser_unet(pipe) is not None:
        return False
    return type(getattr(pipe, "vae", None)).__name__ in _CUDNN_BENCHMARK_DENY_VAES


def _vae_channels_last(
    pipe: Any,
    logger: Any,
    *,
    fused: bool = False,
) -> bool:
    vae = getattr(pipe, "vae", None)
    if vae is None or not hasattr(vae, "to"):
        return False
    if type(vae).__name__ in _VAE_CHANNELS_LAST_DENY and not fused:
        return False
    try:
        import torch

        if _has_conv3d(vae):
            return _vae_channels_last_3d(vae, logger, fused = fused)
        vae.to(memory_format = torch.channels_last)
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "channels_last", exc)
        return False


VAE_CHANNELS_LAST_3D_ENV = "UNSLOTH_VAE_CHANNELS_LAST_3D"


def _has_conv3d(vae: Any) -> bool:
    modules = getattr(vae, "modules", None)
    if not callable(modules):
        return False
    import torch

    return any(isinstance(m, torch.nn.Conv3d) for m in modules())


def _vae_channels_last_3d(vae: Any, logger: Any, *, fused: bool) -> bool:
    """Per-rank channels_last(_3d) on post_quant_conv + decoder; ``Module.to(channels_last)`` raises at the first 5D
    weight. Skipped without the fused passes (slower there); the encoder is kept (channels_last encoded slower)."""
    import torch

    name = type(vae).__name__
    if os.environ.get(VAE_CHANNELS_LAST_3D_ENV, "").strip().lower() in _VAE_FALSE_TOKENS:
        reason = f"{VAE_CHANNELS_LAST_3D_ENV}=0"
    elif not fused:
        reason = "3D-conv VAE without the fused VAE passes (channels_last_3d measured slower there)"
    else:
        reason = None
    if reason is not None:
        if logger is not None:
            logger.info("diffusion.speed: channels_last skipped for %s: %s", name, reason)
        return False
    relaid = 0
    touched: list = []
    for part in (getattr(vae, "post_quant_conv", None), getattr(vae, "decoder", None)):
        if part is None or not callable(getattr(part, "modules", None)):
            continue
        for module in part.modules():
            weight = getattr(module, "weight", None)
            if not isinstance(module, (torch.nn.Conv2d, torch.nn.Conv3d)) or not isinstance(
                weight, torch.Tensor
            ):
                continue
            fmt = torch.channels_last if weight.dim() == 4 else torch.channels_last_3d
            if not weight.is_contiguous(memory_format = fmt):
                weight.data = weight.data.contiguous(memory_format = fmt)
                touched.append(module)
            relaid += 1
    if touched:
        # Relaid because fusion is planned; fusion matched by structure may still be rejected at its first decode,
        # which restores these weights' layout (diffusion_vae_fused._restore_contiguous_layout).
        vae.__dict__.setdefault("_unsloth_channels_last_modules", []).extend(touched)
    if logger is not None:
        logger.info(
            "diffusion.speed: channels_last(_3d) decode weights on %s (%d convs)", name, relaid
        )
    return relaid > 0


VIDEO_VAE_HALF_ENV = "UNSLOTH_VIDEO_VAE_HALF"
# Kill switch: "0" reruns a non-finite fp16 decode in fp32 even when its latents are already non-finite (as before).
VIDEO_VAE_NONFINITE_INPUT_ENV = "UNSLOTH_VIDEO_VAE_NONFINITE_INPUT_KEEP_FP16"


def _vae_input_nonfinite_skips_fp32() -> bool:
    return os.environ.get(VIDEO_VAE_NONFINITE_INPUT_ENV, "").strip().lower() not in (
        "0",
        "false",
        "no",
        "off",
    )


def _video_vae_half_decode(pipe: Any, target: Any, family: Any, logger: Any) -> bool:
    """fp16 channels_last(_3d) decode for fp32-pinned video VAEs (Wan) on NVIDIA sm75+; non-finite output reruns fp32.

    fp16, not bf16: bf16 is 9 dB further from fp32. ``UNSLOTH_VIDEO_VAE_HALF=0`` keeps fp32."""
    if not getattr(family, "vae_force_fp32", False) or getattr(target, "device", None) != "cuda":
        return False
    if os.environ.get(VIDEO_VAE_HALF_ENV, "").strip().lower() in _VAE_FALSE_TOKENS:
        if logger is not None:
            logger.info("diffusion.speed: video VAE decode kept in fp32 (%s=0)", VIDEO_VAE_HALF_ENV)
        return False
    vae = getattr(pipe, "vae", None)
    decoder = getattr(vae, "decoder", None)
    original = getattr(vae, "decode", None)
    if decoder is None or not callable(original):
        return False
    # A dual-DiT family calls apply_speed_optims once per expert over the same VAE.
    if getattr(vae, "_unsloth_half_decode", False):
        return True
    if getattr(target, "backend", None) != "cuda" or not _fp16_compile_capable(target):
        return False
    try:
        import functools

        import torch

        parts = [m for m in (getattr(vae, "post_quant_conv", None), decoder) if m is not None]
        for part in parts:
            part.to(torch.float16)
            for module in part.modules():
                weight = getattr(module, "weight", None)
                if isinstance(module, (torch.nn.Conv2d, torch.nn.Conv3d)) and weight is not None:
                    fmt = torch.channels_last if weight.dim() == 4 else torch.channels_last_3d
                    weight.data = weight.data.contiguous(memory_format = fmt)
        fell_back: list = []

        @functools.wraps(original)
        def decode(z: Any, *args: Any, **kwargs: Any) -> Any:
            if fell_back or not torch.is_tensor(z):
                return original(z, *args, **kwargs)
            out = original(z.to(torch.float16), *args, **kwargs)
            sample = out[0] if isinstance(out, tuple) else getattr(out, "sample", out)
            if not torch.is_tensor(sample) or bool(torch.isfinite(sample).all()):
                return out
            # Non-finite latents stay non-finite in fp32; the rerun only doubles memory (T4 OOM).
            if _vae_input_nonfinite_skips_fp32() and not bool(torch.isfinite(z).all()):
                if logger is not None:
                    logger.warning(
                        "diffusion.speed: the latents reaching the VAE are not finite; fp16 decode kept"
                    )
                return out
            if logger is not None:
                logger.warning(
                    "diffusion.speed: fp16 VAE decode was not finite; decoding in fp32 from now on"
                )
            for part in parts:
                part.to(torch.float32)
            fell_back.append(True)
            return original(z, *args, **kwargs)

        vae.decode = decode
        vae._unsloth_half_decode = True
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        try:
            import torch
            for part in (getattr(vae, "post_quant_conv", None), decoder):
                if part is not None:
                    part.to(torch.float32)
        except Exception:  # noqa: BLE001, S110 - best-effort restore of the fp32 pin
            pass
        _warn(logger, "video vae fp16 decode", exc)
        return False


# U-Net denoisers ship no ``_repeated_blocks``, so the regional compile cannot reach them; these classes get a
# WHOLE-module STATIC compile. SDXL (B200, 30 steps / 1024px): 26.9 vs 45.9 ms/step, 1.61x end-to-end at LPIPS 0.034.
# Static means a recompile per (height, width, batch).
_UNET_WHOLE_COMPILE: frozenset[str] = _targets.UNET_WHOLE_COMPILE


def _denoiser_unet(pipe: Any) -> Any:
    """The pipe's U-Net denoiser when its class is on the whole-compile list, else None."""
    unet = getattr(pipe, "unet", None)
    if unet is not None and type(unet).__name__ in _UNET_WHOLE_COMPILE:
        return unet
    return None


def fused_qkv_added_bytes(pipe: Any) -> int:
    """Device bytes the default tier's U-Net QKV fuse adds. diffusers' ``fuse_projections`` builds ``to_qkv`` /
    ``to_kv`` beside the ``to_q`` / ``to_k`` / ``to_v`` it keeps, so a U-Net that moves whole (model offload) or stays
    resident carries both (SDXL: 4897 MiB grows to 6132 MiB). 0 when the tier would not fuse."""
    unet = _denoiser_unet(pipe)
    if unet is None:
        return 0
    try:
        import torch

        if _quantised_projections(pipe):
            return 0
        if unfused_projections_will_drop(unet):
            return 0  # the fused weights replace the projections they were built from
        total = 0
        for module in unet.modules():
            if not callable(getattr(module, "fuse_projections", None)) or getattr(
                module, "fused_projections", False
            ):
                continue
            names = (
                ("to_k", "to_v")
                if getattr(module, "is_cross_attention", False)
                else ("to_q", "to_k", "to_v")
            )
            if getattr(module, "add_q_proj", None) is not None:
                names += ("add_q_proj", "add_k_proj", "add_v_proj")
            for name in names:
                linear = getattr(module, name, None)
                for t in (getattr(linear, "weight", None), getattr(linear, "bias", None)):
                    if isinstance(t, torch.Tensor):
                        total += t.numel() * t.element_size()
        return total
    except Exception:  # noqa: BLE001 - unreadable: no extra bytes claimed
        return 0


def compiled_shapes_are_static(pipe: Any, speed_mode: Optional[str]) -> bool:
    """Whether this load's compiled artifacts are per-(width, height, batch); the compile cache keys on it."""
    mode = normalize_speed_mode(speed_mode)
    if mode == SPEED_MAX:
        # Auto-dynamic DiT reuses its graph for new sizes; the Dynamo graph-count delta marks compiles.
        return _denoiser_unet(pipe) is not None or not auto_dynamic_active(pipe)
    if mode != SPEED_DEFAULT:
        return False
    return (
        _denoiser_unet(pipe) is not None
        or any(getattr(t, "_unsloth_auto_dynamic", False) for t in _denoiser_dits(pipe))
        or _dits_merge_streams(_denoiser_dits(pipe))
    )


def _denoiser_dits(pipe: Any) -> list:
    """Every DiT the denoise loop runs: the primary ``transformer`` plus a second expert some
    families carry (Ideogram's ``unconditional_transformer``, an MoE ``transformer_2``). Speed /
    attention optims must reach ALL of them (mirroring the offload path), else the second DiT runs
    eager / native while status over-reports the optim as engaged."""
    return _targets.denoiser_dits(pipe)


def int8_gemm_live(pipe: Any, optims: Any) -> list:
    """``speed_optims`` with ``int8_gemm`` dropped once a deferred install's first-forward probe swapped nothing."""
    optims = list(optims or ())
    if "int8_gemm" in optims and not any(
        bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)
    ):
        optims.remove("int8_gemm")
    return optims


# Blocks MEASURED to raise inductor CantSplit under dynamic = True; measure before adding.
_STREAM_MERGING_BLOCKS: frozenset[str] = frozenset({"FluxSingleTransformerBlock"})

_STREAM_MERGE_DETECT_ENV = "UNSLOTH_STATIC_STREAM_MERGE_DETECT"
_STREAM_MERGE_SOURCE = re.compile(
    r"torch\.cat\(\s*\[\s*(?:encoder_hidden_states\s*,\s*hidden_states"
    r"|hidden_states\s*,\s*encoder_hidden_states)\s*\]"
)


@lru_cache(maxsize = None)
def _class_merges_streams(cls: type, broad: bool = False) -> bool:
    """Whether one repeated-block CLASS needs a static compile; ``broad`` is an argument so the memo keys on it."""
    if cls.__name__ in _STREAM_MERGING_BLOCKS:
        return True
    if not broad:
        return False
    try:
        import inspect  # noqa: PLC0415 - only reached under the opt-in broad sweep
        source = inspect.getsource(cls.forward)
    except Exception:  # noqa: BLE001 - no source (frozen / C ext) means fall back to the name list
        return False
    return bool(_STREAM_MERGE_SOURCE.search(source))


def _divisibility_proof_available() -> bool:
    """Whether inductor proves ``(k*a + k*b) % (a + b) == 0`` (torch 2.14+, or the backport; see diffusion_inductor_backports)."""
    try:
        from . import diffusion_inductor_backports  # noqa: PLC0415 - imports torch
        return diffusion_inductor_backports.proof_available()
    except Exception:  # noqa: BLE001 - unanswerable: keep the static fallback
        return False


def _dits_merge_streams(dits: list) -> bool:
    """Whether a stream-merging block must compile static: not once inductor can prove the CantSplit split."""
    if _divisibility_proof_available():
        return False
    broad = os.environ.get(_STREAM_MERGE_DETECT_ENV) == "1"
    seen: set[type] = set()
    for transformer in dits:
        names = set(getattr(transformer, "_repeated_blocks", ()) or ())
        if not names:
            continue
        try:
            modules = list(transformer.named_modules())
        except Exception:  # noqa: BLE001 - a probe, never a failed load
            continue
        for _name, sub in modules:
            cls = type(sub)
            if cls.__name__ not in names or cls in seen:
                continue
            seen.add(cls)
            if _class_merges_streams(cls, broad):
                return True
    return False


def _compile_repeated_blocks(
    pipe: Any,
    logger: Any,
    *,
    max_autotune: bool = False,
    cache_active: bool = False,
    offload_active: bool = False,
    denoiser_offloaded: Optional[bool] = None,
    onload_device: Any = None,
    filter_reductions: bool = False,
) -> bool:
    dits = [
        t for t in _denoiser_dits(pipe) if callable(getattr(t, "compile_repeated_blocks", None))
    ]
    unet = _denoiser_unet(pipe) if not dits else None
    if not dits and unet is None:
        return False
    _install_inductor_backports(logger)
    # Before the stream-merge probe below, which reads the same block names.
    for transformer in dits:
        try:
            from .diffusion_regional_compile import ensure_repeated_blocks
            ensure_repeated_blocks(transformer)
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "repeated block discovery", exc)
    # dynamic=False recompiled per prompt length on text-token DiTs.
    # Inductor cudagraph modes fail on regional blocks, so capture happens at the denoiser.
    static_shapes = (not max_autotune) and _dits_merge_streams(dits)
    if static_shapes and logger is not None:
        logger.info(
            "diffusion.speed: regional compile is static; this DiT merges the text and image streams "
            "inside its repeated block and cannot be codegen'd with dynamic sequence lengths",
        )
    kwargs: dict[str, Any] = {
        "fullgraph": not (cache_active or offload_active),
        "dynamic": None if max_autotune else (not static_shapes),
    }
    if max_autotune:
        kwargs["mode"] = "max-autotune-no-cudagraphs"
    if filter_reductions:
        pin_reduction_configs(kwargs, logger)
    try:
        import torch

        # Via compile_config: the render thread on torch 2.12+ cannot see these writes.
        # recompile_limit 64: heterogeneous-block DiTs exceed the default 8 under fullgraph.
        # Not force_parameter_static_shapes=False: no win and ~6x slower.
        dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
        if dynamo_cfg is not None:
            for _limit_attr in ("recompile_limit", "cache_size_limit"):  # name varies by torch ver
                if hasattr(dynamo_cfg, _limit_attr):
                    current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0
                    compile_config.set_knob(_DYNAMO_MODULE, _limit_attr, max(current, 64))
        # Match eager bf16 rounding in fused pointwise kernels (multi-step denoise amplifies drift).
        # Process-global: snapshot_backend_flags restores it on unload.
        inductor_cfg = _inductor_config()
        if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):
            compile_config.set_knob(_INDUCTOR_MODULE, "emulate_precision_casts", True)
            # Before the first compiled forward: otherwise every restart misses the FX graph cache on libdevice.
            try:
                from . import diffusion_aot_cache
                diffusion_aot_cache.pin_triton_libdevice(logger)
            except Exception as exc:  # noqa: BLE001 - optimisation only
                _warn(logger, "libdevice pin", exc)
        # One R0_BLOCK per reduction: see diffusion_compile_config.DYNAMIC_SCALE_RBLOCK_ENV.
        if (
            inductor_cfg is not None
            and hasattr(inductor_cfg, "dynamic_scale_rblock")
            and compile_config.reduction_blocks_pinned()
        ):
            compile_config.set_knob(_INDUCTOR_MODULE, "dynamic_scale_rblock", False)
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "compile_repeated_blocks", exc)
        return False
    # A block whose attention runs inside sdpa_kernel would otherwise bypass AOTAutogradCache on every start.
    try:
        from . import diffusion_aot_cache
        diffusion_aot_cache.install(logger)
        diffusion_aot_cache.pin_autotune_key(logger)
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "sdpa aot cache", exc)
    if unet is not None:
        # dynamic always False, so each new (height, width, batch) recompiles.
        unet_kwargs: dict[str, Any] = {"fullgraph": kwargs["fullgraph"], "dynamic": False}
        if max_autotune:
            unet_kwargs["mode"] = "max-autotune-no-cudagraphs"
        if filter_reductions:
            pin_reduction_configs(unet_kwargs, logger)
        if unet_compiles_regionally(unet, max_autotune = max_autotune):
            return _compile_unet_blocks(unet, unet_kwargs, logger)
        try:
            unet.compile(**unet_kwargs)
            return True
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "unet whole-module compile", exc)
            return False
    # Compile every denoiser DiT (dual-DiT families run both); a per-DiT failure degrades only that one to eager.
    engaged = False
    for transformer in dits:
        dit_kwargs = dict(kwargs)
        dit_kwargs["dynamic"] = compile_dynamic(transformer, kwargs["dynamic"])
        if (
            max_autotune
            and dit_kwargs["dynamic"] is None
            and _carries_torchao_weights(transformer)
            and _dits_merge_streams([transformer])
        ):
            # Auto dynamic hits CantSplit on torchao stream-merging blocks and drops to eager; static recompiles.
            dit_kwargs["dynamic"] = False
        # Read by auto_dynamic_active: the generalising recompile on a new text length must reach the bundle.
        transformer._unsloth_auto_dynamic = dit_kwargs["dynamic"] is None
        # Registered per-denoiser patches (diffusion_autoopt.patches), class gated, before the int8 swaps.
        _install_compile_patches(
            "pre_int8",
            transformer,
            logger,
            offload_active = offload_active,
            denoiser_offloaded = denoiser_offloaded,
        )
        # Before the compile: the regional compile traces whatever forward the blocks carry at the first call.
        try:
            from .diffusion_int8_fused import install as install_int8_fused
            install_int8_fused(transformer, logger, offload_active = offload_active)
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "int8 fused mlp", exc)
        # After the fused MLP (its down projection calls the same GEMM), before the compile traces the Linears.
        try:
            from .diffusion_int8_gemm import install as install_int8_gemm

            # Keyed on the DENOISER's placement; a weight elsewhere at call time keeps the stock path.
            moved = offload_active if denoiser_offloaded is None else bool(denoiser_offloaded)
            if moved and onload_device is not None:
                transformer._unsloth_int8_gemm = install_int8_gemm(
                    transformer, logger, device = onload_device
                )
            else:
                transformer._unsloth_int8_gemm = install_int8_gemm(
                    transformer, logger, offload_active = moved
                )
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "int8 fused-dequant gemm", exc)
        # Patches run after the int8 GEMM swap; ConvRot sharing reads the swap's leftovers.
        _install_compile_patches(
            "pre_compile",
            transformer,
            logger,
            offload_active = offload_active,
            denoiser_offloaded = denoiser_offloaded,
        )
        try:
            transformer.compile_repeated_blocks(**dit_kwargs)
            engaged = True
            # Kept so a caller compiling below an offload hook (MiniMax-H3) uses these exact compile settings.
            try:
                transformer._unsloth_regional_compile_kwargs = dict(dit_kwargs)
            except Exception:  # noqa: BLE001 - not a settable module (tests/fakes)
                pass
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "compile_repeated_blocks", exc)
            continue
        # dynamic=True (dense H3) still needs the unbacked temb, else step 1 compiles a second graph.
        try:
            from . import diffusion_dynamic_text
            diffusion_dynamic_text.install(transformer, logger, dynamic = dit_kwargs["dynamic"])
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "dynamic text dims", exc)
        # Compile is lazy (first forward in generate()): guard blocks so failures drop to eager.
        guard_compiled_blocks(transformer, logger)
        # After the guard (it wraps the guarded call): FLUX.1's first single block otherwise compiles a second graph.
        try:
            from . import diffusion_block_restride
            diffusion_block_restride.install(transformer, logger)
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "block restride", exc)
        # Inductor turns the prefix KV cache's clone into a view of the full K/V buffer, which pins it for the render.
        try:
            from .diffusion_prefix_kv import install_prefix_kv_compaction
            install_prefix_kv_compaction(transformer, logger)
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "prefix kv compaction", exc)
        # A step cache engaged earlier wrapped block forwards; re-point them at compiled wrappers.
        try:
            from .diffusion_cache import _compile_hooked_block_inners
            _compile_hooked_block_inners(transformer, logger)
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "cache-hook inner compile", exc)
    return engaged


def _unet_blocks(unet: Any) -> list:
    names = set(_targets.UNET_REGIONAL_BLOCKS)
    try:
        return [m for m in unet.modules() if m is not unet and type(m).__name__ in names]
    except Exception:  # noqa: BLE001 - not a torch module
        return []


def unet_compiles_regionally(unet: Any, *, max_autotune: bool = False) -> bool:
    return (
        unet is not None
        and not max_autotune
        and _targets.unet_regional_enabled()
        and bool(_unet_blocks(unet))
    )


_UNET_RECOMPILE_LIMIT = 256


def _compile_unet_blocks(unet: Any, unet_kwargs: dict[str, Any], logger: Any) -> bool:
    """Compile each repeated U-Net block (guarded; the outer forward stays eager for the CUDA graph)."""
    count = 0
    try:
        # One ResnetBlock2D forward serves ~14 graphs per (height, width, batch): 64 would stop at the 5th size.
        import torch
        dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
        for _limit_attr in ("recompile_limit", "cache_size_limit"):
            if dynamo_cfg is not None and hasattr(dynamo_cfg, _limit_attr):
                current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0
                compile_config.set_knob(
                    _DYNAMO_MODULE, _limit_attr, max(current, _UNET_RECOMPILE_LIMIT)
                )
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "unet recompile limit", exc)
    try:
        for module in _unet_blocks(unet):
            module.compile(**unet_kwargs)
            count += 1
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "unet regional compile", exc)
        for module in unet.modules():
            if module is not unet and getattr(module, "_compiled_call_impl", None) is not None:
                module._compiled_call_impl = None
        return False
    if not count:
        return False
    guard_compiled_blocks(unet, logger)
    try:
        unet._unsloth_regional_compile_kwargs = dict(unet_kwargs)
        unet._unsloth_bg_yield = True
    except Exception:  # noqa: BLE001 - not a settable module (tests/fakes)
        pass
    if logger is not None:
        logger.info(
            "diffusion.speed: %s compiles per block (%d blocks: %s); set %s=0 for the whole-module compile",
            type(unet).__name__,
            count,
            ", ".join(_targets.UNET_REGIONAL_BLOCKS),
            _targets.UNET_REGIONAL_ENV,
        )
    return True


def _install_compile_patches(
    stage: str,
    transformer: Any,
    logger: Any,
    *,
    offload_active: bool,
    denoiser_offloaded: Optional[bool],
) -> None:
    try:
        from .diffusion_autoopt.apply import install_compile_patches
        install_compile_patches(
            stage,
            transformer,
            logger,
            offload_active = offload_active,
            denoiser_offloaded = denoiser_offloaded,
        )
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, f"{stage} patches", exc)


def pin_reduction_configs(kwargs: dict[str, Any], logger: Any = None) -> bool:
    """Pins one config per reduction via per-compile options, never the global knob; no benchmark pick."""
    if not compile_config.reduction_config_filter_available():
        return False
    try:
        options = dict(kwargs.get("options") or {})
        mode = kwargs.get("mode")
        if mode is not None:
            from torch._inductor import list_mode_options
            options = {**list_mode_options(mode, kwargs.get("dynamic")), **options}
        options[REDUCTION_FILTER_OPTION] = True
        # Max's coordinate descent retimes the filtered config in every process (its guard reads deterministic).
        if options.get("coordinate_descent_tuning"):
            options["coordinate_descent_tuning"] = False
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "reduction config filter", exc)
        return False
    kwargs.pop("mode", None)
    kwargs["options"] = options
    return True


def _install_inductor_backports(logger: Any) -> bool:
    """Probe-gated backport of torch 2.14's CantSplit divisibility proof for 2.12 / 2.13. Never fails a load."""
    try:
        from . import diffusion_inductor_backports
        return diffusion_inductor_backports.install(logger)
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "inductor backports", exc)
        return False


def compile_dynamic(transformer: Any, dynamic: Optional[bool]) -> Optional[bool]:
    """dynamic=True breaks torchao's per-row quant fusion on torch 2.12/2.13 unpatched; None is faster."""
    # Static kernels for video DiTs: dynamic shapes made LTX-2.3's QK-norm + RoPE ~3x slower.
    if transformer is not None and getattr(transformer, "_unsloth_compile_static", False):
        return False
    if dynamic and transformer is not None and _carries_torchao_weights(transformer):
        return None
    return dynamic


def _carries_torchao_weights(module: Any) -> bool:
    """Only torchao's own tensor types count; GGUF or bitsandbytes subclasses keep their compile policy."""

    return any(True for _ in torchao_weight_tensors(module))


def torchao_weight_tensors(module: Any) -> Iterator[Any]:
    """Each torchao tensor subclass holding a parameter of ``module``: the parameter itself, or its ``.data`` when a
    plain ``nn.Parameter`` wraps the subclass. Yields nothing for a module that cannot be read (tests/fakes)."""

    def _torchao(t: Any) -> bool:
        return t is not None and type(t).__module__.startswith("torchao")

    try:
        parameters = list(module.parameters())
    except Exception:  # noqa: BLE001 - not a torch module (tests/fakes)
        return
    for p in parameters:
        if _torchao(p):
            yield p
        else:
            data = getattr(p, "data", None)
            if _torchao(data):
                yield data


def is_compile_failure(exc: BaseException) -> bool:
    """True for an error torch.compile raises while BUILDING a graph (dynamo tracing, inductor lowering / codegen), as
    opposed to one raised by kernels that already built. Only the former is safe to answer with an eager retry: nothing
    ran yet. An OOM is never one, even when inductor wraps it (autotune ran out), so the OOM backoff still sees it."""
    try:
        import torch
    except Exception:  # noqa: BLE001 - no torch, nothing compiled
        return False
    from .diffusion_batched import is_oom_error

    # OOM anywhere in the cause chain goes to the batch backoff, not the eager retry.
    if is_oom_error(exc):
        return False
    kinds: list = []
    dynamo_exc = getattr(getattr(torch, "_dynamo", None), "exc", None)
    for name in (
        "BackendCompilerFailed",
        "Unsupported",
        "TorchRuntimeError",
        "InternalTorchDynamoError",
    ):
        kind = getattr(dynamo_exc, name, None) if dynamo_exc is not None else None
        if isinstance(kind, type):
            kinds.append(kind)
    try:
        from torch._inductor.exc import (
            InductorError,
        )  # torch 2.7+; older torch wraps it in BackendCompilerFailed
        kinds.append(InductorError)
    except Exception:  # noqa: BLE001
        pass
    return bool(kinds) and isinstance(exc, tuple(kinds))


def _drop_tracebacks(exc: BaseException) -> None:
    from .diffusion_vae_oom import drop_tracebacks  # noqa: PLC0415
    drop_tracebacks(exc)


class _CompileGuard:
    """Per-DiT record of a compile that failed at runtime. The first failure flips every guarded block of that DiT to
    eager (one shared decision: a half-compiled stack would retry the same broken lowering per block)."""

    def __init__(self, logger: Any) -> None:
        self.logger = logger
        self.error: Optional[str] = None
        self.restores: list = []

    def fail(self, exc: BaseException, owner: Any) -> None:
        if self.error is None:
            self.error = f"{type(exc).__name__}: {str(exc).splitlines()[0] if str(exc) else ''}"[
                :300
            ]
            for restore in self.restores:
                try:
                    restore()
                except Exception:  # noqa: BLE001 - per-block best-effort; the guard still routes to eager
                    pass
            if self.logger is not None:
                self.logger.warning(
                    "diffusion.speed: torch.compile failed on %s at the first forward (%s); this load runs eager",
                    type(owner).__name__,
                    self.error,
                )

    def wrap(self, compiled: Any, eager: Any, owner: Any) -> Any:
        guard = self

        def guarded(*args: Any, **kwargs: Any) -> Any:
            # Compile still in flight in the background: entering it here would compile inline.
            if guard.error is None and not _bg_eager_forced():
                _bg_yield_point()
                compile_config.apply()
                try:
                    return compiled(*args, **kwargs)
                except Exception as exc:  # noqa: BLE001 - reraised unless a compile-time failure
                    if not is_compile_failure(exc):
                        raise
                    guard.fail(exc, owner)
                    # The handled exception pins inductor's frames (and the traced fake tensors) through the retry.
                    _drop_tracebacks(exc)
            return eager(*args, **kwargs)

        guarded._unsloth_compile_guard = guard
        return guarded


def guard_compiled_blocks(transformer: Any, logger: Any = None) -> int:
    """Wrap each ``Module.compile``d submodule's ``_compiled_call_impl`` so a compile-time failure on the first forward
    (e.g. inductor ``CantSplit`` on a torchao-quantised Qwen-Image-2.1 block) falls back to that module's eager
    ``_call_impl`` for this and every later call. Idempotent. Returns the number of modules guarded."""
    guard = getattr(transformer, "_unsloth_compile_guard", None)
    if guard is None:
        guard = _CompileGuard(logger)
        try:
            transformer._unsloth_compile_guard = guard
        except Exception:  # noqa: BLE001 - not a settable module (tests/fakes)
            return 0
    count = 0
    try:
        modules = list(transformer.modules())
    except Exception:  # noqa: BLE001 - not a torch module
        return 0
    for module in modules:
        compiled = getattr(module, "_compiled_call_impl", None)
        if compiled is None or getattr(compiled, "_unsloth_compile_guard", None) is not None:
            continue
        eager = getattr(module, "_call_impl", None)
        if eager is None:
            continue

        def restore(m: Any = module) -> None:
            m._compiled_call_impl = None

        guard.restores.append(restore)
        module._compiled_call_impl = guard.wrap(compiled, eager, transformer)
        count += 1
    return count


def _guarded_dits(pipe: Any) -> list:
    """Every DiT a compile guard can sit on: the standard denoisers plus a modular workflow's named partition
    (MiniMax-H3's ``transformer_ref`` compiles through a view, so the bare pipe's ``_denoiser_dits`` misses it)."""
    dits = _denoiser_dits(pipe)
    for attr in _targets.PARTITION_ATTRS:
        extra = getattr(pipe, attr, None)
        if extra is not None and extra not in dits:
            dits.append(extra)
    unet = _denoiser_unet(pipe)
    if (
        unet is not None
        and unet not in dits
        and getattr(unet, "_unsloth_compile_guard", None) is not None
    ):
        dits.append(unet)
    return dits


def compiled_dits_active(pipe: Any) -> bool:
    """True while at least one guarded DiT still runs its compiled blocks (dual-DiT pipelines fall back per DiT)."""
    for transformer in _guarded_dits(pipe):
        guard = getattr(transformer, "_unsloth_compile_guard", None)
        if guard is not None and not guard.error:
            return True
    return False


def settle_compile_fallback(
    state: Any,
    pipe: Any,
    logger: Any = None,
) -> Optional[str]:
    """Adds compile_fallback_eager once; drops compiled only when no guarded DiT still runs compiled."""
    dit_error = compile_fallback_error(pipe)
    vae_error = _vae_compile_error(getattr(pipe, "vae", None))
    dequant_error = gguf_compile.compile_error()
    fallback = dit_error or vae_error or dequant_error
    if not fallback:
        return None
    optims = tuple(getattr(state, "speed_optims", None) or ())
    updated = list(optims)
    if "compile_fallback_eager" not in updated:
        updated.append("compile_fallback_eager")
        if logger is not None:
            logger.warning("diffusion.speed: regional compile fell back to eager: %s", fallback)
    if dit_error and "compiled" in updated and not compiled_dits_active(pipe):
        updated.remove("compiled")
    if vae_error and "compiled_vae_decode" in updated:
        updated.remove("compiled_vae_decode")
    if dequant_error and "compiled_dequant" in updated:
        updated.remove("compiled_dequant")
    if tuple(updated) != optims:
        # The load states are frozen dataclasses; the status must still reflect what actually runs.
        object.__setattr__(state, "speed_optims", tuple(updated))
    return fallback


def auto_dynamic_active(pipe: Any) -> bool:
    """Whether any denoiser DiT compiled with automatic dynamic (the max tier, or torchao weights on the default)."""
    return any(getattr(t, "_unsloth_auto_dynamic", False) for t in _guarded_dits(pipe))


def dynamo_graph_count() -> int:
    """Graphs dynamo has compiled in this process (0 when unavailable); a delta across a render means it compiled."""
    try:
        from torch._dynamo.utils import counters
        return int(counters["stats"]["unique_graphs"])
    except Exception:  # noqa: BLE001
        return 0


def fresh_compile_count() -> int:
    """FX graph cache misses. Not ``dynamo_graph_count``: it also grows on cache-served retraces, rewriting bundles."""
    try:
        from torch._dynamo.utils import counters
        return int(counters["inductor"]["fxgraph_cache_miss"])
    except Exception:  # noqa: BLE001
        return 0


def compile_fallback_error(pipe: Any) -> Optional[str]:
    """The compile failure a guarded DiT fell back from, or None while every compiled DiT still runs compiled."""
    for transformer in _guarded_dits(pipe):
        guard = getattr(transformer, "_unsloth_compile_guard", None)
        if guard is not None and guard.error:
            return guard.error
    return None


COMPILE_VAE_ENV = "UNSLOTH_DIFFUSION_COMPILE_VAE"

_VAE_TRUE_TOKENS = ("1", "true", "yes", "on")
_VAE_FALSE_TOKENS = ("0", "false", "no", "off")

# Not worth compiling (Qwen-Image VAE). Keyed by class: the marker comes after fingerprinting.
_VAE_COMPILE_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage", "AutoencoderKLWan"})

# ``auto`` compiles only measured VAEs: video DiTs also pass through apply_speed_optims.
_VAE_COMPILE_ALLOW: frozenset[str] = frozenset({"AutoencoderKL", "AutoencoderKLFlux2"})


def vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:
    """Compile-cache key input; False for a U-Net so its existing bundles keep their key."""
    return _denoiser_unet(pipe) is None and _vae_decode_compile_allowed(pipe, speed_mode)


def _install_fused_vae(
    pipe: Any,
    logger: Any,
    encoder: bool = True,
) -> bool:
    """Triton-fused VAE norm passes (diffusion_vae_fused), unless the env forces the VAE decode compile instead.
    ``encoder=False`` leaves the VAE encoder stock (the family's ``vae_fused_encoder``)."""
    if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:
        return False
    try:
        from . import diffusion_vae_fused  # noqa: PLC0415 - Triton import only on CUDA loads
        return diffusion_vae_fused.install(getattr(pipe, "vae", None), logger, encoder = encoder) > 0
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "fused vae", exc)
        return False


def _fused_vae_planned(pipe: Any) -> bool:
    if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:
        return False
    try:
        from . import diffusion_vae_fused  # noqa: PLC0415
        return diffusion_vae_fused.will_install(getattr(pipe, "vae", None))
    except Exception:  # noqa: BLE001
        return False


def _vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:
    """U-Nets always; a DiT only in max mode or when the env forces it; never with the fused eager VAE."""
    if _fused_vae_planned(pipe):
        return False
    if _denoiser_unet(pipe) is not None:
        return True
    # H3's fused decoder never calls the blocks, so compiling them would be reported but never run.
    if getattr(getattr(pipe, "vae", None), "_unsloth_decode_blocks_bypassed", False):
        return False
    raw = os.environ.get(COMPILE_VAE_ENV, "").strip().lower()
    if raw in _VAE_FALSE_TOKENS:
        return False
    if raw in _VAE_TRUE_TOKENS:
        return True
    if speed_mode != SPEED_MAX:
        return False
    name = type(getattr(pipe, "vae", None)).__name__
    return name in _VAE_COMPILE_ALLOW and name not in _VAE_COMPILE_DENY


def _vae_eager_when_tiled(pipe: Any) -> bool:
    """A DiT decode not forced on by the env runs eager while tiled; U-Nets keep their measured recipe."""
    if pipe is not None and _denoiser_unet(pipe) is not None:
        return False
    return os.environ.get(COMPILE_VAE_ENV, "").strip().lower() not in _VAE_TRUE_TOKENS


def _vae_contiguous(pipe: Any, logger: Any) -> bool:
    try:
        import torch
        pipe.vae.to(memory_format = torch.contiguous_format)
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "contiguous vae", exc)
        return False


# VAE classes whose decode layout was measured; any other keeps channels_last.
_VAE_LAYOUT_MEASURED: frozenset[str] = frozenset({"AutoencoderKL", "AutoencoderKLFlux2"})


def _channels_last_decode_wins(
    pipe: Any,
    target: Any,
    compiled_decode: bool,
    offload_active: bool = False,
) -> bool:
    """NVIDIA-measured: eager 16-bit is faster contiguous (no NHWC GroupNorm) but offload keeps channels_last (lower peak); compiled 16-bit wins channels_last; fp32 wins contiguous."""
    vae = getattr(pipe, "vae", None)
    if (
        getattr(target, "backend", "cuda") != "cuda"
        or getattr(target, "device", None) != "cuda"
        or type(vae).__name__ not in _VAE_LAYOUT_MEASURED
    ):
        return True
    config = getattr(vae, "config", None)
    # SDXL pipelines upcast a force_upcast fp16 VAE to fp32 for the decode.
    fp32_decode = str(getattr(vae, "dtype", "")).endswith("float32") or (
        bool(getattr(config, "force_upcast", False))
        and _is_float16(getattr(vae, "dtype", None))
        and _denoiser_unet(pipe) is not None
    )
    if fp32_decode:
        return False
    return compiled_decode or offload_active


def _guard_compiled_decode(
    vae: Any,
    compiled: Any,
    eager: Any,
    logger: Any,
    eager_when_tiled: bool = False,
    owner: Any = None,
) -> Any:
    """Lazy compile fails on first call; OOMs still raise; eager_when_tiled sends tiled decode eager."""
    owner = vae if owner is None else owner
    had_own = "decode" in getattr(owner, "__dict__", {})
    failed: list = []

    def guarded(*args: Any, **kwargs: Any) -> Any:
        if eager_when_tiled and getattr(vae, "use_tiling", False):
            return eager(*args, **kwargs)
        # the one force-eager signal (a background compile in flight, the OOM scope's eager retry)
        if _bg_eager_forced():
            return eager(*args, **kwargs)
        if not failed:
            try:
                return compiled(*args, **kwargs)
            except Exception as exc:  # noqa: BLE001 - reraised unless a compile-time failure
                if not is_compile_failure(exc):
                    raise
                error = f"{type(exc).__name__}: {str(exc).splitlines()[0] if str(exc) else ''}"[
                    :300
                ]
                failed.append(error)
                try:
                    vae._unsloth_compile_decode_error = error
                    vae._unsloth_compiled_decode = False
                    # Only while the slot still holds this guard: a layer installed over it since (the OOM scope)
                    # stays, and this guard (``failed``) runs eager under it from now on.
                    if getattr(owner, "__dict__", {}).get("decode") is guarded:
                        if had_own:
                            owner.decode = eager
                        else:
                            del owner.decode
                except Exception:  # noqa: BLE001 - `failed` still routes this wrapper to eager
                    pass
                if getattr(vae, "_unsloth_eager_contiguous", False):
                    try:
                        import torch
                        vae.to(memory_format = torch.contiguous_format)
                    except Exception:  # noqa: BLE001 - optimisation only
                        pass
                if logger is not None:
                    logger.warning(
                        "diffusion.speed: torch.compile failed on the VAE decode (%s); decoding eager",
                        error,
                    )
                # The handled exception pins inductor's frames (and the traced fake tensors) through the retry.
                _drop_tracebacks(exc)
        return eager(*args, **kwargs)

    # diffusion_vae_oom reads these to retry a runtime OOM of the compiled decode on the eager path
    guarded._unsloth_compiled_decode = True
    guarded._unsloth_eager_decode = eager
    return guarded


def _decode_slot_owner(fn: Any) -> Any:
    """The mutable slot an eager outer wrapper (fp16 non-finite check, ROCm bf16 cast) calls through, found under
    any layer installed over it since (the OOM scope), or None."""
    from .decode_layers import next_layer  # noqa: PLC0415

    for _ in range(32):
        if fn is None:
            return None
        slot = getattr(fn, "_unsloth_decode_slot", None)
        if slot is not None:
            return slot
        fn = next_layer(fn)
    return None


def _compile_vae_decode(
    pipe: Any,
    logger: Any,
    max_autotune: bool = False,
    eager_when_tiled: bool = False,
) -> bool:
    """torch.compile the VAE ``decode`` in place; no cudagraphs, whose capture would pin decode activations."""
    vae = getattr(pipe, "vae", None)
    # An outer wrapper that must stay eager (fp16 non-finite check) exposes the slot it calls through.
    outer = getattr(vae, "__dict__", {}).get("decode") if vae is not None else None
    owner = _decode_slot_owner(outer) or vae
    decode = getattr(owner, "decode", None) if vae is not None else None
    if not callable(decode):
        return False
    # A regional block guard does not clear _unsloth_compiled_decode when it falls back, so the error is read first.
    if _vae_compile_error(vae):
        return False
    # A dual-DiT family calls apply_speed_optims twice over the same pipe.
    if getattr(vae, "_unsloth_compiled_decode", False):
        return True
    if getattr(vae, "_unsloth_compile_decode_error", None):
        return False
    _install_inductor_backports(logger)
    if _vae_declares_repeated_blocks(vae):
        return _compile_vae_regionally(vae, logger, max_autotune = max_autotune)
    try:
        import torch

        # dynamic=True: automatic dynamic pays a second compile on the next resolution for no steady-state gain.
        kwargs: dict[str, Any] = {"fullgraph": False, "dynamic": True}
        if max_autotune:
            kwargs["mode"] = "max-autotune-no-cudagraphs"
        compiled = torch.compile(decode, **kwargs)
        owner.decode = _guard_compiled_decode(
            vae, compiled, decode, logger, eager_when_tiled = eager_when_tiled, owner = owner
        )
        vae._unsloth_compiled_decode = True
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "vae decode compile", exc)
        return False


def _vae_declares_repeated_blocks(vae: Any) -> bool:
    try:
        return bool(getattr(vae, "_repeated_blocks", None)) and callable(
            getattr(vae, "compile_repeated_blocks", None)
        )
    except Exception:  # noqa: BLE001
        return False


def _compile_vae_regionally(
    vae: Any,
    logger: Any,
    max_autotune: bool = False,
) -> bool:
    """Compile the VAE's repeated block, not ``decode``: a tiled decode unrolls its Python tile loop into one graph
    (RecursionError static, CantSplit dynamic on MiniMax-H3); a block sees one fixed-shape tile."""
    try:
        kwargs: dict[str, Any] = {"fullgraph": False, "dynamic": False}
        if max_autotune:
            kwargs["mode"] = "max-autotune-no-cudagraphs"
        vae.compile_repeated_blocks(**kwargs)
        guard_compiled_blocks(vae, logger)
        vae._unsloth_compiled_decode = True
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "vae regional compile", exc)
        return False


def _vae_compile_error(vae: Any) -> Optional[str]:
    """The decode-compile failure a VAE fell back from: the whole-decode wrapper's, or a regional block guard's."""
    error = getattr(vae, "_unsloth_compile_decode_error", None)
    if error:
        return error
    guard = getattr(vae, "_unsloth_compile_guard", None)
    return getattr(guard, "error", None) if guard is not None else None


def _enable_cudnn_benchmark(logger: Any) -> bool:
    try:
        import torch

        from core._torchao_stub import _module_is_rocm

        # On ROCm this is MIOpen's exhaustive search: a first VAE decode tuned for 10 to 23 minutes and crashed a gfx1030.
        if _module_is_rocm(torch):
            return False
        _write_cudnn_benchmark(torch.backends.cudnn, True)
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "cudnn_benchmark", exc)
        return False


def _enable_miopen_immediate(logger: Any) -> bool:
    """ROCm: ask MIOpen for a conv solution instead of miopenFind*, whose trial launches on unseen shapes
    are where #9130's uncatchable abort came from. MIOPEN_FIND_MODE / MIOPEN_FIND_ENFORCE keep Find. Not in
    snapshot_backend_flags: an unload restoring it would reopen Find under a co-resident pipeline."""
    if os.environ.get("MIOPEN_FIND_MODE") or os.environ.get("MIOPEN_FIND_ENFORCE"):
        return False
    try:
        import torch

        from core._torchao_stub import _module_is_rocm

        miopen = getattr(torch.backends, "miopen", None)
        if not _module_is_rocm(torch) or miopen is None or not hasattr(miopen, "immediate"):
            return False
        if not miopen.immediate:
            miopen.immediate = True
            if logger is not None:
                logger.info("diffusion.speed: MIOpen immediate mode on (no Find trial launches)")
        return True
    except Exception as exc:  # noqa: BLE001 - keep torch's Find path
        _warn(logger, "miopen immediate", exc)
        return False


TF32_ENV = "UNSLOTH_DIFFUSION_TF32"


def tf32_enabled() -> bool:
    import os
    return (os.environ.get(TF32_ENV) or "").strip().lower() not in ("0", "off", "false", "no")


def _enable_tf32(logger: Any) -> bool:
    try:
        import torch

        torch.backends.cuda.matmul.allow_tf32 = True
        torch.backends.cudnn.allow_tf32 = True
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "tf32", exc)
        return False


# Deny list of families that went non-finite under fp16 accumulation; empty, none of six overflowed.
_FP16_ACCUM_DENY: frozenset[str] = frozenset()


def _enable_fp16_accumulation(
    family: Any,
    logger: Any,
    *,
    dtype: Any = None,
    speed_mode: Optional[str] = None,
) -> bool:
    """Consumer GPUs only; fp16 compute (results change) needs the max tier; bf16 runs on any tier."""
    import os

    if os.environ.get("UNSLOTH_DISABLE_FP16_ACCUM", "").strip().lower() in (
        "1",
        "true",
        "yes",
        "on",
    ):
        return False
    name = str(getattr(family, "name", family or "")).lower()
    if name in _FP16_ACCUM_DENY:
        return False
    if str(dtype).replace("torch.", "") == "float16" and speed_mode != SPEED_MAX:
        return False
    try:
        import torch

        matmul = torch.backends.cuda.matmul
        if not hasattr(matmul, "allow_fp16_accumulation"):
            return False
        from .diffusion_transformer_quant import _is_consumer_gpu

        if not _is_consumer_gpu():
            return False
        _write_fp16_accum(matmul, True)
        return True
    except Exception as exc:  # noqa: BLE001 - optimisation only
        _warn(logger, "fp16_accum", exc)
        return False


def _quantised_projections(pipe: Any) -> bool:
    """Whether a denoiser keeps native int8 / fp8 or packed nvfp4 / mxfp8 Linears, whose ``.weight`` is a dense
    widen: diffusers' fuse would concatenate those into a persistent dense ``to_qkv`` beside the compact buffers."""
    from .diffusion_native_quant import is_native_linear
    from .diffusion_packed_block import is_packed_block_linear

    # the pipe-level fuse reaches any U-Net, not only the whole-compile ones
    for model in (*_denoiser_dits(pipe), getattr(pipe, "unet", None)):
        if model is None or not callable(getattr(model, "modules", None)):
            continue
        try:
            if any(is_native_linear(m) or is_packed_block_linear(m) for m in model.modules()):
                return True
        except Exception:  # noqa: BLE001 -- a probe must never replace the failure it describes
            continue
    return False


def _fuse_qkv(pipe: Any, logger: Any) -> bool:
    # Fusing reads each projection's .weight, which a quantised layer dequantises: a dense to_qkv beside its buffers.
    if _quantised_projections(pipe):
        if logger is not None:
            logger.info("diffusion.speed: fuse_qkv skipped (quantised projections stay unfused)")
        return False
    from . import diffusion_lora_quant_merge as lora_quant_merge
    from .diffusion_manual_cast import manual_cast_info

    # A fused to_qkv is a plain Linear at the stored dtype, which a manual-cast (float32) forward cannot feed.
    if any(manual_cast_info(m) for m in _denoiser_dits(pipe)):
        if logger is not None:
            logger.info("diffusion.speed: fuse_qkv skipped (manual-cast weights stay unfused)")
        return False
    # A fused to_qkv copy would miss in-place re-merges of merged adapter weights.
    if lora_quant_merge.merged(pipe):
        if logger is not None:
            logger.info("diffusion.speed: fuse_qkv skipped (merged LoRA weights change in place)")
        return False
    # Prefer pipe-level fuse (covers every component); else fuse each DiT so dual-DiT families fuse both.
    fn = getattr(pipe, "fuse_qkv_projections", None)
    if callable(fn):
        try:
            vae = getattr(pipe, "vae", None)
            fused_vae_attn = callable(getattr(vae, "modules", None)) and any(
                type(getattr(m, "processor", None)).__name__ == "FusedSingleHeadProcessor"
                for m in vae.modules()
            )
            fn()
            if (
                fused_vae_attn
            ):  # the pipe-level fuse resets every VAE processor to FusedAttnProcessor2_0
                from . import diffusion_vae_fused  # noqa: PLC0415
                diffusion_vae_fused.install_attention_processors(vae)
            drop_unfused_projections(_denoiser_unet(pipe), logger)
            return True
        except Exception as exc:  # noqa: BLE001 - optimisation only
            _warn(logger, "fuse_qkv_projections", exc)
            return False
    engaged = False
    for transformer in _denoiser_dits(pipe):
        tfn = getattr(transformer, "fuse_qkv_projections", None)
        if callable(tfn):
            try:
                tfn()
                engaged = True
            except Exception as exc:  # noqa: BLE001 - optimisation only
                _warn(logger, "fuse_qkv_projections", exc)
    return engaged


DROP_UNFUSED_QKV_ENV = "UNSLOTH_DIFFUSION_DROP_UNFUSED_QKV"
_DROPPED_ATTR = "_unsloth_dropped_projections"


def _fused_attention_drops(module: Any) -> tuple:
    """The projections diffusers' fused processor no longer calls: self attention runs ``to_qkv`` (q, k, v unused),
    cross attention ``to_q`` + ``to_kv`` (k, v unused)."""
    if not getattr(module, "fused_projections", False):
        return ()
    if getattr(module, "is_cross_attention", False):
        return ("to_k", "to_v") if getattr(module, "to_kv", None) is not None else ()
    return ("to_q", "to_k", "to_v") if getattr(module, "to_qkv", None) is not None else ()


def _drop_unfused_enabled() -> bool:
    return (os.environ.get(DROP_UNFUSED_QKV_ENV) or "").strip().lower() not in (
        "0",
        "off",
        "false",
        "no",
    )


def _droppable_projection(module: Any, name: str) -> bool:
    """A projection the drop may delete: a plain Linear without hooks (quantised / wrapped ones are left alone)."""
    from torch import nn

    p = getattr(module, name, None)
    return type(p) is nn.Linear and not (
        p._forward_pre_hooks or p._forward_hooks or hasattr(p, "_hf_hook")
    )


def _drop_plan(unet: Any) -> list:
    """``(module, names)`` the drop would delete on a fused ``unet``; empty when it is off, a PEFT adapter is attached
    (it targets to_q / to_k / to_v) or any projection is not a plain hook-free Linear (all or nothing)."""
    if unet is None or not _drop_unfused_enabled():
        return []
    if any(hasattr(m, "lora_A") for m in unet.modules()):
        return []
    plan = []
    for module in unet.modules():
        names = _fused_attention_drops(module)
        if not names:
            continue
        if not all(_droppable_projection(module, n) for n in names):
            return []
        plan.append((module, names))
    return plan


def unfused_projections_will_drop(unet: Any) -> bool:
    """Whether the fuse's unfused copies will be dropped on this (not yet fused) U-Net: drop on, no PEFT adapter, and
    every projection the fused processor stops calling is a plain hook-free Linear."""
    if unet is None or not _drop_unfused_enabled():
        return False
    try:
        if any(hasattr(m, "lora_A") for m in unet.modules()):
            return False
        seen = False
        for module in unet.modules():
            if not callable(getattr(module, "fuse_projections", None)) or getattr(
                module, "fused_projections", False
            ):
                continue
            if getattr(module, "add_q_proj", None) is not None:
                return False  # the added projections are fused too and never dropped
            names = (
                ("to_k", "to_v")
                if getattr(module, "is_cross_attention", False)
                else ("to_q", "to_k", "to_v")
            )
            if not all(_droppable_projection(module, n) for n in names):
                return False
            seen = True
        return seen
    except Exception:  # noqa: BLE001 - unreadable: count the copies
        return False


def drop_unfused_projections(unet: Any, logger: Any = None) -> int:
    """Delete the unfused projections a U-Net QKV fuse left beside ``to_qkv`` / ``to_kv`` (SDXL: 1235 MiB). Kept when
    a PEFT adapter is attached (it targets them) or the modules carry hooks; ``restore_unfused_projections`` rebuilds
    them bit-exactly from the fused weights. UNSLOTH_DIFFUSION_DROP_UNFUSED_QKV=0 keeps them."""
    try:
        import torch

        plan = _drop_plan(unet)
        freed = 0
        for module, names in plan:
            shapes = []
            for n in names:
                p = getattr(module, n)
                freed += sum(t.numel() * t.element_size() for t in p.parameters())
                shapes.append((n, p.in_features, p.out_features, p.bias is not None))
                delattr(module, n)
            setattr(module, _DROPPED_ATTR, tuple(shapes))
        if plan and torch.cuda.is_available():
            torch.cuda.empty_cache()
        if plan and logger is not None:
            logger.info(
                "diffusion.speed: dropped the unfused projections beside the fused QKV on %d attention modules "
                "(%d MiB)",
                len(plan),
                freed >> 20,
            )
        return freed
    except Exception as exc:  # noqa: BLE001 - memory only: a failed drop keeps what is left
        _warn(logger, "dropping the unfused projections", exc)
        return 0


def restore_unfused_projections(unet: Any) -> int:
    """Rebuild the projections ``drop_unfused_projections`` deleted, row slices of the fused weights (bit-exact: the
    fuse concatenated them unchanged). Returns the modules restored."""
    if unet is None:
        return 0
    import torch
    from torch import nn

    restored = 0
    for module in unet.modules():
        shapes = getattr(module, _DROPPED_ATTR, None)
        if not shapes:
            continue
        fused = module.to_kv if getattr(module, "is_cross_attention", False) else module.to_qkv
        rows = 0
        for name, fin, fout, has_bias in shapes:
            lin = nn.Linear(fin, fout, bias = has_bias, device = "meta")
            with torch.no_grad():
                grad = bool(fused.weight.requires_grad)
                lin.weight = nn.Parameter(
                    fused.weight[rows : rows + fout].clone(), requires_grad = grad
                )
                if has_bias:
                    lin.bias = nn.Parameter(
                        fused.bias[rows : rows + fout].clone(), requires_grad = grad
                    )
            setattr(module, name, lin)
            rows += fout
        delattr(module, _DROPPED_ATTR)
        restored += 1
    return restored


def _warn(logger: Any, what: str, exc: Exception) -> None:
    if logger is not None:
        logger.warning("diffusion.speed: %s failed: %s", what, exc)