Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Opt-in speed modes off/eager/default/max (off by default); TF32 and cudnn.benchmark are process-
wide, restore at unload; torch imported lazily"""
from __future__ import annotations
import os
import re
import sys
import threading
from contextlib import contextmanager, nullcontext
from functools import lru_cache
from typing import Any, Callable, Iterator, Optional
from . import diffusion_compile_config as compile_config
from .diffusion_bg_compile import eager_forced as _bg_eager_forced
from .diffusion_bg_compile import yield_point as _bg_yield_point
from . import diffusion_gguf_compile as gguf_compile
from .diffusion_autoopt import targets as _targets
SPEED_OFF = "off"
SPEED_EAGER = "eager"
SPEED_DEFAULT = "default"
SPEED_MAX = "max"
SPEED_MODES = (SPEED_OFF, SPEED_EAGER, SPEED_DEFAULT, SPEED_MAX)
# Retaining its full size means it never rotates: the write index resets past the buffer end and updates stop landing.
_INDUCTOR_FLAGS = (
("emulate_precision_casts", "inductor_emulate_precision_casts"),
("coordinate_descent_tuning", "inductor_coordinate_descent_tuning"),
("coordinate_descent_check_all_directions", "inductor_coordinate_descent_check_all_directions"),
("force_fuse_int_mm_with_mul", "inductor_force_fuse_int_mm_with_mul"),
("fx_graph_cache", "inductor_fx_graph_cache"),
("dynamic_scale_rblock", "inductor_dynamic_scale_rblock"),
("memory_planning", "inductor_memory_planning"),
)
_INDUCTOR_TRITON_FLAGS = (("unique_kernel_names", "inductor_triton_unique_kernel_names"),)
_DYNAMO_FLAGS = (("suppress_errors", "dynamo_suppress_errors"),)
_DYNAMO_MODULE = "torch._dynamo.config"
REDUCTION_FILTER_OPTION = "test_configs.force_filter_reduction_configs"
_INDUCTOR_MODULE = "torch._inductor.config"
_INDUCTOR_TRITON_MODULE = "torch._inductor.config.triton"
# allow_fp16_accumulation has one owner: writes in an open scope go to the recorded value.
_FP16_ACCUM_LOCK = threading.RLock()
_fp16_accum_scopes: list = []
_fp16_accum_base: Optional[bool] = None
def _read_fp16_accum(matmul: Any) -> bool:
with _FP16_ACCUM_LOCK:
if _fp16_accum_scopes:
return bool(_fp16_accum_base)
return bool(matmul.allow_fp16_accumulation)
def _write_fp16_accum(matmul: Any, value: Any) -> None:
global _fp16_accum_base
with _FP16_ACCUM_LOCK:
if _fp16_accum_scopes:
_fp16_accum_base = bool(value)
else:
matmul.allow_fp16_accumulation = bool(value)
@contextmanager
def fp16_accumulation_scope(value: bool) -> Iterator[None]:
"""Hold ``allow_fp16_accumulation`` at ``value`` for the body, then restore the latest process value."""
global _fp16_accum_base
import torch
matmul = torch.backends.cuda.matmul
token = object()
with _FP16_ACCUM_LOCK:
if not _fp16_accum_scopes:
_fp16_accum_base = bool(matmul.allow_fp16_accumulation)
_fp16_accum_scopes.append((token, bool(value)))
matmul.allow_fp16_accumulation = bool(value)
try:
yield
finally:
with _FP16_ACCUM_LOCK:
_fp16_accum_scopes[:] = [e for e in _fp16_accum_scopes if e[0] is not token]
matmul.allow_fp16_accumulation = (
_fp16_accum_scopes[-1][1] if _fp16_accum_scopes else bool(_fp16_accum_base)
)
_CUDNN_BENCH_LOCK = threading.RLock()
_cudnn_bench_scopes: list = []
_cudnn_bench_base: Optional[bool] = None
def _read_cudnn_benchmark(cudnn: Any) -> bool:
with _CUDNN_BENCH_LOCK:
if _cudnn_bench_scopes:
return bool(_cudnn_bench_base)
return bool(cudnn.benchmark)
def _write_cudnn_benchmark(cudnn: Any, value: Any) -> None:
global _cudnn_bench_base
with _CUDNN_BENCH_LOCK:
if _cudnn_bench_scopes:
_cudnn_bench_base = bool(value)
else:
cudnn.benchmark = bool(value)
@contextmanager
def cudnn_benchmark_scope(value: bool) -> Iterator[None]:
"""Hold ``cudnn.benchmark`` at ``value`` for the body, then restore the latest process value."""
global _cudnn_bench_base
import torch
cudnn = torch.backends.cudnn
token = object()
with _CUDNN_BENCH_LOCK:
if not _cudnn_bench_scopes:
_cudnn_bench_base = bool(cudnn.benchmark)
_cudnn_bench_scopes.append((token, bool(value)))
cudnn.benchmark = bool(value)
try:
yield
finally:
with _CUDNN_BENCH_LOCK:
_cudnn_bench_scopes[:] = [e for e in _cudnn_bench_scopes if e[0] is not token]
cudnn.benchmark = (
_cudnn_bench_scopes[-1][1] if _cudnn_bench_scopes else bool(_cudnn_bench_base)
)
def family_render_cudnn_scope(family: Any, device: Any) -> Any:
"""Hold ``cudnn.benchmark`` off for a render of a family that opts out of it. The flag is process-wide, so a
co-resident or earlier load can have left it on, leaving the VAE convs on a per-process pick."""
if getattr(family, "cudnn_benchmark", True) or not str(device or "").startswith("cuda"):
return nullcontext()
try:
import torch
if getattr(getattr(torch, "backends", None), "cudnn", None) is None:
return nullcontext()
except Exception: # noqa: BLE001 - no torch: nothing to hold
return nullcontext()
return cudnn_benchmark_scope(False)
def snapshot_backend_flags() -> Optional[dict]:
"""Capture the process-wide torch backend flags this layer may mutate, for restore on unload. None
without torch. Each flag is read defensively: a build missing one still captures the rest."""
try:
import torch
except Exception: # noqa: BLE001 - no torch -> nothing to snapshot/restore
return None
state: dict[str, Any] = {}
matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
if matmul is not None and hasattr(matmul, "allow_tf32"):
state["matmul_tf32"] = bool(matmul.allow_tf32)
if matmul is not None and hasattr(matmul, "allow_fp16_accumulation"):
state["matmul_fp16_accum"] = _read_fp16_accum(matmul)
cudnn = getattr(torch.backends, "cudnn", None)
if cudnn is not None:
if hasattr(cudnn, "allow_tf32"):
state["cudnn_tf32"] = bool(cudnn.allow_tf32)
if hasattr(cudnn, "benchmark"):
state["cudnn_benchmark"] = _read_cudnn_benchmark(cudnn)
inductor_cfg = _inductor_config()
if inductor_cfg is not None:
# Process-wide value: on torch 2.12+ a knob set on another thread is invisible here.
for attr, key in _INDUCTOR_FLAGS:
if hasattr(inductor_cfg, attr):
state[key] = bool(compile_config.get_knob(_INDUCTOR_MODULE, attr))
triton_cfg = getattr(inductor_cfg, "triton", None)
if triton_cfg is not None:
for attr, key in _INDUCTOR_TRITON_FLAGS:
if hasattr(triton_cfg, attr):
state[key] = bool(compile_config.get_knob(_INDUCTOR_TRITON_MODULE, attr))
dynamo_cfg = _dynamo_config()
if dynamo_cfg is not None:
for attr, key in _DYNAMO_FLAGS:
if hasattr(dynamo_cfg, attr):
state[key] = bool(compile_config.get_knob(_DYNAMO_MODULE, attr))
getter = getattr(torch, "get_float32_matmul_precision", None)
if callable(getter):
try:
state["matmul_precision"] = str(getter())
except Exception: # noqa: BLE001 - unreadable on this build: restore the rest
pass
return state
def release_rocm_tunableop(pipe: Any) -> None:
"""Unload: drop this pipe's TunableOp claim (process-global, so not in snapshot_backend_flags)."""
mod = sys.modules.get(f"{__package__}.diffusion_rocm_tunableop")
if mod is None or pipe is None:
return
try:
mod.release(pipe)
except Exception: # noqa: BLE001 - teardown is best effort
pass
def restore_backend_flags(state: Optional[dict]) -> None:
"""Restore the flags captured by ``snapshot_backend_flags``. No-op on None. Each flag is
restored independently so one failure can't leak the others."""
if not state:
return
try:
import torch
except Exception: # noqa: BLE001 - no torch -> nothing to restore
return
def _set(obj: Any, attr: str, key: str) -> None:
if obj is not None and key in state and hasattr(obj, attr):
try:
setattr(obj, attr, state[key])
except Exception: # noqa: BLE001 - best-effort per-flag restore
pass
# FIRST: on some builds set_float32_matmul_precision also writes matmul.allow_tf32.
setter = getattr(torch, "set_float32_matmul_precision", None)
if state.get("matmul_precision") and callable(setter):
try:
setter(state["matmul_precision"])
except Exception: # noqa: BLE001 - best-effort per-flag restore
pass
matmul = getattr(getattr(torch.backends, "cuda", None), "matmul", None)
_set(matmul, "allow_tf32", "matmul_tf32")
if (
matmul is not None
and "matmul_fp16_accum" in state
and hasattr(matmul, "allow_fp16_accumulation")
):
try:
_write_fp16_accum(matmul, state["matmul_fp16_accum"])
except Exception: # noqa: BLE001 - best-effort per-flag restore
pass
cudnn = getattr(torch.backends, "cudnn", None)
_set(cudnn, "allow_tf32", "cudnn_tf32")
if cudnn is not None and "cudnn_benchmark" in state and hasattr(cudnn, "benchmark"):
try:
_write_cudnn_benchmark(cudnn, state["cudnn_benchmark"])
except Exception: # noqa: BLE001 - best-effort per-flag restore
pass
def _set_knob(obj: Any, module_name: str, attr: str, key: str) -> None:
if key in state and compile_config.is_recorded(module_name, attr):
compile_config.set_knob(module_name, attr, state[key])
else:
_set(obj, attr, key)
inductor_cfg = _inductor_config()
for attr, key in _INDUCTOR_FLAGS:
_set_knob(inductor_cfg, _INDUCTOR_MODULE, attr, key)
triton_cfg = getattr(inductor_cfg, "triton", None) if inductor_cfg is not None else None
for attr, key in _INDUCTOR_TRITON_FLAGS:
_set_knob(triton_cfg, _INDUCTOR_TRITON_MODULE, attr, key)
dynamo_cfg = _dynamo_config()
for attr, key in _DYNAMO_FLAGS:
_set_knob(dynamo_cfg, _DYNAMO_MODULE, attr, key)
def _gguf_dequant_compilable(
is_gguf: bool,
on_cuda: bool,
family_allows_compile: bool,
target: Any = None,
) -> bool:
return (
is_gguf
and on_cuda
and family_allows_compile
and not rocm_compile_switched_off(target)
and torch_compile_runtime_available()
)
def _pin_compile_defaults() -> None:
"""torch's defaults for every diffusion compile, against Unsloth's training config, which sets both process-wide
on torch < 2.12 (#9897). ``memory_planning``: its pooled buffers come back from ``inductor::_alloc_from_pool`` as
views AOTAutograd's custom-op aliasing check rejects at the first call (every gfx1151 render under ``CI``, a
deprecation warning elsewhere). ``suppress_errors``: a failed compile would silently run eager; Studio's own
guards fall back with a warning instead, as on torch 2.12+ where the render thread never saw the training config."""
inductor_cfg = _inductor_config()
if inductor_cfg is not None and hasattr(inductor_cfg, "memory_planning"):
compile_config.set_knob(_INDUCTOR_MODULE, "memory_planning", False)
dynamo_cfg = _dynamo_config()
if dynamo_cfg is not None and hasattr(dynamo_cfg, "suppress_errors"):
compile_config.set_knob(_DYNAMO_MODULE, "suppress_errors", False)
def _dynamo_config() -> Any:
try:
import torch
return getattr(getattr(torch, "_dynamo", None), "config", None)
except Exception: # noqa: BLE001 - no dynamo -> nothing to snapshot/set
return None
def _inductor_config() -> Any:
"""``torch._inductor.config`` or None. Read as attributes off the imported torch (not a
submodule import) so a stubbed/partial torch reports None instead of a stale sys.modules hit."""
try:
import torch
return getattr(getattr(torch, "_inductor", None), "config", None)
except Exception: # noqa: BLE001 - no inductor -> nothing to snapshot/set
return None
def normalize_speed_mode(value: Optional[str]) -> str:
"""Lower/strip a requested speed mode (dashes ok); None / "" -> off."""
if value is None:
return SPEED_OFF
normalized = str(value).strip().lower().replace("-", "_")
if not normalized:
return SPEED_OFF
if normalized not in SPEED_MODES:
raise ValueError(
f"Unsupported diffusion speed_mode '{value}'. Use one of: {', '.join(SPEED_MODES)}."
)
return normalized
def resolve_speed_mode(
value: Optional[str],
*,
is_gguf: bool,
dense_default: str = SPEED_OFF,
) -> str:
"""Unset resolves to default for GGUF and dense_default otherwise; an explicit value is honored."""
if value is None:
return SPEED_DEFAULT if is_gguf else dense_default
return normalize_speed_mode(value)
@lru_cache(maxsize = 1)
def torch_compile_runtime_available() -> bool:
"""Triton has no Windows wheel; TORCHDYNAMO_DISABLE is honored on every platform."""
if os.environ.get("TORCHDYNAMO_DISABLE", "").strip() not in ("", "0"):
return False
if sys.platform != "win32":
return True
try:
import triton # noqa: F401, PLC0415
except Exception: # noqa: BLE001 -- absent or broken Triton means eager, never a failed load
return False
try:
from .._msvc_env import crt_headers_reachable # noqa: PLC0415
return crt_headers_reachable()
except Exception: # noqa: BLE001 -- this runs during load; never fail it over a probe
return True
def compile_eligible(
target: Any,
*,
is_gguf: bool,
family: Any,
probe: bool = True,
) -> bool:
"""Whether the repeated block is regionally compiled: CUDA or ROCm bf16, or fp16 on NVIDIA
sm_75+, compile-friendly family with inductor; probe=False never runs the ROCm probe,
unprobed counts eligible, a recorded failure does not."""
del is_gguf
if not torch_compile_runtime_available():
return False
if not bool(getattr(target, "supports_default_torch_compile", False)):
return False
if not bool(getattr(family, "supports_torch_compile", True)):
return False
dtype = getattr(target, "dtype", None)
if _is_bfloat16(dtype):
return rocm_compile_probe_passes(target, probe = probe)
# fp16-incompatible families run in fp32 (unmeasured under compile) or in fp16 behind diffusion_fp16_guard, which
# must stay eager: the int8 fused MLP installed for compile replaces the guard's FFN forward.
if bool(getattr(family, "fp16_incompatible", False)):
return False
return (
_is_float16(dtype)
and _fp16_compile_capable(target)
and rocm_compile_probe_passes(target, probe = probe)
)
def rocm_compile_probe_passes(target: Any, *, probe: bool = True) -> bool:
if getattr(target, "backend", None) != "rocm":
return True
try:
from . import diffusion_rocm_compile as rc
if not probe:
return rc.rocm_block_compile_known(getattr(target, "ordinal", None)) is not False
return rc.rocm_block_compile_works(getattr(target, "ordinal", None))
except Exception: # noqa: BLE001
return False
def rocm_probe_load_scoped(fn: Callable[..., Any]) -> Callable[..., Any]:
"""Wrap a load so a ROCm probe OOM is remembered for that load only (no re-probe within it, a retry in the next)."""
from .diffusion_rocm_compile import load_scoped
return load_scoped(fn)
def rocm_compile_switched_off(target: Any) -> bool:
"""``UNSLOTH_DIFFUSION_ROCM_COMPILE=0`` on a ROCm target: no compile at all, the GGUF dequant one included."""
if getattr(target, "backend", None) != "rocm":
return False
try:
from .diffusion_rocm_compile import rocm_compile_allowed
return not rocm_compile_allowed()
except Exception: # noqa: BLE001
return True
def fp16_compile_explicit_only(target: Any) -> bool:
"""fp16 compiles only on an explicit tier, never the deferred profile (T4 SDXL-Turbo: ~245 s for ~0.1 s/image)."""
return _is_float16(getattr(target, "dtype", None))
def family_compiles_regionally(family: Any) -> bool:
"""False only for an EMPTY ``_repeated_blocks`` Studio cannot supply; unknown reads True."""
if getattr(family, "denoiser_attr", "transformer") != "transformer":
return True
name = getattr(family, "transformer_class", None)
if not isinstance(name, str) or not name:
return True
# Called before load_pipeline's guard; `import diffusers` imports torch._dynamo, so wait for the torch warm.
try:
from loggers import get_logger
from utils.torch_warmup import close_dynamo_import_window
close_dynamo_import_window(get_logger(__name__))
except Exception: # noqa: BLE001, S110 - optimisation only
pass
try:
import diffusers
cls = getattr(diffusers, name, None)
except Exception: # noqa: BLE001 - an unanswerable probe keeps today's behaviour
return True
if cls is None or not hasattr(cls, "_repeated_blocks"):
return True
if cls._repeated_blocks:
return True
from .diffusion_regional_compile import verified_repeated_blocks
return bool(verified_repeated_blocks(name))
def _is_bfloat16(dtype: Any) -> bool:
try:
import torch
return dtype is torch.bfloat16
except Exception:
return str(dtype).endswith("bfloat16")
def _is_float16(dtype: Any) -> bool:
try:
import torch
return dtype is torch.float16
except Exception:
return str(dtype).endswith("float16") and not str(dtype).endswith("bfloat16")
# Only sm_75 (T4) was measured; Volta and older stay eager.
_FP16_COMPILE_MIN_CAPABILITY = (7, 5)
def _fp16_compile_capable(target: Any) -> bool:
if getattr(target, "backend", "cuda") != "cuda":
return False
try:
import torch
ordinal = getattr(target, "ordinal", None)
cap = (
torch.cuda.get_device_capability()
if ordinal is None
else torch.cuda.get_device_capability(ordinal)
)
return tuple(cap)[:2] >= _FP16_COMPILE_MIN_CAPABILITY
except Exception: # noqa: BLE001 - an unanswerable probe keeps today's eager path
return False
def apply_speed_optims(
pipe: Any,
target: Any,
*,
is_gguf: bool,
family: Any,
speed_mode: str = SPEED_OFF,
cache_active: bool = False,
offload_active: bool = False,
cuda_graph_default: bool = True,
cache_engaged: Optional[bool] = None,
denoiser_offloaded: Optional[bool] = None,
stream_int8_gemm: bool = False,
logger: Any = None,
) -> dict[str, bool]:
"""Apply speed optims BEFORE placement/offload; offload hooks need compile without fullgraph."""
applied = {
"channels_last": False,
"vae_fp16_decode": False,
"vae_single_frame": False,
"vae_fused": False,
"cudnn_benchmark": False,
"tf32": False,
"fp16_accum": False,
"fused_qkv": False,
"compiled": False,
"compiled_dequant": False,
"rocm_query_chunks": False,
"compiled_vae_decode": False,
"cuda_graph": False,
"int8_gemm": False,
}
mode = normalize_speed_mode(speed_mode)
if getattr(target, "backend", None) in ("rocm", "mps"):
# Memory, not speed, so before the `off` return: ROCm has no fused SDPA for the VAE's head dim 384 / 512.
try:
from .diffusion_vae_attn_chunked import install as install_vae_attn_chunks
install_vae_attn_chunks(getattr(pipe, "vae", None), target, logger)
except Exception as exc: # noqa: BLE001 - keep stock attention
_warn(logger, "vae attention chunks", exc)
# Stability, not speed, so also before the `off` return.
_enable_miopen_immediate(logger)
if getattr(target, "backend", None) == "rocm":
try:
from . import diffusion_rocm_tunableop as rocm_tunableop
if mode == SPEED_OFF:
rocm_tunableop.uninstall(pipe)
else:
rocm_tunableop.install_for_pipe(pipe, getattr(target, "device", None), logger)
except Exception as exc: # noqa: BLE001 - keep torch's default GEMM pick
_warn(logger, "rocm tunableop", exc)
# TF32 (max) and cudnn.benchmark (any non-off CUDA load) are process-global; the caller restores them so a later
# `off` load never inherits them.
if mode == SPEED_OFF:
return applied
if mode in (SPEED_DEFAULT, SPEED_MAX):
_pin_compile_defaults()
if is_gguf:
_prepare_lowbit_iq_tables(pipe, target)
if getattr(target, "backend", None) == "rocm":
try:
from .diffusion_qwenimage21_rocm import install as install_rocm_query_chunks
applied["rocm_query_chunks"] = install_rocm_query_chunks(pipe, target, logger)
except Exception as exc: # noqa: BLE001 - keep stock attention on unsupported installs
_warn(logger, "qwen-image-2.1 ROCm query chunks", exc)
on_cuda = getattr(target, "device", None) == "cuda"
family_allows_compile = bool(getattr(family, "supports_torch_compile", True))
filter_reductions = compile_config.family_filters_reductions(family)
applied["vae_single_frame"] = _vae_single_frame(pipe, logger)
# Lossless: a channels-last VAE speeds up its convs with no numeric change.
applied["channels_last"] = _vae_channels_last(
pipe, logger, fused = on_cuda and _fused_vae_planned(pipe)
)
# Near-lossless, not bit-identical, so never on "off" (returned above).
applied["vae_fp16_decode"] = _video_vae_half_decode(pipe, target, family, logger)
if on_cuda:
applied["vae_fused"] = _install_fused_vae(
pipe, logger, encoder = getattr(family, "vae_fused_encoder", True)
)
if (
on_cuda
and getattr(family, "cudnn_benchmark", True)
and not _cudnn_benchmark_pointless(pipe)
):
applied["cudnn_benchmark"] = _enable_cudnn_benchmark(logger)
if on_cuda:
applied["fp16_accum"] = _enable_fp16_accumulation(
family, logger, dtype = getattr(target, "dtype", None), speed_mode = mode
)
# default: GGUF compiles only the dequant chain, dense uses regional compile; max: autotune.
if mode == SPEED_DEFAULT:
# Asked directly: this arm never reaches compile_eligible(), unlike the dense arm below.
if _gguf_dequant_compilable(is_gguf, on_cuda, family_allows_compile, target):
applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)
elif compile_eligible(target, is_gguf = is_gguf, family = family) and not fp16_unet_offloaded(
target, pipe, offload_active = offload_active
):
# A U-Net fuses QKV before its whole-module compile; DiTs fuse on max only.
if _denoiser_unet(pipe) is not None:
applied["fused_qkv"] = _fuse_qkv(pipe, logger)
applied["compiled"] = _compile_repeated_blocks(
pipe,
logger,
max_autotune = False,
cache_active = cache_active,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
onload_device = _onload_device(target) if stream_int8_gemm else None,
filter_reductions = filter_reductions,
)
elif mode == SPEED_MAX:
if compile_eligible(target, is_gguf = is_gguf, family = family) and not fp16_unet_offloaded(
target, pipe, offload_active = offload_active
):
applied["compiled"] = _compile_repeated_blocks(
pipe,
logger,
max_autotune = True,
cache_active = cache_active,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
onload_device = _onload_device(target) if stream_int8_gemm else None,
filter_reductions = filter_reductions,
)
if not applied["compiled"] and _gguf_dequant_compilable(
is_gguf, on_cuda, family_allows_compile, target
):
# No block compile (ROCm, or its setup failed): keep default's dequant compile rather than run slower.
applied["compiled_dequant"] = gguf_compile.install_compiled_dequant(logger)
if applied["compiled"]:
applied["int8_gemm"] = any(
bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)
)
if applied["compiled"] and _vae_decode_compile_allowed(pipe, mode):
# A U-Net keeps the decode recipe its whole-module compile was measured with.
applied["compiled_vae_decode"] = _compile_vae_decode(
pipe,
logger,
max_autotune = mode == SPEED_MAX and _denoiser_unet(pipe) is None,
eager_when_tiled = _vae_eager_when_tiled(pipe),
)
# Fused passes feed cuDNN channels-last activations; contiguous weights would be relaid out per call.
fused_cl = bool(getattr(getattr(pipe, "vae", None), "_unsloth_vae_fused_cl_weights", False))
if (
applied["channels_last"]
and not fused_cl
and not _channels_last_decode_wins(
pipe, target, applied["compiled_vae_decode"], offload_active
)
):
applied["channels_last"] = not _vae_contiguous(pipe, logger)
elif applied["compiled_vae_decode"] and not _channels_last_decode_wins(
pipe, target, False, offload_active
):
# A decode compile that fails at first call falls back eager: relayout then.
try:
pipe.vae._unsloth_eager_contiguous = True
except Exception: # noqa: BLE001
pass
if mode in (SPEED_DEFAULT, SPEED_MAX) and on_cuda and tf32_enabled():
applied["tf32"] = _enable_tf32(logger)
if mode == SPEED_MAX:
applied["fused_qkv"] = _fuse_qkv(pipe, logger)
# Deliberately NOT gated on applied["compiled"]: a launch-bound step survives the compile.
if mode in (SPEED_DEFAULT, SPEED_MAX):
cuda_graph = None
ok, reason = False, "cuda graph layer unavailable"
moved = bool(offload_active if denoiser_offloaded is None else denoiser_offloaded)
try:
from . import diffusion_cuda_graph as cuda_graph # noqa: PLC0415 - import cycle
after_placement = moved and cuda_graph.offload_graphs_enabled()
ok, reason = cuda_graph.graph_eligible(
target,
family = family,
pipe = pipe,
offload_active = moved and not after_placement,
cache_active = cache_active if cache_engaged is None else bool(cache_engaged),
speed_mode = mode,
family_default = cuda_graph_default,
logger = logger,
offloaded = after_placement,
)
if ok and after_placement:
# The slot and the recorded copies depend on the hooks placement installs: arm_graphs_after_placement.
ok, reason = False, "armed after placement"
try:
pipe._unsloth_cuda_graph_after_placement = True
# A family that opts in only offloaded must not get a resident graph if the denoiser stays put.
pipe._unsloth_cuda_graph_offload_only = not bool(
getattr(family, "supports_cuda_graph", cuda_graph_default)
)
except Exception: # noqa: BLE001
pass
except Exception as exc: # noqa: BLE001 - an unimportable graph layer means eager, never a failed load
_warn(logger, "cuda graph eligibility", exc)
# Stashed either way: status reports WHY graphs are off, not just that they are.
try:
pipe._unsloth_cuda_graph_reason = reason
except Exception: # noqa: BLE001
pass
if ok and cuda_graph is not None:
try:
applied["cuda_graph"] = bool(cuda_graph.install_cuda_graphs(pipe, logger = logger))
except Exception as exc: # noqa: BLE001 - the load proceeds eager
_warn(logger, "cuda graph capture", exc)
return applied
def _prepare_lowbit_iq_tables(pipe: Any, target: Any) -> None:
"""Low-bit IQ lookup tables on the execution device before compile, so a compiled block reads a device constant."""
device = getattr(target, "torch_device", None)
if not isinstance(device, str) or not device:
return
try:
from .diffusion_gguf_lowbit import prepare_lowbit_iq_tables
modules = list(_denoiser_dits(pipe))
unet = _denoiser_unet(pipe)
if unet is not None:
modules.append(unet)
if modules:
prepare_lowbit_iq_tables(device, *modules)
except Exception: # noqa: BLE001 - optimisation only: the lookup still builds traceably
pass
def _onload_device(target: Any) -> Optional[str]:
"""Onload device of an offloaded denoiser, else None. NVIDIA only: ROCm runs int8 weight-only, never this GEMM."""
if getattr(target, "device", None) != "cuda" or getattr(target, "backend", "cuda") != "cuda":
return None
device = getattr(target, "torch_device", None)
return str(device) if isinstance(device, str) and device.startswith("cuda") else "cuda"
def arm_graphs_after_placement(
pipe: Any,
applied: dict,
logger: Any = None,
) -> dict:
"""Arm the denoiser graphs for the placement that actually happened: replaces a resident-planned graph a hook now
wraps, and arms the one apply_speed_optims deferred for an offloaded denoiser (the copies are recorded in it)."""
pending = bool(getattr(pipe, "_unsloth_cuda_graph_after_placement", False))
hooked = bool(applied.get("cuda_graph")) and _denoiser_moves(pipe)
if not (pending or hooked):
return applied
try:
from . import diffusion_cuda_graph as cuda_graph # noqa: PLC0415 - import cycle
if not cuda_graph.offload_graphs_enabled():
cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
pipe._unsloth_cuda_graphs = ()
pipe._unsloth_cuda_graph_reason = "offload active"
applied["cuda_graph"] = False
return applied
from .diffusion_cfg_batch import cfg_pair_decline # noqa: PLC0415 - import cycle
if cfg_pair_decline(pipe, graphs = False) is None:
cuda_graph.uninstall_all(getattr(pipe, "_unsloth_cuda_graphs", ()) or ())
pipe._unsloth_cuda_graphs = ()
pipe._unsloth_cuda_graph_after_placement = False
pipe._unsloth_cuda_graph_reason = "guidance branches pair per streamed block"
applied["cuda_graph"] = False
if logger is not None:
logger.info(
"diffusion.cuda_graph: off for this load (both guidance branches share each streamed "
"block's upload instead)"
)
return applied
handles, reason = cuda_graph.arm_after_placement(pipe, logger = logger)
pipe._unsloth_cuda_graph_after_placement = False
pipe._unsloth_cuda_graph_reason = reason
applied["cuda_graph"] = bool(handles)
except Exception as exc: # noqa: BLE001 - the load proceeds eager
_warn(logger, "cuda graph arm after placement", exc)
applied["cuda_graph"] = False
return applied
def _denoiser_moves(pipe: Any) -> bool:
for module in _denoiser_dits(pipe) or [m for m in [_denoiser_unet(pipe)] if m is not None]:
if getattr(module, "_hf_hook", None) is not None:
return True
hooks = getattr(getattr(module, "_diffusers_hook", None), "hooks", None) or {}
if any("offload" in str(key) for key in hooks):
return True
try:
if any(getattr(m, "_hf_hook", None) is not None for m in module.modules()):
return True
except Exception: # noqa: BLE001
return True
return False
def engage_pinned_denoisers(
pipe: Any,
applied: dict,
logger: Any = None,
) -> dict:
"""Install the int8 GEMM the plan refused once every denoiser group is pinned (before the first forward). Graphs
stay off: an oversized request can still stream the groups, and the hooks' copy-stream wait breaks capture."""
if getattr(pipe, "_unsloth_cuda_graph_reason", None) == "offload active":
try:
pipe._unsloth_cuda_graph_reason = "denoiser pinned resident under offload hooks"
except Exception: # noqa: BLE001
pass
if not applied.get("compiled") or applied.get("int8_gemm"):
return applied
try:
from .diffusion_int8_gemm import install as install_int8_gemm
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "int8 fused-dequant gemm", exc)
return applied
for transformer in _denoiser_dits(pipe):
try:
transformer._unsloth_int8_gemm = install_int8_gemm(
transformer, logger, offload_active = False
)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "int8 fused-dequant gemm", exc)
applied["int8_gemm"] = any(
bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)
)
return applied
def fp16_unet_offloaded(target: Any, pipe: Any, *, offload_active: bool) -> bool:
"""An offloaded fp16 U-Net stays eager: fused QKV costs more transfer than compile saves (L4: 5.43 vs 4.86 s)."""
return (
bool(offload_active)
and _is_float16(getattr(target, "dtype", None))
and _denoiser_unet(pipe) is not None
)
def _vae_single_frame(pipe: Any, logger: Any) -> bool:
try:
from . import diffusion_vae_single_frame # noqa: PLC0415
return diffusion_vae_single_frame.install(getattr(pipe, "vae", None), logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "vae single-frame path", exc)
return False
# channels_last: slower on the stock path (72.7 vs 89 ms), faster once the fused norms install (104.7 vs 121.0 ms).
_VAE_CHANNELS_LAST_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})
# DiT-only VAEs: cudnn.benchmark saves ~3 ms per decode (fused, 1024) but re-tunes 0.7-2 s at every new resolution.
_CUDNN_BENCHMARK_DENY_VAES: frozenset[str] = frozenset({"AutoencoderKLQwenImage21"})
def _cudnn_benchmark_pointless(pipe: Any) -> bool:
if _denoiser_unet(pipe) is not None:
return False
return type(getattr(pipe, "vae", None)).__name__ in _CUDNN_BENCHMARK_DENY_VAES
def _vae_channels_last(
pipe: Any,
logger: Any,
*,
fused: bool = False,
) -> bool:
vae = getattr(pipe, "vae", None)
if vae is None or not hasattr(vae, "to"):
return False
if type(vae).__name__ in _VAE_CHANNELS_LAST_DENY and not fused:
return False
try:
import torch
if _has_conv3d(vae):
return _vae_channels_last_3d(vae, logger, fused = fused)
vae.to(memory_format = torch.channels_last)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "channels_last", exc)
return False
VAE_CHANNELS_LAST_3D_ENV = "UNSLOTH_VAE_CHANNELS_LAST_3D"
def _has_conv3d(vae: Any) -> bool:
modules = getattr(vae, "modules", None)
if not callable(modules):
return False
import torch
return any(isinstance(m, torch.nn.Conv3d) for m in modules())
def _vae_channels_last_3d(vae: Any, logger: Any, *, fused: bool) -> bool:
"""Per-rank channels_last(_3d) on post_quant_conv + decoder; ``Module.to(channels_last)`` raises at the first 5D
weight. Skipped without the fused passes (slower there); the encoder is kept (channels_last encoded slower)."""
import torch
name = type(vae).__name__
if os.environ.get(VAE_CHANNELS_LAST_3D_ENV, "").strip().lower() in _VAE_FALSE_TOKENS:
reason = f"{VAE_CHANNELS_LAST_3D_ENV}=0"
elif not fused:
reason = "3D-conv VAE without the fused VAE passes (channels_last_3d measured slower there)"
else:
reason = None
if reason is not None:
if logger is not None:
logger.info("diffusion.speed: channels_last skipped for %s: %s", name, reason)
return False
relaid = 0
touched: list = []
for part in (getattr(vae, "post_quant_conv", None), getattr(vae, "decoder", None)):
if part is None or not callable(getattr(part, "modules", None)):
continue
for module in part.modules():
weight = getattr(module, "weight", None)
if not isinstance(module, (torch.nn.Conv2d, torch.nn.Conv3d)) or not isinstance(
weight, torch.Tensor
):
continue
fmt = torch.channels_last if weight.dim() == 4 else torch.channels_last_3d
if not weight.is_contiguous(memory_format = fmt):
weight.data = weight.data.contiguous(memory_format = fmt)
touched.append(module)
relaid += 1
if touched:
# Relaid because fusion is planned; fusion matched by structure may still be rejected at its first decode,
# which restores these weights' layout (diffusion_vae_fused._restore_contiguous_layout).
vae.__dict__.setdefault("_unsloth_channels_last_modules", []).extend(touched)
if logger is not None:
logger.info(
"diffusion.speed: channels_last(_3d) decode weights on %s (%d convs)", name, relaid
)
return relaid > 0
VIDEO_VAE_HALF_ENV = "UNSLOTH_VIDEO_VAE_HALF"
# Kill switch: "0" reruns a non-finite fp16 decode in fp32 even when its latents are already non-finite (as before).
VIDEO_VAE_NONFINITE_INPUT_ENV = "UNSLOTH_VIDEO_VAE_NONFINITE_INPUT_KEEP_FP16"
def _vae_input_nonfinite_skips_fp32() -> bool:
return os.environ.get(VIDEO_VAE_NONFINITE_INPUT_ENV, "").strip().lower() not in (
"0",
"false",
"no",
"off",
)
def _video_vae_half_decode(pipe: Any, target: Any, family: Any, logger: Any) -> bool:
"""fp16 channels_last(_3d) decode for fp32-pinned video VAEs (Wan) on NVIDIA sm75+; non-finite output reruns fp32.
fp16, not bf16: bf16 is 9 dB further from fp32. ``UNSLOTH_VIDEO_VAE_HALF=0`` keeps fp32."""
if not getattr(family, "vae_force_fp32", False) or getattr(target, "device", None) != "cuda":
return False
if os.environ.get(VIDEO_VAE_HALF_ENV, "").strip().lower() in _VAE_FALSE_TOKENS:
if logger is not None:
logger.info("diffusion.speed: video VAE decode kept in fp32 (%s=0)", VIDEO_VAE_HALF_ENV)
return False
vae = getattr(pipe, "vae", None)
decoder = getattr(vae, "decoder", None)
original = getattr(vae, "decode", None)
if decoder is None or not callable(original):
return False
# A dual-DiT family calls apply_speed_optims once per expert over the same VAE.
if getattr(vae, "_unsloth_half_decode", False):
return True
if getattr(target, "backend", None) != "cuda" or not _fp16_compile_capable(target):
return False
try:
import functools
import torch
parts = [m for m in (getattr(vae, "post_quant_conv", None), decoder) if m is not None]
for part in parts:
part.to(torch.float16)
for module in part.modules():
weight = getattr(module, "weight", None)
if isinstance(module, (torch.nn.Conv2d, torch.nn.Conv3d)) and weight is not None:
fmt = torch.channels_last if weight.dim() == 4 else torch.channels_last_3d
weight.data = weight.data.contiguous(memory_format = fmt)
fell_back: list = []
@functools.wraps(original)
def decode(z: Any, *args: Any, **kwargs: Any) -> Any:
if fell_back or not torch.is_tensor(z):
return original(z, *args, **kwargs)
out = original(z.to(torch.float16), *args, **kwargs)
sample = out[0] if isinstance(out, tuple) else getattr(out, "sample", out)
if not torch.is_tensor(sample) or bool(torch.isfinite(sample).all()):
return out
# Non-finite latents stay non-finite in fp32; the rerun only doubles memory (T4 OOM).
if _vae_input_nonfinite_skips_fp32() and not bool(torch.isfinite(z).all()):
if logger is not None:
logger.warning(
"diffusion.speed: the latents reaching the VAE are not finite; fp16 decode kept"
)
return out
if logger is not None:
logger.warning(
"diffusion.speed: fp16 VAE decode was not finite; decoding in fp32 from now on"
)
for part in parts:
part.to(torch.float32)
fell_back.append(True)
return original(z, *args, **kwargs)
vae.decode = decode
vae._unsloth_half_decode = True
return True
except Exception as exc: # noqa: BLE001 - optimisation only
try:
import torch
for part in (getattr(vae, "post_quant_conv", None), decoder):
if part is not None:
part.to(torch.float32)
except Exception: # noqa: BLE001, S110 - best-effort restore of the fp32 pin
pass
_warn(logger, "video vae fp16 decode", exc)
return False
# U-Net denoisers ship no ``_repeated_blocks``, so the regional compile cannot reach them; these classes get a
# WHOLE-module STATIC compile. SDXL (B200, 30 steps / 1024px): 26.9 vs 45.9 ms/step, 1.61x end-to-end at LPIPS 0.034.
# Static means a recompile per (height, width, batch).
_UNET_WHOLE_COMPILE: frozenset[str] = _targets.UNET_WHOLE_COMPILE
def _denoiser_unet(pipe: Any) -> Any:
"""The pipe's U-Net denoiser when its class is on the whole-compile list, else None."""
unet = getattr(pipe, "unet", None)
if unet is not None and type(unet).__name__ in _UNET_WHOLE_COMPILE:
return unet
return None
def fused_qkv_added_bytes(pipe: Any) -> int:
"""Device bytes the default tier's U-Net QKV fuse adds. diffusers' ``fuse_projections`` builds ``to_qkv`` /
``to_kv`` beside the ``to_q`` / ``to_k`` / ``to_v`` it keeps, so a U-Net that moves whole (model offload) or stays
resident carries both (SDXL: 4897 MiB grows to 6132 MiB). 0 when the tier would not fuse."""
unet = _denoiser_unet(pipe)
if unet is None:
return 0
try:
import torch
if _quantised_projections(pipe):
return 0
if unfused_projections_will_drop(unet):
return 0 # the fused weights replace the projections they were built from
total = 0
for module in unet.modules():
if not callable(getattr(module, "fuse_projections", None)) or getattr(
module, "fused_projections", False
):
continue
names = (
("to_k", "to_v")
if getattr(module, "is_cross_attention", False)
else ("to_q", "to_k", "to_v")
)
if getattr(module, "add_q_proj", None) is not None:
names += ("add_q_proj", "add_k_proj", "add_v_proj")
for name in names:
linear = getattr(module, name, None)
for t in (getattr(linear, "weight", None), getattr(linear, "bias", None)):
if isinstance(t, torch.Tensor):
total += t.numel() * t.element_size()
return total
except Exception: # noqa: BLE001 - unreadable: no extra bytes claimed
return 0
def compiled_shapes_are_static(pipe: Any, speed_mode: Optional[str]) -> bool:
"""Whether this load's compiled artifacts are per-(width, height, batch); the compile cache keys on it."""
mode = normalize_speed_mode(speed_mode)
if mode == SPEED_MAX:
# Auto-dynamic DiT reuses its graph for new sizes; the Dynamo graph-count delta marks compiles.
return _denoiser_unet(pipe) is not None or not auto_dynamic_active(pipe)
if mode != SPEED_DEFAULT:
return False
return (
_denoiser_unet(pipe) is not None
or any(getattr(t, "_unsloth_auto_dynamic", False) for t in _denoiser_dits(pipe))
or _dits_merge_streams(_denoiser_dits(pipe))
)
def _denoiser_dits(pipe: Any) -> list:
"""Every DiT the denoise loop runs: the primary ``transformer`` plus a second expert some
families carry (Ideogram's ``unconditional_transformer``, an MoE ``transformer_2``). Speed /
attention optims must reach ALL of them (mirroring the offload path), else the second DiT runs
eager / native while status over-reports the optim as engaged."""
return _targets.denoiser_dits(pipe)
def int8_gemm_live(pipe: Any, optims: Any) -> list:
"""``speed_optims`` with ``int8_gemm`` dropped once a deferred install's first-forward probe swapped nothing."""
optims = list(optims or ())
if "int8_gemm" in optims and not any(
bool(getattr(t, "_unsloth_int8_gemm", 0)) for t in _denoiser_dits(pipe)
):
optims.remove("int8_gemm")
return optims
# Blocks MEASURED to raise inductor CantSplit under dynamic = True; measure before adding.
_STREAM_MERGING_BLOCKS: frozenset[str] = frozenset({"FluxSingleTransformerBlock"})
_STREAM_MERGE_DETECT_ENV = "UNSLOTH_STATIC_STREAM_MERGE_DETECT"
_STREAM_MERGE_SOURCE = re.compile(
r"torch\.cat\(\s*\[\s*(?:encoder_hidden_states\s*,\s*hidden_states"
r"|hidden_states\s*,\s*encoder_hidden_states)\s*\]"
)
@lru_cache(maxsize = None)
def _class_merges_streams(cls: type, broad: bool = False) -> bool:
"""Whether one repeated-block CLASS needs a static compile; ``broad`` is an argument so the memo keys on it."""
if cls.__name__ in _STREAM_MERGING_BLOCKS:
return True
if not broad:
return False
try:
import inspect # noqa: PLC0415 - only reached under the opt-in broad sweep
source = inspect.getsource(cls.forward)
except Exception: # noqa: BLE001 - no source (frozen / C ext) means fall back to the name list
return False
return bool(_STREAM_MERGE_SOURCE.search(source))
def _divisibility_proof_available() -> bool:
"""Whether inductor proves ``(k*a + k*b) % (a + b) == 0`` (torch 2.14+, or the backport; see diffusion_inductor_backports)."""
try:
from . import diffusion_inductor_backports # noqa: PLC0415 - imports torch
return diffusion_inductor_backports.proof_available()
except Exception: # noqa: BLE001 - unanswerable: keep the static fallback
return False
def _dits_merge_streams(dits: list) -> bool:
"""Whether a stream-merging block must compile static: not once inductor can prove the CantSplit split."""
if _divisibility_proof_available():
return False
broad = os.environ.get(_STREAM_MERGE_DETECT_ENV) == "1"
seen: set[type] = set()
for transformer in dits:
names = set(getattr(transformer, "_repeated_blocks", ()) or ())
if not names:
continue
try:
modules = list(transformer.named_modules())
except Exception: # noqa: BLE001 - a probe, never a failed load
continue
for _name, sub in modules:
cls = type(sub)
if cls.__name__ not in names or cls in seen:
continue
seen.add(cls)
if _class_merges_streams(cls, broad):
return True
return False
def _compile_repeated_blocks(
pipe: Any,
logger: Any,
*,
max_autotune: bool = False,
cache_active: bool = False,
offload_active: bool = False,
denoiser_offloaded: Optional[bool] = None,
onload_device: Any = None,
filter_reductions: bool = False,
) -> bool:
dits = [
t for t in _denoiser_dits(pipe) if callable(getattr(t, "compile_repeated_blocks", None))
]
unet = _denoiser_unet(pipe) if not dits else None
if not dits and unet is None:
return False
_install_inductor_backports(logger)
# Before the stream-merge probe below, which reads the same block names.
for transformer in dits:
try:
from .diffusion_regional_compile import ensure_repeated_blocks
ensure_repeated_blocks(transformer)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "repeated block discovery", exc)
# dynamic=False recompiled per prompt length on text-token DiTs.
# Inductor cudagraph modes fail on regional blocks, so capture happens at the denoiser.
static_shapes = (not max_autotune) and _dits_merge_streams(dits)
if static_shapes and logger is not None:
logger.info(
"diffusion.speed: regional compile is static; this DiT merges the text and image streams "
"inside its repeated block and cannot be codegen'd with dynamic sequence lengths",
)
kwargs: dict[str, Any] = {
"fullgraph": not (cache_active or offload_active),
"dynamic": None if max_autotune else (not static_shapes),
}
if max_autotune:
kwargs["mode"] = "max-autotune-no-cudagraphs"
if filter_reductions:
pin_reduction_configs(kwargs, logger)
try:
import torch
# Via compile_config: the render thread on torch 2.12+ cannot see these writes.
# recompile_limit 64: heterogeneous-block DiTs exceed the default 8 under fullgraph.
# Not force_parameter_static_shapes=False: no win and ~6x slower.
dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
if dynamo_cfg is not None:
for _limit_attr in ("recompile_limit", "cache_size_limit"): # name varies by torch ver
if hasattr(dynamo_cfg, _limit_attr):
current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0
compile_config.set_knob(_DYNAMO_MODULE, _limit_attr, max(current, 64))
# Match eager bf16 rounding in fused pointwise kernels (multi-step denoise amplifies drift).
# Process-global: snapshot_backend_flags restores it on unload.
inductor_cfg = _inductor_config()
if inductor_cfg is not None and hasattr(inductor_cfg, "emulate_precision_casts"):
compile_config.set_knob(_INDUCTOR_MODULE, "emulate_precision_casts", True)
# Before the first compiled forward: otherwise every restart misses the FX graph cache on libdevice.
try:
from . import diffusion_aot_cache
diffusion_aot_cache.pin_triton_libdevice(logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "libdevice pin", exc)
# One R0_BLOCK per reduction: see diffusion_compile_config.DYNAMIC_SCALE_RBLOCK_ENV.
if (
inductor_cfg is not None
and hasattr(inductor_cfg, "dynamic_scale_rblock")
and compile_config.reduction_blocks_pinned()
):
compile_config.set_knob(_INDUCTOR_MODULE, "dynamic_scale_rblock", False)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "compile_repeated_blocks", exc)
return False
# A block whose attention runs inside sdpa_kernel would otherwise bypass AOTAutogradCache on every start.
try:
from . import diffusion_aot_cache
diffusion_aot_cache.install(logger)
diffusion_aot_cache.pin_autotune_key(logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "sdpa aot cache", exc)
if unet is not None:
# dynamic always False, so each new (height, width, batch) recompiles.
unet_kwargs: dict[str, Any] = {"fullgraph": kwargs["fullgraph"], "dynamic": False}
if max_autotune:
unet_kwargs["mode"] = "max-autotune-no-cudagraphs"
if filter_reductions:
pin_reduction_configs(unet_kwargs, logger)
if unet_compiles_regionally(unet, max_autotune = max_autotune):
return _compile_unet_blocks(unet, unet_kwargs, logger)
try:
unet.compile(**unet_kwargs)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "unet whole-module compile", exc)
return False
# Compile every denoiser DiT (dual-DiT families run both); a per-DiT failure degrades only that one to eager.
engaged = False
for transformer in dits:
dit_kwargs = dict(kwargs)
dit_kwargs["dynamic"] = compile_dynamic(transformer, kwargs["dynamic"])
if (
max_autotune
and dit_kwargs["dynamic"] is None
and _carries_torchao_weights(transformer)
and _dits_merge_streams([transformer])
):
# Auto dynamic hits CantSplit on torchao stream-merging blocks and drops to eager; static recompiles.
dit_kwargs["dynamic"] = False
# Read by auto_dynamic_active: the generalising recompile on a new text length must reach the bundle.
transformer._unsloth_auto_dynamic = dit_kwargs["dynamic"] is None
# Registered per-denoiser patches (diffusion_autoopt.patches), class gated, before the int8 swaps.
_install_compile_patches(
"pre_int8",
transformer,
logger,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
)
# Before the compile: the regional compile traces whatever forward the blocks carry at the first call.
try:
from .diffusion_int8_fused import install as install_int8_fused
install_int8_fused(transformer, logger, offload_active = offload_active)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "int8 fused mlp", exc)
# After the fused MLP (its down projection calls the same GEMM), before the compile traces the Linears.
try:
from .diffusion_int8_gemm import install as install_int8_gemm
# Keyed on the DENOISER's placement; a weight elsewhere at call time keeps the stock path.
moved = offload_active if denoiser_offloaded is None else bool(denoiser_offloaded)
if moved and onload_device is not None:
transformer._unsloth_int8_gemm = install_int8_gemm(
transformer, logger, device = onload_device
)
else:
transformer._unsloth_int8_gemm = install_int8_gemm(
transformer, logger, offload_active = moved
)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "int8 fused-dequant gemm", exc)
# Patches run after the int8 GEMM swap; ConvRot sharing reads the swap's leftovers.
_install_compile_patches(
"pre_compile",
transformer,
logger,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
)
try:
transformer.compile_repeated_blocks(**dit_kwargs)
engaged = True
# Kept so a caller compiling below an offload hook (MiniMax-H3) uses these exact compile settings.
try:
transformer._unsloth_regional_compile_kwargs = dict(dit_kwargs)
except Exception: # noqa: BLE001 - not a settable module (tests/fakes)
pass
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "compile_repeated_blocks", exc)
continue
# dynamic=True (dense H3) still needs the unbacked temb, else step 1 compiles a second graph.
try:
from . import diffusion_dynamic_text
diffusion_dynamic_text.install(transformer, logger, dynamic = dit_kwargs["dynamic"])
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "dynamic text dims", exc)
# Compile is lazy (first forward in generate()): guard blocks so failures drop to eager.
guard_compiled_blocks(transformer, logger)
# After the guard (it wraps the guarded call): FLUX.1's first single block otherwise compiles a second graph.
try:
from . import diffusion_block_restride
diffusion_block_restride.install(transformer, logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "block restride", exc)
# Inductor turns the prefix KV cache's clone into a view of the full K/V buffer, which pins it for the render.
try:
from .diffusion_prefix_kv import install_prefix_kv_compaction
install_prefix_kv_compaction(transformer, logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "prefix kv compaction", exc)
# A step cache engaged earlier wrapped block forwards; re-point them at compiled wrappers.
try:
from .diffusion_cache import _compile_hooked_block_inners
_compile_hooked_block_inners(transformer, logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "cache-hook inner compile", exc)
return engaged
def _unet_blocks(unet: Any) -> list:
names = set(_targets.UNET_REGIONAL_BLOCKS)
try:
return [m for m in unet.modules() if m is not unet and type(m).__name__ in names]
except Exception: # noqa: BLE001 - not a torch module
return []
def unet_compiles_regionally(unet: Any, *, max_autotune: bool = False) -> bool:
return (
unet is not None
and not max_autotune
and _targets.unet_regional_enabled()
and bool(_unet_blocks(unet))
)
_UNET_RECOMPILE_LIMIT = 256
def _compile_unet_blocks(unet: Any, unet_kwargs: dict[str, Any], logger: Any) -> bool:
"""Compile each repeated U-Net block (guarded; the outer forward stays eager for the CUDA graph)."""
count = 0
try:
# One ResnetBlock2D forward serves ~14 graphs per (height, width, batch): 64 would stop at the 5th size.
import torch
dynamo_cfg = getattr(getattr(torch, "_dynamo", None), "config", None)
for _limit_attr in ("recompile_limit", "cache_size_limit"):
if dynamo_cfg is not None and hasattr(dynamo_cfg, _limit_attr):
current = compile_config.get_knob(_DYNAMO_MODULE, _limit_attr) or 0
compile_config.set_knob(
_DYNAMO_MODULE, _limit_attr, max(current, _UNET_RECOMPILE_LIMIT)
)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "unet recompile limit", exc)
try:
for module in _unet_blocks(unet):
module.compile(**unet_kwargs)
count += 1
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "unet regional compile", exc)
for module in unet.modules():
if module is not unet and getattr(module, "_compiled_call_impl", None) is not None:
module._compiled_call_impl = None
return False
if not count:
return False
guard_compiled_blocks(unet, logger)
try:
unet._unsloth_regional_compile_kwargs = dict(unet_kwargs)
unet._unsloth_bg_yield = True
except Exception: # noqa: BLE001 - not a settable module (tests/fakes)
pass
if logger is not None:
logger.info(
"diffusion.speed: %s compiles per block (%d blocks: %s); set %s=0 for the whole-module compile",
type(unet).__name__,
count,
", ".join(_targets.UNET_REGIONAL_BLOCKS),
_targets.UNET_REGIONAL_ENV,
)
return True
def _install_compile_patches(
stage: str,
transformer: Any,
logger: Any,
*,
offload_active: bool,
denoiser_offloaded: Optional[bool],
) -> None:
try:
from .diffusion_autoopt.apply import install_compile_patches
install_compile_patches(
stage,
transformer,
logger,
offload_active = offload_active,
denoiser_offloaded = denoiser_offloaded,
)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, f"{stage} patches", exc)
def pin_reduction_configs(kwargs: dict[str, Any], logger: Any = None) -> bool:
"""Pins one config per reduction via per-compile options, never the global knob; no benchmark pick."""
if not compile_config.reduction_config_filter_available():
return False
try:
options = dict(kwargs.get("options") or {})
mode = kwargs.get("mode")
if mode is not None:
from torch._inductor import list_mode_options
options = {**list_mode_options(mode, kwargs.get("dynamic")), **options}
options[REDUCTION_FILTER_OPTION] = True
# Max's coordinate descent retimes the filtered config in every process (its guard reads deterministic).
if options.get("coordinate_descent_tuning"):
options["coordinate_descent_tuning"] = False
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "reduction config filter", exc)
return False
kwargs.pop("mode", None)
kwargs["options"] = options
return True
def _install_inductor_backports(logger: Any) -> bool:
"""Probe-gated backport of torch 2.14's CantSplit divisibility proof for 2.12 / 2.13. Never fails a load."""
try:
from . import diffusion_inductor_backports
return diffusion_inductor_backports.install(logger)
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "inductor backports", exc)
return False
def compile_dynamic(transformer: Any, dynamic: Optional[bool]) -> Optional[bool]:
"""dynamic=True breaks torchao's per-row quant fusion on torch 2.12/2.13 unpatched; None is faster."""
# Static kernels for video DiTs: dynamic shapes made LTX-2.3's QK-norm + RoPE ~3x slower.
if transformer is not None and getattr(transformer, "_unsloth_compile_static", False):
return False
if dynamic and transformer is not None and _carries_torchao_weights(transformer):
return None
return dynamic
def _carries_torchao_weights(module: Any) -> bool:
"""Only torchao's own tensor types count; GGUF or bitsandbytes subclasses keep their compile policy."""
return any(True for _ in torchao_weight_tensors(module))
def torchao_weight_tensors(module: Any) -> Iterator[Any]:
"""Each torchao tensor subclass holding a parameter of ``module``: the parameter itself, or its ``.data`` when a
plain ``nn.Parameter`` wraps the subclass. Yields nothing for a module that cannot be read (tests/fakes)."""
def _torchao(t: Any) -> bool:
return t is not None and type(t).__module__.startswith("torchao")
try:
parameters = list(module.parameters())
except Exception: # noqa: BLE001 - not a torch module (tests/fakes)
return
for p in parameters:
if _torchao(p):
yield p
else:
data = getattr(p, "data", None)
if _torchao(data):
yield data
def is_compile_failure(exc: BaseException) -> bool:
"""True for an error torch.compile raises while BUILDING a graph (dynamo tracing, inductor lowering / codegen), as
opposed to one raised by kernels that already built. Only the former is safe to answer with an eager retry: nothing
ran yet. An OOM is never one, even when inductor wraps it (autotune ran out), so the OOM backoff still sees it."""
try:
import torch
except Exception: # noqa: BLE001 - no torch, nothing compiled
return False
from .diffusion_batched import is_oom_error
# OOM anywhere in the cause chain goes to the batch backoff, not the eager retry.
if is_oom_error(exc):
return False
kinds: list = []
dynamo_exc = getattr(getattr(torch, "_dynamo", None), "exc", None)
for name in (
"BackendCompilerFailed",
"Unsupported",
"TorchRuntimeError",
"InternalTorchDynamoError",
):
kind = getattr(dynamo_exc, name, None) if dynamo_exc is not None else None
if isinstance(kind, type):
kinds.append(kind)
try:
from torch._inductor.exc import (
InductorError,
) # torch 2.7+; older torch wraps it in BackendCompilerFailed
kinds.append(InductorError)
except Exception: # noqa: BLE001
pass
return bool(kinds) and isinstance(exc, tuple(kinds))
def _drop_tracebacks(exc: BaseException) -> None:
from .diffusion_vae_oom import drop_tracebacks # noqa: PLC0415
drop_tracebacks(exc)
class _CompileGuard:
"""Per-DiT record of a compile that failed at runtime. The first failure flips every guarded block of that DiT to
eager (one shared decision: a half-compiled stack would retry the same broken lowering per block)."""
def __init__(self, logger: Any) -> None:
self.logger = logger
self.error: Optional[str] = None
self.restores: list = []
def fail(self, exc: BaseException, owner: Any) -> None:
if self.error is None:
self.error = f"{type(exc).__name__}: {str(exc).splitlines()[0] if str(exc) else ''}"[
:300
]
for restore in self.restores:
try:
restore()
except Exception: # noqa: BLE001 - per-block best-effort; the guard still routes to eager
pass
if self.logger is not None:
self.logger.warning(
"diffusion.speed: torch.compile failed on %s at the first forward (%s); this load runs eager",
type(owner).__name__,
self.error,
)
def wrap(self, compiled: Any, eager: Any, owner: Any) -> Any:
guard = self
def guarded(*args: Any, **kwargs: Any) -> Any:
# Compile still in flight in the background: entering it here would compile inline.
if guard.error is None and not _bg_eager_forced():
_bg_yield_point()
compile_config.apply()
try:
return compiled(*args, **kwargs)
except Exception as exc: # noqa: BLE001 - reraised unless a compile-time failure
if not is_compile_failure(exc):
raise
guard.fail(exc, owner)
# The handled exception pins inductor's frames (and the traced fake tensors) through the retry.
_drop_tracebacks(exc)
return eager(*args, **kwargs)
guarded._unsloth_compile_guard = guard
return guarded
def guard_compiled_blocks(transformer: Any, logger: Any = None) -> int:
"""Wrap each ``Module.compile``d submodule's ``_compiled_call_impl`` so a compile-time failure on the first forward
(e.g. inductor ``CantSplit`` on a torchao-quantised Qwen-Image-2.1 block) falls back to that module's eager
``_call_impl`` for this and every later call. Idempotent. Returns the number of modules guarded."""
guard = getattr(transformer, "_unsloth_compile_guard", None)
if guard is None:
guard = _CompileGuard(logger)
try:
transformer._unsloth_compile_guard = guard
except Exception: # noqa: BLE001 - not a settable module (tests/fakes)
return 0
count = 0
try:
modules = list(transformer.modules())
except Exception: # noqa: BLE001 - not a torch module
return 0
for module in modules:
compiled = getattr(module, "_compiled_call_impl", None)
if compiled is None or getattr(compiled, "_unsloth_compile_guard", None) is not None:
continue
eager = getattr(module, "_call_impl", None)
if eager is None:
continue
def restore(m: Any = module) -> None:
m._compiled_call_impl = None
guard.restores.append(restore)
module._compiled_call_impl = guard.wrap(compiled, eager, transformer)
count += 1
return count
def _guarded_dits(pipe: Any) -> list:
"""Every DiT a compile guard can sit on: the standard denoisers plus a modular workflow's named partition
(MiniMax-H3's ``transformer_ref`` compiles through a view, so the bare pipe's ``_denoiser_dits`` misses it)."""
dits = _denoiser_dits(pipe)
for attr in _targets.PARTITION_ATTRS:
extra = getattr(pipe, attr, None)
if extra is not None and extra not in dits:
dits.append(extra)
unet = _denoiser_unet(pipe)
if (
unet is not None
and unet not in dits
and getattr(unet, "_unsloth_compile_guard", None) is not None
):
dits.append(unet)
return dits
def compiled_dits_active(pipe: Any) -> bool:
"""True while at least one guarded DiT still runs its compiled blocks (dual-DiT pipelines fall back per DiT)."""
for transformer in _guarded_dits(pipe):
guard = getattr(transformer, "_unsloth_compile_guard", None)
if guard is not None and not guard.error:
return True
return False
def settle_compile_fallback(
state: Any,
pipe: Any,
logger: Any = None,
) -> Optional[str]:
"""Adds compile_fallback_eager once; drops compiled only when no guarded DiT still runs compiled."""
dit_error = compile_fallback_error(pipe)
vae_error = _vae_compile_error(getattr(pipe, "vae", None))
dequant_error = gguf_compile.compile_error()
fallback = dit_error or vae_error or dequant_error
if not fallback:
return None
optims = tuple(getattr(state, "speed_optims", None) or ())
updated = list(optims)
if "compile_fallback_eager" not in updated:
updated.append("compile_fallback_eager")
if logger is not None:
logger.warning("diffusion.speed: regional compile fell back to eager: %s", fallback)
if dit_error and "compiled" in updated and not compiled_dits_active(pipe):
updated.remove("compiled")
if vae_error and "compiled_vae_decode" in updated:
updated.remove("compiled_vae_decode")
if dequant_error and "compiled_dequant" in updated:
updated.remove("compiled_dequant")
if tuple(updated) != optims:
# The load states are frozen dataclasses; the status must still reflect what actually runs.
object.__setattr__(state, "speed_optims", tuple(updated))
return fallback
def auto_dynamic_active(pipe: Any) -> bool:
"""Whether any denoiser DiT compiled with automatic dynamic (the max tier, or torchao weights on the default)."""
return any(getattr(t, "_unsloth_auto_dynamic", False) for t in _guarded_dits(pipe))
def dynamo_graph_count() -> int:
"""Graphs dynamo has compiled in this process (0 when unavailable); a delta across a render means it compiled."""
try:
from torch._dynamo.utils import counters
return int(counters["stats"]["unique_graphs"])
except Exception: # noqa: BLE001
return 0
def fresh_compile_count() -> int:
"""FX graph cache misses. Not ``dynamo_graph_count``: it also grows on cache-served retraces, rewriting bundles."""
try:
from torch._dynamo.utils import counters
return int(counters["inductor"]["fxgraph_cache_miss"])
except Exception: # noqa: BLE001
return 0
def compile_fallback_error(pipe: Any) -> Optional[str]:
"""The compile failure a guarded DiT fell back from, or None while every compiled DiT still runs compiled."""
for transformer in _guarded_dits(pipe):
guard = getattr(transformer, "_unsloth_compile_guard", None)
if guard is not None and guard.error:
return guard.error
return None
COMPILE_VAE_ENV = "UNSLOTH_DIFFUSION_COMPILE_VAE"
_VAE_TRUE_TOKENS = ("1", "true", "yes", "on")
_VAE_FALSE_TOKENS = ("0", "false", "no", "off")
# Not worth compiling (Qwen-Image VAE). Keyed by class: the marker comes after fingerprinting.
_VAE_COMPILE_DENY: frozenset[str] = frozenset({"AutoencoderKLQwenImage", "AutoencoderKLWan"})
# ``auto`` compiles only measured VAEs: video DiTs also pass through apply_speed_optims.
_VAE_COMPILE_ALLOW: frozenset[str] = frozenset({"AutoencoderKL", "AutoencoderKLFlux2"})
def vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:
"""Compile-cache key input; False for a U-Net so its existing bundles keep their key."""
return _denoiser_unet(pipe) is None and _vae_decode_compile_allowed(pipe, speed_mode)
def _install_fused_vae(
pipe: Any,
logger: Any,
encoder: bool = True,
) -> bool:
"""Triton-fused VAE norm passes (diffusion_vae_fused), unless the env forces the VAE decode compile instead.
``encoder=False`` leaves the VAE encoder stock (the family's ``vae_fused_encoder``)."""
if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:
return False
try:
from . import diffusion_vae_fused # noqa: PLC0415 - Triton import only on CUDA loads
return diffusion_vae_fused.install(getattr(pipe, "vae", None), logger, encoder = encoder) > 0
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fused vae", exc)
return False
def _fused_vae_planned(pipe: Any) -> bool:
if os.environ.get(COMPILE_VAE_ENV, "").strip().lower() in _VAE_TRUE_TOKENS:
return False
try:
from . import diffusion_vae_fused # noqa: PLC0415
return diffusion_vae_fused.will_install(getattr(pipe, "vae", None))
except Exception: # noqa: BLE001
return False
def _vae_decode_compile_allowed(pipe: Any, speed_mode: str) -> bool:
"""U-Nets always; a DiT only in max mode or when the env forces it; never with the fused eager VAE."""
if _fused_vae_planned(pipe):
return False
if _denoiser_unet(pipe) is not None:
return True
# H3's fused decoder never calls the blocks, so compiling them would be reported but never run.
if getattr(getattr(pipe, "vae", None), "_unsloth_decode_blocks_bypassed", False):
return False
raw = os.environ.get(COMPILE_VAE_ENV, "").strip().lower()
if raw in _VAE_FALSE_TOKENS:
return False
if raw in _VAE_TRUE_TOKENS:
return True
if speed_mode != SPEED_MAX:
return False
name = type(getattr(pipe, "vae", None)).__name__
return name in _VAE_COMPILE_ALLOW and name not in _VAE_COMPILE_DENY
def _vae_eager_when_tiled(pipe: Any) -> bool:
"""A DiT decode not forced on by the env runs eager while tiled; U-Nets keep their measured recipe."""
if pipe is not None and _denoiser_unet(pipe) is not None:
return False
return os.environ.get(COMPILE_VAE_ENV, "").strip().lower() not in _VAE_TRUE_TOKENS
def _vae_contiguous(pipe: Any, logger: Any) -> bool:
try:
import torch
pipe.vae.to(memory_format = torch.contiguous_format)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "contiguous vae", exc)
return False
# VAE classes whose decode layout was measured; any other keeps channels_last.
_VAE_LAYOUT_MEASURED: frozenset[str] = frozenset({"AutoencoderKL", "AutoencoderKLFlux2"})
def _channels_last_decode_wins(
pipe: Any,
target: Any,
compiled_decode: bool,
offload_active: bool = False,
) -> bool:
"""NVIDIA-measured: eager 16-bit is faster contiguous (no NHWC GroupNorm) but offload keeps channels_last (lower peak); compiled 16-bit wins channels_last; fp32 wins contiguous."""
vae = getattr(pipe, "vae", None)
if (
getattr(target, "backend", "cuda") != "cuda"
or getattr(target, "device", None) != "cuda"
or type(vae).__name__ not in _VAE_LAYOUT_MEASURED
):
return True
config = getattr(vae, "config", None)
# SDXL pipelines upcast a force_upcast fp16 VAE to fp32 for the decode.
fp32_decode = str(getattr(vae, "dtype", "")).endswith("float32") or (
bool(getattr(config, "force_upcast", False))
and _is_float16(getattr(vae, "dtype", None))
and _denoiser_unet(pipe) is not None
)
if fp32_decode:
return False
return compiled_decode or offload_active
def _guard_compiled_decode(
vae: Any,
compiled: Any,
eager: Any,
logger: Any,
eager_when_tiled: bool = False,
owner: Any = None,
) -> Any:
"""Lazy compile fails on first call; OOMs still raise; eager_when_tiled sends tiled decode eager."""
owner = vae if owner is None else owner
had_own = "decode" in getattr(owner, "__dict__", {})
failed: list = []
def guarded(*args: Any, **kwargs: Any) -> Any:
if eager_when_tiled and getattr(vae, "use_tiling", False):
return eager(*args, **kwargs)
# the one force-eager signal (a background compile in flight, the OOM scope's eager retry)
if _bg_eager_forced():
return eager(*args, **kwargs)
if not failed:
try:
return compiled(*args, **kwargs)
except Exception as exc: # noqa: BLE001 - reraised unless a compile-time failure
if not is_compile_failure(exc):
raise
error = f"{type(exc).__name__}: {str(exc).splitlines()[0] if str(exc) else ''}"[
:300
]
failed.append(error)
try:
vae._unsloth_compile_decode_error = error
vae._unsloth_compiled_decode = False
# Only while the slot still holds this guard: a layer installed over it since (the OOM scope)
# stays, and this guard (``failed``) runs eager under it from now on.
if getattr(owner, "__dict__", {}).get("decode") is guarded:
if had_own:
owner.decode = eager
else:
del owner.decode
except Exception: # noqa: BLE001 - `failed` still routes this wrapper to eager
pass
if getattr(vae, "_unsloth_eager_contiguous", False):
try:
import torch
vae.to(memory_format = torch.contiguous_format)
except Exception: # noqa: BLE001 - optimisation only
pass
if logger is not None:
logger.warning(
"diffusion.speed: torch.compile failed on the VAE decode (%s); decoding eager",
error,
)
# The handled exception pins inductor's frames (and the traced fake tensors) through the retry.
_drop_tracebacks(exc)
return eager(*args, **kwargs)
# diffusion_vae_oom reads these to retry a runtime OOM of the compiled decode on the eager path
guarded._unsloth_compiled_decode = True
guarded._unsloth_eager_decode = eager
return guarded
def _decode_slot_owner(fn: Any) -> Any:
"""The mutable slot an eager outer wrapper (fp16 non-finite check, ROCm bf16 cast) calls through, found under
any layer installed over it since (the OOM scope), or None."""
from .decode_layers import next_layer # noqa: PLC0415
for _ in range(32):
if fn is None:
return None
slot = getattr(fn, "_unsloth_decode_slot", None)
if slot is not None:
return slot
fn = next_layer(fn)
return None
def _compile_vae_decode(
pipe: Any,
logger: Any,
max_autotune: bool = False,
eager_when_tiled: bool = False,
) -> bool:
"""torch.compile the VAE ``decode`` in place; no cudagraphs, whose capture would pin decode activations."""
vae = getattr(pipe, "vae", None)
# An outer wrapper that must stay eager (fp16 non-finite check) exposes the slot it calls through.
outer = getattr(vae, "__dict__", {}).get("decode") if vae is not None else None
owner = _decode_slot_owner(outer) or vae
decode = getattr(owner, "decode", None) if vae is not None else None
if not callable(decode):
return False
# A regional block guard does not clear _unsloth_compiled_decode when it falls back, so the error is read first.
if _vae_compile_error(vae):
return False
# A dual-DiT family calls apply_speed_optims twice over the same pipe.
if getattr(vae, "_unsloth_compiled_decode", False):
return True
if getattr(vae, "_unsloth_compile_decode_error", None):
return False
_install_inductor_backports(logger)
if _vae_declares_repeated_blocks(vae):
return _compile_vae_regionally(vae, logger, max_autotune = max_autotune)
try:
import torch
# dynamic=True: automatic dynamic pays a second compile on the next resolution for no steady-state gain.
kwargs: dict[str, Any] = {"fullgraph": False, "dynamic": True}
if max_autotune:
kwargs["mode"] = "max-autotune-no-cudagraphs"
compiled = torch.compile(decode, **kwargs)
owner.decode = _guard_compiled_decode(
vae, compiled, decode, logger, eager_when_tiled = eager_when_tiled, owner = owner
)
vae._unsloth_compiled_decode = True
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "vae decode compile", exc)
return False
def _vae_declares_repeated_blocks(vae: Any) -> bool:
try:
return bool(getattr(vae, "_repeated_blocks", None)) and callable(
getattr(vae, "compile_repeated_blocks", None)
)
except Exception: # noqa: BLE001
return False
def _compile_vae_regionally(
vae: Any,
logger: Any,
max_autotune: bool = False,
) -> bool:
"""Compile the VAE's repeated block, not ``decode``: a tiled decode unrolls its Python tile loop into one graph
(RecursionError static, CantSplit dynamic on MiniMax-H3); a block sees one fixed-shape tile."""
try:
kwargs: dict[str, Any] = {"fullgraph": False, "dynamic": False}
if max_autotune:
kwargs["mode"] = "max-autotune-no-cudagraphs"
vae.compile_repeated_blocks(**kwargs)
guard_compiled_blocks(vae, logger)
vae._unsloth_compiled_decode = True
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "vae regional compile", exc)
return False
def _vae_compile_error(vae: Any) -> Optional[str]:
"""The decode-compile failure a VAE fell back from: the whole-decode wrapper's, or a regional block guard's."""
error = getattr(vae, "_unsloth_compile_decode_error", None)
if error:
return error
guard = getattr(vae, "_unsloth_compile_guard", None)
return getattr(guard, "error", None) if guard is not None else None
def _enable_cudnn_benchmark(logger: Any) -> bool:
try:
import torch
from core._torchao_stub import _module_is_rocm
# On ROCm this is MIOpen's exhaustive search: a first VAE decode tuned for 10 to 23 minutes and crashed a gfx1030.
if _module_is_rocm(torch):
return False
_write_cudnn_benchmark(torch.backends.cudnn, True)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "cudnn_benchmark", exc)
return False
def _enable_miopen_immediate(logger: Any) -> bool:
"""ROCm: ask MIOpen for a conv solution instead of miopenFind*, whose trial launches on unseen shapes
are where #9130's uncatchable abort came from. MIOPEN_FIND_MODE / MIOPEN_FIND_ENFORCE keep Find. Not in
snapshot_backend_flags: an unload restoring it would reopen Find under a co-resident pipeline."""
if os.environ.get("MIOPEN_FIND_MODE") or os.environ.get("MIOPEN_FIND_ENFORCE"):
return False
try:
import torch
from core._torchao_stub import _module_is_rocm
miopen = getattr(torch.backends, "miopen", None)
if not _module_is_rocm(torch) or miopen is None or not hasattr(miopen, "immediate"):
return False
if not miopen.immediate:
miopen.immediate = True
if logger is not None:
logger.info("diffusion.speed: MIOpen immediate mode on (no Find trial launches)")
return True
except Exception as exc: # noqa: BLE001 - keep torch's Find path
_warn(logger, "miopen immediate", exc)
return False
TF32_ENV = "UNSLOTH_DIFFUSION_TF32"
def tf32_enabled() -> bool:
import os
return (os.environ.get(TF32_ENV) or "").strip().lower() not in ("0", "off", "false", "no")
def _enable_tf32(logger: Any) -> bool:
try:
import torch
torch.backends.cuda.matmul.allow_tf32 = True
torch.backends.cudnn.allow_tf32 = True
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "tf32", exc)
return False
# Deny list of families that went non-finite under fp16 accumulation; empty, none of six overflowed.
_FP16_ACCUM_DENY: frozenset[str] = frozenset()
def _enable_fp16_accumulation(
family: Any,
logger: Any,
*,
dtype: Any = None,
speed_mode: Optional[str] = None,
) -> bool:
"""Consumer GPUs only; fp16 compute (results change) needs the max tier; bf16 runs on any tier."""
import os
if os.environ.get("UNSLOTH_DISABLE_FP16_ACCUM", "").strip().lower() in (
"1",
"true",
"yes",
"on",
):
return False
name = str(getattr(family, "name", family or "")).lower()
if name in _FP16_ACCUM_DENY:
return False
if str(dtype).replace("torch.", "") == "float16" and speed_mode != SPEED_MAX:
return False
try:
import torch
matmul = torch.backends.cuda.matmul
if not hasattr(matmul, "allow_fp16_accumulation"):
return False
from .diffusion_transformer_quant import _is_consumer_gpu
if not _is_consumer_gpu():
return False
_write_fp16_accum(matmul, True)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fp16_accum", exc)
return False
def _quantised_projections(pipe: Any) -> bool:
"""Whether a denoiser keeps native int8 / fp8 or packed nvfp4 / mxfp8 Linears, whose ``.weight`` is a dense
widen: diffusers' fuse would concatenate those into a persistent dense ``to_qkv`` beside the compact buffers."""
from .diffusion_native_quant import is_native_linear
from .diffusion_packed_block import is_packed_block_linear
# the pipe-level fuse reaches any U-Net, not only the whole-compile ones
for model in (*_denoiser_dits(pipe), getattr(pipe, "unet", None)):
if model is None or not callable(getattr(model, "modules", None)):
continue
try:
if any(is_native_linear(m) or is_packed_block_linear(m) for m in model.modules()):
return True
except Exception: # noqa: BLE001 -- a probe must never replace the failure it describes
continue
return False
def _fuse_qkv(pipe: Any, logger: Any) -> bool:
# Fusing reads each projection's .weight, which a quantised layer dequantises: a dense to_qkv beside its buffers.
if _quantised_projections(pipe):
if logger is not None:
logger.info("diffusion.speed: fuse_qkv skipped (quantised projections stay unfused)")
return False
from . import diffusion_lora_quant_merge as lora_quant_merge
from .diffusion_manual_cast import manual_cast_info
# A fused to_qkv is a plain Linear at the stored dtype, which a manual-cast (float32) forward cannot feed.
if any(manual_cast_info(m) for m in _denoiser_dits(pipe)):
if logger is not None:
logger.info("diffusion.speed: fuse_qkv skipped (manual-cast weights stay unfused)")
return False
# A fused to_qkv copy would miss in-place re-merges of merged adapter weights.
if lora_quant_merge.merged(pipe):
if logger is not None:
logger.info("diffusion.speed: fuse_qkv skipped (merged LoRA weights change in place)")
return False
# Prefer pipe-level fuse (covers every component); else fuse each DiT so dual-DiT families fuse both.
fn = getattr(pipe, "fuse_qkv_projections", None)
if callable(fn):
try:
vae = getattr(pipe, "vae", None)
fused_vae_attn = callable(getattr(vae, "modules", None)) and any(
type(getattr(m, "processor", None)).__name__ == "FusedSingleHeadProcessor"
for m in vae.modules()
)
fn()
if (
fused_vae_attn
): # the pipe-level fuse resets every VAE processor to FusedAttnProcessor2_0
from . import diffusion_vae_fused # noqa: PLC0415
diffusion_vae_fused.install_attention_processors(vae)
drop_unfused_projections(_denoiser_unet(pipe), logger)
return True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fuse_qkv_projections", exc)
return False
engaged = False
for transformer in _denoiser_dits(pipe):
tfn = getattr(transformer, "fuse_qkv_projections", None)
if callable(tfn):
try:
tfn()
engaged = True
except Exception as exc: # noqa: BLE001 - optimisation only
_warn(logger, "fuse_qkv_projections", exc)
return engaged
DROP_UNFUSED_QKV_ENV = "UNSLOTH_DIFFUSION_DROP_UNFUSED_QKV"
_DROPPED_ATTR = "_unsloth_dropped_projections"
def _fused_attention_drops(module: Any) -> tuple:
"""The projections diffusers' fused processor no longer calls: self attention runs ``to_qkv`` (q, k, v unused),
cross attention ``to_q`` + ``to_kv`` (k, v unused)."""
if not getattr(module, "fused_projections", False):
return ()
if getattr(module, "is_cross_attention", False):
return ("to_k", "to_v") if getattr(module, "to_kv", None) is not None else ()
return ("to_q", "to_k", "to_v") if getattr(module, "to_qkv", None) is not None else ()
def _drop_unfused_enabled() -> bool:
return (os.environ.get(DROP_UNFUSED_QKV_ENV) or "").strip().lower() not in (
"0",
"off",
"false",
"no",
)
def _droppable_projection(module: Any, name: str) -> bool:
"""A projection the drop may delete: a plain Linear without hooks (quantised / wrapped ones are left alone)."""
from torch import nn
p = getattr(module, name, None)
return type(p) is nn.Linear and not (
p._forward_pre_hooks or p._forward_hooks or hasattr(p, "_hf_hook")
)
def _drop_plan(unet: Any) -> list:
"""``(module, names)`` the drop would delete on a fused ``unet``; empty when it is off, a PEFT adapter is attached
(it targets to_q / to_k / to_v) or any projection is not a plain hook-free Linear (all or nothing)."""
if unet is None or not _drop_unfused_enabled():
return []
if any(hasattr(m, "lora_A") for m in unet.modules()):
return []
plan = []
for module in unet.modules():
names = _fused_attention_drops(module)
if not names:
continue
if not all(_droppable_projection(module, n) for n in names):
return []
plan.append((module, names))
return plan
def unfused_projections_will_drop(unet: Any) -> bool:
"""Whether the fuse's unfused copies will be dropped on this (not yet fused) U-Net: drop on, no PEFT adapter, and
every projection the fused processor stops calling is a plain hook-free Linear."""
if unet is None or not _drop_unfused_enabled():
return False
try:
if any(hasattr(m, "lora_A") for m in unet.modules()):
return False
seen = False
for module in unet.modules():
if not callable(getattr(module, "fuse_projections", None)) or getattr(
module, "fused_projections", False
):
continue
if getattr(module, "add_q_proj", None) is not None:
return False # the added projections are fused too and never dropped
names = (
("to_k", "to_v")
if getattr(module, "is_cross_attention", False)
else ("to_q", "to_k", "to_v")
)
if not all(_droppable_projection(module, n) for n in names):
return False
seen = True
return seen
except Exception: # noqa: BLE001 - unreadable: count the copies
return False
def drop_unfused_projections(unet: Any, logger: Any = None) -> int:
"""Delete the unfused projections a U-Net QKV fuse left beside ``to_qkv`` / ``to_kv`` (SDXL: 1235 MiB). Kept when
a PEFT adapter is attached (it targets them) or the modules carry hooks; ``restore_unfused_projections`` rebuilds
them bit-exactly from the fused weights. UNSLOTH_DIFFUSION_DROP_UNFUSED_QKV=0 keeps them."""
try:
import torch
plan = _drop_plan(unet)
freed = 0
for module, names in plan:
shapes = []
for n in names:
p = getattr(module, n)
freed += sum(t.numel() * t.element_size() for t in p.parameters())
shapes.append((n, p.in_features, p.out_features, p.bias is not None))
delattr(module, n)
setattr(module, _DROPPED_ATTR, tuple(shapes))
if plan and torch.cuda.is_available():
torch.cuda.empty_cache()
if plan and logger is not None:
logger.info(
"diffusion.speed: dropped the unfused projections beside the fused QKV on %d attention modules "
"(%d MiB)",
len(plan),
freed >> 20,
)
return freed
except Exception as exc: # noqa: BLE001 - memory only: a failed drop keeps what is left
_warn(logger, "dropping the unfused projections", exc)
return 0
def restore_unfused_projections(unet: Any) -> int:
"""Rebuild the projections ``drop_unfused_projections`` deleted, row slices of the fused weights (bit-exact: the
fuse concatenated them unchanged). Returns the modules restored."""
if unet is None:
return 0
import torch
from torch import nn
restored = 0
for module in unet.modules():
shapes = getattr(module, _DROPPED_ATTR, None)
if not shapes:
continue
fused = module.to_kv if getattr(module, "is_cross_attention", False) else module.to_qkv
rows = 0
for name, fin, fout, has_bias in shapes:
lin = nn.Linear(fin, fout, bias = has_bias, device = "meta")
with torch.no_grad():
grad = bool(fused.weight.requires_grad)
lin.weight = nn.Parameter(
fused.weight[rows : rows + fout].clone(), requires_grad = grad
)
if has_bias:
lin.bias = nn.Parameter(
fused.bias[rows : rows + fout].clone(), requires_grad = grad
)
setattr(module, name, lin)
rows += fout
delattr(module, _DROPPED_ATTR)
restored += 1
return restored
def _warn(logger: Any, what: str, exc: Exception) -> None:
if logger is not None:
logger.warning("diffusion.speed: %s failed: %s", what, exc)