Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
"""Attention backend: auto picks cuDNN on NVIDIA with a speed profile (Blackwell stays native if
torch picks cuDNN, unless UNSLOTH_DIFFUSION_BLACKWELL_NATIVE_ATTN=0); native, cudnn,
flash/flash3/flash4, sage, xformers, aiter; unavailable falls back to native; torch/diffusers
lazy."""
from __future__ import annotations
import contextlib
import contextvars
import functools
import os
import re
import threading
import weakref
from typing import Any, Optional
ATTN_AUTO = "auto"
ATTN_NATIVE = "native"
# User-facing alias -> the diffusers dispatcher backend name.
_ALIASES: dict[str, str] = {
"native": "native",
"sdpa": "native",
"cudnn": "_native_cudnn",
"flash": "flash",
"flash2": "flash",
"flash3": "_flash_3_hub",
"flash4": "flash_4_hub",
"sage": "sage",
"xformers": "xformers",
"aiter": "aiter",
}
ATTN_ALIASES = (ATTN_AUTO,) + tuple(dict.fromkeys(_ALIASES))
def normalize_attention_backend(value: Optional[str]) -> Optional[str]:
"""Lower/strip a requested backend; None / "" / "auto" -> "auto". Raises ValueError for an
unsupported alias so a bad request is rejected cheaply."""
if value is None:
return ATTN_AUTO
normalized = str(value).strip().lower()
if not normalized:
return ATTN_AUTO
if normalized not in ATTN_ALIASES:
raise ValueError(
f"Unsupported attention_backend '{value}'. Use one of: {', '.join(ATTN_ALIASES)}."
)
return normalized
# Kernels that need a specific CUDA arch at run time: (min, max-exclusive) capability.
_ARCH_CAPABILITY: dict[str, tuple[tuple[int, int], Optional[tuple[int, int]]]] = {
"_flash_3_hub": ((9, 0), (10, 0)), # Hopper (SM90) only
"flash_4_hub": (
(10, 0),
None,
), # SM100+; sm110 / sm120 unverified, so _fa4_kernel_runs also checks each card
}
def _cuda_capability() -> Optional[tuple[int, int]]:
"""(major, minor) compute capability of the active CUDA device, or None if unknown."""
try:
import torch
if not torch.cuda.is_available():
return None
return tuple(torch.cuda.get_device_capability()) # type: ignore[return-value]
except Exception: # noqa: BLE001
return None
def _backend_arch_supported(backend: str) -> bool:
"""False only when ``backend`` needs a CUDA arch outside this device's range. Unknown
capability returns True (never block on a guess; the run-time failure falls back to native)."""
bounds = _ARCH_CAPABILITY.get(backend)
if bounds is None:
return True
have = _cuda_capability()
if have is None:
return True
low, high = bounds
return have >= low and (high is None or have < high)
def _is_cuda_nvidia(target: Any) -> bool:
"""CUDA device on an NVIDIA (non-ROCm) build -- where cuDNN attention applies."""
if getattr(target, "device", None) != "cuda":
return False
try:
import torch
# Shared with stub installer; torch.version.hip alone misreads AMD wheels that tag only __version__.
from core._torchao_stub import _module_is_rocm
return not _module_is_rocm(torch)
except Exception: # noqa: BLE001
return False
# flash_sdp_enabled reports the user toggle, not a working kernel, so probe actual execution.
SDPA_FLASH = "flash"
SDPA_MEM_EFFICIENT = "mem_efficient"
SDPA_CUDNN = "cudnn"
SDPA_MATH = "math"
# Backends whose working set is O(N): they never materialise the score matrix.
_SDPA_SUBQUADRATIC = (SDPA_FLASH, SDPA_MEM_EFFICIENT, SDPA_CUDNN)
_SDPA_PROBE_LOCK = threading.Lock()
# (indexed device, dtype name) -> capability; process flags are applied on every read.
_SDPA_PROBE_CACHE: dict[tuple[str, str], tuple[str, ...]] = {}
# An incomplete ROCm probe is retried, but not by every check of the same load (up to 90 s per attempt).
_ROCM_PROBE_RETRY_S = 120.0
_ROCM_PROBE_INCOMPLETE: dict[tuple[str, str], float] = {}
# Candidate-card probes must not flip the process-wide SDPA switches the running card also uses.
_SDPA_PROBE_READ_ONLY: contextvars.ContextVar[bool] = contextvars.ContextVar(
"_sdpa_probe_read_only", default = False
)
@contextlib.contextmanager
def read_only_sdpa_probe():
token = _SDPA_PROBE_READ_ONLY.set(True)
try:
yield
finally:
_SDPA_PROBE_READ_ONLY.reset(token)
def _probe_rocm_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:
import time
key = (device, str(dtype))
last = _ROCM_PROBE_INCOMPLETE.get(key)
if last is not None and time.monotonic() - last < _ROCM_PROBE_RETRY_S:
return ()
available = _run_rocm_sdpa_children(device, dtype)
if available:
_ROCM_PROBE_INCOMPLETE.pop(key, None)
if (
not _SDPA_PROBE_READ_ONLY.get()
and _rocm_guard_allowed()
and str(dtype) in ("torch.float16", "torch.bfloat16")
):
try:
_apply_rocm_guard(available)
except Exception: # noqa: BLE001 - a diagnostic may never fail a load
pass
else:
_ROCM_PROBE_INCOMPLETE[key] = time.monotonic()
return available
def _run_rocm_sdpa_children(device: str, dtype: Any) -> tuple[str, ...]:
"""Isolate each backend's HIP error state from Studio and from the other probes."""
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import subprocess
import sys
from utils.child_stdio import utf8_child_env
from utils.native_path_leases import child_env_without_native_path_secret
from utils.subprocess_compat import windows_hidden_subprocess_kwargs
from utils.torch_device_probe import ROCM_DLL_DIRS_ENV_VAR, _rocm_dll_directories
from .rocm_sdpa_probe import RESULT_PREFIX
# Same bootstrap as the device probe: a fresh interpreter lacks main.py's ROCm DLL registrations.
child_env = child_env_without_native_path_secret()
dll_directories = _rocm_dll_directories()
if dll_directories:
child_env[ROCM_DLL_DIRS_ENV_VAR] = os.pathsep.join(dll_directories)
def probe(backend: str) -> str:
try:
result = subprocess.run(
[
sys.executable,
str(Path(__file__).with_name("rocm_sdpa_probe.py")),
device,
str(dtype).removeprefix("torch."),
backend,
],
capture_output = True,
text = True,
encoding = "utf-8",
errors = "replace",
timeout = 45,
env = utf8_child_env(child_env),
**windows_hidden_subprocess_kwargs(),
)
replies = [
line[len(RESULT_PREFIX) :]
for line in result.stdout.splitlines()
if line.startswith(RESULT_PREFIX)
]
return json.loads(replies[-1]) if result.returncode == 0 and replies else "unknown"
except Exception:
return "unknown"
if probe("MATH") != "available":
return ()
# One child per backend, run concurrently: an in-process fused failure would poison Studio.
candidates = (
(SDPA_FLASH, "FLASH_ATTENTION"),
(SDPA_MEM_EFFICIENT, "EFFICIENT_ATTENTION"),
)
with ThreadPoolExecutor(max_workers = 2) as executor:
statuses = list(executor.map(probe, (backend for _, backend in candidates)))
available = [SDPA_MATH]
for (name, _), status in zip(candidates, statuses):
if status not in ("available", "unavailable", "failed"):
return ()
if status == "available":
available.append(name)
return tuple(available)
def _enabled_sdpa_kernels(kernels: tuple[str, ...]) -> tuple[str, ...]:
if not kernels:
return ()
import torch
flags = {
SDPA_FLASH: "flash_sdp_enabled",
SDPA_MEM_EFFICIENT: "mem_efficient_sdp_enabled",
SDPA_CUDNN: "cudnn_sdp_enabled",
SDPA_MATH: "math_sdp_enabled",
}
return tuple(k for k in kernels if getattr(torch.backends.cuda, flags[k], lambda: True)())
MPS_PREFILL_ATTENTION_OFF_ENV = "PYTORCH_MPS_DISABLE_PREFILL_ATTENTION"
MPS_PREFILL_HEAD_DIMS = frozenset((32, 64, 72, 80, 96, 128, 256))
def mps_prefill_supported() -> bool:
import torch
if os.environ.get(MPS_PREFILL_ATTENTION_OFF_ENV, "0") != "0":
return False
return _version_tuple(torch.__version__)[:2] >= (2, 13)
def mps_prefill_attention(
query: Any,
key: Any,
value: Any,
attn_mask: Any = None,
seq_dim: int = -2,
) -> bool:
"""Whether torch runs this SDPA call on the MPS prefill kernel rather than the MPSGraph path that materialises the
score matrix. ``seq_dim`` is the query's sequence axis (1 for diffusers' ``[B, S, H, D]``)."""
import torch
if (
getattr(getattr(query, "device", None), "type", None) != "mps"
or not mps_prefill_supported()
):
return False
if query.dtype not in (torch.float32, torch.float16, torch.bfloat16):
return False
# torch converts a bool mask to an additive one in q's dtype; a float mask of another dtype falls back.
if attn_mask is not None and attn_mask.dtype not in (torch.bool, query.dtype):
return False
if torch.is_grad_enabled() and any(t.requires_grad for t in (query, key, value)):
return False
return (
query.shape[seq_dim] > 8
and key.shape[seq_dim] > 0
and query.shape[-1] == value.shape[-1]
and query.shape[-1] in MPS_PREFILL_HEAD_DIMS
)
def _mps_sdpa_kernels() -> tuple[str, ...]:
"""MPS ignores ``sdpa_kernel``, so answer from the build: only torch 2.13+ has a fused (prefill) kernel. Shape-blind:
callers that know the family's head dims pass them to ``sdpa_math_only``."""
return (SDPA_FLASH, SDPA_MATH) if mps_prefill_supported() else (SDPA_MATH,)
def _probe_sdpa_kernels(device: str, dtype: Any) -> tuple[str, ...]:
"""Probe execution; ROCm runs in fresh children so launch errors stay isolated."""
import torch
from torch.nn.attention import SDPBackend, sdpa_kernel
candidates = (
(SDPA_FLASH, getattr(SDPBackend, "FLASH_ATTENTION", None)),
(SDPA_MEM_EFFICIENT, getattr(SDPBackend, "EFFICIENT_ATTENTION", None)),
(SDPA_CUDNN, getattr(SDPBackend, "CUDNN_ATTENTION", None)),
(SDPA_MATH, getattr(SDPBackend, "MATH", None)),
)
from core._torchao_stub import _module_is_rocm
if device.split(":")[0] == "mps":
return _mps_sdpa_kernels()
if device.split(":")[0] == "cuda" and _module_is_rocm(torch):
return _probe_rocm_sdpa_kernels(device, dtype)
# Small enough to be free, but shaped like real attention: the fused kernels reject head_dim they cannot serve, and
# a degenerate 1-element tensor would not exercise that.
q = torch.zeros((1, 2, 8, 64), device = device, dtype = dtype)
available: list[str] = []
for name, backend in candidates:
if backend is None:
continue
try:
with sdpa_kernel([backend]):
out = torch.nn.functional.scaled_dot_product_attention(q, q, q)
# Windows ROCm raises a failed fused launch only at the next checked kernel: take it here.
out.float().sum().item()
available.append(name)
except Exception: # noqa: BLE001 - "No available kernel" is the answer, not an error
continue
return tuple(available)
def available_sdpa_kernels(target: Any) -> tuple[str, ...]:
"""SDPA backends that really run on target; empty when the probe cannot run, never read as math-only."""
kernels = _sdpa_capability(target)
if _target_device(target).split(":")[0] == "mps":
# MPS dispatch ignores the torch.backends.cuda toggles.
return kernels
return _enabled_sdpa_kernels(kernels)
def _target_device(target: Any) -> str:
return str(getattr(target, "torch_device", None) or getattr(target, "device", "") or "")
def _sdpa_capability(target: Any) -> tuple[str, ...]:
device = _target_device(target)
if device == "cuda":
ordinal = getattr(target, "ordinal", None)
device = f"cuda:{ordinal}" if ordinal is not None else _indexed_cuda_device(device)
if not device:
return ()
dtype = getattr(target, "dtype", None)
if dtype is None:
try:
import torch
# Fused kernels are half-precision only.
dtype = torch.float16
except Exception: # noqa: BLE001
return ()
key = (device, str(dtype))
with _SDPA_PROBE_LOCK:
cached = _SDPA_PROBE_CACHE.get(key)
if cached is not None:
return cached
try:
available = _probe_sdpa_kernels(device, dtype)
except Exception: # noqa: BLE001 - a probe is a diagnostic; it may never fail a load
available = ()
# Memoize only an answer: empty means the probe itself failed (transient).
if not available:
return ()
with _SDPA_PROBE_LOCK:
return _SDPA_PROBE_CACHE.setdefault(key, available)
def _family_sdpa_kernels(target: Any, head_dims: Any = None) -> tuple[str, ...]:
kernels = available_sdpa_kernels(target)
if (
head_dims
and _target_device(target).split(":")[0] == "mps"
and not set(head_dims) <= MPS_PREFILL_HEAD_DIMS
):
return tuple(k for k in kernels if k not in _SDPA_SUBQUADRATIC)
return kernels
def sdpa_math_only(target: Any, head_dims: Any = None) -> bool:
"""True only when MATH ran and every sub-quadratic backend refused; an empty probe answer is False."""
available = _family_sdpa_kernels(target, head_dims)
if not available:
return False
return SDPA_MATH in available and not any(k in available for k in _SDPA_SUBQUADRATIC)
def sdpa_subquadratic_confirmed(target: Any, head_dims: Any = None) -> bool:
"""True only when a sub-quadratic SDPA kernel was seen to run; an unanswered probe is False."""
return any(k in _SDPA_SUBQUADRATIC for k in _family_sdpa_kernels(target, head_dims))
SDPA_MATH_ONLY_MESSAGE = (
"attention has no fused kernel on this device, so it will run on the SDPA math backend, "
"which materialises the full attention score matrix (batch x heads x tokens x tokens, 4 bytes "
"per element). Peak VRAM then grows with the SQUARE of the token count -- resolution x frames "
"for video -- and can exceed the card by many times even for a small model. Lower the "
"resolution or the frame count, or install a backend with a working kernel for this GPU."
)
def warn_if_sdpa_math_only(
target: Any,
logger: Any = None,
head_dims: Any = None,
) -> bool:
"""Log math-only at load, not 70 seconds into a doomed generation; returns whether it warned."""
if not sdpa_math_only(target, head_dims):
return False
if logger is not None:
logger.warning(
"diffusion.attention.math_only: device=%s dtype=%s kernels=%s -- %s",
getattr(target, "device", None),
getattr(target, "dtype", None),
",".join(_family_sdpa_kernels(target, head_dims)) or "none",
SDPA_MATH_ONLY_MESSAGE,
)
return True
ROCM_FUSED_SDPA_ALLOW_ENV = "UNSLOTH_ALLOW_ROCM_FUSED_SDPA"
_ROCM_GUARD_DISABLED: set[str] = set()
def guard_rocm_fused_sdpa(target: Any, logger: Any = None) -> tuple[str, ...]:
"""Disable failed ROCm fused backends only after math has been verified in isolation."""
if not _is_cuda_rocm(target):
return ()
if not _rocm_guard_allowed():
return ()
import torch
from types import SimpleNamespace
from .rocm_bf16 import rocm_bf16_supported
device = str(getattr(target, "torch_device", None) or getattr(target, "device", "cuda"))
ordinal = getattr(target, "ordinal", None)
if device == "cuda":
device = f"cuda:{ordinal}" if ordinal is not None else _indexed_cuda_device(device)
dtype = getattr(target, "dtype", None)
if dtype not in (torch.float16, torch.bfloat16):
dtype = (
torch.bfloat16
if rocm_bf16_supported(torch, torch.device(device).index)
else torch.float16
)
available = _sdpa_capability(SimpleNamespace(device = device, dtype = dtype))
return _apply_rocm_guard(available, logger)
def guard_rocm_fused_sdpa_cards(
devices: Any,
dtype: Any,
logger: Any = None,
) -> tuple[str, ...]:
"""Switches are process-wide, so only kernels that ran on ALL cards stay on."""
from types import SimpleNamespace
if not _is_cuda_rocm(SimpleNamespace(device = "cuda")) or not _rocm_guard_allowed():
return ()
if dtype is None:
import torch
dtype = torch.float16 # _sdpa_capability's default
if str(dtype) not in ("torch.float16", "torch.bfloat16"):
return ()
answers = []
with read_only_sdpa_probe():
for device in devices:
available = _sdpa_capability(SimpleNamespace(device = str(device), dtype = dtype))
if available:
answers.append(available)
if not answers:
return ()
common = tuple(k for k in answers[0] if all(k in a for a in answers[1:]))
return _apply_rocm_guard(common, logger)
def _rocm_guard_allowed() -> bool:
return os.environ.get(ROCM_FUSED_SDPA_ALLOW_ENV, "").strip().lower() not in (
"1",
"true",
"yes",
"on",
)
def _apply_rocm_guard(available: tuple[str, ...], logger: Any = None) -> tuple[str, ...]:
if SDPA_MATH not in available:
return ()
import torch
disabled = []
for name, enabled, switch in (
(SDPA_FLASH, torch.backends.cuda.flash_sdp_enabled, torch.backends.cuda.enable_flash_sdp),
(
SDPA_MEM_EFFICIENT,
torch.backends.cuda.mem_efficient_sdp_enabled,
torch.backends.cuda.enable_mem_efficient_sdp,
),
):
if name in available:
# The flags are process-wide: a healthy card gets back only what this guard turned off for another.
if name in _ROCM_GUARD_DISABLED and not enabled():
switch(True)
_ROCM_GUARD_DISABLED.discard(name)
elif enabled():
switch(False)
_ROCM_GUARD_DISABLED.add(name)
disabled.append(name)
if disabled:
(logger or _module_logger()).warning(
"diffusion.attention: disabled unavailable ROCm SDPA kernels %s; using the remaining kernels. "
"If fused kernels are expected on this GPU, run 'unsloth studio update' to repair "
"the AMD device packages. %s=1 skips this check.",
", ".join(disabled),
ROCM_FUSED_SDPA_ALLOW_ENV,
)
return tuple(disabled)
# gfx1151 (Strix Halo): AOTriton's SDPA tuning covers contiguous [B, H, S, D] only, and diffusers' native backend
# hands SDPA permuted [B, S, H, D] views. Once one head of the strided k/v outgrows L2 the strided call is far slower
# than copying k/v first (ComfyUI #16726, pytorch/pytorch#190154). Output is bit-identical. 0 disables.
ROCM_CONTIGUOUS_KV_ENV = "UNSLOTH_DIFFUSION_ROCM_CONTIGUOUS_KV"
ROCM_CONTIGUOUS_KV_ARCHES = ("gfx1151",)
ROCM_CONTIGUOUS_KV_ATTR = "_unsloth_rocm_contiguous_kv"
_ROCM_CONTIGUOUS_KV_L2: dict[int, int] = {} # device index -> L2 bytes, gfx1151 cards only
def _rocm_contiguous_kv_wanted(key: Any, value: Any) -> bool:
"""True when one head of the strided ``[B, S, H, D]`` k/v exceeds L2."""
try:
if key.dim() != 4 or value.dim() != 4:
return False
l2 = _ROCM_CONTIGUOUS_KV_L2.get(key.device.index)
if not l2:
return False
strided = [t for t in (key, value) if not t.permute(0, 2, 1, 3).is_contiguous()]
return sum(t.shape[1] * t.shape[3] for t in strided) * key.element_size() > l2
except Exception: # noqa: BLE001
return False
def _heads_major(t: Any) -> Any:
"""``[B, H, S, D]`` memory so the native backend's permute hands SDPA a dense tensor."""
if t.permute(0, 2, 1, 3).is_contiguous():
return t
return t.permute(0, 2, 1, 3).contiguous().permute(0, 2, 1, 3)
def install_rocm_contiguous_kv(target: Any) -> bool:
"""gfx1151 only: k/v over L2 reach SDPA contiguous (q is not copied). False elsewhere: other GPUs untouched."""
if os.environ.get(ROCM_CONTIGUOUS_KV_ENV, "").strip().lower() in ("0", "false", "no", "off"):
return False
if not _is_cuda_rocm(target) or _rocm_gfx_arch(target) not in ROCM_CONTIGUOUS_KV_ARCHES:
return False
try:
import torch
from diffusers.models.attention_dispatch import (
AttentionBackendName,
_AttentionBackendRegistry,
)
device = torch.device(
_indexed_cuda_device(
str(
getattr(target, "torch_device", None)
or getattr(target, "device", None)
or "cuda"
)
)
)
index = device.index if device.index is not None else torch.cuda.current_device()
l2 = int(getattr(torch.cuda.get_device_properties(index), "L2_cache_size", 0) or 0)
backends = _AttentionBackendRegistry._backends
native_fn = backends[AttentionBackendName.NATIVE]
except Exception: # noqa: BLE001
return False
if l2 <= 0 or not callable(native_fn):
return False
_ROCM_CONTIGUOUS_KV_L2[index] = l2
if getattr(native_fn, ROCM_CONTIGUOUS_KV_ATTR, False):
return True
def _contiguous_kv(*args: Any, **kwargs: Any) -> Any:
args = list(args)
key = args[1] if len(args) > 1 else kwargs.get("key")
value = args[2] if len(args) > 2 else kwargs.get("value")
if kwargs.get("_parallel_config") is None and _rocm_contiguous_kv_wanted(key, value):
key, value = _heads_major(key), _heads_major(value)
if len(args) > 1:
args[1] = key
else:
kwargs["key"] = key
if len(args) > 2:
args[2] = value
else:
kwargs["value"] = value
return native_fn(*args, **kwargs)
setattr(_contiguous_kv, ROCM_CONTIGUOUS_KV_ATTR, True)
_contiguous_kv.__wrapped__ = native_fn # type: ignore[attr-defined]
try:
backends[AttentionBackendName.NATIVE] = _contiguous_kv
except Exception: # noqa: BLE001
return False
return True
def select_attention_backend(
target: Any,
requested: Optional[str],
*,
speed_active: bool,
family: Any = None,
speed_unset: bool = False,
) -> Optional[str]:
"""The dispatcher backend name to apply, or None to leave the diffusers default.
An explicit alias is honored (apply falls back if its kernel is unavailable). ``auto``
upgrades to cuDNN on NVIDIA CUDA only when a speed profile is active (so ``off`` stays
bit-identical), and on ROCm with verified flash_attn to ``ROCM_FLASH_CANDIDATE`` (resolved by timing in
``apply_attention_backend``) unless speed is explicitly ``off``; elsewhere returns None (native)."""
alias = normalize_attention_backend(requested)
if alias != ATTN_AUTO:
backend = _ALIASES[alias]
if backend == "native":
return None
# AITER is the ROCm kernel; the NVIDIA-only guard below would drop it.
if backend == "aiter":
if getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target):
return backend
return None
# cuDNN / flash* / sage are CUDA+NVIDIA-only (elsewhere the first generation crashes), except verified ROCm FA2.
if not _is_cuda_nvidia(target):
if backend == "flash" and _is_cuda_rocm(target) and _rocm_flash_attn_runs(target):
return backend
return None
# An arch-gated kernel (flash3/flash4) on a card that can't run it sets fine then crashes.
if not _backend_arch_supported(backend):
return None
# cuDNN fused SDPA needs Ampere+ (SM80); gate an explicit request like the auto path.
if backend == "_native_cudnn" and not _cudnn_attention_supported():
return None
return backend
if speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():
return AUTO_CUDNN
if (speed_active or speed_unset) and _rocm_auto_flash(target, family):
return ROCM_FLASH_CANDIDATE
return None
def auto_attention_reason(engaged: Optional[str]) -> str:
"""Status reason for an ``auto`` pick that engaged ``engaged`` (None = native)."""
if not engaged:
return "diffusers default"
if "cudnn" in engaged:
return "cuDNN fused attention upgrade"
if engaged == "flash":
return "verified ROCm flash attention"
return f"{engaged} attention upgrade"
def attention_status_reason(requested: Optional[str], engaged: Optional[str]) -> str:
"""Status reason for the load's attention pick: the auto reason for None / "auto", else "requested"."""
try:
is_auto = normalize_attention_backend(requested) == ATTN_AUTO
except ValueError:
is_auto = False
if is_auto or not engaged:
return auto_attention_reason(engaged)
return "requested"
def _is_cuda_rocm(target: Any) -> bool:
return getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target)
ROCM_FLASH_CANDIDATE = "_rocm_flash_candidate"
class _AutoCudnn(str):
"""The ``auto`` cuDNN pick. Equal to ``"_native_cudnn"`` for every caller; ``apply_attention_backend`` alone tells
it from an explicit ``cudnn`` request, so it can leave native where torch already dispatches cuDNN (Blackwell)."""
__slots__ = ()
AUTO_CUDNN = _AutoCudnn("_native_cudnn")
# B200 torch 2.13 bf16: native SDPA already runs cuDNN at head_dim 64/128 (pin = 0.4-22% slower eager wrapper);
# head_dim 256 picks FLASH, hence the per-head-dim probe.
BLACKWELL_NATIVE_ATTN_ENV = (
"UNSLOTH_DIFFUSION_BLACKWELL_NATIVE_ATTN" # auto (default) | 0 keeps the cuDNN pin
)
_NATIVE_DISPATCH_CACHE: dict[tuple[str, str, int], bool] = {}
# Native flash measured faster than the cuDNN pin at every DiT/U-Net shape, bf16+fp16 (torch 2.14.1+cu130); others keep the pin.
FLASH_FASTER_CAPABILITIES: dict[tuple[int, int], str] = {
(
8,
0,
): "A100: cuDNN 1-44% slower than native flash (FLUX 0.92x, Lumina-2 0.74x, SD3.5 0.97x per call)",
(
8,
9,
): "L4: cuDNN never faster than native flash (Lumina-2 0.70x per call, the rest 0.93-1.00x)",
}
NATIVE_FLASH_ATTN_ENV = "UNSLOTH_DIFFUSION_NATIVE_FLASH_ATTN" # auto (default) | 0 keeps the cuDNN pin on FLASH_FASTER_CAPABILITIES cards
_NATIVE_FLASH_CACHE: dict[tuple[str, str, int], bool] = {}
ROCM_FLASH_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH" # auto (timed) | 0 (auto never picks flash) | 1 (verified is enough)
ROCM_FLASH_MIN_GAIN = 0.03
# Accuracy is judged on the raw attention output (flash vs native SDPA, same inputs), not the block: the residual and
# FFN would dilute a wrong attention. Rel L1 over the output, and the largest abs error over the largest abs output.
ROCM_FLASH_BLOCK_TOL = 2e-2 # attention output rel L1
ROCM_FLASH_MAX_TOL = 5e-2 # attention max abs error / max abs output
ROCM_FLASH_PROBE_TOKENS = 4096 # the load-time bucket: a 1024^2 image DiT sequence
# Gated per (query, key) length bucket pair, so a 4096 verdict never decides a 30k-token call.
ROCM_FLASH_MIN_BUCKET = 256
ROCM_FLASH_LONG_PROBE = (
16384 # from here the probe times fewer runs (one native SDPA call is seconds at 64k tokens)
)
ROCM_FLASH_ADMISSIONS: dict[tuple, dict] = {}
# Bump when the block probe changes what it measures: every stored verdict is then re-timed.
# 2: the flash arm runs through diffusers' dispatch (the generation path), not flash_attn_func directly.
# 3: accuracy from the raw attention output (rel L1 + max_rel), not the block output; verdicts per length bucket.
ROCM_FLASH_PROBE_VERSION = 3
ROCM_FLASH_CACHE_DIR_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH_CACHE_DIR"
def _rocm_flash_mode() -> str:
raw = os.environ.get(ROCM_FLASH_ENV, "auto").strip().lower()
if raw in ("0", "false", "no", "off", "never"):
return "0"
if raw in ("1", "true", "yes", "on", "always"):
return "1"
return "auto"
def _rocm_gfx_arch(target: Any) -> str:
try:
import torch
from utils.hardware.hardware import _props_gfx_arch
device = torch.device(
_indexed_cuda_device(
str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
)
)
index = device.index if device.index is not None else torch.cuda.current_device()
return _props_gfx_arch(torch.cuda.get_device_properties(index))
except Exception: # noqa: BLE001
return ""
def _flash_attn_installed() -> bool:
"""Without importing it, so an ``auto`` load on a card without flash_attn logs nothing."""
import sys
if "flash_attn" in sys.modules:
return sys.modules["flash_attn"] is not None
try:
import importlib.util
return importlib.util.find_spec("flash_attn") is not None
except Exception: # noqa: BLE001
return False
def _rocm_auto_flash(target: Any, family: Any = None) -> bool:
if not _is_cuda_rocm(target) or _rocm_flash_mode() == "0":
return False
return _flash_attn_installed() and _rocm_flash_attn_runs(target)
def _dit_attention_geometries(pipe: Any) -> tuple[tuple[int, ...], ...]:
"""Every distinct (hidden, heads, head_dim[, context width]) among the attention modules of every denoiser DiT, read off ``to_q``
+ ``heads`` (or ``num_heads``); flash is set on all of them, so all of them are timed."""
found: set[tuple[int, int, int]] = set()
for dit in _attention_dits(pipe):
modules = getattr(dit, "modules", None)
if not callable(modules):
continue
try:
for module in modules():
heads = getattr(module, "heads", None)
if not isinstance(heads, int) or isinstance(heads, bool):
heads = getattr(module, "num_heads", None)
to_q = getattr(module, "to_q", None)
fin = getattr(to_q, "in_features", None)
fout = getattr(to_q, "out_features", None)
if to_q is None:
# Fused self-attention (MiniMax-H3 resident INT8 deletes to_q / to_k / to_v): Q is a third of to_qkv.
to_qkv = getattr(module, "to_qkv", None)
fin = getattr(to_qkv, "in_features", None)
qkv_out = getattr(to_qkv, "out_features", None)
fout = (
qkv_out // 3
if isinstance(qkv_out, int) and qkv_out > 0 and qkv_out % 3 == 0
else None
)
if not all(
isinstance(v, int) and not isinstance(v, bool) and v > 0
for v in (heads, fin, fout)
):
continue
if fout % heads:
continue
# Cross-attention reads K / V from a context of another width (to_k.in_features): a 4th element.
kin = getattr(getattr(module, "to_k", None), "in_features", None)
if isinstance(kin, int) and not isinstance(kin, bool) and kin > 0 and kin != fin:
found.add((fin, heads, fout // heads, kin))
else:
found.add((fin, heads, fout // heads))
except Exception: # noqa: BLE001
continue
return tuple(sorted(found))
def _time_rocm_flash_block(
device: str,
dtype: Any,
hidden: int,
heads: int,
head_dim: int,
tokens: int = ROCM_FLASH_PROBE_TOKENS,
kv_tokens: Optional[int] = None,
ctx_dim: Optional[int] = None,
batch: int = 1,
) -> dict:
"""``kv_tokens`` / ``ctx_dim`` (default ``tokens`` / ``hidden``): a cross-attention block, queries from the image
stream and keys / values from a ``kv_tokens``-long context ``ctx_dim`` wide. ``batch`` samples per call. Median ms of one synthetic DiT block with native SDPA vs flash_attn (timing only), plus the error of the raw
flash attention output against native SDPA on the same q / k / v (accuracy).
Both arms go through diffusers' ``dispatch_attention_fn`` exactly as generation does (flash behind Studio's mask
guard), so a flash_attn build whose ``flash_attn_func`` rejects the arguments diffusers passes fails here."""
import time
import torch
import torch.nn.functional as F
from diffusers.models.attention_dispatch import AttentionBackendName, dispatch_attention_fn
if not _install_rocm_flash_dispatch_guard():
raise RuntimeError("diffusers flash backend registry not found")
if dtype not in (torch.float16, torch.bfloat16):
dtype = torch.bfloat16
inner = heads * head_dim
# Drawn on the device in the run dtype: a long bucket staged in host float32 would need GiBs of unbudgeted RAM.
gen = torch.Generator(device = device).manual_seed(0)
def rand(*shape):
return torch.randn(shape, generator = gen, device = device, dtype = dtype)
def w(rows: int, cols: int):
return rand(rows, cols) * cols**-0.5
kv = tokens if kv_tokens is None else int(kv_tokens)
width = hidden if ctx_dim is None else int(ctx_dim)
b = max(int(batch), 1)
w_qkv, w_out, w_fc1, w_fc2 = (
w(3 * inner, hidden),
w(hidden, inner),
w(4 * hidden, hidden),
w(hidden, 4 * hidden),
)
x = rand(b, tokens, hidden)
cross = kv != tokens or width != hidden
ctx = rand(b, kv, width) if cross else None
w_kv = w(2 * inner, width) if cross else None
def project():
xn = F.layer_norm(x, (hidden,))
if ctx is None:
h = F.linear(xn, w_qkv).view(b, tokens, 3, heads, head_dim)
return h[:, :, 0], h[:, :, 1], h[:, :, 2]
q = F.linear(xn, w_qkv[:inner]).view(b, tokens, heads, head_dim)
h = F.linear(ctx, w_kv).view(b, kv, 2, heads, head_dim)
return q, h[:, :, 0], h[:, :, 1]
def via(backend):
def attn(q, k, v):
out = dispatch_attention_fn(
q, k, v, attn_mask = None, dropout_p = 0.0, is_causal = False, backend = backend
)
return out[0] if isinstance(out, tuple) else out
return attn
native = via(AttentionBackendName.NATIVE)
def block(attn):
with torch.no_grad():
a = attn(*project()).reshape(b, tokens, inner)
x2 = x + F.linear(a, w_out)
return x2 + F.linear(
F.gelu(F.linear(F.layer_norm(x2, (hidden,)), w_fc1), approximate = "tanh"), w_fc2
)
arms = {"native": native, "flash": via(AttentionBackendName.FLASH)}
with torch.no_grad():
qkv = project()
ref = native(*qkv).float()
got = arms["flash"](*qkv).float()
del qkv
if tuple(got.shape) != tuple(ref.shape):
raise RuntimeError(f"flash output shape {tuple(got.shape)} != {tuple(ref.shape)}")
finite = bool(torch.isfinite(got).all())
err = (got - ref).abs()
rel = float(err.mean() / ref.abs().mean().clamp_min(1e-12))
max_rel = float(err.max() / ref.abs().max().clamp_min(1e-12))
del ref, got, err
# diagnostic only: how much of the attention error survives the residual + FFN
block_rel = None
if tokens < ROCM_FLASH_LONG_PROBE:
bref = block(native).float()
bgot = block(arms["flash"]).float()
block_rel = float((bgot - bref).abs().mean() / bref.abs().mean().clamp_min(1e-12))
del bref, bgot
times: dict[str, list] = {name: [] for name in arms}
long = tokens >= ROCM_FLASH_LONG_PROBE
for _ in range(1 if long else 2):
for fn in arms.values():
block(fn)
from .diffusion_cuda_graph import synchronize_outside_capture
synchronize_outside_capture(device)
for _ in range(3 if long else 5):
for name, fn in arms.items():
from .diffusion_cuda_graph import synchronize_outside_capture
synchronize_outside_capture(device)
t0 = time.perf_counter()
block(fn)
from .diffusion_cuda_graph import synchronize_outside_capture
synchronize_outside_capture(device)
times[name].append((time.perf_counter() - t0) * 1000.0)
med = {name: sorted(v)[len(v) // 2] for name, v in times.items()}
return {
"native_ms": round(med["native"], 3),
"flash_ms": round(med["flash"], 3),
"rel_l1": rel,
"max_rel": max_rel,
"block_rel_l1": block_rel,
"finite": finite,
"tokens": tokens,
**({} if kv == tokens else {"kv_tokens": kv}),
**({} if b == 1 else {"batch": b}),
}
def _rocm_flash_block_admitted(
pipe: Any,
target: Any,
logger: Any = None,
) -> bool:
"""Resolve ``ROCM_FLASH_CANDIDATE``: flash only if it runs and (mode auto) wins the block timing with matching output."""
if target is None or not _rocm_auto_flash(target):
return False
mode = _rocm_flash_mode()
if mode == "1":
return _install_rocm_flash_dispatch_guard()
geometries = _dit_attention_geometries(pipe)
if not geometries:
return False
device = _indexed_cuda_device(
str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
)
dtype = getattr(target, "dtype", None)
for geometry in geometries:
if not _rocm_flash_geometry_admitted(device, dtype, geometry, logger):
return False
# diffusers' flash backend raises on attn_mask: masked calls (e.g. mixed-padding batches) must run native
if not _install_rocm_flash_dispatch_guard():
return False
gate: dict[tuple, frozenset] = {}
for geometry in geometries:
shape = (device, str(dtype), geometry[1], geometry[2])
gate[shape] = gate.get(shape, frozenset()) | {tuple(geometry)}
for dit in _attention_dits(pipe):
try:
setattr(dit, ROCM_FLASH_GATE_ATTR, gate)
except Exception: # noqa: BLE001
pass
return True
# Length gating belongs to the DiT an auto load admitted, not to a shape: the gate {(device, dtype, heads, head_dim):
# hidden sizes} lives on that DiT and is active only while it runs (its forward hooks), so a model loaded with explicit
# flash or UNSLOTH_DIFFUSION_ROCM_FLASH=1 is never gated, whichever loaded first.
ROCM_FLASH_GATE_ATTR = "_unsloth_rocm_flash_gate"
# Per thread (concurrent image and video forwards never see each other's): ``gate`` and ``tally`` of the DiT forward
# running now, and ``probing`` while a timing probe runs. Dynamo traces threading.local reads.
_ROCM_FLASH_LOCAL = threading.local()
# (device, dtype, *geometry, query bucket, key bucket, batch bucket) -> flash serves it, filled on an eager call's
# first use (the probe or a stored verdict). In memory: a bucket the probe rejected stays native.
_ROCM_FLASH_BUCKETS: dict[tuple, bool] = {}
# Unrun probes defer per DiT, not verdicts; a transient shortfall cannot disable a working kernel
ROCM_FLASH_DEFERRED_ATTR = "_unsloth_rocm_flash_deferred"
_ROCM_FLASH_GATE_LOCK = threading.Lock()
# The native SDPA baseline the in-memory verdicts were timed against; when it moves (a later load re-enables or disables
# the fused SDPA kernels), both in-memory caches are dropped and the disk cache, keyed on it, decides again.
_ROCM_FLASH_BASELINE: list = [None]
# ``torch.fx.experimental.symbolic_shapes.statically_known_true`` when this torch's Dynamo traces it in graph (no
# guard), else None. Resolved outside tracing, when the dispatch guard installs.
_ROCM_FLASH_STATICALLY_KNOWN_TRUE: list = [None]
def _resolve_statically_known_true() -> None:
try:
from torch._dynamo import trace_rules
from torch.fx.experimental.symbolic_shapes import statically_known_true
name = "torch.fx.experimental.symbolic_shapes.statically_known_true"
if name in trace_rules.manual_torch_name_rule_map:
_ROCM_FLASH_STATICALLY_KNOWN_TRUE[0] = statically_known_true
except Exception: # noqa: BLE001 - older torch: the direct comparison stays
pass
def _sync_rocm_flash_baseline() -> None:
try:
current = _native_sdpa_baseline()
except Exception: # noqa: BLE001
current = None
if current != _ROCM_FLASH_BASELINE[0]:
ROCM_FLASH_ADMISSIONS.clear()
_ROCM_FLASH_BUCKETS.clear()
_ROCM_FLASH_STORED_MISSES.clear()
_ROCM_FLASH_SEGMENTS_MEMO.clear()
_ROCM_FLASH_TRACED_KEYS.clear()
_ROCM_FLASH_BASELINE[0] = current
def _rocm_flash_bucket(tokens: Any) -> int:
"""Largest power of two <= tokens, at least ROCM_FLASH_MIN_BUCKET, open ended."""
bucket = ROCM_FLASH_MIN_BUCKET
while tokens >= bucket * 2:
bucket *= 2
return bucket
def _rocm_flash_batch_bucket(batch: int) -> int:
"""Largest power of two <= batch (1, 2, 4, ...), open ended."""
bucket = 1
while batch >= bucket * 2:
bucket *= 2
return bucket
def _active_rocm_flash_gate() -> Optional[dict]:
return getattr(_ROCM_FLASH_LOCAL, "gate", None)
def _rocm_flash_capturing() -> bool:
"""A CUDA / HIP graph capture is under way (replays cannot re-route a call)."""
try:
import torch
return bool(torch.cuda.is_current_stream_capturing())
except Exception: # noqa: BLE001
return False
def _rocm_flash_length_reason(query: Any, key: Any) -> Optional[str]:
"""None when flash serves this call, else why not. Only calls made inside a DiT an auto load admitted are gated
(explicit flash and ``UNSLOTH_DIFFUSION_ROCM_FLASH=1`` are not). A call is decided by its (batch, query length, key
length) bucket, timed on first use. A traced (torch.compile) call is decided once, at trace time, by
``_rocm_flash_traced_reason``: its answer is baked into the graph, and the bucket edges compare symbolically, so a
dynamic length guards on its bucket range, not on its value (a new bucket recompiles, a new length inside one
does not). A graph capture runs native."""
gate = _active_rocm_flash_gate()
if not gate:
return None
traced = _compiling()
if not traced and _rocm_flash_capturing():
return "compiled or captured"
if getattr(_ROCM_FLASH_LOCAL, "probing", False):
return None
try:
batch, seq, heads, head_dim = query.shape
kv = key.shape[1]
raw_device, dtype_name = str(query.device), str(query.dtype)
except Exception: # noqa: BLE001 - not an NHD tensor call: leave it to the kernel as before
return None
if traced:
# Nothing here may graph-break: the device index, the verdict tables and any probe are read in plain Python
# by the constant-marked helpers, called with constants only (the gate's own key, never a traced size).
match = None
for shape, found in gate.items():
if shape[1] == dtype_name and heads == shape[2] and head_dim == shape[3]:
match = (shape, found)
break
if match is None:
return None
h, d, geometries = match[0][2], match[0][3], tuple(sorted(match[1]))
b_bucket = _rocm_flash_batch_bucket(batch)
# Self / joint attention: keys and queries share one length symbol. ``statically_known_true`` answers that
# without a guard, so cross attention (an independent key length) never specialises on the query length.
# A torch whose Dynamo cannot trace it (2.6) compares directly, which guards.
known = _ROCM_FLASH_STATICALLY_KNOWN_TRUE[0]
joint = known(kv == seq) if known is not None else kv == seq
kv_bucket = None if joint else _rocm_flash_bucket(kv)
# Guard only where the route changes, so neighbouring same-verdict buckets reuse this graph.
segments = _rocm_flash_traced_segments(
raw_device, dtype_name, h, d, geometries, kv_bucket, b_bucket
)
for end, bucket, known, reason in segments:
if seq < end:
if known:
return reason
return _rocm_flash_traced_reason(
raw_device,
dtype_name,
h,
d,
geometries,
bucket,
bucket if joint else kv_bucket,
b_bucket,
)
q_bucket = _rocm_flash_bucket(seq)
return _rocm_flash_traced_reason(
raw_device,
dtype_name,
h,
d,
geometries,
q_bucket,
q_bucket if joint else kv_bucket,
b_bucket,
)
device = _indexed_cuda_device(raw_device)
try:
geometries = gate.get((device, dtype_name, int(heads), int(head_dim)))
except Exception: # noqa: BLE001
return None
if not geometries:
return None
return _rocm_flash_bucket_reason(
device,
query.dtype,
tuple(sorted(geometries)),
_rocm_flash_bucket(int(seq)),
_rocm_flash_bucket(int(kv)),
_rocm_flash_batch_bucket(int(batch)),
)
def _rocm_flash_traced_reason(
raw_device: str,
dtype_name: str,
heads: int,
head_dim: int,
geometries: tuple,
q_bucket: int,
kv_bucket: int,
b_bucket: int,
) -> Optional[str]:
"""The traced call's route, run in plain Python while Dynamo traces (``torch.compiler.assume_constant_result``,
marked when the guard installs): the same verdict as an eager call of this bucket, timing it now if unknown (no
room beside the live forward -> native for this graph). Never raises: anything odd runs native."""
try:
import torch
device = _indexed_cuda_device(raw_device)
gate = _active_rocm_flash_gate()
if gate and (device, dtype_name, heads, head_dim) not in gate:
return "compiled or captured"
dtype = getattr(torch, dtype_name.rpartition(".")[2], None)
if not isinstance(dtype, torch.dtype):
return "compiled or captured"
before = set(_ROCM_FLASH_BUCKETS)
try:
return _rocm_flash_bucket_reason(
device, dtype, geometries, q_bucket, kv_bucket, b_bucket
)
finally:
_ROCM_FLASH_TRACED_KEYS.update(set(_ROCM_FLASH_BUCKETS) - before)
except Exception: # noqa: BLE001
return "compiled or captured"
# Dynamo replays a frame after a graph break, so segments must answer as on the first pass (else
# SpeculationLogDivergence, Wan2.2 on gfx1151). Key excludes verdicts added by traced calls.
_ROCM_FLASH_TRACED_KEYS: set = set()
_ROCM_FLASH_SEGMENTS_MEMO: dict = {}
def _rocm_flash_segments_key(args: tuple) -> tuple:
known = frozenset(
(k, v) for k, v in _ROCM_FLASH_BUCKETS.items() if k not in _ROCM_FLASH_TRACED_KEYS
)
return (*args, id(_active_rocm_flash_gate()), known)
# Traced query lengths are routed by segments up to this many tokens; longer ones guard on their own bucket.
_ROCM_FLASH_SEGMENT_LIMIT = 1 << 22
# Admission keys with no stored verdict at first stored-only lookup (later verdicts reach the per-bucket route).
_ROCM_FLASH_STORED_MISSES: set = set()
_ROCM_FLASH_SLOWER = "slower or inexact at this size"
def _rocm_flash_bucket_known(
device: str, dtype: Any, geometries: tuple, q_bucket: int, kv_bucket: int, b_bucket: int
) -> Optional[bool]:
"""This bucket's verdict if already known (in memory, this load's admissions or the stored timings), never
probing: True flash, False native, None untimed or deferred."""
deferred = getattr(_ROCM_FLASH_LOCAL, "deferred", None)
verdict: Optional[bool] = True
for g in geometries:
k = (device, str(dtype), *g, q_bucket, kv_bucket, b_bucket)
ok = _ROCM_FLASH_BUCKETS.get(k)
if ok is None and not (isinstance(deferred, set) and k in deferred):
ok = _rocm_flash_geometry_admitted(
device,
dtype,
g,
tokens = q_bucket,
batch = b_bucket,
stored_only = True,
**({} if kv_bucket == q_bucket else {"kv_tokens": kv_bucket}),
)
if ok is not None:
_ROCM_FLASH_BUCKETS[k] = ok
if ok is False:
return False
if ok is None:
verdict = None
return verdict
def _rocm_flash_traced_segments(
raw_device: str,
dtype_name: str,
heads: int,
head_dim: int,
geometries: tuple,
kv_bucket: Optional[int],
b_bucket: int,
) -> tuple:
"""The query-length axis of a traced call as ascending ``(end, bucket, known, reason)`` segments: a run of
neighbouring buckets whose known verdict matches is one segment (``reason`` its route), an untimed bucket is its
own (``known`` False). ``kv_bucket`` None means keys are as long as queries. Run in plain Python while Dynamo
traces (constant-marked like ``_rocm_flash_traced_reason``); ``()`` when anything is odd (the caller falls back
to the per-bucket route)."""
args = (raw_device, dtype_name, heads, head_dim, geometries, kv_bucket, b_bucket)
try:
key = _rocm_flash_segments_key(args)
if key in _ROCM_FLASH_SEGMENTS_MEMO:
return _ROCM_FLASH_SEGMENTS_MEMO[key]
segments = _rocm_flash_traced_segments_uncached(*args)
# Under both keys: reading stored verdicts while building the segments moves the key itself.
_ROCM_FLASH_SEGMENTS_MEMO[key] = _ROCM_FLASH_SEGMENTS_MEMO[
_rocm_flash_segments_key(args)
] = segments
return segments
except Exception: # noqa: BLE001
return ()
def _rocm_flash_traced_segments_uncached(
raw_device: str,
dtype_name: str,
heads: int,
head_dim: int,
geometries: tuple,
kv_bucket: Optional[int],
b_bucket: int,
) -> tuple:
try:
import torch
device = _indexed_cuda_device(raw_device)
gate = _active_rocm_flash_gate()
if gate and (device, dtype_name, heads, head_dim) not in gate:
return ()
dtype = getattr(torch, dtype_name.rpartition(".")[2], None)
if not isinstance(dtype, torch.dtype):
return ()
_sync_rocm_flash_baseline()
segments: list = []
bucket = ROCM_FLASH_MIN_BUCKET
with _ROCM_FLASH_GATE_LOCK:
while bucket < _ROCM_FLASH_SEGMENT_LIMIT:
ok = _rocm_flash_bucket_known(
device, dtype, geometries, bucket, kv_bucket or bucket, b_bucket
)
reason = None if ok else _ROCM_FLASH_SLOWER
if ok is not None and segments and segments[-1][2] and segments[-1][3] == reason:
segments[-1] = (bucket * 2, segments[-1][1], True, reason)
else:
segments.append((bucket * 2, bucket, ok is not None, reason))
bucket *= 2
return tuple(segments)
except Exception: # noqa: BLE001
return ()
def _rocm_flash_bucket_reason(
device: str, dtype: Any, geometries: tuple, q_bucket: int, kv_bucket: int, b_bucket: int
) -> Optional[str]:
_sync_rocm_flash_baseline()
keys = [(device, str(dtype), *g, q_bucket, kv_bucket, b_bucket) for g in geometries]
if all(_ROCM_FLASH_BUCKETS.get(k) is True for k in keys):
return None
if any(_ROCM_FLASH_BUCKETS.get(k) is False for k in keys):
return _ROCM_FLASH_SLOWER
deferred = getattr(_ROCM_FLASH_LOCAL, "deferred", None)
if not isinstance(deferred, set):
deferred = set() # no owning DiT: nothing to remember the deferral on
if any(k in deferred for k in keys):
return "timing deferred"
with _ROCM_FLASH_GATE_LOCK:
verdict = True
for k, g in zip(keys, geometries):
ok = _ROCM_FLASH_BUCKETS.get(k)
if ok is None:
ok = _rocm_flash_geometry_admitted(
device,
dtype,
g,
tokens = q_bucket,
batch = b_bucket,
live = True,
**({} if kv_bucket == q_bucket else {"kv_tokens": kv_bucket}),
)
if ok is None:
deferred.add(k)
return "timing deferred"
_ROCM_FLASH_BUCKETS[k] = ok
verdict = verdict and ok
return None if verdict else _ROCM_FLASH_SLOWER
def _native_sdpa_fused_serves(
device: Any, dtype: Any, heads: int, head_dim: int, tokens: int, kv_tokens: int, batch: int
) -> bool:
"""Whether native SDPA takes an enabled fused (flash / memory-efficient) kernel at exactly this shape, asked of
torch's own dispatcher with zero-size-storage views; otherwise (or when it cannot say) it runs the math backend."""
try:
import torch
from torch.backends.cuda import (
SDPAParams,
can_use_efficient_attention,
can_use_flash_attention,
)
cuda = torch.backends.cuda
def view(n):
return torch.empty((1, 1, 1, head_dim), device = device, dtype = dtype).expand(
batch, heads, n, head_dim
)
q, k = view(tokens), view(kv_tokens)
try:
params = SDPAParams(q, k, k, None, 0.0, False, False)
except TypeError: # torch before enable_gqa
params = SDPAParams(q, k, k, None, 0.0, False)
return bool(
(cuda.flash_sdp_enabled() and can_use_flash_attention(params, False))
or (cuda.mem_efficient_sdp_enabled() and can_use_efficient_attention(params, False))
)
except Exception: # noqa: BLE001
return False
def _rocm_flash_probe_bytes(
geometry: tuple,
tokens: int,
kv_tokens: int,
batch: int,
dtype: Any,
device: Any = "cuda",
) -> int:
"""Upper bound on what ``_time_rocm_flash_block`` allocates: its weights, the block's widest activations and the
fp32 accuracy outputs (measured on gfx1151: 5.46 GiB at Wan's 64k tokens, this gives 8.1 GiB). Unless a fused SDPA
kernel is confirmed for this exact shape (``guard_rocm_fused_sdpa`` may have turned them off, or one may not serve
this length / head size), the native arm runs math SDPA, which also holds fp32 scores and their softmax, batch x
heads x queries x keys each."""
hidden, heads, head_dim = (int(v) for v in geometry[:3])
width = int(geometry[3]) if len(geometry) > 3 else hidden
inner = heads * head_dim
elem = getattr(dtype, "itemsize", 2) or 2
weights = (4 * inner * hidden + 8 * hidden * hidden + 2 * inner * width) * elem
per_query = (11 * hidden + 3 * inner) * elem + 2 * inner * 4
per_key = (width + 2 * inner) * elem
fused = _native_sdpa_fused_serves(device, dtype, heads, head_dim, tokens, kv_tokens, batch)
scores = 0 if fused else 2 * batch * heads * tokens * kv_tokens * 4
return weights + batch * (tokens * per_query + kv_tokens * per_key) + scores
def _rocm_flash_probe_fits(device: str, need: int) -> bool:
"""Probe room: free plus idle cache, less 25% + 256 MiB and the VRAM Budget reserve; unknown is none"""
try:
import torch
from .diffusion_memory import user_budget_reserve_bytes
free, total = torch.cuda.mem_get_info(device)
cached = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)
reserve = user_budget_reserve_bytes(device, total)
return free + max(int(cached), 0) - reserve >= need * 1.25 + (256 << 20)
except Exception: # noqa: BLE001
return False
def _rocm_flash_geometry_admitted(
device: str,
dtype: Any,
geometry: tuple,
logger: Any = None,
tokens: int = ROCM_FLASH_PROBE_TOKENS,
kv_tokens: Optional[int] = None,
batch: int = 1,
live: bool = False,
stored_only: bool = False,
) -> Optional[bool]:
"""Whether flash is admitted for ``geometry`` (hidden, heads, head_dim[, context width]) at ``batch`` x ``tokens``
queries x ``kv_tokens`` keys (default square, batch 1). None when the probe could not run (it raised, or ``live``,
inside a running forward, and there is no room for it beside that forward's tensors): nothing is cached.
``stored_only``: None instead of probing when no verdict is known yet."""
kv = tokens if kv_tokens is None else int(kv_tokens)
# Geometry, lengths and batch as separate fields: a flat tuple aliases (h, H, d, ctx) at n with (h, H, d) at ctx x n.
key = (device, str(dtype), tuple(geometry), tokens, kv, batch)
_sync_rocm_flash_baseline()
cached = ROCM_FLASH_ADMISSIONS.get(key)
if cached is not None:
return bool(cached.get("admitted"))
if stored_only and key in _ROCM_FLASH_STORED_MISSES:
return (
None # traced segments ask for every bucket on every call: read the store once per slot
)
toolchain = _rocm_flash_toolchain_key(device)
path = _rocm_flash_cache_path(toolchain)
size = (str(tokens) if kv == tokens else f"{tokens}x{kv}") + ("" if batch == 1 else f"b{batch}")
slot = "|".join((str(dtype), *(str(v) for v in geometry), size))
stored = _load_rocm_flash_verdicts(path, toolchain).get(slot)
if (
stored is not None
and _rocm_flash_record_ok(stored)
and stored["tokens"] == tokens
and stored.get("kv_tokens", stored["tokens"]) == kv
and stored.get("batch", 1) == batch
):
rec = {**stored, "admitted": _rocm_flash_wins(stored), "cached": True}
ROCM_FLASH_ADMISSIONS[key] = rec
_log_rocm_flash_verdict(logger, device, geometry, rec, "cached")
return rec["admitted"]
if stored_only:
_ROCM_FLASH_STORED_MISSES.add(key)
return None
if live:
need = _rocm_flash_probe_bytes(geometry, tokens, kv, batch, dtype, device)
if not _rocm_flash_probe_fits(device, need):
_module_logger().info(
"diffusion.attention: ROCm flash timing at %s tokens deferred, no room beside this forward "
"(needs about %.2f GiB); native SDPA for now",
size,
need / 2**30,
)
return None
_ROCM_FLASH_LOCAL.probing = True # the probe's own flash calls are neither gated nor counted
try:
shape = {} if kv == tokens else {"kv_tokens": kv}
if len(geometry) > 3:
shape["ctx_dim"] = geometry[3]
if batch != 1:
shape["batch"] = batch
rec = _time_rocm_flash_block(device, dtype, *geometry[:3], tokens = tokens, **shape)
except Exception as exc: # noqa: BLE001 - not cached: retried next load
_module_logger().info(
"diffusion.attention: ROCm flash timing probe could not run (%s); native SDPA", exc
)
return None
finally:
_ROCM_FLASH_LOCAL.probing = False
try:
import torch
torch.cuda.empty_cache()
except Exception: # noqa: BLE001
pass
rec = dict(rec) if isinstance(rec, dict) else {}
rec["geometry"] = {"hidden": geometry[0], "heads": geometry[1], "head_dim": geometry[2]}
if len(geometry) > 3:
rec["geometry"]["context"] = geometry[3]
valid = _rocm_flash_record_ok(rec)
rec["admitted"] = valid and _rocm_flash_wins(rec)
ROCM_FLASH_ADMISSIONS[key] = rec
_ROCM_FLASH_STORED_MISSES.discard(key)
if valid: # an implausible timing is not stored, so it cannot pin a verdict
_save_rocm_flash_verdict(path, toolchain, slot, rec)
_log_rocm_flash_verdict(logger, device, geometry, rec, "probe")
return rec["admitted"]
def _real_number(value: Any) -> bool:
return isinstance(value, (int, float)) and not isinstance(value, bool)
def _rocm_flash_record_ok(rec: Any) -> bool:
"""Every field a verdict is derived from has the type and range the probe writes; anything else is a cache miss."""
import math
try:
if not isinstance(rec, dict) or not isinstance(rec.get("finite"), bool):
return False
for name in ("native_ms", "flash_ms"):
v = rec.get(name)
if not _real_number(v) or not math.isfinite(v) or v <= 0:
return False
for name in ("tokens", "kv_tokens", "batch"):
n = rec.get(name, 1)
if not isinstance(n, int) or isinstance(n, bool) or n <= 0:
return False
rel = rec.get("rel_l1")
if rec["finite"]:
if not _real_number(rel) or not math.isfinite(rel) or rel < 0:
return False
elif rel is not None and not _real_number(rel): # a non-finite output may leave NaN here
return False
max_rel = rec.get("max_rel")
if rec["finite"]:
if not _real_number(max_rel) or not math.isfinite(max_rel) or max_rel < 0:
return False
elif max_rel is not None and not _real_number(max_rel):
return False
if rec.get("block_rel_l1") is not None and not _real_number(rec["block_rel_l1"]):
return False
if "admitted" in rec and not isinstance(rec["admitted"], bool):
return False
geometry = rec.get("geometry")
if geometry is not None and not (
isinstance(geometry, dict)
and all(
isinstance(geometry.get(k), int)
and not isinstance(geometry.get(k), bool)
and geometry.get(k) > 0
for k in ("hidden", "heads", "head_dim")
+ (("context",) if "context" in geometry else ())
)
):
return False
return True
except Exception: # noqa: BLE001
return False
def _rocm_flash_wins(rec: dict) -> bool:
"""Re-derived from the stored numbers on a cache hit, so a threshold change applies without re-timing."""
try:
return bool(
rec["finite"] is True
and float(rec["rel_l1"]) <= ROCM_FLASH_BLOCK_TOL
and float(rec["max_rel"]) <= ROCM_FLASH_MAX_TOL
and float(rec["flash_ms"]) <= float(rec["native_ms"]) * (1.0 - ROCM_FLASH_MIN_GAIN)
)
except Exception: # noqa: BLE001 - malformed record: never flash
return False
def _log_rocm_flash_verdict(
logger: Any, device: str, geometry: tuple, rec: dict, source: str
) -> None:
"""Never raises: the values are formatted defensively, whatever a record holds."""
def num(value: Any, spec: str) -> str:
try:
return format(float(value), spec) if _real_number(value) else "?"
except Exception: # noqa: BLE001
return "?"
try:
(logger or _module_logger()).info(
"diffusion.attention: ROCm flash block %s %s (hidden %s, heads %s, head_dim %s, %s tokens): flash %s ms "
"vs native %s ms, attention rel L1 %s, max %s -> %s",
source,
device,
*(str(g) for g in tuple(geometry)[:3]),
num(rec.get("tokens"), ".0f"),
num(rec.get("flash_ms"), ".2f"),
num(rec.get("native_ms"), ".2f"),
num(rec.get("rel_l1"), ".2e"),
num(rec.get("max_rel"), ".2e"),
"flash" if rec.get("admitted") is True else "native",
)
except Exception: # noqa: BLE001 - logging is diagnostic only
pass
_ROCM_DRIVER_VERSION: list = []
def _rocm_driver_version() -> str:
"""The ROCm stack this process actually runs on, not the wheel's build (``torch.version.hip``): the HIP runtime's
``hipDriverGetVersion`` from the libamdhip64 this process loaded (HIP runtime API), plus the amdgpu kernel driver
(``/sys/module/amdgpu/version`` for the DKMS driver, else the kernel release for the in-tree one). Read once per
process; parts that cannot be read are left out."""
if _ROCM_DRIVER_VERSION:
return _ROCM_DRIVER_VERSION[0]
import ctypes
import platform
parts = []
try:
lib = None
if os.name == "nt":
names = ["amdhip64_7.dll", "amdhip64_6.dll", "amdhip64.dll"]
else:
with open("/proc/self/maps", encoding = "utf-8", errors = "replace") as f:
names = sorted(
{line.split()[-1] for line in f if "libamdhip64" in line and "/" in line}
)
for name in names:
try:
lib = ctypes.CDLL(name)
break
except OSError:
continue
if lib is not None:
version = ctypes.c_int(0)
if lib.hipDriverGetVersion(ctypes.byref(version)) == 0:
parts.append(f"hipdriver{version.value}")
except Exception: # noqa: BLE001
pass
try:
with open("/sys/module/amdgpu/version", encoding = "utf-8") as f:
parts.append(f"amdgpu{f.read().strip()}")
except OSError:
try:
parts.append(f"kernel{platform.release()}")
except Exception: # noqa: BLE001
pass
_ROCM_DRIVER_VERSION.append("driver:" + (",".join(parts) or "?"))
return _ROCM_DRIVER_VERSION[0]
def _diffusers_flash_path_fingerprint() -> str:
"""diffusers version + a hash of the code generation runs flash through (``dispatch_attention_fn`` and the
registered FLASH backend, unwrapped from Studio's guard), so a diffusers change that alters the call re-times."""
import hashlib
import inspect
import diffusers
from diffusers.models.attention_dispatch import (
AttentionBackendName,
_AttentionBackendRegistry,
dispatch_attention_fn,
)
flash = _AttentionBackendRegistry._backends[AttentionBackendName.FLASH]
while (
getattr(flash, _SAGE_GUARD_ATTR, False) and getattr(flash, "__wrapped__", None) is not None
):
flash = flash.__wrapped__
parts = []
for fn in (dispatch_attention_fn, flash):
try:
parts.append(inspect.getsource(fn))
except (OSError, TypeError):
code = getattr(fn, "__code__", None)
parts.append(repr((code.co_code, code.co_consts)) if code is not None else repr(fn))
digest = hashlib.sha1("\n".join(parts).encode("utf-8")).hexdigest()[:12]
return f"diffusers{getattr(diffusers, '__version__', '?')}:{digest}"
def _flash_attn_build_fingerprint() -> str:
"""size + mtime_ns of the flash_attn kernel files this process loaded: the CK extension (``flash_attn_2_cuda``) or,
for the Triton AMD backend, the source files of the package ``flash_attn_interface.flash_attn_gpu`` came from. A
rebuild in place (same version, same path) changes it. Stat only."""
import importlib.util
import sys
files: set[str] = set()
for name in ("flash_attn", "flash_attn.flash_attn_interface", "flash_attn_2_cuda"):
f = getattr(sys.modules.get(name), "__file__", None)
if isinstance(f, str) and f:
files.add(f)
gpu = getattr(sys.modules.get("flash_attn.flash_attn_interface"), "flash_attn_gpu", None)
gpu_file = getattr(gpu, "__file__", None)
if isinstance(gpu_file, str) and gpu_file:
files.add(gpu_file)
if gpu_file.endswith(".py"):
folder = os.path.dirname(gpu_file)
files.update(os.path.join(folder, n) for n in os.listdir(folder) if n.endswith(".py"))
elif "flash_attn_2_cuda" not in sys.modules:
try:
spec = importlib.util.find_spec("flash_attn_2_cuda")
if spec is not None and spec.origin:
files.add(spec.origin)
except Exception: # noqa: BLE001
pass
parts = []
for f in sorted(files):
try:
st = os.stat(f)
parts.append(f"{os.path.realpath(f)}:{st.st_size}:{st.st_mtime_ns}")
except OSError:
parts.append(f"{f}:missing")
import hashlib
return hashlib.sha1("\n".join(parts).encode("utf-8")).hexdigest()[:12]
def _rocm_flash_toolchain_key(device: str) -> Optional[str]:
"""What the block timing depends on: card + gfx arch, torch, HIP, the flash_attn build (version, install path, CK or
Triton backend) and the probe itself. None (in-memory only) when any part is unreadable."""
try:
import hashlib
import importlib.metadata
import inspect
import sys
import torch
from utils.hardware.hardware import _props_gfx_arch
index = torch.device(device).index
index = torch.cuda.current_device() if index is None else index
props = torch.cuda.get_device_properties(index)
fa = sys.modules.get("flash_attn")
if fa is None:
import flash_attn as fa
fa_version = getattr(fa, "__version__", None) or importlib.metadata.version("flash_attn")
# flash_attn_interface reads exactly this to route to the Triton AMD kernels instead of the CK extension, and
# falls back to Triton itself when the CK extension does not import; its USE_TRITON_ROCM is the outcome.
triton_amd = getattr(
sys.modules.get("flash_attn.flash_attn_interface"), "USE_TRITON_ROCM", None
)
if not isinstance(triton_amd, bool):
triton_amd = os.environ.get("FLASH_ATTENTION_TRITON_AMD_ENABLE", "FALSE") == "TRUE"
backend = "ck"
if triton_amd:
import triton
backend = f"triton{triton.__version__}"
src = inspect.getsource(_time_rocm_flash_block)
return "|".join(
(
f"v{ROCM_FLASH_PROBE_VERSION}",
str(props.name),
_props_gfx_arch(props) or "gfx?",
f"torch{torch.__version__}",
f"hip{getattr(torch.version, 'hip', None)}",
_rocm_driver_version(),
_native_sdpa_baseline(),
f"flash_attn{fa_version}",
os.path.dirname(os.path.realpath(fa.__file__)),
backend,
f"build{_flash_attn_build_fingerprint()}",
_diffusers_flash_path_fingerprint(),
hashlib.sha1(src.encode("utf-8")).hexdigest()[:12],
)
)
except Exception: # noqa: BLE001 - unidentifiable toolchain: never persisted
return None
def _native_sdpa_baseline() -> str:
"""The native arm a verdict was timed against: which SDPA backends are on (``guard_rocm_fused_sdpa``, the user, or
``UNSLOTH_ALLOW_ROCM_FUSED_SDPA`` move them) and the AOTriton experimental switch that decides whether they run."""
import torch
cuda = torch.backends.cuda
flags = "".join(
name if enabled() else "-"
for name, enabled in (
("f", cuda.flash_sdp_enabled),
("e", cuda.mem_efficient_sdp_enabled),
("m", cuda.math_sdp_enabled),
)
)
experimental = os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "").strip() or "unset"
return f"sdpa{flags},aotriton-exp{experimental}"
def _rocm_flash_cache_path(toolchain: Optional[str]) -> Optional[str]:
if toolchain is None:
return None
import hashlib
root = (os.environ.get(ROCM_FLASH_CACHE_DIR_ENV) or "").strip()
if not root:
try:
from utils.paths.storage_roots import cache_root
root = str(cache_root() / "diffusion_rocm_flash")
except Exception: # noqa: BLE001 - outside Studio: in-memory only
return None
return os.path.join(
root, "probe_" + hashlib.sha1(toolchain.encode("utf-8")).hexdigest()[:16] + ".json"
)
def _load_rocm_flash_verdicts(path: Optional[str], toolchain: Optional[str]) -> dict:
import json
if path is None or not os.path.isfile(path):
return {}
try:
with open(path, encoding = "utf-8") as f:
data = json.load(f)
if not isinstance(data, dict) or data.get("key") != toolchain:
return {}
verdicts = data.get("verdicts")
if not isinstance(verdicts, dict):
return {}
return {str(slot): rec for slot, rec in verdicts.items() if _rocm_flash_record_ok(rec)}
except Exception: # noqa: BLE001 - a corrupt / unreadable file is re-timed
return {}
def _save_rocm_flash_verdict(
path: Optional[str], toolchain: Optional[str], slot: str, rec: dict
) -> None:
"""Best effort, under a file lock: merge what another process stored since, then write a temp file and replace."""
if path is None:
return
import contextlib
import json
tmp = f"{path}.{os.getpid()}.{threading.get_ident()}.tmp"
try:
os.makedirs(os.path.dirname(path), exist_ok = True)
try:
from filelock import FileLock
lock: Any = FileLock(f"{path}.lock", timeout = 60)
except ImportError:
lock = contextlib.nullcontext()
with lock:
verdicts = _load_rocm_flash_verdicts(path, toolchain)
verdicts[slot] = {k: v for k, v in rec.items() if k not in ("admitted", "cached")}
with open(tmp, "w", encoding = "utf-8") as f:
json.dump({"key": toolchain, "verdicts": verdicts}, f, indent = 1, sort_keys = True)
os.replace(tmp, path)
except Exception: # noqa: BLE001 - read-only cache / lock timeout: timed again next load
try:
os.remove(tmp)
except OSError:
pass
# Max abs error of the probe's flash_attn output against an fp32 reference (bf16 eps is ~4e-3).
_ROCM_FLASH_PROBE_TOL = 2e-2
# (device, dtype) -> whether flash_attn ran and matched. Only answers are cached; an unaskable probe is retried.
_ROCM_FLASH_PROBE_CACHE: dict[tuple[str, str], bool] = {}
_ROCM_FLASH_MISSING_LOGGED: list[bool] = []
_ROCM_FLASH_HINT = (
"the PyPI flash-attn wheel is the NVIDIA CUDA build; on ROCm install the DAO-AILab flash-attention ROCm build "
"(github.com/Dao-AILab/flash-attention, CK backend, or FLASH_ATTENTION_TRITON_AMD_ENABLE=TRUE for Triton) "
"or use attention_backend=aiter"
)
def _module_logger() -> Any:
try:
from loggers import get_logger
return get_logger(__name__)
except Exception: # noqa: BLE001
import logging
return logging.getLogger(__name__)
def _run_rocm_flash_probe(device: str, dtype: Any) -> bool:
"""Tiny ``flash_attn_func`` on ``device`` vs an fp32 reference; False when it raises or disagrees."""
import torch
from flash_attn import flash_attn_func
if dtype not in (torch.float16, torch.bfloat16):
# diffusers' flash backend is half precision only, and the CK build raises on fp32 at the first call
if dtype is not None:
return False
dtype = torch.bfloat16
gen = torch.Generator().manual_seed(0)
q, k, v = (torch.randn((1, 128, 2, 64), generator = gen).to(device, dtype) for _ in range(3))
try:
out = flash_attn_func(q, k, v)
torch.cuda.synchronize(device)
except torch.cuda.OutOfMemoryError:
raise
except Exception: # noqa: BLE001 - no kernel for this arch is the answer
return False
qf, kf, vf = (t.float().transpose(1, 2) for t in (q, k, v))
ref = torch.softmax(qf @ kf.transpose(-1, -2) * qf.shape[-1] ** -0.5, dim = -1) @ vf
err = (out.float() - ref.transpose(1, 2)).abs().max()
return bool(torch.isfinite(err).item() and err.item() <= _ROCM_FLASH_PROBE_TOL)
def _rocm_flash_attn_runs(target: Any) -> bool:
"""True only when flash_attn imports and its kernel ran and matched on this ROCm card at this dtype."""
device = _indexed_cuda_device(
str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
)
dtype = getattr(target, "dtype", None)
key = (device, str(dtype))
cached = _ROCM_FLASH_PROBE_CACHE.get(key)
if cached is not None:
return cached
try:
import flash_attn # noqa: F401
except Exception as exc: # noqa: BLE001
if not _ROCM_FLASH_MISSING_LOGGED:
_ROCM_FLASH_MISSING_LOGGED.append(True)
_module_logger().warning(
"diffusion.attention: flash requested on ROCm but flash_attn is not importable (%s); %s. "
"Using the default backend",
exc,
_ROCM_FLASH_HINT,
)
return False
try:
ok = _run_rocm_flash_probe(device, dtype)
except Exception: # noqa: BLE001 - OOM / device trouble: no answer, keep native this time
return False
if not ok:
_module_logger().warning(
"diffusion.attention: flash_attn did not run or did not match SDPA on %s (%s); using the default backend",
device,
dtype,
)
return _ROCM_FLASH_PROBE_CACHE.setdefault(key, ok)
def _cudnn_attention_supported() -> bool:
"""cuDNN fused SDPA needs Ampere+ (SM80); on pre-SM80 cards (T4/V100) diffusers accepts it
then fails at generation, so gate the upgrade on capability. Unknown capability allows it."""
have = _cuda_capability()
return have is None or have >= (8, 0)
# Arch support varies by sageattn build, so a run per card is the gate. (device, dtype, head_dim) -> "" or why not.
_SAGE_PROBE_CACHE: dict[tuple[str, str, int], str] = {}
_SAGE_MAX_HEAD_DIM = 128 # sageattn raises above 128
# SageAttention 2 lands near cos 0.9999 / rel-L1 0.01-0.03 vs fp32 here; a wrong-arch or broken build is far outside.
_SAGE_MIN_COSINE = 0.99
_SAGE_MAX_REL_L1 = 0.08
def _run_sage_probe(
device: str,
dtype: Any,
head_dim: int = 128,
sageattn: Any = None,
) -> str:
"""Empty when sageattn matches fp32 SDPA at head_dim, else the reason; raises if unaskable."""
import torch
if sageattn is None:
from sageattention import sageattn
if dtype not in (torch.float16, torch.bfloat16):
dtype = torch.float16
shape = (1, 128, 2, int(head_dim))
gen = torch.Generator(device = "cpu").manual_seed(0)
q, k, v = (torch.randn(shape, generator = gen) for _ in range(3))
k = k + 2.0 * torch.randn((1, 1, 1, shape[-1]), generator = gen)
q, k, v = (t.to(device = device, dtype = dtype) for t in (q, k, v))
try:
out = sageattn(q, k, v, tensor_layout = "NHD")
if str(device).startswith("cuda"):
torch.cuda.synchronize(device)
except torch.cuda.OutOfMemoryError:
raise
except Exception as exc: # noqa: BLE001
return f"{type(exc).__name__}: {exc}"
try:
ref = torch.nn.functional.scaled_dot_product_attention(
*(t.float().transpose(1, 2) for t in (q, k, v))
).transpose(1, 2)
got = out.float()
if tuple(got.shape) != tuple(ref.shape):
return f"self-check: output shape {tuple(got.shape)} != {tuple(ref.shape)}"
if not bool(torch.isfinite(got).all()):
return "self-check: non-finite output"
cos = float(torch.nn.functional.cosine_similarity(got.flatten(), ref.flatten(), dim = 0))
rel = float((got - ref).abs().mean() / ref.abs().mean().clamp_min(1e-12))
except torch.cuda.OutOfMemoryError:
raise
except Exception as exc: # noqa: BLE001
return f"self-check: {type(exc).__name__}: {exc}"
if cos < _SAGE_MIN_COSINE or rel > _SAGE_MAX_REL_L1:
return f"self-check: cosine {cos:.4f}, relative L1 {rel:.4f} vs fp32 at head_dim {head_dim}"
return ""
def _indexed_cuda_device(device: str) -> str:
"""Bare "cuda" -> the card pinned on this thread, so one card's verdict is never reused for another."""
if device != "cuda":
return device
try:
import torch
return f"cuda:{torch.cuda.current_device()}"
except Exception: # noqa: BLE001
return device
def _sage_probe_head_dims(head_dims: Any) -> tuple[int, ...]:
"""Head dims Sage serves (<= 128; larger run native per call), 128 when none are known."""
dims = sorted(
{
int(d)
for d in (head_dims or ())
if isinstance(d, int) and not isinstance(d, bool) and 0 < d <= _SAGE_MAX_HEAD_DIM
}
)
return tuple(dims) or (_SAGE_MAX_HEAD_DIM,)
def _sage_kernel_runs(
target: Any,
logger: Any = None,
head_dims: Any = None,
) -> Optional[bool]:
"""True when ``sageattn`` passed its self-check at every head dim it serves; False when it raised, failed, or is
not importable (not cached: a later install may fix it); None (unaskable) keeps the request."""
device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
if not device.startswith("cuda"):
return None
device = _indexed_cuda_device(device)
dtype = getattr(target, "dtype", None)
error = ""
for head_dim in _sage_probe_head_dims(head_dims):
key = (device, str(dtype), head_dim)
error = _SAGE_PROBE_CACHE.get(key)
if error is None:
try:
error = _run_sage_probe(device, dtype, head_dim)
except ImportError as exc:
error = f"sageattention could not be imported ({type(exc).__name__}: {exc})"
break
except Exception: # noqa: BLE001
return None
error = _SAGE_PROBE_CACHE.setdefault(key, error)
if error:
break
if error and logger is not None:
logger.warning(
"diffusion.attention: SageAttention does not run on this GPU (%s); using the default backend",
error,
)
return not error
def _runs_in_float32(target: Any) -> bool:
"""fp32 pipeline (video families promote fp16 on cards without bf16): Sage never applies."""
dtype = getattr(target, "dtype", None)
if dtype is None:
return False
name = str(dtype).replace("torch.", "").lower()
return name in ("float32", "fp32", "float")
# Calls Sage cannot take run native; the checks read only shapes/dtypes/mask, so no graph break.
_SAGE_GUARD_ATTR = "_unsloth_sage_guard"
_SAGE_ROUTED: dict[str, int] = {}
_SAGE_ROUTED_LOGGED: set[str] = set()
def _sage_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any) -> Optional[str]:
import torch
if attn_mask is not None:
return "attn_mask"
if not all(isinstance(t, torch.Tensor) for t in (query, key, value)):
return "inputs"
if (
query.dtype not in (torch.float16, torch.bfloat16)
or key.dtype != query.dtype
or value.dtype != query.dtype
):
return "dtype"
if query.dim() != 4 or key.dim() != 4 or value.dim() != 4:
return "rank"
head_dim = query.shape[-1]
if head_dim > _SAGE_MAX_HEAD_DIM or key.shape[-1] != head_dim or value.shape[-1] != head_dim:
return "head_dim"
if key.shape[2] == 0 or query.shape[2] % key.shape[2] != 0:
return "heads"
if query.device.type != "cuda":
return "device"
return None
def _note_reroute(label: str, reason: str, counts: dict, logged: set) -> None:
"""Count and log once per reason. Eager only: under torch.compile it would be a traced side effect."""
try:
import torch
if torch.compiler.is_compiling():
return
except Exception: # noqa: BLE001
pass
counts[reason] = counts.get(reason, 0) + 1
if reason not in logged:
logged.add(reason)
import logging
logging.getLogger(__name__).warning(
"diffusion.attention: %s cannot take this attention call (%s); running it on the default "
"backend instead",
label,
reason,
)
def _note_sage_reroute(reason: str) -> None:
_note_reroute("SageAttention", reason, _SAGE_ROUTED, _SAGE_ROUTED_LOGGED)
# sageattn reads arch via untraceable torch.cuda calls: opaque op keeps the block compiled.
_SAGE_OP_NAME = "unsloth_studio::sage_attention_nhd"
# Own op for the hub build, so a pip SageAttention 2 installed later never runs through it.
_SAGE_HUB_OP_NAME = "unsloth_studio::sage_hub_attention_nhd"
_SAGE_OPS: dict[str, dict[str, Any]] = {}
_SAGE_OP_LOCK = threading.Lock()
def _sage_custom_op(sage_fn: Any, op_name: str = _SAGE_OP_NAME) -> Optional[Any]:
"""An opaque ``torch.library`` op around a diffusers attention function (NHD), or None. Used for Sage and FA4."""
with _SAGE_OP_LOCK:
slot = _SAGE_OPS.setdefault(op_name, {})
if "op" in slot:
slot["fn"] = sage_fn
return slot["op"]
try:
import torch
slot["fn"] = sage_fn
def _impl(query, key, value, is_causal, scale):
out = slot["fn"](
query = query, key = key, value = value, is_causal = is_causal, scale = scale
)
# Head dims under 64 come back as a padded slice; the fake describes a contiguous tensor.
return out.contiguous()
# custom_op infers the schema from real types, not `from __future__ import annotations` strings.
_impl.__annotations__ = {
"query": torch.Tensor,
"key": torch.Tensor,
"value": torch.Tensor,
"is_causal": bool,
"scale": Optional[float],
"return": torch.Tensor,
}
_op = torch.library.custom_op(op_name, mutates_args = ())(_impl)
@_op.register_fake
def _(query, key, value, is_causal, scale):
return query.new_empty((*query.shape[:-1], value.shape[-1]))
slot["op"] = _op
except Exception: # noqa: BLE001 - old torch: call the function directly
slot["op"] = None
return slot["op"]
def _install_dispatch_guard(
backend: str,
reroute_reason: Any,
note: Any,
make_op: Any = None,
ran: Any = None,
) -> bool:
"""Wrap the registered backend so unsupported calls run native; idempotent; False if registry moved"""
try:
from diffusers.models.attention_dispatch import (
AttentionBackendName,
_AttentionBackendRegistry,
)
backends = _AttentionBackendRegistry._backends
name = AttentionBackendName(backend)
kernel_fn = backends[name]
native_fn = backends[AttentionBackendName.NATIVE]
except Exception: # noqa: BLE001
return False
if getattr(kernel_fn, _SAGE_GUARD_ATTR, False):
return True
if not callable(kernel_fn) or not callable(native_fn):
return False
# Eagerly: the guard runs inside compiled graphs, where a registration lock cannot trace.
op = make_op(kernel_fn) if make_op is not None else None
names = ("query", "key", "value", "attn_mask")
def _guarded(*args: Any, **kwargs: Any) -> Any:
bound = dict(zip(names, args))
bound.update({n: kwargs[n] for n in names if n in kwargs})
reason = reroute_reason(
bound.get("query"), bound.get("key"), bound.get("value"), bound.get("attn_mask")
)
if reason is None:
if ran is not None:
ran()
extra = {k: v for k, v in kwargs.items() if k not in names}
plain = (
op is not None
and len(args) <= 4
and not extra.get("return_lse", False)
and extra.get("_parallel_config") is None
and set(extra) <= {"is_causal", "scale", "return_lse", "_parallel_config"}
)
if plain:
return op(
bound["query"],
bound["key"],
bound["value"],
bool(extra.get("is_causal", False)),
extra.get("scale"),
)
return kernel_fn(*args, **kwargs)
note(reason)
return native_fn(*args, **kwargs)
setattr(_guarded, _SAGE_GUARD_ATTR, True)
_guarded.__wrapped__ = kernel_fn # type: ignore[attr-defined]
try:
backends[name] = _guarded
except Exception: # noqa: BLE001
return False
return True
def _install_sage_dispatch_guard() -> bool:
return _install_dispatch_guard(
"sage", lambda *a: _sage_reroute_reason(*a), _note_sage_reroute, _sage_custom_op
)
# diffusers' flash_4_hub raises on any attn_mask too; head dims are checked at load (_fa4_kernel_runs).
_FA4_ROUTED: dict[str, int] = {}
_FA4_ROUTED_LOGGED: set[str] = set()
def _fa4_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any) -> Optional[str]:
import torch
if attn_mask is not None:
return "attn_mask"
if not all(isinstance(t, torch.Tensor) for t in (query, key, value)):
return "inputs"
if (
query.dtype not in (torch.float16, torch.bfloat16)
or key.dtype != query.dtype
or value.dtype != query.dtype
):
return "dtype"
if query.device.type != "cuda":
return "device"
return None
def _note_fa4_reroute(reason: str) -> None:
_note_reroute("FlashAttention 4", reason, _FA4_ROUTED, _FA4_ROUTED_LOGGED)
# The hub FA4 launch (CuTe DSL, tvm-ffi) breaks a fullgraph compile 5 times per Flux block: keep it opaque, like Sage.
_FA4_OP_NAME = "unsloth_studio::flash_4_hub_attention_nhd"
# diffusers' flash (FA2) backend raises on any attn_mask; on ROCm those calls run native instead.
_ROCM_FLASH_ROUTED: dict[str, int] = {}
_ROCM_FLASH_ROUTED_LOGGED: set[str] = set()
def _rocm_flash_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any) -> Optional[str]:
if attn_mask is not None:
return "attn_mask"
return _rocm_flash_length_reason(query, key)
def _install_rocm_flash_dispatch_guard() -> bool:
_resolve_statically_known_true()
try:
import torch
torch.compiler.assume_constant_result(_rocm_flash_traced_reason)
torch.compiler.assume_constant_result(_rocm_flash_traced_segments)
except Exception: # noqa: BLE001 - torch without it: a traced call graph-breaks into the helper, still correct
pass
return _install_dispatch_guard(
"flash",
_rocm_flash_reroute_reason,
lambda reason: (
_note_rocm_flash_route(reason),
_note_reroute(
"ROCm flash attention", reason, _ROCM_FLASH_ROUTED, _ROCM_FLASH_ROUTED_LOGGED
),
),
ran = _note_rocm_flash_ran,
)
# Per DiT, the routes its attention calls took in the current generation: {"flash": n, "<reroute reason>": m}. Report
# only: nothing reads it to choose a kernel, size memory or key a calibration.
ROCM_FLASH_TALLY_ATTR = "_unsloth_rocm_flash_tally"
_ROCM_FLASH_HOOKS_ATTR = "_unsloth_rocm_flash_hooks"
# On the pipe: every DiT the hooks went on, whatever attribute holds it (a MiniMax-H3 view hooks transformer_ref, which
# _attention_dits does not enumerate). Reset, report and the memory guard iterate this.
_ROCM_FLASH_DITS_ATTR = "_unsloth_rocm_flash_dits"
# On a hooked DiT: its live CFG replicas (weak refs), reset by begin_attention_run with the source.
_ROCM_FLASH_REPLICAS_ATTR = "_unsloth_rocm_flash_replicas"
def _rocm_flash_dits(pipe: Any) -> list:
"""The hooked DiTs of ``pipe`` (registry, then any hooked one _attention_dits still sees), each once."""
found: list = []
registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None) if pipe is not None else None
for dit in list(registry or ()) + list(_attention_dits(pipe) if pipe is not None else ()):
if getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None) and not any(dit is d for d in found):
found.append(dit)
return found
def _with_rocm_flash_replicas(dits: list) -> list:
"""``dits`` followed by their live CFG replicas (``track_rocm_flash_replica``)."""
out = list(dits)
for source in dits:
replicas = getattr(source, _ROCM_FLASH_REPLICAS_ATTR, None)
if isinstance(replicas, weakref.WeakSet):
out.extend(r for r in list(replicas) if not any(r is d for d in out))
return out
def _compiling() -> bool:
try:
import torch
return bool(torch.compiler.is_compiling())
except Exception: # noqa: BLE001
return False
def _note_rocm_flash_route(route: str) -> None:
"""Count one eager call on the running DiT's tally (this thread's). Traced and replayed calls are not counted."""
if _compiling() or getattr(_ROCM_FLASH_LOCAL, "probing", False):
return
tally = getattr(_ROCM_FLASH_LOCAL, "tally", None)
if isinstance(tally, dict):
tally[route] = tally.get(route, 0) + 1
def _note_rocm_flash_ran() -> None:
_note_rocm_flash_route("flash")
def _track_rocm_flash_calls(pipe: Any) -> None:
"""Forward hooks on each DiT running ROCm flash: while it runs, this thread's gate (auto loads only) and tally are
its own. Each hooked DiT is recorded on the pipe (``_rocm_flash_dits``). Idempotent."""
for dit in _attention_dits(pipe):
try:
registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None)
if not isinstance(registry, list):
registry = []
setattr(pipe, _ROCM_FLASH_DITS_ATTR, registry)
if not any(dit is d for d in registry):
registry.append(dit)
except Exception: # noqa: BLE001
pass
_hook_rocm_flash_dit(dit)
def _hook_rocm_flash_dit(dit: Any) -> None:
register_pre = getattr(dit, "register_forward_pre_hook", None)
register_post = getattr(dit, "register_forward_hook", None)
if not callable(register_pre) or not callable(register_post):
return
if getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None):
return
# Dynamo replays these threading.local writes, so traced calls see the gate too.
# Use the hook's ``dit`` arg, never a closure: that cycle pins a dropped CFG replica's weights on its card.
def pre(dit, _args) -> None:
outer = getattr(_ROCM_FLASH_LOCAL, "stack", None)
if outer is None:
outer = _ROCM_FLASH_LOCAL.stack = []
outer.append(
(
getattr(_ROCM_FLASH_LOCAL, "gate", None),
getattr(_ROCM_FLASH_LOCAL, "tally", None),
getattr(_ROCM_FLASH_LOCAL, "deferred", None),
)
)
_ROCM_FLASH_LOCAL.gate = getattr(dit, ROCM_FLASH_GATE_ATTR, None) or None
tally = getattr(dit, ROCM_FLASH_TALLY_ATTR, None)
_ROCM_FLASH_LOCAL.tally = tally if isinstance(tally, dict) else None
deferred = getattr(dit, ROCM_FLASH_DEFERRED_ATTR, None)
_ROCM_FLASH_LOCAL.deferred = deferred if isinstance(deferred, set) else None
def post(_module, _args, _output) -> None:
stack = getattr(_ROCM_FLASH_LOCAL, "stack", None)
gate, tally, deferred = stack.pop() if stack else (None, None, None)
_ROCM_FLASH_LOCAL.gate, _ROCM_FLASH_LOCAL.tally = gate, tally
_ROCM_FLASH_LOCAL.deferred = deferred
try:
try: # also after a forward that raised, so a failed run never leaves its gate active
post_handle = register_post(post, always_call = True)
except TypeError:
post_handle = register_post(post)
handles = [register_pre(pre), post_handle]
setattr(dit, _ROCM_FLASH_HOOKS_ATTR, handles)
setattr(dit, ROCM_FLASH_TALLY_ATTR, {})
setattr(dit, ROCM_FLASH_DEFERRED_ATTR, set())
except Exception: # noqa: BLE001
pass
def track_rocm_flash_replica(source: Any, replica: Any, device: Any) -> None:
"""A CFG replica (``diffusion_multigpu.build_replica``, root hooks cleared) runs its forward on its own worker
thread, where the source's hooks never set the gate: give it its own hooks and the source's gate keyed to its
card, so its calls are gated (and their buckets timed) on that card exactly as the source's are."""
if not getattr(source, _ROCM_FLASH_HOOKS_ATTR, None):
return
gate = getattr(source, ROCM_FLASH_GATE_ATTR, None)
try:
if gate:
card = _indexed_cuda_device(str(device))
gate = {(card, *shape[1:]): found for shape, found in gate.items()}
setattr(replica, ROCM_FLASH_GATE_ATTR, gate or None)
setattr(
replica, _ROCM_FLASH_HOOKS_ATTR, None
) # the copied handles point at the source's hooks
except Exception: # noqa: BLE001
return
_hook_rocm_flash_dit(replica)
try: # the replica outlives a generation: its deferred buckets must be retried at the next one as the source's are
replicas = getattr(source, _ROCM_FLASH_REPLICAS_ATTR, None)
if not isinstance(replicas, weakref.WeakSet):
replicas = weakref.WeakSet()
setattr(source, _ROCM_FLASH_REPLICAS_ATTR, replicas)
replicas.add(replica)
except Exception: # noqa: BLE001
pass
def _untrack_rocm_flash_calls(pipe: Any) -> None:
"""Undo ``_track_rocm_flash_calls`` for this pipe's DiTs once they no longer run ROCm flash (a re-apply picked
another backend), so the badge and the memory guard stop treating them as flash. Only the DiTs this pipe (or view)
owns: an MoE view leaves the other expert's hooks alone."""
dits = _attention_dits(pipe) if pipe is not None else []
for dit in dits:
handles = getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None)
if not handles:
continue
for handle in handles:
try:
handle.remove()
except Exception: # noqa: BLE001
pass
for attr in (_ROCM_FLASH_HOOKS_ATTR, ROCM_FLASH_TALLY_ATTR, ROCM_FLASH_DEFERRED_ATTR):
try:
setattr(dit, attr, None)
except Exception: # noqa: BLE001
pass
registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None) if pipe is not None else None
if isinstance(registry, list) and dits:
registry[:] = [d for d in registry if not any(d is x for x in dits)]
def attention_may_run_native(pipe: Any) -> bool:
"""Whether a call may run native SDPA despite flash; conservative, never from past run reports"""
return bool(_rocm_flash_dits(pipe))
def rocm_flash_ran(pipe: Any) -> Optional[tuple[str, str]]:
"""``(value, reason)`` describing the current generation's routes for a ROCm flash pipe, or None when it is not one.
A run with no counted call (compiled or graphed attention, which the guard cannot see) says so."""
dits = _rocm_flash_dits(pipe)
if not dits:
return None
total: dict[str, int] = {}
for dit in _with_rocm_flash_replicas(dits): # a CFG replica's calls are this generation's too
tally = getattr(dit, ROCM_FLASH_TALLY_ATTR, None)
if isinstance(tally, dict):
for name, count in tally.items():
if isinstance(count, int):
total[name] = total.get(name, 0) + count
flash = total.pop("flash", 0)
native = sum(total.values())
if flash + native == 0:
return "flash", (
"ROCm flash_attn; per-call routing is not tracked for compiled or graphed attention (masked calls and "
"lengths flash does not serve run native SDPA)"
)
why = ", ".join(f"{name}: {count}" for name, count in sorted(total.items()))
if native == 0:
return "flash", "ROCm flash_attn on every attention call of the last generation"
if flash == 0:
return (
"native",
f"ROCm flash_attn engaged, but every attention call of the last generation ran native SDPA ({why})",
)
return "flash", (
f"ROCm flash_attn on {flash} of {flash + native} attention calls of the last generation; the rest ran native "
f"SDPA ({why})"
)
def begin_attention_run(state: Any) -> None:
"""A generation boundary: the per-DiT route tallies restart, so the report after it describes this run alone, and
this run's DiTs retry buckets whose probe could not run."""
for dit in _with_rocm_flash_replicas(_rocm_flash_dits(getattr(state, "pipe", None))):
if isinstance(getattr(dit, ROCM_FLASH_TALLY_ATTR, None), dict):
try:
setattr(dit, ROCM_FLASH_TALLY_ATTR, {})
setattr(dit, ROCM_FLASH_DEFERRED_ATTR, set())
except Exception: # noqa: BLE001
pass
def report_attention_run(state: Any) -> None:
"""After a generation: the ``resolved`` attention badge (value + reason) describes the routes it took. Report only:
``state.attention_backend`` stays the engaged backend, which recipes, calibration keys and compile caches key on."""
try:
ran = rocm_flash_ran(getattr(state, "pipe", None))
resolved = getattr(state, "resolved", None)
entry = resolved.get("attention_backend") if isinstance(resolved, dict) else None
if ran is None or not isinstance(entry, dict):
return
value, reason = ran
if entry.get("value") != value or entry.get("reason") != reason:
entry = {**entry, "value": value, "reason": reason}
object.__setattr__(state, "resolved", {**resolved, "attention_backend": entry})
except Exception: # noqa: BLE001 - reporting only
pass
def _install_fa4_dispatch_guard() -> bool:
return _install_dispatch_guard(
"flash_4_hub",
lambda *a: _fa4_reroute_reason(*a),
_note_fa4_reroute,
lambda fn: _sage_custom_op(fn, _FA4_OP_NAME),
)
# Whether the hub FA4 build runs correctly here, once per (device, dtype, head_dim) through diffusers' dispatch: catches
# cards with no code (sm110 / sm120 unverified) and API drift. An exact kernel, so the bound is tight.
_FA4_PROBE_CACHE: dict[tuple[str, str, int], str] = {}
_FA4_MIN_COSINE = 0.999
_FA4_MAX_REL_L1 = 0.02
def _run_fa4_probe(
device: str,
dtype: Any,
head_dim: int = 128,
) -> str:
"""Empty when flash_4_hub matches fp32 SDPA at ``head_dim``, else why not. Raises on OOM."""
import torch
from diffusers.models.attention_dispatch import AttentionBackendName, dispatch_attention_fn
if dtype not in (torch.float16, torch.bfloat16):
dtype = torch.bfloat16
gen = torch.Generator(device = "cpu").manual_seed(0)
q, k, v = (torch.randn((1, 256, 2, int(head_dim)), generator = gen) for _ in range(3))
q, k, v = (t.to(device = device, dtype = dtype) for t in (q, k, v))
try:
out = dispatch_attention_fn(q, k, v, backend = AttentionBackendName.FLASH_4_HUB)
if str(device).startswith("cuda"):
torch.cuda.synchronize(device)
if isinstance(out, tuple):
out = out[0]
ref = torch.nn.functional.scaled_dot_product_attention(
*(t.float().transpose(1, 2) for t in (q, k, v))
).transpose(1, 2)
got = out.float()
if tuple(got.shape) != tuple(ref.shape):
return f"self-check: output shape {tuple(got.shape)} != {tuple(ref.shape)}"
if not bool(torch.isfinite(got).all()):
return "self-check: non-finite output"
cos = float(torch.nn.functional.cosine_similarity(got.flatten(), ref.flatten(), dim = 0))
rel = float((got - ref).abs().mean() / ref.abs().mean().clamp_min(1e-12))
except torch.cuda.OutOfMemoryError:
raise
except Exception as exc: # noqa: BLE001
return f"{type(exc).__name__}: {exc}"
if cos < _FA4_MIN_COSINE or rel > _FA4_MAX_REL_L1:
return f"self-check: cosine {cos:.5f}, relative L1 {rel:.4f} vs fp32 at head_dim {head_dim}"
return ""
def _fa4_kernel_runs(
target: Any,
logger: Any = None,
head_dims: Any = None,
) -> Optional[bool]:
"""True when flash_4_hub passed at every DiT head dim; False with a logged reason; None unaskable."""
device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
if not device.startswith("cuda"):
return None
device = _indexed_cuda_device(device)
dtype = getattr(target, "dtype", None)
dims = sorted(
{
int(d)
for d in (head_dims or ())
if isinstance(d, int) and not isinstance(d, bool) and d > 0
}
)
error = ""
for head_dim in dims or [128]:
key = (device, str(dtype), head_dim)
error = _FA4_PROBE_CACHE.get(key)
if error is None:
try:
error = _run_fa4_probe(device, dtype, head_dim)
except Exception: # noqa: BLE001
return None
error = _FA4_PROBE_CACHE.setdefault(key, error)
if error:
break
if error and logger is not None:
logger.warning(
"diffusion.attention: FlashAttention 4 does not run correctly on this GPU (%s); using the default backend",
error,
)
return not error
def _version_tuple(text: str) -> tuple[int, ...]:
m = re.match(r"\s*(\d+(?:\.\d+)*)", str(text))
return tuple(int(x) for x in m.group(1).split(".")) if m else ()
def _sage_version_too_old() -> Optional[str]:
"""Why an installed sageattention is below diffusers' floor (PyPI's 1.0.6 is SageAttention 1), else None. Not
cached: a later install may replace it."""
try:
from importlib.metadata import PackageNotFoundError, version
try:
installed = version("sageattention")
except PackageNotFoundError:
return None
floor = "2.1.1"
try:
from diffusers.models.attention_dispatch import _REQUIRED_SAGE_VERSION
if isinstance(_REQUIRED_SAGE_VERSION, str) and _REQUIRED_SAGE_VERSION.strip():
floor = _REQUIRED_SAGE_VERSION.strip()
except Exception: # noqa: BLE001
pass
if _version_tuple(installed) and _version_tuple(installed) < _version_tuple(floor):
return f"sageattention {installed} is older than {floor}, the SageAttention 2 release diffusers needs"
except Exception: # noqa: BLE001
return None
return None
def _sage_usable(
pipe: Any,
target: Any,
logger: Any = None,
) -> bool:
"""Whether an explicit ``sage`` request may engage: not fp32, kernel passes at the DiT head dims, guard in place."""
if target is not None and _runs_in_float32(target):
if logger is not None:
logger.warning(
"diffusion.attention: SageAttention needs fp16 or bf16 and this pipeline runs in float32; "
"using the default backend"
)
return False
too_old = _sage_version_too_old()
if too_old:
if logger is not None:
logger.warning("diffusion.attention: %s; using the default backend", too_old)
return False
head_dims = _dit_head_dims(pipe)
if head_dims and min(head_dims) > _SAGE_MAX_HEAD_DIM:
if logger is not None:
logger.warning(
"diffusion.attention: SageAttention serves head_dim <= %d and this model uses %s; "
"using the default backend",
_SAGE_MAX_HEAD_DIM,
",".join(str(d) for d in sorted(head_dims)),
)
return False
if target is not None and _sage_kernel_runs(target, logger, head_dims) is False:
return False
if not _install_sage_dispatch_guard():
if logger is not None:
logger.warning(
"diffusion.attention: this diffusers build does not expose the sage backend the way Unsloth expects, "
"so masked attention calls could not be routed around it; using the default backend"
)
return False
return True
# PyPI has only SageAttention 1 (diffusers refuses it), so sage uses the kernels-hub build.
# Version 2 first: the version 1 builds diffusers asks for stop at torch 2.10.
SAGE_HUB_BACKEND = "sage_hub"
_SAGE_HUB_REPO = "kernels-community/sage-attention"
_SAGE_HUB_VERSIONS = (2, 1)
_SAGE_HUB_PROBE_CACHE: dict[tuple[str, str, int], str] = {}
_SAGE_HUB_LOCK = threading.Lock()
# A failed fetch, remembered so the in-lock apply does not repeat it. Cleared by a restart, like _INSTALL_ATTEMPTED.
_SAGE_HUB_FAILED: list[str] = []
SAGE_SOURCE_HINT = (
"SageAttention 2 is not published on PyPI (pip only has SageAttention 1.0.6), so on this platform it needs a "
"source build or a community wheel"
)
def _pip_sage2_installed() -> bool:
try:
from importlib.metadata import PackageNotFoundError, version
try:
installed = version("sageattention")
except PackageNotFoundError:
return False
except Exception: # noqa: BLE001
return False
return bool(_version_tuple(installed)) and _sage_version_too_old() is None
def _load_sage_hub_kernel() -> tuple[Any, str]:
"""``(sageattn, "")`` from the hub build, set in diffusers' ``sage_hub`` slot, or ``(None, why)``. Never raises."""
with _SAGE_HUB_LOCK:
try:
from diffusers.models.attention_dispatch import (
_HUB_KERNELS_REGISTRY,
AttentionBackendName,
)
config = _HUB_KERNELS_REGISTRY[AttentionBackendName(SAGE_HUB_BACKEND)]
except Exception as exc: # noqa: BLE001
return None, f"this diffusers build has no sage_hub backend ({type(exc).__name__})"
if callable(getattr(config, "kernel_fn", None)):
return config.kernel_fn, ""
if _SAGE_HUB_FAILED:
return None, _SAGE_HUB_FAILED[0]
try:
from kernels import get_kernel
except Exception as exc: # noqa: BLE001
return None, f"the kernels package is not importable ({type(exc).__name__}: {exc})"
errors = []
versions = list(_SAGE_HUB_VERSIONS)
default = getattr(config, "version", None)
if isinstance(default, int) and default not in versions:
versions.append(default)
for ver in versions:
try:
fn = getattr(get_kernel(_SAGE_HUB_REPO, version = ver), "sageattn", None)
except Exception as exc: # noqa: BLE001
errors.append(f"version {ver}: {type(exc).__name__}: {str(exc)[:200]}")
continue
if callable(fn):
config.kernel_fn = fn
return fn, ""
errors.append(f"version {ver}: no sageattn in the build")
_SAGE_HUB_FAILED.append(f"no {_SAGE_HUB_REPO} build loads here ({'; '.join(errors)})")
return None, _SAGE_HUB_FAILED[0]
def _sage_hub_kernel_runs(
target: Any,
sageattn: Any,
head_dims: Any = None,
) -> tuple[Optional[bool], str]:
"""``(True, "")`` when the hub ``sageattn`` passed at every head dim, ``(False, why)``, or ``(None, "")`` unaskable."""
device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
if not device.startswith("cuda"):
return None, ""
device = _indexed_cuda_device(device)
dtype = getattr(target, "dtype", None)
for head_dim in _sage_probe_head_dims(head_dims):
key = (device, str(dtype), head_dim)
error = _SAGE_HUB_PROBE_CACHE.get(key)
if error is None:
try:
error = _run_sage_probe(device, dtype, head_dim, sageattn = sageattn)
except Exception: # noqa: BLE001
return None, ""
error = _SAGE_HUB_PROBE_CACHE.setdefault(key, error)
if error:
return False, error
return True, ""
def _install_sage_hub_dispatch_guard() -> bool:
return _install_dispatch_guard(
SAGE_HUB_BACKEND,
lambda *a: _sage_reroute_reason(*a),
_note_sage_reroute,
lambda fn: _sage_custom_op(fn, _SAGE_HUB_OP_NAME),
)
def _sage_hub_backend(
pipe: Any,
target: Any,
logger: Any = None,
) -> Optional[str]:
"""``sage_hub`` when the hub build passes its self-check here (before any DiT is switched), else None, logged."""
def _decline(why: str) -> None:
if logger is not None:
logger.warning(
"diffusion.attention: SageAttention unavailable: %s. %s. Using the default backend",
why,
SAGE_SOURCE_HINT,
)
if target is not None and _runs_in_float32(target):
if logger is not None:
logger.warning(
"diffusion.attention: SageAttention needs fp16 or bf16 and this pipeline runs in float32; "
"using the default backend"
)
return None
head_dims = _dit_head_dims(pipe)
if head_dims and min(head_dims) > _SAGE_MAX_HEAD_DIM:
if logger is not None:
logger.warning(
"diffusion.attention: SageAttention serves head_dim <= %d and this model uses %s; "
"using the default backend",
_SAGE_MAX_HEAD_DIM,
",".join(str(d) for d in sorted(head_dims)),
)
return None
sageattn, why = _load_sage_hub_kernel()
if sageattn is None:
_decline(f"the Hugging Face kernels-hub build could not be loaded ({why})")
return None
if target is not None:
runs, why = _sage_hub_kernel_runs(target, sageattn, head_dims)
if runs is False:
_decline(
f"the Hugging Face kernels-hub build does not run correctly on this GPU ({why})"
)
return None
if not _install_sage_hub_dispatch_guard():
if logger is not None:
logger.warning(
"diffusion.attention: this diffusers build does not expose the sage_hub backend the way Unsloth "
"expects, so masked attention calls could not be routed around it; using the default backend"
)
return None
if logger is not None:
logger.info(
"diffusion.attention: SageAttention 2 from the Hugging Face kernels hub (%s) passed its self-check",
_SAGE_HUB_REPO,
)
return SAGE_HUB_BACKEND
# Only cutlass-dsl 4.4 / 4.5 load the hub FA4 build (4.6+ lacks cute.core.ThrMma, 4.3 PipelineClcFetchAsync).
FA4_CUTLASS_DSL_MIN = (4, 4)
FA4_CUTLASS_DSL_MAX_EXCL = (4, 6)
FA4_CUTLASS_DSL_SPEC = "nvidia-cutlass-dsl>=4.4,<4.6"
FA4_TVM_FFI_SPEC = "apache-tvm-ffi>=0.1.6,<0.2,!=0.1.8,!=0.1.8.post0"
FA4_KERNELS_MIN = (0, 12, 3)
FA4_KERNELS_SPEC = "kernels==0.12.3"
def _dist_version(name: str) -> Optional[str]:
try:
from importlib.metadata import PackageNotFoundError, version
try:
return version(name)
except PackageNotFoundError:
return None
except Exception: # noqa: BLE001
return None
def fa4_cutlass_dsl_ok(installed: Optional[str]) -> bool:
have = _version_tuple(installed or "")[:2]
return bool(have) and FA4_CUTLASS_DSL_MIN <= have < FA4_CUTLASS_DSL_MAX_EXCL
def _cutlass_dsl_dependents() -> list[str]:
found = []
try:
from importlib.metadata import distributions
for dist in distributions():
for req in dist.requires or ():
head = (
re.split(r"[\s;\[<>=!~]", req.strip(), maxsplit = 1)[0].lower().replace("_", "-")
)
if head == "nvidia-cutlass-dsl" and "extra ==" not in req:
found.append(
f"{dist.metadata['Name']} {dist.version} ({req.split(';')[0].strip()})"
)
break
except Exception: # noqa: BLE001
pass
return sorted(set(found))
def _fa4_python_deps_plan() -> tuple[list[str], Optional[str]]:
"""``(requirements, refusal)`` for the hub FA4 build. An out-of-range cutlass-dsl is never replaced (a user's quack
or a newer FlashInfer may need it): the request is refused instead."""
reqs: list[str] = []
kernels = _dist_version("kernels")
if (
kernels is not None
and _version_tuple(kernels)
and _version_tuple(kernels) < FA4_KERNELS_MIN
):
# Same dependencies as 0.12.1, so --no-deps is safe.
reqs.append(FA4_KERNELS_SPEC)
cutlass = _dist_version("nvidia-cutlass-dsl")
if cutlass is None:
reqs.append(FA4_CUTLASS_DSL_SPEC)
elif not fa4_cutlass_dsl_ok(cutlass):
users = [d for d in _cutlass_dsl_dependents() if not d.lower().startswith("flash-attn")]
return [], (
f"FlashAttention 4 from the kernels hub loads only with nvidia-cutlass-dsl 4.4.x or 4.5.x and {cutlass} is "
"installed"
+ (f" (required by {', '.join(users)})" if users else "")
+ "; Unsloth does not replace it"
)
if _dist_version("apache-tvm-ffi") is None:
reqs.append(FA4_TVM_FFI_SPEC)
if _dist_version("einops") is None:
reqs.append("einops")
return reqs, None
def _pinned_constraints_file() -> Optional[str]:
"""Every installed distribution pinned, so a dependency install cannot move anything."""
try:
from .diffusion_nvfp4_install import _write_constraints, installed_distributions
return _write_constraints(installed_distributions())
except Exception: # noqa: BLE001
return None
def _refresh_diffusers_kernels_version(logger: Any = None) -> None:
"""diffusers reads the kernels version once at import and gates flash_4_hub on it: refresh it after an upgrade, only
while ``kernels`` is not imported yet (an imported old version runs until a restart)."""
import sys
if "kernels" in sys.modules:
if logger is not None:
logger.warning(
"diffusion.attention: kernels was upgraded to %s for FlashAttention 4 and takes effect after Unsloth "
"restarts; this load uses the default backend",
FA4_KERNELS_SPEC.split("==", 1)[1],
)
return
try:
from diffusers.utils import import_utils
installed = _dist_version("kernels")
if installed and getattr(import_utils, "_kernels_version", None) not in (None, installed):
import_utils._kernels_version = installed
import_utils._kernels_available = True
# diffusers memoizes is_kernels_version, so a check made before the upgrade would still answer False.
cache_clear = getattr(
getattr(import_utils, "is_kernels_version", None), "cache_clear", None
)
if callable(cache_clear):
cache_clear()
except Exception: # noqa: BLE001
pass
# cutlass-dsl reaches sys.path through a .pth file (libs-base in 4.4 / 4.5, the main dist in 4.6+).
_PTH_DISTRIBUTIONS = ("nvidia-cutlass-dsl-libs-base", "nvidia-cutlass-dsl")
def _activate_installed_pth_files(names: tuple[str, ...] = _PTH_DISTRIBUTIONS) -> None:
"""Run the .pth files of ``names`` now: the interpreter reads them only at startup."""
try:
import site
import sys
from importlib.metadata import PackageNotFoundError, distribution
except Exception: # noqa: BLE001
return
for name in names:
try:
dist = distribution(name)
except PackageNotFoundError:
continue
except Exception: # noqa: BLE001
continue
for entry in dist.files or ():
if not str(entry).endswith(".pth") or "/" in str(entry).replace("\\", "/"):
continue
try:
path = os.fspath(dist.locate_file(entry))
site.addpackage(os.path.dirname(path), os.path.basename(path), set(sys.path))
except Exception: # noqa: BLE001
continue
def _ensure_fa4_python_deps(logger: Any = None) -> Optional[str]:
"""Install the hub FA4 build's imports at versions it loads with. Returns the refusal reason, or None."""
import importlib
import subprocess
import sys
reqs, refusal = _fa4_python_deps_plan()
if refusal is not None:
if logger is not None:
logger.warning("diffusion.attention: %s; using the default backend", refusal)
return refusal
if not reqs:
# An NVFP4 FlashInfer install earlier in this process may have added cutlass-dsl without running its .pth.
_activate_installed_pth_files()
return None
key = "fa4-deps:" + ",".join(reqs)
if key in _INSTALL_ATTEMPTED:
return None
_INSTALL_ATTEMPTED.add(key)
kernels_upgrade = [r for r in reqs if r == FA4_KERNELS_SPEC]
deps = [r for r in reqs if r != FA4_KERNELS_SPEC]
if logger is not None:
logger.info(
"diffusion.attention: installing %s for backend=flash_4_hub (wheel-only)",
", ".join(reqs),
)
base = [
sys.executable,
"-m",
"pip",
"install",
"--disable-pip-version-check",
"--only-binary",
":all:",
]
constraints = _pinned_constraints_file() if deps else None
try:
if kernels_upgrade:
subprocess.run(
base + ["--no-deps", *kernels_upgrade], capture_output = True, timeout = 600, check = True
)
_refresh_diffusers_kernels_version(logger)
if deps:
# Resolved (cutlass-dsl needs its libs and cuda-python); the constraints add only new packages.
cmd = base + (["-c", constraints] if constraints else []) + deps
subprocess.run(cmd, capture_output = True, timeout = 600, check = True)
importlib.invalidate_caches()
_activate_installed_pth_files()
except Exception as exc: # noqa: BLE001 - FA4 then falls back at set time
if logger is not None:
stderr = getattr(exc, "stderr", None)
if isinstance(stderr, bytes):
stderr = stderr.decode("utf-8", errors = "replace")
logger.warning(
"diffusion.attention: could not install %s for FlashAttention 4: %s",
", ".join(reqs),
_redacted_for_log((stderr or "").strip()[-1500:]) or _redacted_for_log(str(exc)),
)
finally:
if constraints:
try:
os.unlink(constraints)
except OSError:
pass
return None
# Wheels only (--only-binary=:all:): source builds need a CUDA toolchain the host may lack.
_INSTALLABLE_BACKENDS: dict[str, tuple[str, str]] = {
# Never ``sageattention`` from PyPI (only SageAttention 1 there): the hub build is the installable one.
"sage": ("kernels", "kernels"),
"flash": ("flash_attn", "flash-attn"),
"_flash_3_hub": ("kernels", "kernels"), # FA3/FA4 from the HF kernels hub
"flash_4_hub": ("kernels", "kernels"),
# Never passed to pip by name; see _MATCHED_WHEEL_BACKENDS.
"xformers": ("xformers", "xformers"),
}
# On-demand install gate (mirrors UNSLOTH_DIFFUSION_SD_CPP_INSTALL): auto (default) / 1 installs a missing package
# when a gated backend is requested; 0 never installs and falls back to native.
_ATTENTION_INSTALL_ENV = "UNSLOTH_DIFFUSION_ATTENTION_INSTALL"
# A recorded attempt stops the in-lock apply re-running the 600s install under _generate_lock.
_INSTALL_ATTEMPTED: set[str] = set()
# Wheel resolved against the running torch; probe timeouts are transient and not cached.
_MATCHED_WHEEL_BACKENDS = frozenset({"xformers"})
_XFORMERS_WHEEL_TARGET: Optional[tuple[Optional[str], Optional[str]]] = None
_XFORMERS_WHEEL_LOCK = threading.Lock()
# Scheme-qualified URLs only; nothing else in these messages can carry a credential.
_URL_IN_TEXT = re.compile(r"[a-zA-Z][a-zA-Z0-9+.\-]*://[^\s'\"<>]+")
def _redacted_for_log(text: str) -> str:
"""Redacts every URL in free text, stripping userinfo, query and fragment; pip echoes URLs in stderr."""
try:
from utils.wheel_utils import redact_url_credentials
except Exception: # noqa: BLE001 -- redaction must never be the thing that breaks a log
return text
return _URL_IN_TEXT.sub(lambda m: redact_url_credentials(m.group(0)), text)
def _xformers_wheel_target() -> tuple[Optional[str], Optional[str]]:
"""Exact xFormers wheel for this torch+CUDA pair, or a refusal: a mismatch loads silently broken."""
global _XFORMERS_WHEEL_TARGET
with _XFORMERS_WHEEL_LOCK:
if _XFORMERS_WHEEL_TARGET is not None:
return _XFORMERS_WHEEL_TARGET
try:
from utils.wheel_utils import probe_torch_wheel_env, xformers_wheel_url
# include_windows: this is the one resolver that HAS win_amd64 wheels upstream. timeout matches the other
# probe_torch_wheel_env callers.
env = probe_torch_wheel_env(timeout = 30, include_windows = True)
except Exception as exc: # noqa: BLE001 -- must never break a model load
return (None, f"the xFormers wheel could not be resolved ({exc})")
if env is None:
# Ambiguous: a platform wheel_platform_tag() does not name (macOS, Windows on ARM) which is deterministic,
# or a probe that timed out on a busy box which is transient. Not cached, so the next request can settle
# it. Linux aarch64 is NOT here: it gets a platform_tag and so lands on the branch below.
return (
None,
"torch could not be probed, or this platform has no xFormers wheel "
"(macOS / Windows on ARM)",
)
url = xformers_wheel_url(env)
if url is None:
# Name the platform. Linux aarch64 reaches here with a perfectly ordinary torch, and reporting only the
# torch and CUDA would read as "upstream never built this pair" when the truth is "upstream never built it
# for this arch".
target = (
None,
f"no xFormers wheel is published for torch "
f"{env.get('torch_version') or 'unknown'} with CUDA "
f"{env.get('cuda_version') or 'none'} on "
f"{env.get('platform_tag') or 'this platform'}",
)
else:
target = (url, None)
_XFORMERS_WHEEL_TARGET = target
return target
# kernels >= 0.14.1 requires huggingface-hub >= 1.10.0.
_KERNELS_HUB_FLOOR = (1, 10)
def _kernels_hub_compatible() -> bool:
"""False for hub under 1.10; unknown hub allows install; --no-deps leaves this the only floor check."""
try:
import re
from importlib.metadata import version
m = re.match(r"\s*(\d+)(?:\.(\d+))?", version("huggingface_hub"))
if m is None:
return True
return (int(m.group(1)), int(m.group(2) or 0)) >= _KERNELS_HUB_FLOOR
except Exception: # noqa: BLE001 - unknown hub -> keep the previous permissive behaviour
return True
def _ensure_attention_backend_installed(backend: str, logger: Any = None) -> Optional[str]:
"""Best-effort install of ``backend``'s package, plus the hub FA4 build's dependencies for ``flash_4_hub``. Returns
the refusal reason, or None; see ``_ensure_backend_package``."""
if backend == "sage" and _pip_sage2_installed():
return None
reason = _ensure_backend_package(backend, logger)
if reason is not None or backend not in ("flash_4_hub", "sage"):
return reason
if os.environ.get(_ATTENTION_INSTALL_ENV, "auto").strip().lower() in (
"0",
"false",
"no",
"off",
):
return None
try:
import importlib.util
if importlib.util.find_spec("kernels") is None:
return None
except Exception: # noqa: BLE001
return None
if backend == "sage":
# Fetched here, outside the loader's locks; apply finds it in diffusers' registry.
_load_sage_hub_kernel()
return None
return _ensure_fa4_python_deps(logger)
def _ensure_backend_package(backend: str, logger: Any = None) -> Optional[str]:
"""Best-effort wheel-only install; returns the policy refusal reason, or None, and never raises."""
import importlib.util
import os
import sys
spec = _INSTALLABLE_BACKENDS.get(backend)
if spec is None:
return None
module, package = spec
gate = os.environ.get(_ATTENTION_INSTALL_ENV, "auto").strip().lower()
if gate in ("0", "false", "no", "off"):
return None
# A hidden package stays unusable until restart; skip without recording an attempt.
if module in sys.modules and sys.modules[module] is None:
reason = f"{module} is disabled in this process because it requires a different torch"
if module == "kernels":
from utils.kernels_compat import hidden_reason
reason = hidden_reason() or reason
if logger is not None:
logger.warning(
"diffusion.attention: not installing %s for backend=%s: %s; restart Unsloth "
"once the installer has removed it. Using the default backend",
package,
backend,
reason,
)
return reason
# Policy refusal, checked before _INSTALL_ATTEMPTED so a fixed env can still install.
if package == "kernels" and not _kernels_hub_compatible():
if logger is not None:
logger.warning(
"diffusion.attention: not installing 'kernels' for backend=%s — the resident "
"huggingface_hub is below %d.%d and a kernels install would break every later "
"diffusers pipeline import; using the default backend",
backend,
*_KERNELS_HUB_FLOOR,
)
return "the resident huggingface_hub is too old for the kernels package"
try:
if importlib.util.find_spec(module) is not None:
# Deliberately leave a mismatched xformers alone: install.ps1 repairs it outside any request.
return None
except Exception: # noqa: BLE001 - a broken install probes as missing; try the install
pass
# PyPI flash-attn is the CUDA build: never pip it onto a ROCm torch. Policy, so no _INSTALL_ATTEMPTED record.
if backend == "flash" and _torch_is_rocm():
reason = "flash-attn on PyPI is the NVIDIA CUDA build"
if logger is not None:
logger.warning(
"diffusion.attention: not installing %s for backend=%s on ROCm: %s; %s. Using the default backend",
package,
backend,
reason,
_ROCM_FLASH_HINT,
)
return reason
# xformers binds one exact (torch, CUDA) pair: resolve a matching wheel URL or refuse.
if backend in _MATCHED_WHEEL_BACKENDS:
wheel_url, refusal = _xformers_wheel_target()
if wheel_url is None:
if logger is not None:
logger.warning(
"diffusion.attention: not installing %s for backend=%s — %s; an unpinned "
"install would land an extension that cannot load next to the resident "
"torch and would disable memory-efficient attention silently. Using the "
"default backend",
package,
backend,
refusal,
)
return refusal
package = wheel_url
# Pip gets the real URL; the log gets a redacted one, since the mirror URL may carry a token.
display = _redacted_for_log(package)
# Once per process, else the in-lock apply re-runs it under _generate_lock.
if package in _INSTALL_ATTEMPTED:
return None
_INSTALL_ATTEMPTED.add(package)
import subprocess
requirement = package
previous_kernels = None
if package == "kernels":
# Unpinned, pip takes the newest kernels, which an older transformers cannot import next to.
from utils.kernels_compat import current_kernels_install_spec
requirement = current_kernels_install_spec()
previous_kernels = _dist_version("kernels")
display = requirement
if logger is not None:
logger.info(
"diffusion.attention: installing %s for backend=%s (wheel-only)", display, backend
)
try:
subprocess.run(
# --no-deps: wheels pin an exact torch that would replace the running one; no ABI check here.
[
sys.executable,
"-m",
"pip",
"install",
"--only-binary",
":all:",
"--no-deps",
requirement,
],
capture_output = True,
timeout = 600,
check = True,
)
# The import system caches directory listings, so invalidate the finder caches or the next find_spec can miss
# the new wheel.
importlib.invalidate_caches()
if package == "kernels":
from utils.kernels_compat import check_kernels_install
# A kernels transformers cannot import breaks every later process: roll it back instead.
if check_kernels_install(previous_kernels, logger):
_refresh_diffusers_kernels_version(logger)
except Exception as exc: # noqa: BLE001 - no wheel / no network -> native fallback
if logger is not None:
# CalledProcessError.str() shows only the exit code; surface stderr so the fallback is diagnosable.
stderr = getattr(exc, "stderr", None)
if stderr:
if isinstance(stderr, bytes):
stderr = stderr.decode("utf-8", errors = "replace")
logger.warning(
"diffusion.attention: could not install %s; pip failed with: %s",
display,
# pip echoes the URL it was given, so redact the body too rather than only the name in front of it.
_redacted_for_log(stderr.strip()) or str(exc),
)
else:
logger.warning(
"diffusion.attention: could not install %s (%s); falling back to default",
display,
_redacted_for_log(str(exc)),
)
return None
def _torch_is_rocm() -> bool:
try:
import torch
from core._torchao_stub import _module_is_rocm
return _module_is_rocm(torch)
except Exception: # noqa: BLE001
return False
def _attention_dits(pipe: Any) -> list:
"""Every DiT the denoise loop runs: the primary ``transformer`` plus a second expert some
families carry (Ideogram's ``unconditional_transformer``, an MoE ``transformer_2``). The
backend must be set on ALL of them, else the second DiT keeps the native default."""
from .diffusion_autoopt.targets import denoiser_dits
return denoiser_dits(pipe)
# head_dim 256 (Ideogram 4) has no cuDNN kernel on torch 2.11-2.13 (cuDNN 9.19/9.20); a pinned cuDNN then has no fallback.
_CUDNN_HEAD_DIM_CACHE: dict[tuple[str, str, int], bool] = {}
def _dit_head_dims(pipe: Any) -> set[int]:
"""The attention head dims of every denoiser DiT: ``config.attention_head_dim``, else any module's ``head_dim``."""
dims: set[int] = set()
for dit in _attention_dits(pipe):
config = getattr(dit, "config", None)
value = None
try:
value = config.get("attention_head_dim") if hasattr(config, "get") else None
except Exception: # noqa: BLE001
value = None
if isinstance(value, int) and not isinstance(value, bool):
dims.add(value)
continue
if isinstance(value, (list, tuple)) and value and all(isinstance(v, int) for v in value):
dims.update(value)
continue
modules = getattr(dit, "modules", None)
if not callable(modules):
continue
try:
for module in modules():
hd = getattr(module, "head_dim", None)
if isinstance(hd, int) and not isinstance(hd, bool) and hd > 0:
dims.add(hd)
except Exception: # noqa: BLE001
continue
return dims
CUDNN_HEAD_DIM_PROBE_ENV = "UNSLOTH_DIFFUSION_CUDNN_HEAD_DIM_PROBE"
def _run_cudnn_head_dim_probe(device: str, dtype: Any, head_dim: int) -> bool:
"""True when cuDNN attention serves ``head_dim`` per SDPA's own gate (launches no kernel); raises when unaskable."""
import torch
if dtype not in (torch.float16, torch.bfloat16):
dtype = torch.bfloat16
q = torch.empty((1, 2, 8, int(head_dim)), device = device, dtype = dtype)
try:
from torch.backends.cuda import SDPAParams, can_use_cudnn_attention
except ImportError:
SDPAParams = can_use_cudnn_attention = None
if SDPAParams is not None and can_use_cudnn_attention is not None:
try:
params = SDPAParams(q, q, q, None, 0.0, False, False)
except TypeError: # torch < 2.5 has no enable_gqa argument
params = SDPAParams(q, q, q, None, 0.0, False)
return bool(can_use_cudnn_attention(params, False))
from torch.nn.attention import SDPBackend, sdpa_kernel
try:
with sdpa_kernel([SDPBackend.CUDNN_ATTENTION]):
torch.nn.functional.scaled_dot_product_attention(q, q, q)
except torch.cuda.OutOfMemoryError:
raise
except Exception: # noqa: BLE001 - "No available kernel" is the answer
return False
return True
def _cudnn_runs_head_dim(target: Any, head_dim: int) -> Optional[bool]:
"""Whether cuDNN attention runs at ``head_dim`` on ``target``; None when unaskable (not cached)."""
device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
if not device.startswith("cuda"):
return None
device = _indexed_cuda_device(device)
dtype = getattr(target, "dtype", None)
key = (device, str(dtype), int(head_dim))
cached = _CUDNN_HEAD_DIM_CACHE.get(key)
if cached is not None:
return cached
try:
ran = _run_cudnn_head_dim_probe(device, dtype, head_dim)
except Exception: # noqa: BLE001 - no torch, no device, allocator trouble: no answer about the kernel
return None
return _CUDNN_HEAD_DIM_CACHE.setdefault(key, bool(ran))
def _native_sdp_choice(device: str, dtype: Any, head_dim: int) -> int:
"""torch's own SDPA dispatch choice (an ``SDPBackend`` value) at ``head_dim`` / ``dtype`` on ``device``."""
import torch
q = torch.zeros(1, 2, 256, int(head_dim), device = device, dtype = dtype)
return int(torch._fused_sdp_choice(q, q, q))
def _device_capability(device: str) -> tuple[int, int]:
import torch
return tuple(torch.cuda.get_device_capability(device)) # type: ignore[return-value]
def _native_dispatches_cudnn(pipe: Any, target: Any) -> bool:
"""True on a Blackwell (SM100+) NVIDIA card when torch's own SDPA dispatch picks cuDNN for every DiT head dim at the
run dtype, so the ``auto`` pin would change no kernel. Unknown head dims, dtype or capability keep the pin."""
if (os.environ.get(BLACKWELL_NATIVE_ATTN_ENV) or "").strip().lower() in (
"0",
"off",
"false",
"no",
):
return False
if not _is_cuda_nvidia(target):
return False
device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
if not device.startswith("cuda"):
return False
dims = _dit_head_dims(pipe)
if not dims:
return False
try:
import torch
from torch.nn.attention import SDPBackend
device = _indexed_cuda_device(device)
if _device_capability(device) < (10, 0):
return False
dtype = getattr(target, "dtype", None)
if dtype not in (torch.bfloat16, torch.float16):
return False
for dim in sorted(dims):
key = (device, str(dtype), int(dim))
hit = _NATIVE_DISPATCH_CACHE.get(key)
if hit is None:
hit = _native_sdp_choice(device, dtype, dim) == int(SDPBackend.CUDNN_ATTENTION)
_NATIVE_DISPATCH_CACHE[key] = hit
if not hit:
return False
return True
except Exception: # noqa: BLE001 - unaskable: keep the pin
return False
def _native_flash_measured_faster(pipe: Any, target: Any) -> bool:
"""True on a ``FLASH_FASTER_CAPABILITIES`` card whose native SDPA picks FLASH for every DiT head dim; else keep the pin."""
if (os.environ.get(NATIVE_FLASH_ATTN_ENV) or "").strip().lower() in ("0", "off", "false", "no"):
return False
if not _is_cuda_nvidia(target):
return False
device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
if not device.startswith("cuda"):
return False
dims = _dit_head_dims(pipe)
if not dims:
return False
try:
import torch
from torch.nn.attention import SDPBackend
device = _indexed_cuda_device(device)
if tuple(_device_capability(device)) not in FLASH_FASTER_CAPABILITIES:
return False
dtype = getattr(target, "dtype", None)
if dtype not in (torch.bfloat16, torch.float16):
return False
for dim in sorted(dims):
key = (device, str(dtype), int(dim))
hit = _NATIVE_FLASH_CACHE.get(key)
if hit is None:
hit = _native_sdp_choice(device, dtype, dim) == int(SDPBackend.FLASH_ATTENTION)
_NATIVE_FLASH_CACHE[key] = hit
if not hit:
return False
return True
except Exception: # noqa: BLE001 - unaskable: keep the pin
return False
def _cudnn_serves_pipe(
pipe: Any,
target: Any,
logger: Any = None,
) -> bool:
"""False only when cuDNN demonstrably lacks a DiT head_dim. ``UNSLOTH_DIFFUSION_CUDNN_HEAD_DIM_PROBE=0`` skips it."""
if (os.environ.get(CUDNN_HEAD_DIM_PROBE_ENV) or "").strip().lower() in (
"0",
"off",
"false",
"no",
):
return True
missing = sorted(d for d in _dit_head_dims(pipe) if _cudnn_runs_head_dim(target, d) is False)
if missing and logger is not None:
logger.warning(
"diffusion.attention: cuDNN attention has no kernel for head_dim %s with this torch/cuDNN build; "
"keeping the default SDPA dispatch",
",".join(str(d) for d in missing),
)
return not missing
def _configure_native_attention(pipe: Any, target: Any, logger: Any) -> None:
rocm = _is_cuda_rocm(target)
if rocm:
try:
guard_rocm_fused_sdpa(target, logger)
except Exception as exc:
_warn(logger, "ROCm fused SDPA check", exc)
mps = getattr(target, "device", None) == "mps"
if mps or (rocm and sdpa_math_only(target)):
try:
from .diffusion_qwenimage21_math import install
if install(pipe, target, logger):
if logger is not None and not mps:
logger.warning(
"diffusion.attention: Qwen-Image-2.1 is using bounded math attention; "
"check the AMD device packages if fused kernels are expected on this GPU"
)
return
except Exception as exc:
_warn(logger, "bounded Qwen-Image-2.1 math attention", exc)
warn_if_sdpa_math_only(target, logger, _dit_head_dims(pipe))
def apply_attention_backend(
pipe: Any,
backend: Optional[str],
*,
logger: Any = None,
target: Any = None,
) -> Optional[str]:
"""Sets the backend on every DiT (None means native, set explicitly so an earlier pin cannot leak)."""
if target is not None:
try:
guard_rocm_fused_sdpa(target, logger)
except Exception as exc:
_warn(logger, "ROCm fused SDPA check", exc)
install_rocm_contiguous_kv(target)
setters = [
s
for s in (getattr(t, "set_attention_backend", None) for t in _attention_dits(pipe))
if callable(s)
]
if not setters:
# SDXL U-Net has no backend setter but still runs SDPA: report the math-only diagnosis.
if target is not None:
_configure_native_attention(pipe, target, logger)
return None
for dit in _attention_dits(pipe): # only an auto admission below re-arms per-length gating
try:
setattr(dit, ROCM_FLASH_GATE_ATTR, None)
except Exception: # noqa: BLE001
pass
if backend == ROCM_FLASH_CANDIDATE:
backend = "flash" if _rocm_flash_block_admitted(pipe, target, logger) else None
if backend is not None:
_ensure_attention_backend_installed(backend, logger)
if backend == "sage":
if _pip_sage2_installed():
if not _sage_usable(pipe, target, logger):
backend = None
else:
too_old = _sage_version_too_old()
if too_old and logger is not None:
logger.info("diffusion.attention: ignoring %s", too_old)
backend = _sage_hub_backend(pipe, target, logger)
# Re-verify at the dtype the pipeline runs in: selection may have seen the pre-promotion (fp16) dtype.
if (
backend == "flash"
and target is not None
and _is_cuda_rocm(target)
and not _rocm_flash_attn_runs(target)
):
backend = None
if (
backend == "flash"
and target is not None
and _is_cuda_rocm(target)
and not _install_rocm_flash_dispatch_guard()
):
# An explicit request too: without the guard a masked call would raise mid-generation, not run native.
if logger is not None:
logger.warning(
"diffusion.attention: this diffusers build does not expose the flash backend the way Unsloth "
"expects, so masked attention calls could not be routed around it; using the default backend"
)
backend = None
if (
backend == "_native_cudnn"
and target is not None
and not _cudnn_serves_pipe(pipe, target, logger)
):
backend = None
if backend == "_native_flash" and target is not None:
# H3 Auto can pick flash after select(); refuse exclusive flash when this build has no kernel (#16294).
try:
kernels = available_sdpa_kernels(target)
except Exception: # noqa: BLE001 -- an unreadable probe must not force flash
kernels = ()
if SDPA_FLASH not in kernels:
# A probe that answered without cuDNN keeps the default dispatch; an empty one keeps Auto's cuDNN.
cudnn_ok = (not kernels or SDPA_CUDNN in kernels) and _cudnn_serves_pipe(
pipe, target, logger
)
if logger is not None:
logger.info(
"diffusion.attention: flash SDPA unavailable on this torch build; using %s",
"cuDNN" if cudnn_ok else "the default SDPA dispatch",
)
backend = AUTO_CUDNN if cudnn_ok else None
if (
isinstance(backend, _AutoCudnn)
and target is not None
and _native_dispatches_cudnn(pipe, target)
):
if logger is not None:
logger.info(
"diffusion.attention: auto keeps native SDPA (it already dispatches cuDNN on this Blackwell card; "
"%s=0 pins cuDNN)",
BLACKWELL_NATIVE_ATTN_ENV,
)
backend = None
if (
isinstance(backend, _AutoCudnn)
and target is not None
and _native_flash_measured_faster(pipe, target)
):
if logger is not None:
logger.info(
"diffusion.attention: auto keeps native SDPA (flash attention measured faster than the cuDNN "
"pin on this card; %s=0 pins cuDNN)",
NATIVE_FLASH_ATTN_ENV,
)
backend = None
if backend is not None:
backend = str(backend)
if backend == "flash_4_hub" and not _install_fa4_dispatch_guard():
if logger is not None:
logger.warning(
"diffusion.attention: this diffusers build does not expose the flash_4_hub backend the way "
"Unsloth expects, so masked attention calls could not be routed around it; using the default "
"backend"
)
backend = None
if backend is not None:
engaged = False
for fn in setters:
try:
fn(backend)
engaged = True
except Exception as exc: # noqa: BLE001 - unavailable kernel -> restore native below
_warn(logger, backend, exc)
if (
engaged
and backend == "flash_4_hub"
and target is not None
and _fa4_kernel_runs(target, logger, _dit_head_dims(pipe)) is False
):
# set_attention_backend fetches the kernel, so probe after it and undo the pin on failure.
for fn in setters:
try:
fn(ATTN_NATIVE)
except Exception as exc: # noqa: BLE001
_warn(logger, ATTN_NATIVE, exc)
engaged = False
if engaged:
_tag_dits(pipe, backend)
if backend == "_native_cudnn" and target is not None and _is_cuda_nvidia(target):
from .diffusion_flux3_attn import install as install_flux3_strided
install_flux3_strided(pipe, backend, target, logger)
if backend == "flash" and target is not None and _is_cuda_rocm(target):
_track_rocm_flash_calls(pipe)
else:
_untrack_rocm_flash_calls(pipe)
# set_attention_backend also pins the backend process-wide. Each DiT's processors keep it locally, so reset
# the global to native ONCE, else a later component inherits this kernel.
_reset_global_backend_to_native(logger)
if logger is not None:
logger.info("diffusion.attention: backend=%s", backend)
return backend
# No backend requested or all failed: pin native so a stale process-wide backend cannot leak in.
_tag_dits(pipe, None)
_untrack_rocm_flash_calls(pipe)
_restore_native_backend(setters[0], logger)
# Native means torch's SDPA dispatch decides per call, and on a device with no fused kernel that decision is MATH.
# Say so now; the flags this would otherwise be read off lie (#8225).
if target is not None:
_configure_native_attention(pipe, target, logger)
return None
# Read by graph_eligible: Sage under a replayed graph renders noise.
ATTENTION_BACKEND_ATTR = "_unsloth_attention_backend"
def _tag_dits(pipe: Any, backend: Optional[str]) -> None:
for dit in _attention_dits(pipe):
try:
setattr(dit, ATTENTION_BACKEND_ATTR, backend)
except Exception: # noqa: BLE001
pass
def _active_attention_backend() -> Optional[str]:
"""The diffusers process-wide active attention backend name, or None if undeterminable."""
try:
from diffusers.models.attention_dispatch import _AttentionBackendRegistry
# get_active_backend() returns (AttentionBackendName, fn) or None; read element 0's .value
active = _AttentionBackendRegistry.get_active_backend()
if active is None:
return None
name = active[0] if isinstance(active, tuple) else active
return getattr(name, "value", str(name))
except Exception: # noqa: BLE001
return None
def _reset_global_backend_to_native(logger: Any) -> None:
"""Reset the process-wide active backend to native after a successful per-transformer set, so
a later unconfigured component doesn't inherit this kernel (the DiT's own processors keep it).
Best-effort: if the diffusers internals move, the prior (leaking) behavior is unchanged."""
if _active_attention_backend() == ATTN_NATIVE:
return
try:
from diffusers.models.attention_dispatch import (
AttentionBackendName,
_AttentionBackendRegistry,
)
_AttentionBackendRegistry.set_active_backend(AttentionBackendName.NATIVE)
except Exception: # noqa: BLE001 - best-effort; leave the global as-is on any change
pass
def _restore_native_backend(set_backend_fn: Any, logger: Any) -> None:
"""Force the native default when the global active backend isn't already native."""
if _active_attention_backend() == ATTN_NATIVE:
return # already native -> avoid redundant work and an extra dispatcher warning
try:
set_backend_fn(ATTN_NATIVE)
except Exception as exc: # noqa: BLE001 - best-effort restore
_warn(logger, ATTN_NATIVE, exc)
def _warn(logger: Any, what: str, exc: Exception) -> None:
if logger is not None:
logger.warning("diffusion.attention: %s unavailable (%s); using default", what, exc)
# HunyuanVideo-1.5: trim padded text before joint attention so fused SDPA runs without a dense mask
# (no information lost, though not bit-identical; ~10x faster). Trimmed length varies per
# prompt, so compile must stay dynamic=None.
_HUNYUAN15_TRANSFORMER_CLS = "HunyuanVideo15Transformer3DModel"
_HUNYUAN15_PROCESSOR_CLS = "HunyuanVideo15AttnProcessor2_0"
_NULL_ATTN_FLAG = "_unsloth_null_attn_mask"
_NULL_PROCESSOR_CACHE: dict = {}
def _set_hunyuan_null_mask(module: Any, enabled: bool) -> None:
"""Null-mask flag on block attention, valid only for the forward whose pre-hook removed the padding."""
for blk in getattr(module, "transformer_blocks", []):
attn = getattr(blk, "attn", None)
if attn is not None:
setattr(attn, _NULL_ATTN_FLAG, enabled)
def _hunyuan_null_mask_state(module: Any) -> bool:
"""The null-mask flag this call's pre-hook set (read by the graph layer as part of its key)."""
for blk in getattr(module, "transformer_blocks", []):
attn = getattr(blk, "attn", None)
if attn is not None:
return bool(getattr(attn, _NULL_ATTN_FLAG, False))
return False
def _null_mask_processor_cls():
"""Skips attn_mask only when the DiT is flagged; otherwise delegates to the stock processor."""
cached = _NULL_PROCESSOR_CACHE.get("cls")
if cached is not None:
return cached
import torch
from diffusers.models.attention_dispatch import dispatch_attention_fn
from diffusers.models.transformers.transformer_hunyuan_video15 import (
HunyuanVideo15AttnProcessor2_0,
)
class _HunyuanNullMaskProcessor(HunyuanVideo15AttnProcessor2_0):
def __call__(
self,
attn,
hidden_states,
encoder_hidden_states = None,
attention_mask = None,
image_rotary_emb = None,
):
# Fast path only if the pre-hook removed all padding; a constant bool so torch.compile folds it.
if not getattr(attn, _NULL_ATTN_FLAG, False):
return super().__call__(
attn,
hidden_states,
encoder_hidden_states = encoder_hidden_states,
attention_mask = attention_mask,
image_rotary_emb = image_rotary_emb,
)
# Null path = the stock body with the mask block removed and attn_mask=None.
query = attn.to_q(hidden_states)
key = attn.to_k(hidden_states)
value = attn.to_v(hidden_states)
query = query.unflatten(2, (attn.heads, -1))
key = key.unflatten(2, (attn.heads, -1))
value = value.unflatten(2, (attn.heads, -1))
query = attn.norm_q(query)
key = attn.norm_k(key)
if image_rotary_emb is not None:
from diffusers.models.embeddings import apply_rotary_emb
query = apply_rotary_emb(query, image_rotary_emb, sequence_dim = 1)
key = apply_rotary_emb(key, image_rotary_emb, sequence_dim = 1)
if encoder_hidden_states is not None:
encoder_query = attn.add_q_proj(encoder_hidden_states)
encoder_key = attn.add_k_proj(encoder_hidden_states)
encoder_value = attn.add_v_proj(encoder_hidden_states)
encoder_query = encoder_query.unflatten(2, (attn.heads, -1))
encoder_key = encoder_key.unflatten(2, (attn.heads, -1))
encoder_value = encoder_value.unflatten(2, (attn.heads, -1))
if attn.norm_added_q is not None:
encoder_query = attn.norm_added_q(encoder_query)
if attn.norm_added_k is not None:
encoder_key = attn.norm_added_k(encoder_key)
query = torch.cat([query, encoder_query], dim = 1)
key = torch.cat([key, encoder_key], dim = 1)
value = torch.cat([value, encoder_value], dim = 1)
hidden_states = dispatch_attention_fn(
query,
key,
value,
attn_mask = None,
dropout_p = 0.0,
is_causal = False,
backend = self._attention_backend,
parallel_config = self._parallel_config,
)
hidden_states = hidden_states.flatten(2, 3)
hidden_states = hidden_states.to(query.dtype)
if encoder_hidden_states is not None:
enc_len = encoder_hidden_states.shape[1]
hidden_states, encoder_hidden_states = (
hidden_states[:, :-enc_len],
hidden_states[:, -enc_len:],
)
if getattr(attn, "to_out", None) is not None:
hidden_states = attn.to_out[0](hidden_states)
hidden_states = attn.to_out[1](hidden_states)
if getattr(attn, "to_add_out", None) is not None:
encoder_hidden_states = attn.to_add_out(encoder_hidden_states)
# Always the 2-tuple, matching the stock processor's return contract.
return hidden_states, encoder_hidden_states
_NULL_PROCESSOR_CACHE["cls"] = _HunyuanNullMaskProcessor
return _HunyuanNullMaskProcessor
def _trim_stream(states, mask):
"""Drops globally padded columns; all_valid (no partly padded column left) means no mask needed."""
if states is None or mask is None or mask.dim() != 2:
return states, mask, True # nothing to mask -> treat as no-padding
mb = mask.bool()
keep = mb.any(dim = 0) # column valid for at least one batch element
if not bool(keep.all()):
states = states[:, keep]
mask = mask[:, keep]
mb = mb[:, keep]
# Vacuously True for a 0-length stream, e.g. the unused byt5 stream in t2v; all slots valid.
all_valid = bool(mb.all().item())
return states, mask, all_valid
_TRIM_MEMO_ATTR = "_unsloth_trim_memo"
def _trim_plan(module: Any, kwargs: dict) -> dict:
"""Trim plan per input set, reused across steps; keyed on tensors and versions, so edits replan."""
import torch
names = ("image_embeds", "encoder_attention_mask", "encoder_attention_mask_2")
srcs = tuple(kwargs.get(n) for n in names)
# Inference tensors have no version counter: keyed on identity alone.
versions = tuple(
("inference" if t.is_inference() else t._version) if torch.is_tensor(t) else None
for t in srcs
)
memo = module.__dict__.setdefault(_TRIM_MEMO_ATTR, [])
for held, held_versions, plan in memo:
if held_versions == versions and all(a is b for a, b in zip(held, srcs)):
return plan
image = srcs[0]
plan = {
"t2v": bool(image is not None and image.numel() > 0 and bool(torch.all(image == 0).item()))
}
for name, mask in zip(names[1:], srcs[1:]):
if mask is None or not torch.is_tensor(mask) or mask.dim() != 2:
plan[name] = (None, True)
continue
mb = mask.bool()
keep = mb.any(dim = 0) # column valid for at least one batch element
index = None
if not bool(keep.all()):
index = keep.nonzero().squeeze(1)
mb = mb.index_select(1, index)
# vacuously True for a 0-length stream, fine for an unused secondary stream (byt5 in t2v)
plan[name] = (index, bool(mb.all().item()))
memo.insert(0, (srcs, versions, plan))
del memo[4:] # the CFG branches of one render
return plan
def _apply_trim(states: Any, mask: Any, decided: tuple) -> tuple:
"""``_trim_stream`` with its decisions already made: the same columns, gathered on the device."""
if states is None or mask is None or mask.dim() != 2:
return states, mask, True
index, all_valid = decided
if index is not None:
states = states.index_select(1, index)
mask = mask.index_select(1, index)
return states, mask, all_valid
def _hunyuan_trim_pre_hook(module, args, kwargs):
"""Strips padding so attention runs fused; call the module, never module.forward, which skips it."""
import torch
original = dict(kwargs)
try:
null_ok = True
plan = _trim_plan(module, kwargs)
image = kwargs.get("image_embeds")
if plan["t2v"]:
# is_t2v stays True in forward: all() of an empty tensor is True.
kwargs["image_embeds"] = image[:, :0]
for skey, mkey, required in (
("encoder_hidden_states", "encoder_attention_mask", True),
("encoder_hidden_states_2", "encoder_attention_mask_2", False),
):
# Only keyword streams: a positional encoder_hidden_states would collide.
if skey not in kwargs:
null_ok = null_ok and not required
continue
states, mask, all_valid = _apply_trim(kwargs.get(skey), kwargs.get(mkey), plan[mkey])
kwargs[skey] = states
kwargs[mkey] = mask
null_ok = null_ok and all_valid
# The TokenRefiner pooling divides by the mask sum: never pass a 0-length sequence.
primary = kwargs.get("encoder_hidden_states")
if primary is not None and primary.dim() == 3 and primary.shape[1] == 0:
kwargs.clear()
kwargs.update(original)
null_ok = False
_set_hunyuan_null_mask(module, null_ok)
return args, kwargs
except Exception: # noqa: BLE001 - optimisation only; never break the forward
# Different arch, not a refreshed Qwen-Image: 14.2 GB DiT, a larger Qwen3-VL 8B encoder; bf16, so resident equals on-disk; sizes from Hub metadata
kwargs.clear()
kwargs.update(original)
_set_hunyuan_null_mask(module, False)
return args, kwargs
def _hunyuan_trim_post_hook(module, _args, output):
"""Clears the null-mask flag after every forward, including ones that raise, so it cannot latch on."""
_set_hunyuan_null_mask(module, False)
return output
def _install_null_processors(dit: Any, logger: Any) -> bool:
"""Swap stock block processors for the null-mask subclass; keep any pinned attention backend"""
try:
cls = _null_mask_processor_cls()
except Exception as exc: # noqa: BLE001 - diffusers moved / unavailable -> skip
_warn(logger, "hunyuan_attn_trim", exc)
return False
installed = 0
for blk in getattr(dit, "transformer_blocks", []):
attn = getattr(blk, "attn", None)
proc = getattr(attn, "processor", None) if attn is not None else None
if proc is None:
continue
if isinstance(proc, cls):
installed += 1
continue
if type(proc).__name__ != _HUNYUAN15_PROCESSOR_CLS:
continue
new = cls()
# carry over any backend/parallel config the stock processor already held
new._attention_backend = getattr(proc, "_attention_backend", None)
new._parallel_config = getattr(proc, "_parallel_config", None)
try:
attn.set_processor(new)
except Exception: # noqa: BLE001 - fall back to direct assignment
attn.processor = new
installed += 1
return installed > 0
def install_hunyuan_attention_trim(
pipe: Any,
family: Any,
*,
logger: Any = None,
) -> bool:
"""HunyuanVideo-1.5 only; call before apply_attention_backend, and not under static-shape compile."""
if getattr(family, "transformer_class", None) != _HUNYUAN15_TRANSFORMER_CLS:
return False
engaged = False
for dit in _attention_dits(pipe):
if type(dit).__name__ != _HUNYUAN15_TRANSFORMER_CLS:
continue
if not _install_null_processors(dit, logger):
continue
# installation and every idle period start in the conservative state
_set_hunyuan_null_mask(dit, False)
# The flag picks the attention branch inside the forward, so a CUDA graph keys on it (diffusion_cuda_graph).
dit._unsloth_graph_key_extra = functools.partial(_hunyuan_null_mask_state, dit)
if getattr(dit, "_unsloth_trim_hook", None) is None:
pre_handle = None
try:
pre_handle = dit.register_forward_pre_hook(_hunyuan_trim_pre_hook, with_kwargs = True)
# always_call: never leave the null-mask flag latched after a raise.
post_handle = dit.register_forward_hook(_hunyuan_trim_post_hook, always_call = True)
dit._unsloth_trim_hook = (pre_handle, post_handle)
except Exception as exc: # noqa: BLE001 - optimisation only
if pre_handle is not None:
pre_handle.remove()
_set_hunyuan_null_mask(dit, False)
_warn(logger, "hunyuan_attn_trim", exc)
continue
engaged = True
if engaged and logger is not None:
logger.info("diffusion.attention: hunyuan padded-text trim engaged")
return engaged
# declared checkpoint attention -> Studio backend of the same arithmetic (still probe-gated)
CHECKPOINT_ATTENTION_BACKENDS = {"int8": "sage"}
def checkpoint_attention_backend(declared: Any) -> Optional[str]:
if not isinstance(declared, str):
return None
return CHECKPOINT_ATTENTION_BACKENDS.get(declared.strip().lower())