Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: never fail, slow down or render noise on an explicit SageAttention or FlashAttention 4 request (#12641)

Daniel Han23308cd8084a
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 755be7d36e0..e3361b21e03 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -72,7 +72,10 @@ def normalize_attention_backend(value: Optional[str]) -> Optional[str]:
# (min, max-exclusive) capability range: FA3 is Hopper-SM90 only, FA4 is Blackwell+.
_ARCH_CAPABILITY: dict[str, tuple[tuple[int, int], Optional[tuple[int, int]]]] = {
"_flash_3_hub": ((9, 0), (10, 0)), # Hopper (SM90) only
- "flash_4_hub": ((10, 0), None), # Blackwell (SM100)+
+ "flash_4_hub": (
+ (10, 0),
+ None,
+ ), # SM100+; sm110 / sm120 unverified, so _fa4_kernel_runs also checks each card
}
@@ -282,26 +285,59 @@ def _cudnn_attention_supported() -> bool:
return have is None or have >= (8, 0)
-# sageattn dispatches on an exact arch match whose set varies by build (2.2.0: sm80/86/89/90/120; community builds add
-# sm75/87/100) and diffusers only checks the version, so ask the kernel. (device, dtype) -> "" or its error.
-_SAGE_PROBE_CACHE: dict[tuple[str, str], str] = {}
+# Arch support varies by sageattn build, so a run per card is the gate. (device, dtype, head_dim) -> "" or why not.
+_SAGE_PROBE_CACHE: dict[tuple[str, str, int], str] = {}
+_SAGE_MAX_HEAD_DIM = 128 # sageattn raises above 128
+_SAGE_DTYPE_NAMES = ("float16", "bfloat16", "fp16", "bf16", "half")
+# SageAttention 2 lands near cos 0.9999 / rel-L1 0.01-0.03 vs fp32 here; a wrong-arch or broken build is far outside.
+_SAGE_MIN_COSINE = 0.99
+_SAGE_MAX_REL_L1 = 0.08
-def _run_sage_probe(device: str, dtype: Any) -> str:
- """Empty when a tiny ``sageattn`` ran on ``device``, else its error; raises when unaskable (import, device, OOM)."""
+
+def _run_sage_probe(
+ device: str,
+ dtype: Any,
+ head_dim: int = 128,
+) -> str:
+ """Empty when ``sageattn`` matches fp32 SDPA at ``head_dim``, else why not. Raises when unaskable (import, OOM).
+
+ Random inputs with a per-channel K offset like real keys: a zero tensor proves the launch, not the numbers."""
import torch
from sageattention import sageattn
if dtype not in (torch.float16, torch.bfloat16):
dtype = torch.float16
- q = torch.zeros((1, 128, 2, 128), device = device, dtype = dtype)
+ shape = (1, 128, 2, int(head_dim))
+ gen = torch.Generator(device = "cpu").manual_seed(0)
+ q, k, v = (torch.randn(shape, generator = gen) for _ in range(3))
+ k = k + 2.0 * torch.randn((1, 1, 1, shape[-1]), generator = gen)
+ q, k, v = (t.to(device = device, dtype = dtype) for t in (q, k, v))
try:
- sageattn(q, q, q, tensor_layout = "NHD")
- torch.cuda.synchronize(device)
+ out = sageattn(q, k, v, tensor_layout = "NHD")
+ if str(device).startswith("cuda"):
+ torch.cuda.synchronize(device)
except torch.cuda.OutOfMemoryError:
raise
except Exception as exc: # noqa: BLE001
return f"{type(exc).__name__}: {exc}"
+ try:
+ ref = torch.nn.functional.scaled_dot_product_attention(
+ *(t.float().transpose(1, 2) for t in (q, k, v))
+ ).transpose(1, 2)
+ got = out.float()
+ if tuple(got.shape) != tuple(ref.shape):
+ return f"self-check: output shape {tuple(got.shape)} != {tuple(ref.shape)}"
+ if not bool(torch.isfinite(got).all()):
+ return "self-check: non-finite output"
+ cos = float(torch.nn.functional.cosine_similarity(got.flatten(), ref.flatten(), dim = 0))
+ rel = float((got - ref).abs().mean() / ref.abs().mean().clamp_min(1e-12))
+ except torch.cuda.OutOfMemoryError:
+ raise
+ except Exception as exc: # noqa: BLE001
+ return f"self-check: {type(exc).__name__}: {exc}"
+ if cos < _SAGE_MIN_COSINE or rel > _SAGE_MAX_REL_L1:
+ return f"self-check: cosine {cos:.4f}, relative L1 {rel:.4f} vs fp32 at head_dim {head_dim}"
return ""
@@ -316,21 +352,45 @@ def _indexed_cuda_device(device: str) -> str:
return device
-def _sage_kernel_runs(target: Any, logger: Any = None) -> Optional[bool]:
- """False only when the kernel raised; None (unaskable, not cached) keeps the requested backend."""
+def _sage_probe_head_dims(head_dims: Any) -> tuple[int, ...]:
+ """Head dims Sage serves (<= 128; larger run native per call), 128 when none are known."""
+ dims = sorted(
+ {
+ int(d)
+ for d in (head_dims or ())
+ if isinstance(d, int) and not isinstance(d, bool) and 0 < d <= _SAGE_MAX_HEAD_DIM
+ }
+ )
+ return tuple(dims) or (_SAGE_MAX_HEAD_DIM,)
+
+
+def _sage_kernel_runs(
+ target: Any,
+ logger: Any = None,
+ head_dims: Any = None,
+) -> Optional[bool]:
+ """True when ``sageattn`` passed its self-check at every head dim it serves; False when it raised, failed, or is
+ not importable (not cached: a later install may fix it); None (unaskable) keeps the request."""
device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
if not device.startswith("cuda"):
return None
device = _indexed_cuda_device(device)
dtype = getattr(target, "dtype", None)
- key = (device, str(dtype))
- error = _SAGE_PROBE_CACHE.get(key)
- if error is None:
- try:
- error = _run_sage_probe(device, dtype)
- except Exception: # noqa: BLE001
- return None
- error = _SAGE_PROBE_CACHE.setdefault(key, error)
+ error = ""
+ for head_dim in _sage_probe_head_dims(head_dims):
+ key = (device, str(dtype), head_dim)
+ error = _SAGE_PROBE_CACHE.get(key)
+ if error is None:
+ try:
+ error = _run_sage_probe(device, dtype, head_dim)
+ except ImportError as exc:
+ error = f"sageattention could not be imported ({type(exc).__name__}: {exc})"
+ break
+ except Exception: # noqa: BLE001
+ return None
+ error = _SAGE_PROBE_CACHE.setdefault(key, error)
+ if error:
+ break
if error and logger is not None:
logger.warning(
"diffusion.attention: SageAttention does not run on this GPU (%s); using the default backend",
@@ -339,6 +399,376 @@ def _sage_kernel_runs(target: Any, logger: Any = None) -> Optional[bool]:
return not error
+def _runs_in_float32(target: Any) -> bool:
+ """fp32 pipeline (video families promote fp16 on cards without bf16): Sage never applies."""
+ dtype = getattr(target, "dtype", None)
+ if dtype is None:
+ return False
+ name = str(dtype).replace("torch.", "").lower()
+ return name in ("float32", "fp32", "float")
+
+
+# Calls Sage cannot take (attn_mask, head_dim > 128, dtype) run native instead of raising mid-generation. The checks
+# read only shapes, dtypes and mask presence, so torch.compile specialises on them without a graph break.
+_SAGE_GUARD_ATTR = "_unsloth_sage_guard"
+_SAGE_ROUTED: dict[str, int] = {}
+_SAGE_ROUTED_LOGGED: set[str] = set()
+
+
+def _sage_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any) -> Optional[str]:
+ import torch
+
+ if attn_mask is not None:
+ return "attn_mask"
+ if not all(isinstance(t, torch.Tensor) for t in (query, key, value)):
+ return "inputs"
+ if (
+ query.dtype not in (torch.float16, torch.bfloat16)
+ or key.dtype != query.dtype
+ or value.dtype != query.dtype
+ ):
+ return "dtype"
+ if query.dim() != 4 or key.dim() != 4 or value.dim() != 4:
+ return "rank"
+ head_dim = query.shape[-1]
+ if head_dim > _SAGE_MAX_HEAD_DIM or key.shape[-1] != head_dim or value.shape[-1] != head_dim:
+ return "head_dim"
+ if key.shape[2] == 0 or query.shape[2] % key.shape[2] != 0:
+ return "heads"
+ if query.device.type != "cuda":
+ return "device"
+ return None
+
+
+def _note_reroute(label: str, reason: str, counts: dict, logged: set) -> None:
+ """Count and log once per reason. Eager only: under torch.compile it would be a traced side effect."""
+ try:
+ import torch
+ if torch.compiler.is_compiling():
+ return
+ except Exception: # noqa: BLE001
+ pass
+ counts[reason] = counts.get(reason, 0) + 1
+ if reason not in logged:
+ logged.add(reason)
+ import logging
+ logging.getLogger(__name__).warning(
+ "diffusion.attention: %s cannot take this attention call (%s); running it on the default "
+ "backend instead",
+ label,
+ reason,
+ )
+
+
+def _note_sage_reroute(reason: str) -> None:
+ _note_reroute("SageAttention", reason, _SAGE_ROUTED, _SAGE_ROUTED_LOGGED)
+
+
+# sageattn() reads the arch via torch.cuda calls Dynamo cannot trace, so a fullgraph compile failed and the load ran
+# eager. An opaque custom op keeps the block compiled.
+_SAGE_OP_NAME = "unsloth_studio::sage_attention_nhd"
+_SAGE_OP: dict[str, Any] = {}
+_SAGE_OP_LOCK = threading.Lock()
+
+
+def _sage_custom_op(sage_fn: Any) -> Optional[Any]:
+ """An opaque ``torch.library`` op around diffusers' sage function (NHD), or None."""
+ with _SAGE_OP_LOCK:
+ if "op" in _SAGE_OP:
+ _SAGE_OP["fn"] = sage_fn
+ return _SAGE_OP["op"]
+ try:
+ import torch
+
+ _SAGE_OP["fn"] = sage_fn
+
+ def _impl(query, key, value, is_causal, scale):
+ out = _SAGE_OP["fn"](
+ query = query, key = key, value = value, is_causal = is_causal, scale = scale
+ )
+ # Head dims under 64 come back as a padded slice; the fake describes a contiguous tensor.
+ return out.contiguous()
+
+ # custom_op infers the schema from real types, not `from __future__ import annotations` strings.
+ _impl.__annotations__ = {
+ "query": torch.Tensor,
+ "key": torch.Tensor,
+ "value": torch.Tensor,
+ "is_causal": bool,
+ "scale": Optional[float],
+ "return": torch.Tensor,
+ }
+ _op = torch.library.custom_op(_SAGE_OP_NAME, mutates_args = ())(_impl)
+
+ @_op.register_fake
+ def _(query, key, value, is_causal, scale):
+ return query.new_empty((*query.shape[:-1], value.shape[-1]))
+
+ _SAGE_OP["op"] = _op
+ except Exception: # noqa: BLE001 - old torch: call the function directly
+ _SAGE_OP["op"] = None
+ return _SAGE_OP["op"]
+
+
+def _install_dispatch_guard(
+ backend: str,
+ reroute_reason: Any,
+ note: Any,
+ make_op: Any = None,
+) -> bool:
+ """Wrap diffusers' registered ``backend`` so calls its kernel cannot take run native. Idempotent.
+
+ False when the registry is not where expected: the caller then does not engage the backend. ``make_op(fn)``
+ optionally returns an opaque op for plain (no lse, no context-parallel) calls."""
+ try:
+ from diffusers.models.attention_dispatch import (
+ AttentionBackendName,
+ _AttentionBackendRegistry,
+ )
+
+ backends = _AttentionBackendRegistry._backends
+ name = AttentionBackendName(backend)
+ kernel_fn = backends[name]
+ native_fn = backends[AttentionBackendName.NATIVE]
+ except Exception: # noqa: BLE001
+ return False
+ if getattr(kernel_fn, _SAGE_GUARD_ATTR, False):
+ return True
+ if not callable(kernel_fn) or not callable(native_fn):
+ return False
+
+ # Eagerly: the guard runs inside compiled graphs, where a registration lock cannot trace.
+ op = make_op(kernel_fn) if make_op is not None else None
+ names = ("query", "key", "value", "attn_mask")
+
+ def _guarded(*args: Any, **kwargs: Any) -> Any:
+ bound = dict(zip(names, args))
+ bound.update({n: kwargs[n] for n in names if n in kwargs})
+ reason = reroute_reason(
+ bound.get("query"), bound.get("key"), bound.get("value"), bound.get("attn_mask")
+ )
+ if reason is None:
+ extra = {k: v for k, v in kwargs.items() if k not in names}
+ plain = (
+ op is not None
+ and len(args) <= 4
+ and not extra.get("return_lse", False)
+ and extra.get("_parallel_config") is None
+ and set(extra) <= {"is_causal", "scale", "return_lse", "_parallel_config"}
+ )
+ if plain:
+ return op(
+ bound["query"],
+ bound["key"],
+ bound["value"],
+ bool(extra.get("is_causal", False)),
+ extra.get("scale"),
+ )
+ return kernel_fn(*args, **kwargs)
+ note(reason)
+ return native_fn(*args, **kwargs)
+
+ setattr(_guarded, _SAGE_GUARD_ATTR, True)
+ _guarded.__wrapped__ = kernel_fn # type: ignore[attr-defined]
+ try:
+ backends[name] = _guarded
+ except Exception: # noqa: BLE001
+ return False
+ return True
+
+
+def _install_sage_dispatch_guard() -> bool:
+ return _install_dispatch_guard(
+ "sage", lambda *a: _sage_reroute_reason(*a), _note_sage_reroute, _sage_custom_op
+ )
+
+
+# diffusers' flash_4_hub raises on any attn_mask too; head dims are checked at load (_fa4_kernel_runs).
+_FA4_ROUTED: dict[str, int] = {}
+_FA4_ROUTED_LOGGED: set[str] = set()
+
+
+def _fa4_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any) -> Optional[str]:
+ import torch
+
+ if attn_mask is not None:
+ return "attn_mask"
+ if not all(isinstance(t, torch.Tensor) for t in (query, key, value)):
+ return "inputs"
+ if (
+ query.dtype not in (torch.float16, torch.bfloat16)
+ or key.dtype != query.dtype
+ or value.dtype != query.dtype
+ ):
+ return "dtype"
+ if query.device.type != "cuda":
+ return "device"
+ return None
+
+
+def _note_fa4_reroute(reason: str) -> None:
+ _note_reroute("FlashAttention 4", reason, _FA4_ROUTED, _FA4_ROUTED_LOGGED)
+
+
+def _install_fa4_dispatch_guard() -> bool:
+ return _install_dispatch_guard(
+ "flash_4_hub", lambda *a: _fa4_reroute_reason(*a), _note_fa4_reroute
+ )
+
+
+# Whether the hub FA4 build runs correctly here, once per (device, dtype, head_dim) through diffusers' dispatch: catches
+# cards with no code (sm110 / sm120 unverified) and API drift. An exact kernel, so the bound is tight.
+_FA4_PROBE_CACHE: dict[tuple[str, str, int], str] = {}
+_FA4_MIN_COSINE = 0.999
+_FA4_MAX_REL_L1 = 0.02
+
+
+def _run_fa4_probe(
+ device: str,
+ dtype: Any,
+ head_dim: int = 128,
+) -> str:
+ """Empty when flash_4_hub matches fp32 SDPA at ``head_dim``, else why not. Raises on OOM."""
+ import torch
+ from diffusers.models.attention_dispatch import AttentionBackendName, dispatch_attention_fn
+
+ if dtype not in (torch.float16, torch.bfloat16):
+ dtype = torch.bfloat16
+ gen = torch.Generator(device = "cpu").manual_seed(0)
+ q, k, v = (torch.randn((1, 256, 2, int(head_dim)), generator = gen) for _ in range(3))
+ q, k, v = (t.to(device = device, dtype = dtype) for t in (q, k, v))
+ try:
+ out = dispatch_attention_fn(q, k, v, backend = AttentionBackendName.FLASH_4_HUB)
+ if str(device).startswith("cuda"):
+ torch.cuda.synchronize(device)
+ if isinstance(out, tuple):
+ out = out[0]
+ ref = torch.nn.functional.scaled_dot_product_attention(
+ *(t.float().transpose(1, 2) for t in (q, k, v))
+ ).transpose(1, 2)
+ got = out.float()
+ if tuple(got.shape) != tuple(ref.shape):
+ return f"self-check: output shape {tuple(got.shape)} != {tuple(ref.shape)}"
+ if not bool(torch.isfinite(got).all()):
+ return "self-check: non-finite output"
+ cos = float(torch.nn.functional.cosine_similarity(got.flatten(), ref.flatten(), dim = 0))
+ rel = float((got - ref).abs().mean() / ref.abs().mean().clamp_min(1e-12))
+ except torch.cuda.OutOfMemoryError:
+ raise
+ except Exception as exc: # noqa: BLE001
+ return f"{type(exc).__name__}: {exc}"
+ if cos < _FA4_MIN_COSINE or rel > _FA4_MAX_REL_L1:
+ return f"self-check: cosine {cos:.5f}, relative L1 {rel:.4f} vs fp32 at head_dim {head_dim}"
+ return ""
+
+
+def _fa4_kernel_runs(
+ target: Any,
+ logger: Any = None,
+ head_dims: Any = None,
+) -> Optional[bool]:
+ """True when flash_4_hub passed at every DiT head dim; False with a logged reason; None unaskable."""
+ device = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
+ if not device.startswith("cuda"):
+ return None
+ device = _indexed_cuda_device(device)
+ dtype = getattr(target, "dtype", None)
+ dims = sorted(
+ {
+ int(d)
+ for d in (head_dims or ())
+ if isinstance(d, int) and not isinstance(d, bool) and d > 0
+ }
+ )
+ error = ""
+ for head_dim in dims or [128]:
+ key = (device, str(dtype), head_dim)
+ error = _FA4_PROBE_CACHE.get(key)
+ if error is None:
+ try:
+ error = _run_fa4_probe(device, dtype, head_dim)
+ except Exception: # noqa: BLE001
+ return None
+ error = _FA4_PROBE_CACHE.setdefault(key, error)
+ if error:
+ break
+ if error and logger is not None:
+ logger.warning(
+ "diffusion.attention: FlashAttention 4 does not run correctly on this GPU (%s); using the default backend",
+ error,
+ )
+ return not error
+
+
+def _version_tuple(text: str) -> tuple[int, ...]:
+ m = re.match(r"\s*(\d+(?:\.\d+)*)", str(text))
+ return tuple(int(x) for x in m.group(1).split(".")) if m else ()
+
+
+def _sage_version_too_old() -> Optional[str]:
+ """Why an installed sageattention is below diffusers' floor (PyPI's 1.0.6 is SageAttention 1), else None. Not
+ cached: a later install may replace it."""
+ try:
+ from importlib.metadata import PackageNotFoundError, version
+
+ try:
+ installed = version("sageattention")
+ except PackageNotFoundError:
+ return None
+ floor = "2.1.1"
+ try:
+ from diffusers.models.attention_dispatch import _REQUIRED_SAGE_VERSION
+ if isinstance(_REQUIRED_SAGE_VERSION, str) and _REQUIRED_SAGE_VERSION.strip():
+ floor = _REQUIRED_SAGE_VERSION.strip()
+ except Exception: # noqa: BLE001
+ pass
+ if _version_tuple(installed) and _version_tuple(installed) < _version_tuple(floor):
+ return f"sageattention {installed} is older than {floor}, the SageAttention 2 release diffusers needs"
+ except Exception: # noqa: BLE001
+ return None
+ return None
+
+
+def _sage_usable(
+ pipe: Any,
+ target: Any,
+ logger: Any = None,
+) -> bool:
+ """Whether an explicit ``sage`` request may engage: not fp32, kernel passes at the DiT head dims, guard in place."""
+ if target is not None and _runs_in_float32(target):
+ if logger is not None:
+ logger.warning(
+ "diffusion.attention: SageAttention needs fp16 or bf16 and this pipeline runs in float32; "
+ "using the default backend"
+ )
+ return False
+ too_old = _sage_version_too_old()
+ if too_old:
+ if logger is not None:
+ logger.warning("diffusion.attention: %s; using the default backend", too_old)
+ return False
+ head_dims = _dit_head_dims(pipe)
+ if head_dims and min(head_dims) > _SAGE_MAX_HEAD_DIM:
+ if logger is not None:
+ logger.warning(
+ "diffusion.attention: SageAttention serves head_dim <= %d and this model uses %s; "
+ "using the default backend",
+ _SAGE_MAX_HEAD_DIM,
+ ",".join(str(d) for d in sorted(head_dims)),
+ )
+ return False
+ if target is not None and _sage_kernel_runs(target, logger, head_dims) is False:
+ return False
+ if not _install_sage_dispatch_guard():
+ if logger is not None:
+ logger.warning(
+ "diffusion.attention: this diffusers build does not expose the sage backend the way Unsloth expects, "
+ "so masked attention calls could not be routed around it; using the default backend"
+ )
+ return False
+ return True
+
+
# Optional kernels installable on demand: dispatcher name -> (probe module, pip package). Wheels only
# (--only-binary=:all:), since a source build needs a CUDA toolchain the host may lack.
_INSTALLABLE_BACKENDS: dict[str, tuple[str, str]] = {
@@ -787,7 +1217,7 @@ def apply_attention_backend(
return None
if backend is not None:
_ensure_attention_backend_installed(backend, logger)
- if backend == "sage" and target is not None and _sage_kernel_runs(target, logger) is False:
+ if backend == "sage" and not _sage_usable(pipe, target, logger):
backend = None
if (
backend == "_native_cudnn"
@@ -795,6 +1225,14 @@ def apply_attention_backend(
and not _cudnn_serves_pipe(pipe, target, logger)
):
backend = None
+ if backend == "flash_4_hub" and not _install_fa4_dispatch_guard():
+ if logger is not None:
+ logger.warning(
+ "diffusion.attention: this diffusers build does not expose the flash_4_hub backend the way "
+ "Unsloth expects, so masked attention calls could not be routed around it; using the default "
+ "backend"
+ )
+ backend = None
if backend is not None:
engaged = False
for fn in setters:
@@ -803,7 +1241,21 @@ def apply_attention_backend(
engaged = True
except Exception as exc: # noqa: BLE001 - unavailable kernel -> restore native below
_warn(logger, backend, exc)
+ if (
+ engaged
+ and backend == "flash_4_hub"
+ and target is not None
+ and _fa4_kernel_runs(target, logger, _dit_head_dims(pipe)) is False
+ ):
+ # set_attention_backend fetches the kernel, so probe after it and undo the pin on failure.
+ for fn in setters:
+ try:
+ fn(ATTN_NATIVE)
+ except Exception as exc: # noqa: BLE001
+ _warn(logger, ATTN_NATIVE, exc)
+ engaged = False
if engaged:
+ _tag_dits(pipe, backend)
# set_attention_backend also pins the backend process-wide. Each DiT's processors keep it locally, so reset
# the global to native ONCE, else a later component inherits this kernel.
_reset_global_backend_to_native(logger)
@@ -812,6 +1264,7 @@ def apply_attention_backend(
return backend
# No backend requested, or every set failed: pin native so a stale process-wide backend cannot leak in. One reset
# covers every fresh DiT.
+ _tag_dits(pipe, None)
_restore_native_backend(setters[0], logger)
# Native means torch's SDPA dispatch decides per call, and on a device with no fused kernel that decision is MATH.
# Say so now; the flags this would otherwise be read off lie (#8225).
@@ -820,6 +1273,18 @@ def apply_attention_backend(
return None
+# Read by graph_eligible: Sage under a replayed graph renders noise.
+ATTENTION_BACKEND_ATTR = "_unsloth_attention_backend"
+
+
+def _tag_dits(pipe: Any, backend: Optional[str]) -> None:
+ for dit in _attention_dits(pipe):
+ try:
+ setattr(dit, ATTENTION_BACKEND_ATTR, backend)
+ except Exception: # noqa: BLE001
+ pass
+
+
def _active_attention_backend() -> Optional[str]:
"""The diffusers process-wide active attention backend name, or None if undeterminable."""
try:
diff --git a/studio/backend/core/inference/diffusion_cuda_graph.py b/studio/backend/core/inference/diffusion_cuda_graph.py
index 5537d4311f7..94db1fece10 100644
--- a/studio/backend/core/inference/diffusion_cuda_graph.py
+++ b/studio/backend/core/inference/diffusion_cuda_graph.py
@@ -672,6 +672,10 @@ def graph_eligible(
if not _denoiser_dits(pipe) and _denoiser_unet(pipe) is None:
return False, "no denoiser transformer"
+ # Sage under a replayed graph renders noise (FLUX.1-schnell, A100), while the ungraphed render is correct.
+ if any(getattr(m, "_unsloth_attention_backend", None) == "sage" for m in _denoiser_dits(pipe)):
+ return False, "SageAttention is not CUDA-graph safe"
+
try:
cuda = getattr(_torch(), "cuda", None)
if cuda is None or not hasattr(cuda, "CUDAGraph") or not cuda.is_available():
diff --git a/studio/backend/tests/test_diffusion_attention.py b/studio/backend/tests/test_diffusion_attention.py
index 6c5beffeae3..41192ee0751 100644
--- a/studio/backend/tests/test_diffusion_attention.py
+++ b/studio/backend/tests/test_diffusion_attention.py
@@ -206,6 +206,9 @@ def test_apply_falls_back_on_unavailable_kernel(monkeypatch):
def test_apply_failed_kernel_restores_native_when_polluted(monkeypatch):
# Requested kernel fails AND the global is polluted: restore native before returning.
monkeypatch.setattr(att, "_active_attention_backend", lambda: "_native_cudnn")
+ # The set itself must be what fails here, not the sageattention version floor in front of it.
+ monkeypatch.setattr(att, "_sage_version_too_old", lambda: None)
+ monkeypatch.setattr(att, "_install_sage_dispatch_guard", lambda: True)
class _FailOnceTransformer:
def __init__(self):
diff --git a/studio/backend/tests/test_diffusion_attention_sage_gate.py b/studio/backend/tests/test_diffusion_attention_sage_gate.py
new file mode 100644
index 00000000000..4abe5fbb802
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_attention_sage_gate.py
@@ -0,0 +1,610 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An explicit ``sage`` request never raises mid-generation: calls Sage cannot take run native, and the load-time check
+verifies the kernel's numbers at the DiT's head dims, not just that it launched."""
+
+from __future__ import annotations
+
+import sys
+import types
+
+import pytest
+
+import core.inference.diffusion_attention as att
+from core.inference.diffusion_attention import apply_attention_backend
+
+torch = pytest.importorskip("torch")
+dispatch = pytest.importorskip("diffusers.models.attention_dispatch")
+F = torch.nn.functional
+
+
+class _Transformer:
+ def __init__(self, head_dim = None):
+ self.calls: list = []
+ self.config = {"attention_head_dim": head_dim} if head_dim is not None else {}
+
+ def set_attention_backend(self, name):
+ self.calls.append(name)
+
+
+class _Logger:
+ def __init__(self):
+ self.warnings: list = []
+
+ def warning(self, msg, *args):
+ self.warnings.append(msg % args if args else msg)
+
+ def info(self, *_a, **_k):
+ pass
+
+
+def _target(dtype = "bf16", device = "cuda"):
+ return types.SimpleNamespace(device = device, dtype = dtype, torch_device = device)
+
+
[email protected](autouse = True)
+def _isolated(monkeypatch):
+ monkeypatch.setattr(att, "_SAGE_PROBE_CACHE", {})
+ monkeypatch.setattr(att, "_ensure_attention_backend_installed", lambda *a, **k: None)
+ monkeypatch.setattr(att, "_active_attention_backend", lambda: "native")
+ monkeypatch.setattr(att, "warn_if_sdpa_math_only", lambda *a, **k: False)
+ monkeypatch.setattr(att, "_indexed_cuda_device", lambda device: device)
+ monkeypatch.setattr(att, "_sage_version_too_old", lambda: None, raising = False)
+ backends = dispatch._AttentionBackendRegistry._backends
+ saved = backends[dispatch.AttentionBackendName.SAGE]
+ saved_fa4 = backends[dispatch.AttentionBackendName.FLASH_4_HUB]
+ monkeypatch.setattr(att, "_FA4_PROBE_CACHE", {}, raising = False)
+ yield
+ backends[dispatch.AttentionBackendName.SAGE] = saved
+ backends[dispatch.AttentionBackendName.FLASH_4_HUB] = saved_fa4
+
+
+def _qkv(
+ head_dim = 64,
+ dtype = None,
+ heads = 2,
+ tokens = 16,
+):
+ gen = torch.Generator().manual_seed(0)
+ return tuple(
+ torch.randn((1, tokens, heads, head_dim), generator = gen, dtype = dtype or torch.float32)
+ for _ in range(3)
+ )
+
+
+def _native(
+ q,
+ k,
+ v,
+ mask = None,
+):
+ return F.scaled_dot_product_attention(
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2), attn_mask = mask
+ ).transpose(1, 2)
+
+
+def _dispatch_sage(
+ q,
+ k,
+ v,
+ mask = None,
+):
+ return dispatch.dispatch_attention_fn(
+ q, k, v, attn_mask = mask, backend = dispatch.AttentionBackendName.SAGE
+ )
+
+
+def _engage_sage(monkeypatch, head_dim = 64):
+ monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: "")
+ t = _Transformer(head_dim)
+ engaged = apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", target = _target()
+ )
+ return engaged, t
+
+
+def test_masked_call_runs_native_instead_of_raising(monkeypatch):
+ engaged, t = _engage_sage(monkeypatch)
+ assert engaged == "sage" and t.calls == ["sage"]
+ q, k, v = _qkv()
+ mask = torch.ones((1, 1, 1, q.shape[1]), dtype = torch.bool)
+ mask[..., -3:] = False
+ out = _dispatch_sage(q, k, v, mask)
+ torch.testing.assert_close(out, _native(q, k, v, mask))
+
+
+ "kind, qkv",
+ [
+ ("head_dim", lambda: _qkv(head_dim = 256, dtype = torch.bfloat16)),
+ ("dtype", lambda: _qkv(head_dim = 64, dtype = torch.float32)),
+ ],
+)
+def test_unservable_shape_or_dtype_runs_native(monkeypatch, kind, qkv):
+ _engage_sage(monkeypatch)
+ q, k, v = qkv()
+ assert att._sage_reroute_reason(q, k, v, None) == kind
+ out = _dispatch_sage(q, k, v)
+ torch.testing.assert_close(out, _native(q, k, v))
+
+
+def test_servable_call_reaches_the_sage_kernel(monkeypatch):
+ backends = dispatch._AttentionBackendRegistry._backends
+ calls: list = []
+
+ def _fake_sage(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ is_causal = False,
+ scale = None,
+ return_lse = False,
+ _parallel_config = None,
+ ):
+ calls.append(tuple(query.shape))
+ return torch.full_like(query, 7.0)
+
+ backends[dispatch.AttentionBackendName.SAGE] = _fake_sage
+ assert att._install_sage_dispatch_guard() is True
+ q, k, v = _qkv(head_dim = 128, dtype = torch.bfloat16)
+ # Only the device reason fires on CPU ...
+ assert att._sage_reroute_reason(q, k, v, None) == "device"
+ torch.testing.assert_close(_dispatch_sage(q, k, v), _native(q, k, v))
+ assert calls == []
+
+ # ... and the same call on a CUDA tensor is the one Sage takes.
+ class _OnCuda(torch.Tensor):
+ @property
+ def device(self):
+ return types.SimpleNamespace(type = "cuda")
+
+ qc, kc, vc = (t.as_subclass(_OnCuda) for t in (q, k, v))
+ assert att._sage_reroute_reason(qc, kc, vc, None) is None
+ monkeypatch.setattr(att, "_sage_reroute_reason", lambda *a: None)
+ out = _dispatch_sage(q, k, v)
+ assert calls == [tuple(q.shape)] and bool((out == 7.0).all())
+
+
+def test_guard_is_installed_once(monkeypatch):
+ _engage_sage(monkeypatch)
+ first = dispatch._AttentionBackendRegistry._backends[dispatch.AttentionBackendName.SAGE]
+ assert att._install_sage_dispatch_guard() is True
+ assert dispatch._AttentionBackendRegistry._backends[dispatch.AttentionBackendName.SAGE] is first
+
+
+def test_rerouted_calls_are_counted_and_logged_once(monkeypatch, caplog):
+ monkeypatch.setattr(att, "_SAGE_ROUTED", {})
+ monkeypatch.setattr(att, "_SAGE_ROUTED_LOGGED", set())
+ _engage_sage(monkeypatch)
+ q, k, v = _qkv()
+ mask = torch.ones((1, 1, 1, q.shape[1]), dtype = torch.bool)
+ with caplog.at_level("WARNING"):
+ _dispatch_sage(q, k, v, mask)
+ _dispatch_sage(q, k, v, mask)
+ assert att._SAGE_ROUTED == {"attn_mask": 2}
+ assert (
+ sum("cannot take this attention call (attn_mask)" in r.getMessage() for r in caplog.records)
+ == 1
+ )
+
+
+def test_guarded_masked_call_traces_without_a_graph_break(monkeypatch):
+ _engage_sage(monkeypatch)
+ q, k, v = _qkv()
+ mask = torch.ones((1, 1, 1, q.shape[1]), dtype = torch.bool)
+ mask[..., :2] = False
+ # Not dispatch_attention_fn: Dynamo cannot trace diffusers' AttentionBackendName(...) lookup on torch 2.6 / 2.11.
+ guarded = dispatch._AttentionBackendRegistry._backends[dispatch.AttentionBackendName.SAGE]
+ torch._dynamo.reset()
+ compiled = torch.compile(
+ lambda q, k, v, m: guarded(query = q, key = k, value = v, attn_mask = m),
+ backend = "eager",
+ fullgraph = True,
+ )
+ torch.testing.assert_close(compiled(q, k, v, mask), _native(q, k, v, mask))
+ torch._dynamo.reset()
+
+
+def test_without_the_guard_sage_is_not_engaged(monkeypatch):
+ monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: "")
+ monkeypatch.setattr(att, "_install_sage_dispatch_guard", lambda: False)
+ t, log = _Transformer(64), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target()
+ )
+ is None
+ )
+ assert "sage" not in t.calls and any("masked attention" in w for w in log.warnings)
+
+
[email protected]("dtype", ["float32", "torch.float32", "fp32"])
+def test_float32_pipeline_never_engages_sage(monkeypatch, dtype):
+ seen: list = []
+ monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: seen.append(hd) or "")
+ t, log = _Transformer(128), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target(dtype)
+ )
+ is None
+ )
+ assert "sage" not in t.calls and seen == []
+ assert any("float32" in w for w in log.warnings)
+
+
+def test_head_dims_all_above_128_never_engage_sage(monkeypatch):
+ monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: "")
+ t, log = _Transformer(256), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target()
+ )
+ is None
+ )
+ assert "sage" not in t.calls and any("head_dim <= 128" in w for w in log.warnings)
+
+
+def test_probe_runs_at_the_dit_head_dims(monkeypatch):
+ seen: list = []
+ monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: seen.append(hd) or "")
+ t = _Transformer(64)
+ assert (
+ apply_attention_backend(types.SimpleNamespace(transformer = t), "sage", target = _target())
+ == "sage"
+ )
+ assert seen == [64]
+
+
+def test_missing_package_falls_back_with_a_reason_and_is_not_cached(monkeypatch):
+ calls: list = []
+
+ def _probe(
+ d,
+ dt,
+ hd = 128,
+ ):
+ calls.append(hd)
+ raise ImportError("No module named 'sageattention'")
+
+ monkeypatch.setattr(att, "_run_sage_probe", _probe)
+ t, log = _Transformer(128), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target()
+ )
+ is None
+ )
+ assert "sage" not in t.calls
+ assert any("could not be imported" in w for w in log.warnings)
+ assert att._SAGE_PROBE_CACHE == {}
+ assert att._sage_kernel_runs(_target()) is False and len(calls) == 2
+
+
+def _fake_sage(monkeypatch, fn):
+ monkeypatch.setitem(sys.modules, "sageattention", types.SimpleNamespace(sageattn = fn))
+
+
+def _exact_sageattn(
+ q,
+ k,
+ v,
+ tensor_layout = "NHD",
+):
+ assert tensor_layout == "NHD"
+ return _native(q.float(), k.float(), v.float()).to(q.dtype)
+
+
[email protected]("head_dim", [64, 128])
+def test_self_check_passes_an_accurate_kernel(monkeypatch, head_dim):
+ _fake_sage(monkeypatch, _exact_sageattn)
+ assert att._run_sage_probe("cpu", torch.bfloat16, head_dim) == ""
+
+
+ "wrong",
+ [
+ lambda q, k, v, tensor_layout = "NHD": torch.zeros_like(q),
+ lambda q, k, v, tensor_layout = "NHD": v,
+ lambda q, k, v, tensor_layout = "NHD": torch.full_like(q, float("nan")),
+ lambda q, k, v, tensor_layout = "NHD": q[:, :1],
+ ],
+)
+def test_self_check_rejects_a_kernel_that_runs_but_is_wrong(monkeypatch, wrong):
+ _fake_sage(monkeypatch, wrong)
+ error = att._run_sage_probe("cpu", torch.bfloat16, 128)
+ assert error.startswith("self-check")
+
+
+def test_self_check_failure_falls_back_at_load(monkeypatch):
+ _fake_sage(monkeypatch, lambda q, k, v, tensor_layout = "NHD": torch.zeros_like(q))
+ monkeypatch.setattr(att, "_indexed_cuda_device", lambda device: "cpu")
+ t, log = _Transformer(128), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target()
+ )
+ is None
+ )
+ assert "sage" not in t.calls and any("self-check" in w for w in log.warnings)
+
+
+def test_probe_head_dims_ignore_dims_sage_cannot_serve():
+ assert att._sage_probe_head_dims({64, 256}) == (64,)
+ assert att._sage_probe_head_dims(set()) == (128,)
+ assert att._sage_probe_head_dims({True, 96, 128}) == (96, 128)
+
+
+# Sage is CUDA-only and slower than AOTriton SDPA on gfx1151:
+# https://rocm.blogs.amd.com/software-tools-optimization/comfyui-fa-backends/README.html
+
+
+ "hip, version", [("7.2.1", "2.9.1+rocm7.2.1"), (None, "2.10.0a0+rocm7.10.0a20251116")]
+)
+def test_sage_is_never_selected_on_rocm(monkeypatch, hip, version):
+ from core.inference.diffusion_attention import select_attention_backend
+
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ monkeypatch.setattr(torch, "__version__", version)
+ rocm = types.SimpleNamespace(device = "cuda", dtype = torch.bfloat16)
+ for speed in (True, False):
+ assert select_attention_backend(rocm, "sage", speed_active = speed) is None
+ assert select_attention_backend(rocm, "auto", speed_active = speed) != "sage"
+
+
+def test_auto_never_selects_sage_on_nvidia(monkeypatch):
+ from core.inference.diffusion_attention import select_attention_backend
+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: True)
+ for cap in ((7, 5), (8, 0), (8, 9), (9, 0), (10, 0), (12, 0)):
+ monkeypatch.setattr(att, "_cuda_capability", lambda cap = cap: cap)
+ for speed in (True, False):
+ assert select_attention_backend(_target(), "auto", speed_active = speed) != "sage"
+
+
+def _arch_reading_sage(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ is_causal = False,
+ scale = None,
+ return_lse = False,
+ _parallel_config = None,
+):
+ torch._dynamo.graph_break() # stands in for sageattn's get_cuda_arch_versions(): Python Dynamo cannot trace
+ return _native(query.float(), key.float(), value.float()).to(query.dtype)
+
+
[email protected]("head_dim", [64, 128])
+def test_sage_call_compiles_fullgraph_through_the_guard(monkeypatch, head_dim):
+ backends = dispatch._AttentionBackendRegistry._backends
+ backends[dispatch.AttentionBackendName.SAGE] = _arch_reading_sage
+ assert att._install_sage_dispatch_guard() is True
+ monkeypatch.setattr(att, "_sage_reroute_reason", lambda *a: None)
+ guarded = backends[dispatch.AttentionBackendName.SAGE]
+ q, k, v = _qkv(head_dim = head_dim, dtype = torch.bfloat16)
+
+ def block(q, k, v):
+ return guarded(query = q * 1.0, key = k, value = v).sum(-1)
+
+ torch._dynamo.reset()
+ compiled = torch.compile(block, backend = "aot_eager", fullgraph = True)
+ torch.testing.assert_close(
+ compiled(q, k, v), _native(q.float(), k.float(), v.float()).to(q.dtype).sum(-1)
+ )
+ torch._dynamo.reset()
+
+
+def test_unguarded_sage_call_breaks_a_fullgraph_compile():
+ """Negative control: the same function called directly cannot be traced."""
+ q, k, v = _qkv(head_dim = 64, dtype = torch.bfloat16)
+ torch._dynamo.reset()
+ compiled = torch.compile(
+ lambda q, k, v: _arch_reading_sage(q, k, v), backend = "aot_eager", fullgraph = True
+ )
+ with pytest.raises(Exception):
+ compiled(q, k, v)
+ torch._dynamo.reset()
+
+
+def test_engaged_backend_is_tagged_on_every_dit(monkeypatch):
+ monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: "")
+ t, t2 = _Transformer(128), _Transformer(128)
+ pipe = types.SimpleNamespace(transformer = t, transformer_2 = t2)
+ assert apply_attention_backend(pipe, "sage", target = _target()) == "sage"
+ assert t._unsloth_attention_backend == "sage" and t2._unsloth_attention_backend == "sage"
+ # A later load on the same modules that falls back clears the tag.
+ monkeypatch.setattr(
+ att,
+ "_run_sage_probe",
+ lambda d, dt, hd = 128: "ValueError: Unsupported CUDA architecture: sm100",
+ )
+ monkeypatch.setattr(att, "_SAGE_PROBE_CACHE", {})
+ assert apply_attention_backend(pipe, "sage", target = _target()) is None
+ assert t._unsloth_attention_backend is None and t2._unsloth_attention_backend is None
+
+
+_REAL_SAGE_VERSION_TOO_OLD = getattr(att, "_sage_version_too_old", None)
+
+
+ "installed, refused", [("1.0.6", True), ("2.0.1", True), ("2.1.1", False), ("2.2.0", False)]
+)
+def test_sage_version_floor(monkeypatch, installed, refused):
+ import importlib.metadata as md
+
+ real = md.version
+ monkeypatch.setattr(
+ md, "version", lambda name: installed if name == "sageattention" else real(name)
+ )
+ monkeypatch.setattr(dispatch, "_REQUIRED_SAGE_VERSION", "2.1.1", raising = False)
+ assert callable(_REAL_SAGE_VERSION_TOO_OLD)
+ reason = _REAL_SAGE_VERSION_TOO_OLD()
+ assert bool(reason) is refused
+ if refused:
+ assert f"sageattention {installed} is older than 2.1.1" in reason
+
+
+def test_old_sageattention_falls_back_with_a_reason(monkeypatch):
+ seen: list = []
+ monkeypatch.setattr(att, "_run_sage_probe", lambda d, dt, hd = 128: seen.append(hd) or "")
+ monkeypatch.setattr(
+ att,
+ "_sage_version_too_old",
+ lambda: "sageattention 1.0.6 is older than 2.1.1, the "
+ "SageAttention 2 release diffusers needs",
+ )
+ t, log = _Transformer(128), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target()
+ )
+ is None
+ )
+ assert "sage" not in t.calls and seen == []
+ assert any("1.0.6 is older than 2.1.1" in w for w in log.warnings)
+
+
+def test_sm100_explicit_sage_falls_back_and_auto_never_picks_it(monkeypatch):
+ from core.inference.diffusion_attention import select_attention_backend
+
+ monkeypatch.setattr(att, "_is_cuda_nvidia", lambda target: True)
+ monkeypatch.setattr(att, "_cuda_capability", lambda: (10, 0))
+ for speed in (True, False):
+ assert select_attention_backend(_target(), "auto", speed_active = speed) == (
+ "_native_cudnn" if speed else None
+ )
+ # Every released SageAttention 2 raises on sm100; the load keeps the default backend and says why.
+ monkeypatch.setattr(
+ att,
+ "_run_sage_probe",
+ lambda d, dt, hd = 128: "ValueError: Unsupported CUDA architecture: sm100",
+ )
+ t, log = _Transformer(128), _Logger()
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "sage", logger = log, target = _target()
+ )
+ is None
+ )
+ assert any("sm100" in w for w in log.warnings)
+
+
+def _engage_fa4(monkeypatch, probe = ""):
+ monkeypatch.setattr(att, "_run_fa4_probe", lambda d, dt, hd = 128: probe)
+ t, log = _Transformer(128), _Logger()
+ engaged = apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "flash_4_hub", logger = log, target = _target()
+ )
+ return engaged, t, log
+
+
+def test_fa4_masked_call_runs_native_instead_of_raising(monkeypatch):
+ engaged, t, _ = _engage_fa4(monkeypatch)
+ assert engaged == "flash_4_hub" and t.calls == ["flash_4_hub"]
+ q, k, v = _qkv(dtype = torch.bfloat16)
+ mask = torch.ones((1, 1, 1, q.shape[1]), dtype = torch.bool)
+ mask[..., -3:] = False
+ out = dispatch.dispatch_attention_fn(
+ q, k, v, attn_mask = mask, backend = dispatch.AttentionBackendName.FLASH_4_HUB
+ )
+ torch.testing.assert_close(out, _native(q, k, v, mask))
+ assert att._fa4_reroute_reason(q, k, v, mask) == "attn_mask"
+
+
+ "error",
+ [
+ "ValueError: too many values to unpack (expected 2)",
+ "RuntimeError: CUDA error: no kernel image is available for execution on the device",
+ "self-check: cosine 0.10000, relative L1 0.9000 vs fp32 at head_dim 128",
+ ],
+)
+def test_fa4_failed_probe_unpins_and_logs(monkeypatch, error):
+ engaged, t, log = _engage_fa4(monkeypatch, error)
+ assert engaged is None
+ assert t.calls == ["flash_4_hub", "native"]
+ assert getattr(t, "_unsloth_attention_backend", "unset") is None
+ assert any("FlashAttention 4 does not run correctly" in w and error in w for w in log.warnings)
+
+
+def test_fa4_probe_runs_at_dit_head_dims_and_is_cached(monkeypatch):
+ seen: list = []
+ monkeypatch.setattr(att, "_run_fa4_probe", lambda d, dt, hd = 128: seen.append(hd) or "")
+ for _ in range(2):
+ t = _Transformer(64)
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "flash_4_hub", target = _target()
+ )
+ == "flash_4_hub"
+ )
+ assert seen == [64]
+
+
+def test_fa4_unaskable_probe_keeps_the_request(monkeypatch):
+ def _oom(
+ d,
+ dt,
+ hd = 128,
+ ):
+ raise RuntimeError("CUDA out of memory")
+
+ monkeypatch.setattr(att, "_run_fa4_probe", _oom)
+ t = _Transformer(128)
+ assert (
+ apply_attention_backend(
+ types.SimpleNamespace(transformer = t), "flash_4_hub", target = _target()
+ )
+ == "flash_4_hub"
+ )
+
+
+def test_fa4_real_probe_reports_an_api_mismatch(monkeypatch):
+ """The probe goes through diffusers' dispatch, so a kernel returning a shape the caller does not expect (the
+ torch 2.12.1 + flash-attn-4 4.0.0b33 class of break) is an answer, not a crash."""
+
+ def _five_values(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ scale = None,
+ is_causal = False,
+ return_lse = False,
+ _parallel_config = None,
+ ):
+ a, b = (query, key, value, None, None) # noqa: F841 - unpacks 5 into 2, as torch's FA4 hook does
+ return a
+
+ dispatch._AttentionBackendRegistry._backends[dispatch.AttentionBackendName.FLASH_4_HUB] = (
+ _five_values
+ )
+ monkeypatch.setattr(
+ dispatch, "_check_attention_backend_requirements", lambda *a, **k: None, raising = False
+ )
+ error = att._run_fa4_probe("cpu", torch.bfloat16, 64)
+ assert error.startswith("ValueError") and "unpack" in error
+
+
+def test_fa4_real_probe_passes_an_exact_kernel(monkeypatch):
+ def _exact(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ scale = None,
+ is_causal = False,
+ return_lse = False,
+ _parallel_config = None,
+ ):
+ return _native(query.float(), key.float(), value.float()).to(query.dtype)
+
+ dispatch._AttentionBackendRegistry._backends[dispatch.AttentionBackendName.FLASH_4_HUB] = _exact
+ monkeypatch.setattr(
+ dispatch, "_check_attention_backend_requirements", lambda *a, **k: None, raising = False
+ )
+ assert att._run_fa4_probe("cpu", torch.bfloat16, 128) == ""
diff --git a/studio/backend/tests/test_diffusion_attention_sage_probe.py b/studio/backend/tests/test_diffusion_attention_sage_probe.py
index 0212113adb1..6f5ab38316b 100644
--- a/studio/backend/tests/test_diffusion_attention_sage_probe.py
+++ b/studio/backend/tests/test_diffusion_attention_sage_probe.py
@@ -49,6 +49,10 @@ def _isolated(monkeypatch):
monkeypatch.setattr(att, "_active_attention_backend", lambda: "native")
monkeypatch.setattr(att, "warn_if_sdpa_math_only", lambda *a, **k: False)
monkeypatch.setattr(att, "_indexed_cuda_device", lambda device: device)
+ monkeypatch.setattr(att, "_install_sage_dispatch_guard", lambda: True)
+ monkeypatch.setattr(att, "_sage_version_too_old", lambda: None)
+ monkeypatch.setattr(att, "_install_fa4_dispatch_guard", lambda: True)
+ monkeypatch.setattr(att, "_fa4_kernel_runs", lambda *a, **k: True)
def _stub_probe(
@@ -56,7 +60,11 @@ def _stub_probe(
result,
seen = None,
):
- def _probe(device, dtype):
+ def _probe(
+ device,
+ dtype,
+ head_dim = 128,
+ ):
if seen is not None:
seen.append((device, dtype))
if isinstance(result, BaseException):
@@ -89,12 +97,9 @@ def test_sage_stays_engaged_where_its_kernel_runs(monkeypatch):
assert t.calls == ["sage"]
- "exc", [ImportError("no sageattention"), RuntimeError("CUDA out of memory")]
-)
-def test_an_unanswerable_probe_keeps_the_request(monkeypatch, exc):
+def test_an_unanswerable_probe_keeps_the_request(monkeypatch):
seen: list = []
- _stub_probe(monkeypatch, exc, seen)
+ _stub_probe(monkeypatch, RuntimeError("CUDA out of memory"), seen)
t = _Transformer()
assert (
apply_attention_backend(types.SimpleNamespace(transformer = t), "sage", target = _target())
@@ -172,7 +177,7 @@ def test_bare_cuda_is_keyed_by_the_card_pinned_on_this_thread(monkeypatch):
monkeypatch.setattr(
att,
"_run_sage_probe",
- lambda d, dt: seen.append(d) or ("" if d == "cuda:1" else _UNSUPPORTED),
+ lambda d, dt, hd = 128: seen.append(d) or ("" if d == "cuda:1" else _UNSUPPORTED),
)
bare = types.SimpleNamespace(device = "cuda", dtype = "bf16")
assert att._sage_kernel_runs(bare) is False
diff --git a/studio/backend/tests/test_diffusion_cuda_graph.py b/studio/backend/tests/test_diffusion_cuda_graph.py
index 386fa8fb78a..65e0302bdd1 100644
--- a/studio/backend/tests/test_diffusion_cuda_graph.py
+++ b/studio/backend/tests/test_diffusion_cuda_graph.py
@@ -1134,3 +1134,18 @@ def test_real_cuda_compiled_module_capture_matches_the_compiled_call():
finally:
cg.uninstall_all(handles)
assert unet._compiled_call_impl is compiled
+
+
+def test_graph_eligible_declines_a_sage_denoiser(stub_torch, monkeypatch):
+ """SageAttention under a replayed graph renders noise (FLUX.1-schnell, A100), so a Sage load stays ungraphed."""
+ dit = _FakeDiT()
+ dit._unsloth_attention_backend = "sage"
+ ok, reason = _eligible(monkeypatch, pipe = types.SimpleNamespace(transformer = dit))
+ assert (ok, reason) == (False, "SageAttention is not CUDA-graph safe")
+
+
[email protected]("backend", [None, "_native_cudnn", "flash", "native"])
+def test_graph_eligible_keeps_other_backends(stub_torch, monkeypatch, backend):
+ dit = _FakeDiT()
+ dit._unsloth_attention_backend = backend
+ assert _eligible(monkeypatch, pipe = types.SimpleNamespace(transformer = dit)) == (True, "eligible")