Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: pick ROCm flash attention by a whole-block timing probe instead of a model table (#13792)

Daniel Han3c14a2b3359b
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index c8c35d6f365..7d1c35c10a7 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -250,10 +250,14 @@ from .diffusion_vae_fp16 import enable_fp16_vae_decode
from .diffusion_vae_tiling import install as install_wide_vae_tiles
from .diffusion_attention import (
apply_attention_backend,
+ attention_status_reason,
auto_attention_reason,
normalize_attention_backend,
sdpa_math_only,
sdpa_subquadratic_confirmed,
+ attention_may_run_native,
+ begin_attention_run,
+ report_attention_run,
select_attention_backend,
_ensure_attention_backend_installed,
)
@@ -2295,9 +2299,14 @@ def _quadratic_attention(
pipe: Any = None,
) -> bool:
"""Whether attention can only run on SDPA math (quadratic memory); False when unknown.
- Any engaged non-native backend is a fused kernel, so the SDPA probe only speaks for native."""
+ An engaged non-native backend is a fused kernel, so the SDPA probe only speaks for native, unless that backend can
+ hand calls to native SDPA (ROCm flash: masked calls, lengths it does not serve): then budget as native."""
if engaged_backend is not None and str(engaged_backend) != "native":
- return False
+ try:
+ if pipe is None or not attention_may_run_native(pipe):
+ return False
+ except Exception: # noqa: BLE001
+ return False
try:
if not sdpa_math_only(target):
return False
@@ -9588,11 +9597,7 @@ class DiffusionBackend:
"attention_backend": (
attention_backend,
attention_engaged or "native",
- auto_attention_reason(attention_engaged)
- if attention_engaged and attention_backend is None
- else "diffusers default"
- if attention_engaged is None
- else "requested",
+ attention_status_reason(attention_backend, attention_engaged),
),
"memory_mode": (
memory_mode,
@@ -11924,6 +11929,7 @@ class DiffusionBackend:
# Publish an active (step 0) state before the slow pre-denoise setup so a reload mount probe does not
# read idle.
self._gen = _GenState(total_steps = steps)
+ begin_attention_run(state)
# Reset in the finally, so a failed or cancelled generation frees its reused outputs.
static_skip_pipe = None
restore_scheduler: Optional[tuple[Any, Any]] = None
@@ -12759,6 +12765,9 @@ class DiffusionBackend:
compile_cache.save_async(state.compile_cache_ctx, logger = logger)
except Exception: # noqa: BLE001 - cache persistence is best-effort
pass
+ # The status badge reports the attention routes this run took (report only: the recipe keeps the
+ # engaged backend).
+ report_attention_run(state)
# Last word on cancellation, AFTER the post-denoise work: the event stays registered through the
# compile-cache bookkeeping and the page still shows Stop for as long as progress reads active, so a Stop
# landing there was answered cancelled = true and then contradicted by the image the route persisted.
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 33d84bc5b54..ffc128b8850 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -24,7 +24,9 @@ Attention is bandwidth-bound, so a better kernel is a real win orthogonal to wei
sage - SageAttention 2 (INT8 QK); quantized, small quality cost. A pip SageAttention 2 when installed, else the
kernels-community build from the Hugging Face kernels hub (``sage_hub``, Ampere / Ada / Hopper).
xformers / aiter - memory-efficient (NVIDIA) / AITER (AMD ROCm).
- flash on ROCm - only when the DAO-AILab ROCm build imports and passes an on-card check; ``auto`` stays native.
+ flash on ROCm - only when the DAO-AILab ROCm build passes an on-card check; ``auto`` also needs a block timing win
+ (``UNSLOTH_DIFFUSION_ROCM_FLASH``: auto | 0 | 1). The timing persists per toolchain and geometry under
+ the Studio cache (``UNSLOTH_DIFFUSION_ROCM_FLASH_CACHE_DIR``), so a restart reads it instead of re-timing.
Best-effort: an unavailable backend falls back to the diffusers default. torch/diffusers lazy.
"""
@@ -445,8 +447,8 @@ def select_attention_backend(
An explicit alias is honored (apply falls back if its kernel is unavailable). ``auto``
upgrades to cuDNN on NVIDIA CUDA only when a speed profile is active (so ``off`` stays
- bit-identical), and to verified ROCm flash for ``ROCM_AUTO_FLASH_FAMILIES`` on gfx11 unless speed is
- explicitly ``off``; elsewhere returns None (native)."""
+ bit-identical), and on ROCm with verified flash_attn to ``ROCM_FLASH_CANDIDATE`` (resolved by timing in
+ ``apply_attention_backend``) unless speed is explicitly ``off``; elsewhere returns None (native)."""
alias = normalize_attention_backend(requested)
if alias != ATTN_AUTO:
backend = _ALIASES[alias]
@@ -472,10 +474,8 @@ def select_attention_backend(
return backend
if speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():
return "_native_cudnn"
- if speed_active and ROCM_AUTO_FLASH and _is_cuda_rocm(target) and _rocm_flash_attn_runs(target):
- return "flash"
if (speed_active or speed_unset) and _rocm_auto_flash(target, family):
- return "flash"
+ return ROCM_FLASH_CANDIDATE
return None
@@ -490,16 +490,51 @@ def auto_attention_reason(engaged: Optional[str]) -> str:
return f"{engaged} attention upgrade"
+def attention_status_reason(requested: Optional[str], engaged: Optional[str]) -> str:
+ """Status reason for the load's attention pick: the auto reason for None / "auto", else "requested"."""
+ try:
+ is_auto = normalize_attention_backend(requested) == ATTN_AUTO
+ except ValueError:
+ is_auto = False
+ if is_auto or not engaged:
+ return auto_attention_reason(engaged)
+ return "requested"
+
+
def _is_cuda_rocm(target: Any) -> bool:
return getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target)
-# ``auto`` -> verified ROCm flash under a speed profile. Off: gfx1151 gains over AOTriton SDPA were too small/narrow.
-ROCM_AUTO_FLASH = False
-# Also with speed unset (the UI default). gfx1151 klein 1024^2: 7.37-7.40 s vs SDPA 8.04-8.09 s, LPIPS 0.008.
-# FLUX.1 measured LPIPS 0.048 vs SDPA (over the 0.02 bar): stays native.
-ROCM_AUTO_FLASH_FAMILIES = frozenset({"flux.2-klein"})
-ROCM_AUTO_FLASH_ARCHES = ("gfx11",)
+ROCM_FLASH_CANDIDATE = "_rocm_flash_candidate"
+ROCM_FLASH_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH" # auto (timed) | 0 (auto never picks flash) | 1 (verified is enough)
+ROCM_FLASH_MIN_GAIN = 0.03
+# Accuracy is judged on the raw attention output (flash vs native SDPA, same inputs), not the block: the residual and
+# FFN would dilute a wrong attention. Rel L1 over the output, and the largest abs error over the largest abs output.
+ROCM_FLASH_BLOCK_TOL = 2e-2 # attention output rel L1
+ROCM_FLASH_MAX_TOL = 5e-2 # attention max abs error / max abs output
+ROCM_FLASH_PROBE_TOKENS = 4096 # the load-time bucket: a 1024^2 image DiT sequence
+# Calls are gated per (query, key) length bucket pair (powers of two from 256 up, open ended): a pair is timed at its
+# lower edges on first use, so the 4096 verdict never decides a 30k-token video call, and a long-video x short-text
+# cross-attention is timed as that rectangle, not as square self-attention.
+ROCM_FLASH_MIN_BUCKET = 256
+ROCM_FLASH_LONG_PROBE = (
+ 16384 # from here the probe times fewer runs (one native SDPA call is seconds at 64k tokens)
+)
+ROCM_FLASH_ADMISSIONS: dict[tuple, dict] = {}
+# Bump when the block probe changes what it measures: every stored verdict is then re-timed.
+# 2: the flash arm runs through diffusers' dispatch (the generation path), not flash_attn_func directly.
+# 3: accuracy from the raw attention output (rel L1 + max_rel), not the block output; verdicts per length bucket.
+ROCM_FLASH_PROBE_VERSION = 3
+ROCM_FLASH_CACHE_DIR_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH_CACHE_DIR"
+
+
+def _rocm_flash_mode() -> str:
+ raw = os.environ.get(ROCM_FLASH_ENV, "auto").strip().lower()
+ if raw in ("0", "false", "no", "off", "never"):
+ return "0"
+ if raw in ("1", "true", "yes", "on", "always"):
+ return "1"
+ return "auto"
def _rocm_gfx_arch(target: Any) -> str:
@@ -531,15 +566,889 @@ def _flash_attn_installed() -> bool:
return False
-def _rocm_auto_flash(target: Any, family: Any) -> bool:
- if not _is_cuda_rocm(target):
+def _rocm_auto_flash(target: Any, family: Any = None) -> bool:
+ if not _is_cuda_rocm(target) or _rocm_flash_mode() == "0":
return False
- name = family if isinstance(family, str) else getattr(family, "name", None)
- if name not in ROCM_AUTO_FLASH_FAMILIES:
+ return _flash_attn_installed() and _rocm_flash_attn_runs(target)
+
+
+def _dit_attention_geometries(pipe: Any) -> tuple[tuple[int, ...], ...]:
+ """Every distinct (hidden, heads, head_dim[, context width]) among the attention modules of every denoiser DiT, read off ``to_q``
+ + ``heads`` (or ``num_heads``); flash is set on all of them, so all of them are timed."""
+ found: set[tuple[int, int, int]] = set()
+ for dit in _attention_dits(pipe):
+ modules = getattr(dit, "modules", None)
+ if not callable(modules):
+ continue
+ try:
+ for module in modules():
+ heads = getattr(module, "heads", None)
+ if not isinstance(heads, int) or isinstance(heads, bool):
+ heads = getattr(module, "num_heads", None)
+ to_q = getattr(module, "to_q", None)
+ fin = getattr(to_q, "in_features", None)
+ fout = getattr(to_q, "out_features", None)
+ if to_q is None:
+ # Fused self-attention (MiniMax-H3 resident INT8 deletes to_q / to_k / to_v): Q is a third of to_qkv.
+ to_qkv = getattr(module, "to_qkv", None)
+ fin = getattr(to_qkv, "in_features", None)
+ qkv_out = getattr(to_qkv, "out_features", None)
+ fout = (
+ qkv_out // 3
+ if isinstance(qkv_out, int) and qkv_out > 0 and qkv_out % 3 == 0
+ else None
+ )
+ if not all(
+ isinstance(v, int) and not isinstance(v, bool) and v > 0
+ for v in (heads, fin, fout)
+ ):
+ continue
+ if fout % heads:
+ continue
+ # Cross-attention reads K / V from a context of another width (to_k.in_features): a 4th element.
+ kin = getattr(getattr(module, "to_k", None), "in_features", None)
+ if isinstance(kin, int) and not isinstance(kin, bool) and kin > 0 and kin != fin:
+ found.add((fin, heads, fout // heads, kin))
+ else:
+ found.add((fin, heads, fout // heads))
+ except Exception: # noqa: BLE001
+ continue
+ return tuple(sorted(found))
+
+
+def _time_rocm_flash_block(
+ device: str,
+ dtype: Any,
+ hidden: int,
+ heads: int,
+ head_dim: int,
+ tokens: int = ROCM_FLASH_PROBE_TOKENS,
+ kv_tokens: Optional[int] = None,
+ ctx_dim: Optional[int] = None,
+ batch: int = 1,
+) -> dict:
+ """``kv_tokens`` / ``ctx_dim`` (default ``tokens`` / ``hidden``): a cross-attention block, queries from the image
+ stream and keys / values from a ``kv_tokens``-long context ``ctx_dim`` wide. ``batch`` samples per call. Median ms of one synthetic DiT block with native SDPA vs flash_attn (timing only), plus the error of the raw
+ flash attention output against native SDPA on the same q / k / v (accuracy).
+
+ Both arms go through diffusers' ``dispatch_attention_fn`` exactly as generation does (flash behind Studio's mask
+ guard), so a flash_attn build whose ``flash_attn_func`` rejects the arguments diffusers passes fails here."""
+ import time
+
+ import torch
+ import torch.nn.functional as F
+ from diffusers.models.attention_dispatch import AttentionBackendName, dispatch_attention_fn
+
+ if not _install_rocm_flash_dispatch_guard():
+ raise RuntimeError("diffusers flash backend registry not found")
+ if dtype not in (torch.float16, torch.bfloat16):
+ dtype = torch.bfloat16
+ inner = heads * head_dim
+ # Drawn on the device in the run dtype: a long bucket staged in host float32 would need GiBs of unbudgeted RAM.
+ gen = torch.Generator(device = device).manual_seed(0)
+
+ def rand(*shape):
+ return torch.randn(shape, generator = gen, device = device, dtype = dtype)
+
+ def w(rows: int, cols: int):
+ return rand(rows, cols) * cols**-0.5
+
+ kv = tokens if kv_tokens is None else int(kv_tokens)
+ width = hidden if ctx_dim is None else int(ctx_dim)
+ b = max(int(batch), 1)
+ w_qkv, w_out, w_fc1, w_fc2 = (
+ w(3 * inner, hidden),
+ w(hidden, inner),
+ w(4 * hidden, hidden),
+ w(hidden, 4 * hidden),
+ )
+ x = rand(b, tokens, hidden)
+ cross = kv != tokens or width != hidden
+ ctx = rand(b, kv, width) if cross else None
+ w_kv = w(2 * inner, width) if cross else None
+
+ def project():
+ xn = F.layer_norm(x, (hidden,))
+ if ctx is None:
+ h = F.linear(xn, w_qkv).view(b, tokens, 3, heads, head_dim)
+ return h[:, :, 0], h[:, :, 1], h[:, :, 2]
+ q = F.linear(xn, w_qkv[:inner]).view(b, tokens, heads, head_dim)
+ h = F.linear(ctx, w_kv).view(b, kv, 2, heads, head_dim)
+ return q, h[:, :, 0], h[:, :, 1]
+
+ def via(backend):
+ def attn(q, k, v):
+ out = dispatch_attention_fn(
+ q, k, v, attn_mask = None, dropout_p = 0.0, is_causal = False, backend = backend
+ )
+ return out[0] if isinstance(out, tuple) else out
+
+ return attn
+
+ native = via(AttentionBackendName.NATIVE)
+
+ def block(attn):
+ with torch.no_grad():
+ a = attn(*project()).reshape(b, tokens, inner)
+ x2 = x + F.linear(a, w_out)
+ return x2 + F.linear(
+ F.gelu(F.linear(F.layer_norm(x2, (hidden,)), w_fc1), approximate = "tanh"), w_fc2
+ )
+
+ arms = {"native": native, "flash": via(AttentionBackendName.FLASH)}
+ with torch.no_grad():
+ qkv = project()
+ ref = native(*qkv).float()
+ got = arms["flash"](*qkv).float()
+ del qkv
+ if tuple(got.shape) != tuple(ref.shape):
+ raise RuntimeError(f"flash output shape {tuple(got.shape)} != {tuple(ref.shape)}")
+ finite = bool(torch.isfinite(got).all())
+ err = (got - ref).abs()
+ rel = float(err.mean() / ref.abs().mean().clamp_min(1e-12))
+ max_rel = float(err.max() / ref.abs().max().clamp_min(1e-12))
+ del ref, got, err
+ # diagnostic only: how much of the attention error survives the residual + FFN
+ block_rel = None
+ if tokens < ROCM_FLASH_LONG_PROBE:
+ bref = block(native).float()
+ bgot = block(arms["flash"]).float()
+ block_rel = float((bgot - bref).abs().mean() / bref.abs().mean().clamp_min(1e-12))
+ del bref, bgot
+ times: dict[str, list] = {name: [] for name in arms}
+ long = tokens >= ROCM_FLASH_LONG_PROBE
+ for _ in range(1 if long else 2):
+ for fn in arms.values():
+ block(fn)
+ torch.cuda.synchronize(device)
+ for _ in range(3 if long else 5):
+ for name, fn in arms.items():
+ torch.cuda.synchronize(device)
+ t0 = time.perf_counter()
+ block(fn)
+ torch.cuda.synchronize(device)
+ times[name].append((time.perf_counter() - t0) * 1000.0)
+ med = {name: sorted(v)[len(v) // 2] for name, v in times.items()}
+ return {
+ "native_ms": round(med["native"], 3),
+ "flash_ms": round(med["flash"], 3),
+ "rel_l1": rel,
+ "max_rel": max_rel,
+ "block_rel_l1": block_rel,
+ "finite": finite,
+ "tokens": tokens,
+ **({} if kv == tokens else {"kv_tokens": kv}),
+ **({} if b == 1 else {"batch": b}),
+ }
+
+
+def _rocm_flash_block_admitted(
+ pipe: Any,
+ target: Any,
+ logger: Any = None,
+) -> bool:
+ """Resolve ``ROCM_FLASH_CANDIDATE``: flash only if it runs and (mode auto) wins the block timing with matching output."""
+ if target is None or not _rocm_auto_flash(target):
return False
- if not _rocm_gfx_arch(target).startswith(ROCM_AUTO_FLASH_ARCHES):
+ mode = _rocm_flash_mode()
+ if mode == "1":
+ return _install_rocm_flash_dispatch_guard()
+ geometries = _dit_attention_geometries(pipe)
+ if not geometries:
return False
- return _flash_attn_installed() and _rocm_flash_attn_runs(target)
+ device = _indexed_cuda_device(
+ str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")
+ )
+ dtype = getattr(target, "dtype", None)
+ for geometry in geometries:
+ if not _rocm_flash_geometry_admitted(device, dtype, geometry, logger):
+ return False
+ # diffusers' flash backend raises on attn_mask: masked calls (e.g. mixed-padding batches) must run native
+ if not _install_rocm_flash_dispatch_guard():
+ return False
+ gate: dict[tuple, frozenset] = {}
+ for geometry in geometries:
+ shape = (device, str(dtype), geometry[1], geometry[2])
+ gate[shape] = gate.get(shape, frozenset()) | {tuple(geometry)}
+ for dit in _attention_dits(pipe):
+ try:
+ setattr(dit, ROCM_FLASH_GATE_ATTR, gate)
+ except Exception: # noqa: BLE001
+ pass
+ return True
+
+
+# Length gating belongs to the DiT an auto load admitted, not to a shape: the gate {(device, dtype, heads, head_dim):
+# hidden sizes} lives on that DiT and is active only while it runs (its forward hooks), so a model loaded with explicit
+# flash or UNSLOTH_DIFFUSION_ROCM_FLASH=1 is never gated, whichever loaded first.
+ROCM_FLASH_GATE_ATTR = "_unsloth_rocm_flash_gate"
+# Per thread (concurrent image and video forwards never see each other's): ``gate`` and ``tally`` of the DiT forward
+# running now, and ``probing`` while a timing probe runs. Dynamo traces threading.local reads.
+_ROCM_FLASH_LOCAL = threading.local()
+# (device, dtype, *geometry, query bucket, key bucket, batch bucket) -> flash serves it, filled on an eager call's
+# first use (the probe or a stored verdict). In memory: a bucket the probe rejected stays native.
+_ROCM_FLASH_BUCKETS: dict[tuple, bool] = {}
+# On each hooked DiT: bucket keys whose probe could not run (no room beside the live forward, or it raised), native
+# until that DiT's next generation retries them. Never a verdict, so a transient shortfall cannot disable a working
+# kernel; per DiT, so another pipeline's generation boundary does not clear it.
+ROCM_FLASH_DEFERRED_ATTR = "_unsloth_rocm_flash_deferred"
+_ROCM_FLASH_GATE_LOCK = threading.Lock()
+# The native SDPA baseline the in-memory verdicts were timed against; when it moves (a later load re-enables or disables
+# the fused SDPA kernels), both in-memory caches are dropped and the disk cache, keyed on it, decides again.
+_ROCM_FLASH_BASELINE: list = [None]
+
+
+def _sync_rocm_flash_baseline() -> None:
+ try:
+ current = _native_sdpa_baseline()
+ except Exception: # noqa: BLE001
+ current = None
+ if current != _ROCM_FLASH_BASELINE[0]:
+ ROCM_FLASH_ADMISSIONS.clear()
+ _ROCM_FLASH_BUCKETS.clear()
+ _ROCM_FLASH_BASELINE[0] = current
+
+
+def _rocm_flash_bucket(tokens: Any) -> int:
+ """Largest power of two <= tokens, at least ROCM_FLASH_MIN_BUCKET, open ended."""
+ bucket = ROCM_FLASH_MIN_BUCKET
+ while tokens >= bucket * 2:
+ bucket *= 2
+ return bucket
+
+
+def _rocm_flash_batch_bucket(batch: int) -> int:
+ """Largest power of two <= batch (1, 2, 4, ...), open ended."""
+ bucket = 1
+ while batch >= bucket * 2:
+ bucket *= 2
+ return bucket
+
+
+def _active_rocm_flash_gate() -> Optional[dict]:
+ return getattr(_ROCM_FLASH_LOCAL, "gate", None)
+
+
+def _rocm_flash_capturing() -> bool:
+ """A CUDA / HIP graph capture is under way (replays cannot re-route a call)."""
+ try:
+ import torch
+ return bool(torch.cuda.is_current_stream_capturing())
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _rocm_flash_length_reason(query: Any, key: Any) -> Optional[str]:
+ """None when flash serves this call, else why not. Only calls made inside a DiT an auto load admitted are gated
+ (explicit flash and ``UNSLOTH_DIFFUSION_ROCM_FLASH=1`` are not). A call is decided by its (batch, query length, key
+ length) bucket, timed on first use. A traced (torch.compile) call is decided once, at trace time, by
+ ``_rocm_flash_traced_reason``: its answer is baked into the graph, and the bucket edges compare symbolically, so a
+ dynamic length guards on its bucket range, not on its value (a new bucket recompiles, a new length inside one
+ does not). A graph capture runs native."""
+ gate = _active_rocm_flash_gate()
+ if not gate:
+ return None
+ traced = _compiling()
+ if not traced and _rocm_flash_capturing():
+ return "compiled or captured"
+ if getattr(_ROCM_FLASH_LOCAL, "probing", False):
+ return None
+ try:
+ batch, seq, heads, head_dim = query.shape
+ kv = key.shape[1]
+ raw_device, dtype_name = str(query.device), str(query.dtype)
+ except Exception: # noqa: BLE001 - not an NHD tensor call: leave it to the kernel as before
+ return None
+ if traced:
+ # Nothing here may graph-break: the device index, the verdict tables and any probe are read in plain Python
+ # by the constant-marked helper, called with constants only (the gate's own key, never a traced size).
+ match = None
+ for shape, found in gate.items():
+ if shape[1] == dtype_name and heads == shape[2] and head_dim == shape[3]:
+ match = (shape, found)
+ break
+ if match is None:
+ return None
+ return _rocm_flash_traced_reason(
+ raw_device,
+ dtype_name,
+ match[0][2],
+ match[0][3],
+ tuple(sorted(match[1])),
+ _rocm_flash_bucket(seq),
+ _rocm_flash_bucket(kv),
+ _rocm_flash_batch_bucket(batch),
+ )
+ device = _indexed_cuda_device(raw_device)
+ try:
+ geometries = gate.get((device, dtype_name, int(heads), int(head_dim)))
+ except Exception: # noqa: BLE001
+ return None
+ if not geometries:
+ return None
+ return _rocm_flash_bucket_reason(
+ device,
+ query.dtype,
+ tuple(sorted(geometries)),
+ _rocm_flash_bucket(int(seq)),
+ _rocm_flash_bucket(int(kv)),
+ _rocm_flash_batch_bucket(int(batch)),
+ )
+
+
+def _rocm_flash_traced_reason(
+ raw_device: str,
+ dtype_name: str,
+ heads: int,
+ head_dim: int,
+ geometries: tuple,
+ q_bucket: int,
+ kv_bucket: int,
+ b_bucket: int,
+) -> Optional[str]:
+ """The traced call's route, run in plain Python while Dynamo traces (``torch.compiler.assume_constant_result``,
+ marked when the guard installs): the same verdict as an eager call of this bucket, timing it now if unknown (no
+ room beside the live forward -> native for this graph). Never raises: anything odd runs native."""
+ try:
+ import torch
+
+ device = _indexed_cuda_device(raw_device)
+ gate = _active_rocm_flash_gate()
+ if gate and (device, dtype_name, heads, head_dim) not in gate:
+ return "compiled or captured"
+ dtype = getattr(torch, dtype_name.rpartition(".")[2], None)
+ if not isinstance(dtype, torch.dtype):
+ return "compiled or captured"
+ return _rocm_flash_bucket_reason(device, dtype, geometries, q_bucket, kv_bucket, b_bucket)
+ except Exception: # noqa: BLE001
+ return "compiled or captured"
+
+
+def _rocm_flash_bucket_reason(
+ device: str, dtype: Any, geometries: tuple, q_bucket: int, kv_bucket: int, b_bucket: int
+) -> Optional[str]:
+ _sync_rocm_flash_baseline()
+ keys = [(device, str(dtype), *g, q_bucket, kv_bucket, b_bucket) for g in geometries]
+ if all(_ROCM_FLASH_BUCKETS.get(k) is True for k in keys):
+ return None
+ if any(_ROCM_FLASH_BUCKETS.get(k) is False for k in keys):
+ return "slower or inexact at this size"
+ deferred = getattr(_ROCM_FLASH_LOCAL, "deferred", None)
+ if not isinstance(deferred, set):
+ deferred = set() # no owning DiT: nothing to remember the deferral on
+ if any(k in deferred for k in keys):
+ return "timing deferred"
+ with _ROCM_FLASH_GATE_LOCK:
+ verdict = True
+ for k, g in zip(keys, geometries):
+ ok = _ROCM_FLASH_BUCKETS.get(k)
+ if ok is None:
+ ok = _rocm_flash_geometry_admitted(
+ device,
+ dtype,
+ g,
+ tokens = q_bucket,
+ batch = b_bucket,
+ live = True,
+ **({} if kv_bucket == q_bucket else {"kv_tokens": kv_bucket}),
+ )
+ if ok is None:
+ deferred.add(k)
+ return "timing deferred"
+ _ROCM_FLASH_BUCKETS[k] = ok
+ verdict = verdict and ok
+ return None if verdict else "slower or inexact at this size"
+
+
+def _native_sdpa_fused_serves(
+ device: Any, dtype: Any, heads: int, head_dim: int, tokens: int, kv_tokens: int, batch: int
+) -> bool:
+ """Whether native SDPA takes an enabled fused (flash / memory-efficient) kernel at exactly this shape, asked of
+ torch's own dispatcher with zero-size-storage views; otherwise (or when it cannot say) it runs the math backend."""
+ try:
+ import torch
+ from torch.backends.cuda import (
+ SDPAParams,
+ can_use_efficient_attention,
+ can_use_flash_attention,
+ )
+
+ cuda = torch.backends.cuda
+
+ def view(n):
+ return torch.empty((1, 1, 1, head_dim), device = device, dtype = dtype).expand(
+ batch, heads, n, head_dim
+ )
+
+ q, k = view(tokens), view(kv_tokens)
+ try:
+ params = SDPAParams(q, k, k, None, 0.0, False, False)
+ except TypeError: # torch before enable_gqa
+ params = SDPAParams(q, k, k, None, 0.0, False)
+ return bool(
+ (cuda.flash_sdp_enabled() and can_use_flash_attention(params, False))
+ or (cuda.mem_efficient_sdp_enabled() and can_use_efficient_attention(params, False))
+ )
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _rocm_flash_probe_bytes(
+ geometry: tuple,
+ tokens: int,
+ kv_tokens: int,
+ batch: int,
+ dtype: Any,
+ device: Any = "cuda",
+) -> int:
+ """Upper bound on what ``_time_rocm_flash_block`` allocates: its weights, the block's widest activations and the
+ fp32 accuracy outputs (measured on gfx1151: 5.46 GiB at Wan's 64k tokens, this gives 8.1 GiB). Unless a fused SDPA
+ kernel is confirmed for this exact shape (``guard_rocm_fused_sdpa`` may have turned them off, or one may not serve
+ this length / head size), the native arm runs math SDPA, which also holds fp32 scores and their softmax, batch x
+ heads x queries x keys each."""
+ hidden, heads, head_dim = (int(v) for v in geometry[:3])
+ width = int(geometry[3]) if len(geometry) > 3 else hidden
+ inner = heads * head_dim
+ elem = getattr(dtype, "itemsize", 2) or 2
+ weights = (4 * inner * hidden + 8 * hidden * hidden + 2 * inner * width) * elem
+ per_query = (11 * hidden + 3 * inner) * elem + 2 * inner * 4
+ per_key = (width + 2 * inner) * elem
+ fused = _native_sdpa_fused_serves(device, dtype, heads, head_dim, tokens, kv_tokens, batch)
+ scores = 0 if fused else 2 * batch * heads * tokens * kv_tokens * 4
+ return weights + batch * (tokens * per_query + kv_tokens * per_key) + scores
+
+
+def _rocm_flash_probe_fits(device: str, need: int) -> bool:
+ """Room for the probe beside the live forward: device free plus the allocator's cached-but-unused blocks, with
+ 25% + 256 MiB to spare. Unknown memory state counts as no room."""
+ try:
+ import torch
+
+ free, _total = torch.cuda.mem_get_info(device)
+ cached = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)
+ return free + max(int(cached), 0) >= need * 1.25 + (256 << 20)
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _rocm_flash_geometry_admitted(
+ device: str,
+ dtype: Any,
+ geometry: tuple,
+ logger: Any = None,
+ tokens: int = ROCM_FLASH_PROBE_TOKENS,
+ kv_tokens: Optional[int] = None,
+ batch: int = 1,
+ live: bool = False,
+) -> Optional[bool]:
+ """Whether flash is admitted for ``geometry`` (hidden, heads, head_dim[, context width]) at ``batch`` x ``tokens``
+ queries x ``kv_tokens`` keys (default square, batch 1). None when the probe could not run (it raised, or ``live``,
+ inside a running forward, and there is no room for it beside that forward's tensors): nothing is cached."""
+ kv = tokens if kv_tokens is None else int(kv_tokens)
+ # Geometry, lengths and batch as separate fields: a flat tuple aliases (h, H, d, ctx) at n with (h, H, d) at ctx x n.
+ key = (device, str(dtype), tuple(geometry), tokens, kv, batch)
+ _sync_rocm_flash_baseline()
+ cached = ROCM_FLASH_ADMISSIONS.get(key)
+ if cached is not None:
+ return bool(cached.get("admitted"))
+ toolchain = _rocm_flash_toolchain_key(device)
+ path = _rocm_flash_cache_path(toolchain)
+ size = (str(tokens) if kv == tokens else f"{tokens}x{kv}") + ("" if batch == 1 else f"b{batch}")
+ slot = "|".join((str(dtype), *(str(v) for v in geometry), size))
+ stored = _load_rocm_flash_verdicts(path, toolchain).get(slot)
+ if (
+ stored is not None
+ and _rocm_flash_record_ok(stored)
+ and stored["tokens"] == tokens
+ and stored.get("kv_tokens", stored["tokens"]) == kv
+ and stored.get("batch", 1) == batch
+ ):
+ rec = {**stored, "admitted": _rocm_flash_wins(stored), "cached": True}
+ ROCM_FLASH_ADMISSIONS[key] = rec
+ _log_rocm_flash_verdict(logger, device, geometry, rec, "cached")
+ return rec["admitted"]
+ if live:
+ need = _rocm_flash_probe_bytes(geometry, tokens, kv, batch, dtype, device)
+ if not _rocm_flash_probe_fits(device, need):
+ _module_logger().info(
+ "diffusion.attention: ROCm flash timing at %s tokens deferred, no room beside this forward "
+ "(needs about %.2f GiB); native SDPA for now",
+ size,
+ need / 2**30,
+ )
+ return None
+ _ROCM_FLASH_LOCAL.probing = True # the probe's own flash calls are neither gated nor counted
+ try:
+ shape = {} if kv == tokens else {"kv_tokens": kv}
+ if len(geometry) > 3:
+ shape["ctx_dim"] = geometry[3]
+ if batch != 1:
+ shape["batch"] = batch
+ rec = _time_rocm_flash_block(device, dtype, *geometry[:3], tokens = tokens, **shape)
+ except Exception as exc: # noqa: BLE001 - not cached: retried next load
+ _module_logger().info(
+ "diffusion.attention: ROCm flash timing probe could not run (%s); native SDPA", exc
+ )
+ return None
+ finally:
+ _ROCM_FLASH_LOCAL.probing = False
+ try:
+ import torch
+ torch.cuda.empty_cache()
+ except Exception: # noqa: BLE001
+ pass
+ rec = dict(rec) if isinstance(rec, dict) else {}
+ rec["geometry"] = {"hidden": geometry[0], "heads": geometry[1], "head_dim": geometry[2]}
+ if len(geometry) > 3:
+ rec["geometry"]["context"] = geometry[3]
+ valid = _rocm_flash_record_ok(rec)
+ rec["admitted"] = valid and _rocm_flash_wins(rec)
+ ROCM_FLASH_ADMISSIONS[key] = rec
+ if valid: # an implausible timing is not stored, so it cannot pin a verdict
+ _save_rocm_flash_verdict(path, toolchain, slot, rec)
+ _log_rocm_flash_verdict(logger, device, geometry, rec, "probe")
+ return rec["admitted"]
+
+
+def _real_number(value: Any) -> bool:
+ return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def _rocm_flash_record_ok(rec: Any) -> bool:
+ """Every field a verdict is derived from has the type and range the probe writes; anything else is a cache miss."""
+ import math
+ try:
+ if not isinstance(rec, dict) or not isinstance(rec.get("finite"), bool):
+ return False
+ for name in ("native_ms", "flash_ms"):
+ v = rec.get(name)
+ if not _real_number(v) or not math.isfinite(v) or v <= 0:
+ return False
+ for name in ("tokens", "kv_tokens", "batch"):
+ n = rec.get(name, 1)
+ if not isinstance(n, int) or isinstance(n, bool) or n <= 0:
+ return False
+ rel = rec.get("rel_l1")
+ if rec["finite"]:
+ if not _real_number(rel) or not math.isfinite(rel) or rel < 0:
+ return False
+ elif rel is not None and not _real_number(rel): # a non-finite output may leave NaN here
+ return False
+ max_rel = rec.get("max_rel")
+ if rec["finite"]:
+ if not _real_number(max_rel) or not math.isfinite(max_rel) or max_rel < 0:
+ return False
+ elif max_rel is not None and not _real_number(max_rel):
+ return False
+ if rec.get("block_rel_l1") is not None and not _real_number(rec["block_rel_l1"]):
+ return False
+ if "admitted" in rec and not isinstance(rec["admitted"], bool):
+ return False
+ geometry = rec.get("geometry")
+ if geometry is not None and not (
+ isinstance(geometry, dict)
+ and all(
+ isinstance(geometry.get(k), int)
+ and not isinstance(geometry.get(k), bool)
+ and geometry.get(k) > 0
+ for k in ("hidden", "heads", "head_dim")
+ + (("context",) if "context" in geometry else ())
+ )
+ ):
+ return False
+ return True
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _rocm_flash_wins(rec: dict) -> bool:
+ """Re-derived from the stored numbers on a cache hit, so a threshold change applies without re-timing."""
+ try:
+ return bool(
+ rec["finite"] is True
+ and float(rec["rel_l1"]) <= ROCM_FLASH_BLOCK_TOL
+ and float(rec["max_rel"]) <= ROCM_FLASH_MAX_TOL
+ and float(rec["flash_ms"]) <= float(rec["native_ms"]) * (1.0 - ROCM_FLASH_MIN_GAIN)
+ )
+ except Exception: # noqa: BLE001 - malformed record: never flash
+ return False
+
+
+def _log_rocm_flash_verdict(
+ logger: Any, device: str, geometry: tuple, rec: dict, source: str
+) -> None:
+ """Never raises: the values are formatted defensively, whatever a record holds."""
+
+ def num(value: Any, spec: str) -> str:
+ try:
+ return format(float(value), spec) if _real_number(value) else "?"
+ except Exception: # noqa: BLE001
+ return "?"
+
+ try:
+ (logger or _module_logger()).info(
+ "diffusion.attention: ROCm flash block %s %s (hidden %s, heads %s, head_dim %s, %s tokens): flash %s ms "
+ "vs native %s ms, attention rel L1 %s, max %s -> %s",
+ source,
+ device,
+ *(str(g) for g in tuple(geometry)[:3]),
+ num(rec.get("tokens"), ".0f"),
+ num(rec.get("flash_ms"), ".2f"),
+ num(rec.get("native_ms"), ".2f"),
+ num(rec.get("rel_l1"), ".2e"),
+ num(rec.get("max_rel"), ".2e"),
+ "flash" if rec.get("admitted") is True else "native",
+ )
+ except Exception: # noqa: BLE001 - logging is diagnostic only
+ pass
+
+
+_ROCM_DRIVER_VERSION: list = []
+
+
+def _rocm_driver_version() -> str:
+ """The ROCm stack this process actually runs on, not the wheel's build (``torch.version.hip``): the HIP runtime's
+ ``hipDriverGetVersion`` from the libamdhip64 this process loaded (HIP runtime API), plus the amdgpu kernel driver
+ (``/sys/module/amdgpu/version`` for the DKMS driver, else the kernel release for the in-tree one). Read once per
+ process; parts that cannot be read are left out."""
+ if _ROCM_DRIVER_VERSION:
+ return _ROCM_DRIVER_VERSION[0]
+ import ctypes
+ import platform
+
+ parts = []
+ try:
+ lib = None
+ if os.name == "nt":
+ names = ["amdhip64_7.dll", "amdhip64_6.dll", "amdhip64.dll"]
+ else:
+ with open("/proc/self/maps", encoding = "utf-8", errors = "replace") as f:
+ names = sorted(
+ {line.split()[-1] for line in f if "libamdhip64" in line and "/" in line}
+ )
+ for name in names:
+ try:
+ lib = ctypes.CDLL(name)
+ break
+ except OSError:
+ continue
+ if lib is not None:
+ version = ctypes.c_int(0)
+ if lib.hipDriverGetVersion(ctypes.byref(version)) == 0:
+ parts.append(f"hipdriver{version.value}")
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ with open("/sys/module/amdgpu/version", encoding = "utf-8") as f:
+ parts.append(f"amdgpu{f.read().strip()}")
+ except OSError:
+ try:
+ parts.append(f"kernel{platform.release()}")
+ except Exception: # noqa: BLE001
+ pass
+ _ROCM_DRIVER_VERSION.append("driver:" + (",".join(parts) or "?"))
+ return _ROCM_DRIVER_VERSION[0]
+
+
+def _diffusers_flash_path_fingerprint() -> str:
+ """diffusers version + a hash of the code generation runs flash through (``dispatch_attention_fn`` and the
+ registered FLASH backend, unwrapped from Studio's guard), so a diffusers change that alters the call re-times."""
+ import hashlib
+ import inspect
+
+ import diffusers
+ from diffusers.models.attention_dispatch import (
+ AttentionBackendName,
+ _AttentionBackendRegistry,
+ dispatch_attention_fn,
+ )
+
+ flash = _AttentionBackendRegistry._backends[AttentionBackendName.FLASH]
+ while (
+ getattr(flash, _SAGE_GUARD_ATTR, False) and getattr(flash, "__wrapped__", None) is not None
+ ):
+ flash = flash.__wrapped__
+ parts = []
+ for fn in (dispatch_attention_fn, flash):
+ try:
+ parts.append(inspect.getsource(fn))
+ except (OSError, TypeError):
+ code = getattr(fn, "__code__", None)
+ parts.append(repr((code.co_code, code.co_consts)) if code is not None else repr(fn))
+ digest = hashlib.sha1("\n".join(parts).encode("utf-8")).hexdigest()[:12]
+ return f"diffusers{getattr(diffusers, '__version__', '?')}:{digest}"
+
+
+def _flash_attn_build_fingerprint() -> str:
+ """size + mtime_ns of the flash_attn kernel files this process loaded: the CK extension (``flash_attn_2_cuda``) or,
+ for the Triton AMD backend, the source files of the package ``flash_attn_interface.flash_attn_gpu`` came from. A
+ rebuild in place (same version, same path) changes it. Stat only."""
+ import importlib.util
+ import sys
+
+ files: set[str] = set()
+ for name in ("flash_attn", "flash_attn.flash_attn_interface", "flash_attn_2_cuda"):
+ f = getattr(sys.modules.get(name), "__file__", None)
+ if isinstance(f, str) and f:
+ files.add(f)
+ gpu = getattr(sys.modules.get("flash_attn.flash_attn_interface"), "flash_attn_gpu", None)
+ gpu_file = getattr(gpu, "__file__", None)
+ if isinstance(gpu_file, str) and gpu_file:
+ files.add(gpu_file)
+ if gpu_file.endswith(".py"):
+ folder = os.path.dirname(gpu_file)
+ files.update(os.path.join(folder, n) for n in os.listdir(folder) if n.endswith(".py"))
+ elif "flash_attn_2_cuda" not in sys.modules:
+ try:
+ spec = importlib.util.find_spec("flash_attn_2_cuda")
+ if spec is not None and spec.origin:
+ files.add(spec.origin)
+ except Exception: # noqa: BLE001
+ pass
+ parts = []
+ for f in sorted(files):
+ try:
+ st = os.stat(f)
+ parts.append(f"{os.path.realpath(f)}:{st.st_size}:{st.st_mtime_ns}")
+ except OSError:
+ parts.append(f"{f}:missing")
+ import hashlib
+
+ return hashlib.sha1("\n".join(parts).encode("utf-8")).hexdigest()[:12]
+
+
+def _rocm_flash_toolchain_key(device: str) -> Optional[str]:
+ """What the block timing depends on: card + gfx arch, torch, HIP, the flash_attn build (version, install path, CK or
+ Triton backend) and the probe itself. None (in-memory only) when any part is unreadable."""
+ try:
+ import hashlib
+ import importlib.metadata
+ import inspect
+ import sys
+
+ import torch
+ from utils.hardware.hardware import _props_gfx_arch
+
+ index = torch.device(device).index
+ index = torch.cuda.current_device() if index is None else index
+ props = torch.cuda.get_device_properties(index)
+ fa = sys.modules.get("flash_attn")
+ if fa is None:
+ import flash_attn as fa
+ fa_version = getattr(fa, "__version__", None) or importlib.metadata.version("flash_attn")
+ # flash_attn_interface reads exactly this to route to the Triton AMD kernels instead of the CK extension, and
+ # falls back to Triton itself when the CK extension does not import; its USE_TRITON_ROCM is the outcome.
+ triton_amd = getattr(
+ sys.modules.get("flash_attn.flash_attn_interface"), "USE_TRITON_ROCM", None
+ )
+ if not isinstance(triton_amd, bool):
+ triton_amd = os.environ.get("FLASH_ATTENTION_TRITON_AMD_ENABLE", "FALSE") == "TRUE"
+ backend = "ck"
+ if triton_amd:
+ import triton
+ backend = f"triton{triton.__version__}"
+ src = inspect.getsource(_time_rocm_flash_block)
+ return "|".join(
+ (
+ f"v{ROCM_FLASH_PROBE_VERSION}",
+ str(props.name),
+ _props_gfx_arch(props) or "gfx?",
+ f"torch{torch.__version__}",
+ f"hip{getattr(torch.version, 'hip', None)}",
+ _rocm_driver_version(),
+ _native_sdpa_baseline(),
+ f"flash_attn{fa_version}",
+ os.path.dirname(os.path.realpath(fa.__file__)),
+ backend,
+ f"build{_flash_attn_build_fingerprint()}",
+ _diffusers_flash_path_fingerprint(),
+ hashlib.sha1(src.encode("utf-8")).hexdigest()[:12],
+ )
+ )
+ except Exception: # noqa: BLE001 - unidentifiable toolchain: never persisted
+ return None
+
+
+def _native_sdpa_baseline() -> str:
+ """The native arm a verdict was timed against: which SDPA backends are on (``guard_rocm_fused_sdpa``, the user, or
+ ``UNSLOTH_ALLOW_ROCM_FUSED_SDPA`` move them) and the AOTriton experimental switch that decides whether they run."""
+ import torch
+
+ cuda = torch.backends.cuda
+ flags = "".join(
+ name if enabled() else "-"
+ for name, enabled in (
+ ("f", cuda.flash_sdp_enabled),
+ ("e", cuda.mem_efficient_sdp_enabled),
+ ("m", cuda.math_sdp_enabled),
+ )
+ )
+ experimental = os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "").strip() or "unset"
+ return f"sdpa{flags},aotriton-exp{experimental}"
+
+
+def _rocm_flash_cache_path(toolchain: Optional[str]) -> Optional[str]:
+ if toolchain is None:
+ return None
+ import hashlib
+
+ root = (os.environ.get(ROCM_FLASH_CACHE_DIR_ENV) or "").strip()
+ if not root:
+ try:
+ from utils.paths.storage_roots import cache_root
+ root = str(cache_root() / "diffusion_rocm_flash")
+ except Exception: # noqa: BLE001 - outside Studio: in-memory only
+ return None
+ return os.path.join(
+ root, "probe_" + hashlib.sha1(toolchain.encode("utf-8")).hexdigest()[:16] + ".json"
+ )
+
+
+def _load_rocm_flash_verdicts(path: Optional[str], toolchain: Optional[str]) -> dict:
+ import json
+ if path is None or not os.path.isfile(path):
+ return {}
+ try:
+ with open(path, encoding = "utf-8") as f:
+ data = json.load(f)
+ if not isinstance(data, dict) or data.get("key") != toolchain:
+ return {}
+ verdicts = data.get("verdicts")
+ if not isinstance(verdicts, dict):
+ return {}
+ return {str(slot): rec for slot, rec in verdicts.items() if _rocm_flash_record_ok(rec)}
+ except Exception: # noqa: BLE001 - a corrupt / unreadable file is re-timed
+ return {}
+
+
+def _save_rocm_flash_verdict(
+ path: Optional[str], toolchain: Optional[str], slot: str, rec: dict
+) -> None:
+ """Best effort, under a file lock: merge what another process stored since, then write a temp file and replace."""
+ if path is None:
+ return
+ import contextlib
+ import json
+
+ tmp = f"{path}.{os.getpid()}.{threading.get_ident()}.tmp"
+ try:
+ os.makedirs(os.path.dirname(path), exist_ok = True)
+ try:
+ from filelock import FileLock
+ lock: Any = FileLock(f"{path}.lock", timeout = 60)
+ except ImportError:
+ lock = contextlib.nullcontext()
+ with lock:
+ verdicts = _load_rocm_flash_verdicts(path, toolchain)
+ verdicts[slot] = {k: v for k, v in rec.items() if k not in ("admitted", "cached")}
+ with open(tmp, "w", encoding = "utf-8") as f:
+ json.dump({"key": toolchain, "verdicts": verdicts}, f, indent = 1, sort_keys = True)
+ os.replace(tmp, path)
+ except Exception: # noqa: BLE001 - read-only cache / lock timeout: timed again next load
+ try:
+ os.remove(tmp)
+ except OSError:
+ pass
# Max abs error of the probe's flash_attn output against an fp32 reference (bf16 eps is ~4e-3).
@@ -868,11 +1777,13 @@ def _install_dispatch_guard(
reroute_reason: Any,
note: Any,
make_op: Any = None,
+ ran: Any = None,
) -> bool:
"""Wrap diffusers' registered ``backend`` so calls its kernel cannot take run native. Idempotent.
False when the registry is not where expected: the caller then does not engage the backend. ``make_op(fn)``
- optionally returns an opaque op for plain (no lse, no context-parallel) calls."""
+ optionally returns an opaque op for plain (no lse, no context-parallel) calls. ``ran()`` is called for each call
+ the kernel itself takes."""
try:
from diffusers.models.attention_dispatch import (
AttentionBackendName,
@@ -901,6 +1812,8 @@ def _install_dispatch_guard(
bound.get("query"), bound.get("key"), bound.get("value"), bound.get("attn_mask")
)
if reason is None:
+ if ran is not None:
+ ran()
extra = {k: v for k, v in kwargs.items() if k not in names}
plain = (
op is not None
@@ -967,6 +1880,232 @@ def _note_fa4_reroute(reason: str) -> None:
_FA4_OP_NAME = "unsloth_studio::flash_4_hub_attention_nhd"
+# diffusers' flash (FA2) backend raises on any attn_mask; on ROCm those calls run native instead.
+_ROCM_FLASH_ROUTED: dict[str, int] = {}
+_ROCM_FLASH_ROUTED_LOGGED: set[str] = set()
+
+
+def _rocm_flash_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any) -> Optional[str]:
+ if attn_mask is not None:
+ return "attn_mask"
+ return _rocm_flash_length_reason(query, key)
+
+
+def _install_rocm_flash_dispatch_guard() -> bool:
+ try:
+ import torch
+ torch.compiler.assume_constant_result(_rocm_flash_traced_reason)
+ except Exception: # noqa: BLE001 - torch without it: a traced call graph-breaks into the helper, still correct
+ pass
+ return _install_dispatch_guard(
+ "flash",
+ _rocm_flash_reroute_reason,
+ lambda reason: (
+ _note_rocm_flash_route(reason),
+ _note_reroute(
+ "ROCm flash attention", reason, _ROCM_FLASH_ROUTED, _ROCM_FLASH_ROUTED_LOGGED
+ ),
+ ),
+ ran = _note_rocm_flash_ran,
+ )
+
+
+# Per DiT, the routes its attention calls took in the current generation: {"flash": n, "<reroute reason>": m}. Report
+# only: nothing reads it to choose a kernel, size memory or key a calibration.
+ROCM_FLASH_TALLY_ATTR = "_unsloth_rocm_flash_tally"
+_ROCM_FLASH_HOOKS_ATTR = "_unsloth_rocm_flash_hooks"
+# On the pipe: every DiT the hooks went on, whatever attribute holds it (a MiniMax-H3 view hooks transformer_ref, which
+# _attention_dits does not enumerate). Reset, report and the memory guard iterate this.
+_ROCM_FLASH_DITS_ATTR = "_unsloth_rocm_flash_dits"
+
+
+def _rocm_flash_dits(pipe: Any) -> list:
+ """The hooked DiTs of ``pipe`` (registry, then any hooked one _attention_dits still sees), each once."""
+ found: list = []
+ registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None) if pipe is not None else None
+ for dit in list(registry or ()) + list(_attention_dits(pipe) if pipe is not None else ()):
+ if getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None) and not any(dit is d for d in found):
+ found.append(dit)
+ return found
+
+
+def _compiling() -> bool:
+ try:
+ import torch
+ return bool(torch.compiler.is_compiling())
+ except Exception: # noqa: BLE001
+ return False
+
+
+def _note_rocm_flash_route(route: str) -> None:
+ """Count one eager call on the running DiT's tally (this thread's). Traced and replayed calls are not counted."""
+ if _compiling() or getattr(_ROCM_FLASH_LOCAL, "probing", False):
+ return
+ tally = getattr(_ROCM_FLASH_LOCAL, "tally", None)
+ if isinstance(tally, dict):
+ tally[route] = tally.get(route, 0) + 1
+
+
+def _note_rocm_flash_ran() -> None:
+ _note_rocm_flash_route("flash")
+
+
+def _track_rocm_flash_calls(pipe: Any) -> None:
+ """Forward hooks on each DiT running ROCm flash: while it runs, this thread's gate (auto loads only) and tally are
+ its own. Each hooked DiT is recorded on the pipe (``_rocm_flash_dits``). Idempotent."""
+ for dit in _attention_dits(pipe):
+ try:
+ registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None)
+ if not isinstance(registry, list):
+ registry = []
+ setattr(pipe, _ROCM_FLASH_DITS_ATTR, registry)
+ if not any(dit is d for d in registry):
+ registry.append(dit)
+ except Exception: # noqa: BLE001
+ pass
+ register_pre = getattr(dit, "register_forward_pre_hook", None)
+ register_post = getattr(dit, "register_forward_hook", None)
+ if not callable(register_pre) or not callable(register_post):
+ continue
+ if getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None):
+ continue
+
+ # Also while Dynamo traces a compiled DiT (it replays these threading.local writes), so a traced call sees the
+ # gate and runs native.
+ def pre(
+ _module,
+ _args,
+ dit = dit,
+ ) -> None:
+ outer = getattr(_ROCM_FLASH_LOCAL, "stack", None)
+ if outer is None:
+ outer = _ROCM_FLASH_LOCAL.stack = []
+ outer.append(
+ (
+ getattr(_ROCM_FLASH_LOCAL, "gate", None),
+ getattr(_ROCM_FLASH_LOCAL, "tally", None),
+ getattr(_ROCM_FLASH_LOCAL, "deferred", None),
+ )
+ )
+ _ROCM_FLASH_LOCAL.gate = getattr(dit, ROCM_FLASH_GATE_ATTR, None) or None
+ tally = getattr(dit, ROCM_FLASH_TALLY_ATTR, None)
+ _ROCM_FLASH_LOCAL.tally = tally if isinstance(tally, dict) else None
+ deferred = getattr(dit, ROCM_FLASH_DEFERRED_ATTR, None)
+ _ROCM_FLASH_LOCAL.deferred = deferred if isinstance(deferred, set) else None
+
+ def post(_module, _args, _output) -> None:
+ stack = getattr(_ROCM_FLASH_LOCAL, "stack", None)
+ gate, tally, deferred = stack.pop() if stack else (None, None, None)
+ _ROCM_FLASH_LOCAL.gate, _ROCM_FLASH_LOCAL.tally = gate, tally
+ _ROCM_FLASH_LOCAL.deferred = deferred
+
+ try:
+ try: # also after a forward that raised, so a failed run never leaves its gate active
+ post_handle = register_post(post, always_call = True)
+ except TypeError:
+ post_handle = register_post(post)
+ handles = [register_pre(pre), post_handle]
+ setattr(dit, _ROCM_FLASH_HOOKS_ATTR, handles)
+ setattr(dit, ROCM_FLASH_TALLY_ATTR, {})
+ setattr(dit, ROCM_FLASH_DEFERRED_ATTR, set())
+ except Exception: # noqa: BLE001
+ pass
+
+
+def _untrack_rocm_flash_calls(pipe: Any) -> None:
+ """Undo ``_track_rocm_flash_calls`` for this pipe's DiTs once they no longer run ROCm flash (a re-apply picked
+ another backend), so the badge and the memory guard stop treating them as flash. Only the DiTs this pipe (or view)
+ owns: an MoE view leaves the other expert's hooks alone."""
+ dits = _attention_dits(pipe) if pipe is not None else []
+ for dit in dits:
+ handles = getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None)
+ if not handles:
+ continue
+ for handle in handles:
+ try:
+ handle.remove()
+ except Exception: # noqa: BLE001
+ pass
+ for attr in (_ROCM_FLASH_HOOKS_ATTR, ROCM_FLASH_TALLY_ATTR, ROCM_FLASH_DEFERRED_ATTR):
+ try:
+ setattr(dit, attr, None)
+ except Exception: # noqa: BLE001
+ pass
+ registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None) if pipe is not None else None
+ if isinstance(registry, list) and dits:
+ registry[:] = [d for d in registry if not any(d is x for x in dits)]
+
+
+def attention_may_run_native(pipe: Any) -> bool:
+ """Whether some attention call of this pipe can run native SDPA although flash is engaged: ROCm flash routes masked
+ calls (explicit or auto), and in auto lengths flash does not win (or could not time beside a live forward), to native. For
+ memory budgets: conservative, never from what a past run reported."""
+ return bool(_rocm_flash_dits(pipe))
+
+
+def rocm_flash_ran(pipe: Any) -> Optional[tuple[str, str]]:
+ """``(value, reason)`` describing the current generation's routes for a ROCm flash pipe, or None when it is not one.
+ A run with no counted call (compiled or graphed attention, which the guard cannot see) says so."""
+ dits = _rocm_flash_dits(pipe)
+ if not dits:
+ return None
+ total: dict[str, int] = {}
+ for dit in dits:
+ tally = getattr(dit, ROCM_FLASH_TALLY_ATTR, None)
+ if isinstance(tally, dict):
+ for name, count in tally.items():
+ if isinstance(count, int):
+ total[name] = total.get(name, 0) + count
+ flash = total.pop("flash", 0)
+ native = sum(total.values())
+ if flash + native == 0:
+ return "flash", (
+ "ROCm flash_attn; per-call routing is not tracked for compiled or graphed attention (masked calls and "
+ "lengths flash does not serve run native SDPA)"
+ )
+ why = ", ".join(f"{name}: {count}" for name, count in sorted(total.items()))
+ if native == 0:
+ return "flash", "ROCm flash_attn on every attention call of the last generation"
+ if flash == 0:
+ return (
+ "native",
+ f"ROCm flash_attn engaged, but every attention call of the last generation ran native SDPA ({why})",
+ )
+ return "flash", (
+ f"ROCm flash_attn on {flash} of {flash + native} attention calls of the last generation; the rest ran native "
+ f"SDPA ({why})"
+ )
+
+
+def begin_attention_run(state: Any) -> None:
+ """A generation boundary: the per-DiT route tallies restart, so the report after it describes this run alone, and
+ this run's DiTs retry buckets whose probe could not run."""
+ for dit in _rocm_flash_dits(getattr(state, "pipe", None)):
+ if isinstance(getattr(dit, ROCM_FLASH_TALLY_ATTR, None), dict):
+ try:
+ setattr(dit, ROCM_FLASH_TALLY_ATTR, {})
+ setattr(dit, ROCM_FLASH_DEFERRED_ATTR, set())
+ except Exception: # noqa: BLE001
+ pass
+
+
+def report_attention_run(state: Any) -> None:
+ """After a generation: the ``resolved`` attention badge (value + reason) describes the routes it took. Report only:
+ ``state.attention_backend`` stays the engaged backend, which recipes, calibration keys and compile caches key on."""
+ try:
+ ran = rocm_flash_ran(getattr(state, "pipe", None))
+ resolved = getattr(state, "resolved", None)
+ entry = resolved.get("attention_backend") if isinstance(resolved, dict) else None
+ if ran is None or not isinstance(entry, dict):
+ return
+ value, reason = ran
+ if entry.get("value") != value or entry.get("reason") != reason:
+ entry = {**entry, "value": value, "reason": reason}
+ object.__setattr__(state, "resolved", {**resolved, "attention_backend": entry})
+ except Exception: # noqa: BLE001 - reporting only
+ pass
+
+
def _install_fa4_dispatch_guard() -> bool:
return _install_dispatch_guard(
"flash_4_hub",
@@ -1998,6 +3137,13 @@ def apply_attention_backend(
if target is not None:
_configure_native_attention(pipe, target, logger)
return None
+ for dit in _attention_dits(pipe): # only an auto admission below re-arms per-length gating
+ try:
+ setattr(dit, ROCM_FLASH_GATE_ATTR, None)
+ except Exception: # noqa: BLE001
+ pass
+ if backend == ROCM_FLASH_CANDIDATE:
+ backend = "flash" if _rocm_flash_block_admitted(pipe, target, logger) else None
if backend is not None:
_ensure_attention_backend_installed(backend, logger)
if backend == "sage":
@@ -2017,6 +3163,19 @@ def apply_attention_backend(
and not _rocm_flash_attn_runs(target)
):
backend = None
+ if (
+ backend == "flash"
+ and target is not None
+ and _is_cuda_rocm(target)
+ and not _install_rocm_flash_dispatch_guard()
+ ):
+ # An explicit request too: without the guard a masked call would raise mid-generation, not run native.
+ if logger is not None:
+ logger.warning(
+ "diffusion.attention: this diffusers build does not expose the flash backend the way Unsloth "
+ "expects, so masked attention calls could not be routed around it; using the default backend"
+ )
+ backend = None
if (
backend == "_native_cudnn"
and target is not None
@@ -2054,6 +3213,10 @@ def apply_attention_backend(
engaged = False
if engaged:
_tag_dits(pipe, backend)
+ if backend == "flash" and target is not None and _is_cuda_rocm(target):
+ _track_rocm_flash_calls(pipe)
+ else:
+ _untrack_rocm_flash_calls(pipe)
# set_attention_backend also pins the backend process-wide. Each DiT's processors keep it locally, so reset
# the global to native ONCE, else a later component inherits this kernel.
_reset_global_backend_to_native(logger)
@@ -2063,6 +3226,7 @@ def apply_attention_backend(
# No backend requested, or every set failed: pin native so a stale process-wide backend cannot leak in. One reset
# covers every fresh DiT.
_tag_dits(pipe, None)
+ _untrack_rocm_flash_calls(pipe)
_restore_native_backend(setters[0], logger)
# Native means torch's SDPA dispatch decides per call, and on a device with no fused kernel that decision is MATH.
# Say so now; the flags this would otherwise be read off lie (#8225).
diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.py
index eb7f2d6182e..f97c417db6e 100644
--- a/studio/backend/core/inference/video.py
+++ b/studio/backend/core/inference/video.py
@@ -52,8 +52,11 @@ from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_m
from .diffusion_attention import (
SDPA_MATH_ONLY_MESSAGE,
apply_attention_backend,
+ attention_status_reason,
install_hunyuan_attention_trim,
sdpa_math_only,
+ begin_attention_run,
+ report_attention_run,
select_attention_backend,
)
from .diffusion_flow_shift import apply_comfy_flow_shift
@@ -9453,7 +9456,7 @@ class VideoBackend:
"attention_backend": (
attention_backend,
attention_engaged or "native",
- "cuDNN fused attention on NVIDIA when a speed profile is active",
+ attention_status_reason(attention_backend, attention_engaged),
),
"transformer_cache": (
None if cache_auto else transformer_cache,
@@ -10761,7 +10764,7 @@ class VideoBackend:
"attention_backend": (
attention_backend,
attention_engaged or "native",
- "cuDNN fused attention on NVIDIA when a speed profile is active",
+ attention_status_reason(attention_backend, attention_engaged),
),
"transformer_cache": (
(
@@ -11446,6 +11449,7 @@ class VideoBackend:
if _resolved_inputs is not None and _resolved_inputs.state is not state:
raise RuntimeError(VIDEO_CANCELLED_MSG)
self._active_generate_cancel = cancel
+ begin_attention_run(state)
# Bound below, once the request is resolved. None means the failure beat the resolution, and there is
# nothing truthful to report.
request_shape: Optional[dict[str, Any]] = None
@@ -12234,6 +12238,8 @@ class VideoBackend:
# A cancel during the blocking export/mux must still discard the clip; re-check before it is persisted.
if cancel.is_set():
raise RuntimeError(VIDEO_CANCELLED_MSG)
+ # The status badge reports the attention routes this run took (report only, keys are unchanged).
+ report_attention_run(state)
if len(video_frames) and not fam.modular_workflow:
from . import video_stream_residency
diff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
index 2c8e59475fa..22c87d263a3 100644
--- a/studio/backend/tests/test_diffusion_attention_rocm_flash.py
+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py
@@ -6,6 +6,8 @@ NVIDIA path unchanged; the CUDA flash-attn wheel is never pip installed onto ROC
from __future__ import annotations
+import importlib
+import os
import subprocess
import sys
import types
@@ -15,15 +17,31 @@ import pytest
import core.inference.diffusion_attention as att
from core.inference.diffusion_attention import select_attention_backend
+try: # before any test puts a bare flash_attn module in sys.modules (diffusers' import probes its spec)
+ importlib.import_module("diffusers.models.attention_dispatch")
+except Exception: # noqa: BLE001
+ pass
+
+_REAL_TOOLCHAIN_KEY = att._rocm_flash_toolchain_key
+_REAL_PROBE_FITS = att._rocm_flash_probe_fits
+_REAL_GUARD = att._install_rocm_flash_dispatch_guard
+
@pytest.fixture(autouse = True)
-def _clean(monkeypatch):
+def _clean(monkeypatch, tmp_path):
monkeypatch.setattr(att, "_ROCM_FLASH_PROBE_CACHE", {}, raising = False)
+ monkeypatch.setenv(att.ROCM_FLASH_CACHE_DIR_ENV, str(tmp_path / "rocm_flash"))
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: "tc-A", raising = False)
monkeypatch.setattr(
att, "_indexed_cuda_device", lambda d: "cuda:0" if d == "cuda" else d, raising = False
)
monkeypatch.setattr(att, "_cuda_capability", lambda: (9, 0))
monkeypatch.setattr(att, "_INSTALL_ATTEMPTED", set())
+ monkeypatch.setattr(att, "_ROCM_FLASH_LOCAL", __import__("threading").local(), raising = False)
+ monkeypatch.setattr(att, "_ROCM_FLASH_BUCKETS", {}, raising = False)
+ monkeypatch.setattr(att, "_ROCM_FLASH_BASELINE", [None], raising = False)
+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True, raising = False)
+ monkeypatch.setattr(att, "_ROCM_FLASH_ROUTED", {})
def _target(device = "cuda", dtype = "torch.bfloat16"):
@@ -96,11 +114,13 @@ def test_rocm_unaskable_check_is_native_and_not_cached(monkeypatch):
assert att._ROCM_FLASH_PROBE_CACHE == {}
-def test_rocm_auto_unchanged(monkeypatch):
+def test_rocm_auto_is_a_timed_candidate_not_flash(monkeypatch):
calls = _rocm(monkeypatch)
- assert select_attention_backend(_target(), "auto", speed_active = True) is None
+ assert (
+ select_attention_backend(_target(), "auto", speed_active = True) == att.ROCM_FLASH_CANDIDATE
+ )
assert select_attention_backend(_target(), "auto", speed_active = False) is None
- assert calls == []
+ assert calls == [("cuda:0", "torch.bfloat16")]
@pytest.mark.parametrize("alias", ["flash3", "flash4", "sage", "cudnn", "xformers"])
@@ -242,54 +262,57 @@ def test_cudnn_head_dim_probe_still_answers_fp32_at_bf16(monkeypatch):
assert seen == [torch.bfloat16]
-# ``auto`` -> flash for FLUX.2-klein on ROCm gfx11 (measured on gfx1151); every other family and arch unchanged.
-
-
def _klein():
return types.SimpleNamespace(name = "flux.2-klein")
-def _arch(monkeypatch, arch = "gfx1151"):
- monkeypatch.setattr(att, "_rocm_gfx_arch", lambda target: arch)
-
-
-def test_rocm_auto_flash_for_klein_on_gfx11(monkeypatch):
- calls = _rocm(monkeypatch)
- _arch(monkeypatch)
[email protected]("family", ["flux.2-klein", "flux.1", "z-image", "qwen-image", None])
+def test_rocm_auto_candidate_for_every_family(monkeypatch, family):
+ _rocm(monkeypatch)
+ fam = None if family is None else types.SimpleNamespace(name = family)
assert (
- select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) == "flash"
+ select_attention_backend(_target(), "auto", speed_active = True, family = fam)
+ == att.ROCM_FLASH_CANDIDATE
)
assert (
- select_attention_backend(_target(), None, speed_active = True, family = "flux.2-klein")
- == "flash"
+ select_attention_backend(_target(), None, speed_active = False, family = fam, speed_unset = True)
+ == att.ROCM_FLASH_CANDIDATE
)
- # unset speed (ROCm resolves it to `off`): the default path, so flash too
- assert (
- select_attention_backend(
- _target(), "auto", speed_active = False, family = _klein(), speed_unset = True
- )
- == "flash"
+ assert select_attention_backend(_target(), "auto", speed_active = False, family = fam) is None
+
+
+def test_rocm_auto_without_flash_attn_is_silent(monkeypatch):
+ calls = _rocm(monkeypatch, flash_attn = False)
+ logged = []
+ monkeypatch.setattr(
+ att, "_module_logger", lambda: types.SimpleNamespace(warning = lambda *a: logged.append(a))
)
- # an explicit `off` stays native (the bit-identical reference path)
- assert select_attention_backend(_target(), "auto", speed_active = False, family = _klein()) is None
- assert calls == [("cuda:0", "torch.bfloat16")]
+ assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None
+ assert calls == [] and logged == []
[email protected]("family", ["flux.1", "flux.2", "z-image", "qwen-image", "sdxl", None])
-def test_rocm_auto_stays_native_for_unmeasured_families(monkeypatch, family):
+def test_rocm_auto_failed_probe_stays_native(monkeypatch):
+ _rocm(monkeypatch, probe = False)
+ assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None
+
+
+def test_rocm_auto_explicit_native_wins(monkeypatch):
calls = _rocm(monkeypatch)
- _arch(monkeypatch)
- fam = None if family is None else types.SimpleNamespace(name = family)
- assert select_attention_backend(_target(), "auto", speed_active = True, family = fam) is None
- assert (
- select_attention_backend(
- _target(), "auto", speed_active = False, family = fam, speed_unset = True
+ for alias in ("native", "sdpa"):
+ assert (
+ select_attention_backend(_target(), alias, speed_active = True, family = _klein()) is None
)
- is None
- )
assert calls == []
+def test_rocm_kill_switch_zero_keeps_auto_native(monkeypatch):
+ calls = _rocm(monkeypatch)
+ monkeypatch.setenv(att.ROCM_FLASH_ENV, "0")
+ assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None
+ assert calls == []
+ assert select_attention_backend(_target(), "flash", speed_active = True) == "flash"
+
+
def test_rocm_gfx_arch_reads_the_selected_card(monkeypatch):
import torch
@@ -306,39 +329,173 @@ def test_rocm_gfx_arch_reads_the_selected_card(monkeypatch):
assert asked == [1]
[email protected]("arch", ["gfx942", "gfx90a", "gfx1201", "gfx1030", ""])
-def test_rocm_auto_klein_flash_gfx11_only(monkeypatch, arch):
- calls = _rocm(monkeypatch)
- _arch(monkeypatch, arch)
- assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None
- assert calls == []
+def _pipe_with_attention(
+ hidden = 3072,
+ heads = 24,
+ head_dim = 128,
+ n = 3,
+):
+ set_to = []
+ mods = [
+ types.SimpleNamespace(
+ heads = heads,
+ to_q = types.SimpleNamespace(in_features = hidden, out_features = heads * head_dim),
+ )
+ for _ in range(n)
+ ]
+ dit = types.SimpleNamespace(set_attention_backend = set_to.append, modules = lambda: iter(mods))
+ return types.SimpleNamespace(transformer = dit), set_to
-def test_rocm_auto_klein_without_flash_attn_is_silent(monkeypatch, caplog):
- calls = _rocm(monkeypatch, flash_attn = False)
- _arch(monkeypatch)
- logged = []
+def _apply_env(monkeypatch):
+ monkeypatch.setattr(att, "_active_attention_backend", lambda: att.ATTN_NATIVE)
+ monkeypatch.setattr(att, "warn_if_sdpa_math_only", lambda *a, **k: False)
+ monkeypatch.setattr(att, "_configure_native_attention", lambda *a, **k: None)
+ monkeypatch.setattr(att, "ROCM_FLASH_ADMISSIONS", {})
+ monkeypatch.setattr(att, "_install_rocm_flash_dispatch_guard", lambda: True, raising = False)
+
+
+ "flash_ms, rel, engaged",
+ [(70.0, 1e-3, "flash"), (78.0, 1e-3, None), (90.0, 1e-3, None), (60.0, 0.5, None)],
+)
+def test_apply_resolves_candidate_by_block_timing(monkeypatch, flash_ms, rel, engaged):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ ):
+ timed.append((device, hidden, heads, head_dim))
+ return {
+ "native_ms": 80.0,
+ "flash_ms": flash_ms,
+ "rel_l1": rel,
+ "max_rel": rel,
+ "finite": True,
+ "tokens": 4096,
+ }
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
+ pipe, set_to = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged
+ assert timed == [("cuda:0", 3072, 24, 128)]
+ assert set_to == (["flash"] if engaged else [])
+ pipe2, _ = _pipe_with_attention()
+ att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target())
+ assert len(timed) == 1
+ rec = next(iter(att.ROCM_FLASH_ADMISSIONS.values()))
+ assert rec["admitted"] is (engaged == "flash") and rec["geometry"] == {
+ "hidden": 3072,
+ "heads": 24,
+ "head_dim": 128,
+ }
+
+
+def test_apply_candidate_kill_switch_one_skips_timing(monkeypatch):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setenv(att.ROCM_FLASH_ENV, "1")
monkeypatch.setattr(
- att, "_module_logger", lambda: types.SimpleNamespace(warning = lambda *a: logged.append(a))
+ att, "_time_rocm_flash_block", lambda *a, **k: pytest.fail("timed under =1")
)
- assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None
- assert calls == [] and logged == []
+ pipe, set_to = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
-def test_rocm_auto_klein_failed_probe_stays_native(monkeypatch):
- _rocm(monkeypatch, probe = False)
- _arch(monkeypatch)
- assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None
+def test_apply_candidate_probe_error_is_native_and_retried(monkeypatch):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ n = []
+ def boom(*a, **k):
+ n.append(1)
+ raise RuntimeError("out of memory")
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", boom)
+ pipe, set_to = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None
+ assert len(n) == 2 and set_to == []
+
+
+def test_apply_candidate_without_geometry_is_native(monkeypatch):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(
+ att, "_time_rocm_flash_block", lambda *a, **k: pytest.fail("no geometry, nothing to time")
+ )
+ set_to = []
+ dit = types.SimpleNamespace(set_attention_backend = set_to.append, modules = lambda: iter(()))
+ pipe = types.SimpleNamespace(transformer = dit)
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None
+
+
+def test_candidate_never_pip_installs(monkeypatch):
+ assert att._ensure_backend_package(att.ROCM_FLASH_CANDIDATE) is None
-def test_rocm_auto_klein_explicit_native_wins(monkeypatch):
- calls = _rocm(monkeypatch)
- _arch(monkeypatch)
- for alias in ("native", "sdpa"):
- assert (
- select_attention_backend(_target(), alias, speed_active = True, family = _klein()) is None
- )
- assert calls == []
+
+def _diffusers_flash(monkeypatch, flash_attn_func):
+ """diffusers' real FLASH backend (unguarded, restored after the test) calling ``flash_attn_func``."""
+ ad = pytest.importorskip("diffusers.models.attention_dispatch")
+ backends = ad._AttentionBackendRegistry._backends
+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, ad._flash_attention)
+ monkeypatch.setattr(ad, "flash_attn_func", flash_attn_func)
+ return ad
+
+
+def test_block_timer_runs_on_cpu_shapes(monkeypatch):
+ torch = pytest.importorskip("torch")
+ seen = []
+
+ def flash_attn_func(
+ q,
+ k,
+ v,
+ dropout_p = 0.0,
+ softmax_scale = None,
+ causal = False,
+ window_size = (-1, -1),
+ return_attn_probs = False,
+ ):
+ seen.append((dropout_p, softmax_scale, causal, tuple(window_size), return_attn_probs))
+ return torch.nn.functional.scaled_dot_product_attention(
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
+ ).transpose(1, 2)
+
+ _diffusers_flash(monkeypatch, flash_attn_func)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
+ att._ROCM_FLASH_LOCAL.tally = {}
+ rec = att._time_rocm_flash_block("cpu", torch.bfloat16, 64, 4, 16, tokens = 32)
+ assert rec["finite"] and rec["rel_l1"] == 0.0 and rec["native_ms"] > 0 and rec["flash_ms"] > 0
+ assert len(seen) == 9 and set(seen) == {(0.0, None, False, (-1, -1), False)}
+ assert att._ROCM_FLASH_LOCAL.tally == {"flash": 9}
+
+
+def test_flash_attn_rejecting_the_diffusers_arguments_is_not_admitted(monkeypatch, tmp_path):
+ torch = pytest.importorskip("torch")
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_indexed_cuda_device", lambda d: "cpu")
+
+ def old_api(q, k, v):
+ return q
+
+ _diffusers_flash(monkeypatch, old_api)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None)
+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_TOKENS", 32)
+ pipe, set_to = _pipe_with_attention(hidden = 64, heads = 4, head_dim = 16)
+ with pytest.raises(TypeError):
+ att._time_rocm_flash_block("cpu", torch.bfloat16, 64, 4, 16, tokens = 32)
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None
+ assert set_to == [] and att.ROCM_FLASH_ADMISSIONS == {} and _cache_files(tmp_path) == []
def test_nvidia_auto_klein_unchanged(monkeypatch):
@@ -381,8 +538,1736 @@ def test_auto_attention_reason_names_the_engaged_kernel():
def test_status_reason_is_derived_from_the_engaged_backend():
- src = open(
- att.__file__.replace("diffusion_attention.py", "diffusion.py"), encoding = "utf-8"
- ).read()
- assert src.count("auto_attention_reason(attention_engaged)") == 2
- assert '"cuDNN fused attention upgrade"' not in src
+ def read(name):
+ return open(att.__file__.replace("diffusion_attention.py", name), encoding = "utf-8").read()
+
+ image, video = read("diffusion.py"), read("video.py")
+ assert image.count("attention_status_reason(attention_backend, attention_engaged)") == 1
+ assert image.count("auto_attention_reason(attention_engaged)") == 1
+ assert '"cuDNN fused attention upgrade"' not in image
+ assert video.count("attention_status_reason(attention_backend, attention_engaged)") == 2
+ assert "cuDNN fused attention on NVIDIA" not in video
+
+
+# Masked attention (HunyuanVideo 1.5 mixed padding) must never reach diffusers' flash backend, which raises on
+# attn_mask; and every attention geometry that receives the backend must pass the timing, not only the most common one.
+
+
+def test_auto_flash_engages_only_with_the_mask_guard(monkeypatch):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(
+ att,
+ "_time_rocm_flash_block",
+ lambda *a, **k: {
+ "native_ms": 80.0,
+ "flash_ms": 60.0,
+ "rel_l1": 1e-3,
+ "max_rel": 2e-3,
+ "finite": True,
+ "tokens": 4096,
+ },
+ )
+ installed = []
+ monkeypatch.setattr(
+ att, "_install_rocm_flash_dispatch_guard", lambda: installed.append(1) or True
+ )
+ pipe, set_to = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
+ assert installed and set_to == ["flash"]
+ # a diffusers build whose registry cannot be guarded: auto stays native rather than risk a masked call
+ monkeypatch.setattr(att, "_install_rocm_flash_dispatch_guard", lambda: False)
+ pipe2, set_to2 = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) is None
+ assert set_to2 == []
+ pipe3, set_to3 = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe3, "flash", target = _target()) is None
+ assert set_to3 == []
+
+
+def test_rocm_flash_guard_routes_masked_calls_to_native(monkeypatch):
+ torch = pytest.importorskip("torch")
+ ad = pytest.importorskip("diffusers.models.attention_dispatch")
+ backends = ad._AttentionBackendRegistry._backends
+ calls = []
+
+ def fake_flash(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ **kw,
+ ):
+ if attn_mask is not None:
+ raise ValueError("`attn_mask` is not supported for flash-attn 2.")
+ calls.append("flash")
+ return query
+
+ def fake_native(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ **kw,
+ ):
+ calls.append("native")
+ return query
+
+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, fake_flash)
+ monkeypatch.setitem(backends, ad.AttentionBackendName.NATIVE, fake_native)
+ assert att._install_rocm_flash_dispatch_guard() is True
+ guarded = backends[ad.AttentionBackendName.FLASH]
+ q = torch.zeros(1, 8, 2, 16, dtype = torch.bfloat16)
+ mask = torch.ones(1, 1, 8, 8, dtype = torch.bool)
+ guarded(q, q, q, attn_mask = mask)
+ guarded(q, q, q, None)
+ assert calls == ["native", "flash"]
+ assert att._install_rocm_flash_dispatch_guard() is True
+ assert backends[ad.AttentionBackendName.FLASH] is guarded
+
+
+def _two_geometry_pipe():
+ set_to = []
+
+ def dit(hidden, heads, head_dim, n):
+ mods = [
+ types.SimpleNamespace(
+ heads = heads,
+ to_q = types.SimpleNamespace(in_features = hidden, out_features = heads * head_dim),
+ )
+ for _ in range(n)
+ ]
+ return types.SimpleNamespace(
+ set_attention_backend = set_to.append, modules = lambda: iter(mods)
+ )
+
+ first, second = dit(3072, 24, 128, 5), dit(1536, 12, 128, 1)
+ monkey_dits = [first, second]
+ return types.SimpleNamespace(transformer = first, transformer_2 = second), set_to, monkey_dits
+
+
[email protected]("minority_wins, engaged", [(True, "flash"), (False, None)])
+def test_every_attention_geometry_is_timed(monkeypatch, minority_wins, engaged):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ pipe, set_to, dits = _two_geometry_pipe()
+ monkeypatch.setattr(att, "_attention_dits", lambda p: dits)
+ timed = []
+
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ ):
+ timed.append((hidden, heads, head_dim))
+ flash = 60.0 if (hidden == 3072 or minority_wins) else 85.0
+ return {
+ "native_ms": 80.0,
+ "flash_ms": flash,
+ "rel_l1": 1e-3,
+ "max_rel": 2e-3,
+ "finite": True,
+ "tokens": 4096,
+ }
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged
+ assert (1536, 12, 128) in timed
+ if engaged:
+ assert sorted(timed) == [(1536, 12, 128), (3072, 24, 128)]
+ assert set_to == ["flash", "flash"] and len(att.ROCM_FLASH_ADMISSIONS) == 2
+ else:
+ assert set_to == []
+ n = len(timed)
+ pipe2, set_to2, dits2 = _two_geometry_pipe()
+ monkeypatch.setattr(att, "_attention_dits", lambda p: dits2)
+ assert (
+ att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) is None
+ )
+ assert len(timed) == n
+
+
+def test_num_heads_attention_modules_are_measured():
+ # Krea 2's attention exposes num_heads, not heads: without the fallback auto could never time it.
+ import types
+
+ from torch import nn
+
+ class Krea2LikeAttention(nn.Module):
+ def __init__(self) -> None:
+ super().__init__()
+ self.num_heads = 24
+ self.to_q = nn.Linear(3072, 3072, bias = False)
+
+ dit = nn.Module()
+ dit.attn = Krea2LikeAttention()
+ pipe = types.SimpleNamespace()
+ original = att._attention_dits
+ try:
+ att._attention_dits = lambda p: [dit]
+ assert att._dit_attention_geometries(pipe) == ((3072, 24, 128),)
+ finally:
+ att._attention_dits = original
+
+
+def _timer(timed, flash_ms = 70.0):
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ ):
+ timed.append((hidden, heads, head_dim))
+ return {
+ "native_ms": 80.0,
+ "flash_ms": flash_ms,
+ "rel_l1": 1e-3,
+ "max_rel": 2e-3,
+ "finite": True,
+ "tokens": 4096,
+ }
+
+ return fake_time
+
+
+def _restart(monkeypatch):
+ """A new process: the in-memory verdicts are gone, the cache dir is not."""
+ monkeypatch.setattr(att, "ROCM_FLASH_ADMISSIONS", {})
+ monkeypatch.setattr(att, "_ROCM_FLASH_PROBE_CACHE", {})
+
+
+def _cache_files(tmp_path):
+ root = tmp_path / "rocm_flash"
+ return sorted(root.glob("probe_*.json")) if root.is_dir() else []
+
+
[email protected]("flash_ms, engaged", [(70.0, "flash"), (90.0, None)])
+def test_block_verdict_persists_across_processes(monkeypatch, tmp_path, flash_ms, engaged):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed, flash_ms))
+ pipe, _ = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged
+ assert len(timed) == 1 and len(_cache_files(tmp_path)) == 1
+ _restart(monkeypatch)
+ monkeypatch.setattr(
+ att,
+ "_time_rocm_flash_block",
+ lambda *a, **k: pytest.fail("re-timed despite a stored verdict"),
+ )
+ pipe2, set_to2 = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged
+ assert set_to2 == (["flash"] if engaged else [])
+ rec = next(iter(att.ROCM_FLASH_ADMISSIONS.values()))
+ assert rec["cached"] is True and rec["flash_ms"] == flash_ms
+
+
[email protected]("change", ["torch", "flash_attn", "probe_version", "geometry", "dtype"])
+def test_block_verdict_key_mismatch_retimes(monkeypatch, tmp_path, change):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))
+ toolchain = {"torch": "torch2.9", "flash_attn": "flash_attn2.8.3"}
+ monkeypatch.setattr(
+ att,
+ "_rocm_flash_toolchain_key",
+ lambda device: f"v{att.ROCM_FLASH_PROBE_VERSION}|{toolchain['torch']}|{toolchain['flash_attn']}",
+ )
+ pipe, _ = _pipe_with_attention()
+ att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target())
+ assert len(timed) == 1
+ _restart(monkeypatch)
+ target, geometry = _target(), {}
+ if change == "torch":
+ toolchain["torch"] = "torch2.10"
+ elif change == "flash_attn":
+ toolchain["flash_attn"] = "flash_attn2.8.4"
+ elif change == "probe_version":
+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_VERSION", att.ROCM_FLASH_PROBE_VERSION + 1)
+ elif change == "geometry":
+ geometry = {"hidden": 2048, "heads": 16, "head_dim": 128}
+ else:
+ target = _target(dtype = "torch.float16")
+ pipe2, _ = _pipe_with_attention(**geometry)
+ att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = target)
+ assert len(timed) == 2
+
+
+ "payload", ["{not json", "", "[]", '{"key": "tc-A", "verdicts": {"x": 1}}']
+)
+def test_corrupt_cache_file_retimes(monkeypatch, tmp_path, payload):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))
+ pipe, _ = _pipe_with_attention()
+ att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target())
+ (path,) = _cache_files(tmp_path)
+ path.write_text(payload)
+ _restart(monkeypatch)
+ pipe2, _ = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
+ assert len(timed) == 2
+ _restart(monkeypatch)
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ assert len(timed) == 2
+
+
+def test_probe_error_is_not_persisted(monkeypatch, tmp_path):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+
+ def boom(*a, **k):
+ raise RuntimeError("out of memory")
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", boom)
+ pipe, _ = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None
+ assert _cache_files(tmp_path) == []
+
+
+def test_unidentifiable_toolchain_stays_in_memory(monkeypatch, tmp_path):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: None)
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))
+ pipe, _ = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
+ assert _cache_files(tmp_path) == [] and not (tmp_path / "rocm_flash").exists()
+
+
+def test_unwritable_cache_dir_still_admits(monkeypatch, tmp_path):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ blocker = tmp_path / "file"
+ blocker.write_text("")
+ monkeypatch.setenv(att.ROCM_FLASH_CACHE_DIR_ENV, str(blocker / "sub"))
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+ pipe, _ = _pipe_with_attention()
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
+
+
[email protected]("mode, engaged", [("0", None), ("1", "flash")])
+def test_forced_modes_neither_read_nor_write_the_cache(monkeypatch, tmp_path, mode, engaged):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ # a stored verdict that would say the opposite of the forced mode
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed, 70.0 if mode == "0" else 90.0))
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ (path,) = _cache_files(tmp_path)
+ before = path.read_bytes()
+ _restart(monkeypatch)
+ monkeypatch.setenv(att.ROCM_FLASH_ENV, mode)
+ monkeypatch.setattr(att, "_load_rocm_flash_verdicts", lambda *a: pytest.fail("cache read"))
+ monkeypatch.setattr(att, "_save_rocm_flash_verdict", lambda *a: pytest.fail("cache write"))
+ monkeypatch.setattr(att, "_time_rocm_flash_block", lambda *a, **k: pytest.fail("timed"))
+ pipe, _ = _pipe_with_attention(hidden = 2048, heads = 16)
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged
+ assert path.read_bytes() == before and att.ROCM_FLASH_ADMISSIONS == {}
+
+
+def test_cache_dir_defaults_under_the_studio_cache(monkeypatch, tmp_path):
+ monkeypatch.delenv(att.ROCM_FLASH_CACHE_DIR_ENV)
+ import utils.paths.storage_roots as roots
+
+ monkeypatch.setattr(roots, "cache_root", lambda: tmp_path / "cache")
+ path = att._rocm_flash_cache_path("tc-A")
+ assert path.startswith(str(tmp_path / "cache" / "diffusion_rocm_flash" / "probe_"))
+ assert att._rocm_flash_cache_path("tc-B") != path and att._rocm_flash_cache_path(None) is None
+
+
+def test_toolchain_key_names_the_flash_attn_build(monkeypatch):
+ pytest.importorskip("diffusers.models.attention_dispatch")
+ import torch
+
+ props = types.SimpleNamespace(name = "AMD Radeon 8060S", gcnArchName = "gfx1151:sramecc-:xnack-")
+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda i: props)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(torch.version, "hip", "7.1.0", raising = False)
+ fa = types.ModuleType("flash_attn")
+ fa.__version__ = "2.8.3"
+ fa.__file__ = "/x/site-packages/flash_attn/__init__.py"
+ monkeypatch.setitem(sys.modules, "flash_attn", fa)
+ # A real flash_attn build already imported (a ROCm host) would answer the backend through its USE_TRITON_ROCM.
+ monkeypatch.delitem(sys.modules, "flash_attn.flash_attn_interface", raising = False)
+ monkeypatch.delenv("FLASH_ATTENTION_TRITON_AMD_ENABLE", raising = False)
+ ck = _REAL_TOOLCHAIN_KEY("cuda:0")
+ assert ck is not None and ck.startswith(f"v{att.ROCM_FLASH_PROBE_VERSION}|")
+ for part in (
+ "AMD Radeon 8060S",
+ "|gfx1151|",
+ "|hip7.1.0|",
+ "|flash_attn2.8.3|",
+ os.sep + "flash_attn|ck|",
+ ):
+ assert part in ck, part
+ fa.__version__ = "2.8.4"
+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != ck
+ fa.__version__ = "2.8.3"
+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_VERSION", att.ROCM_FLASH_PROBE_VERSION + 1)
+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != ck
+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_VERSION", att.ROCM_FLASH_PROBE_VERSION - 1)
+ assert _REAL_TOOLCHAIN_KEY("cuda:0") == ck
+ pytest.importorskip("triton")
+ monkeypatch.setenv("FLASH_ATTENTION_TRITON_AMD_ENABLE", "TRUE")
+ assert "|triton" in _REAL_TOOLCHAIN_KEY("cuda:0")
+
+
+# A flash_attn rebuilt in place (same version, same path) must re-time: the key fingerprints the loaded kernel files.
+
+
+def _fake_build(
+ monkeypatch,
+ tmp_path,
+ *,
+ triton = False,
+):
+ import torch
+
+ props = types.SimpleNamespace(name = "AMD Radeon 8060S", gcnArchName = "gfx1151")
+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda i: props)
+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)
+ monkeypatch.setattr(torch.version, "hip", "7.1.0", raising = False)
+ pkg = tmp_path / "site" / "flash_attn"
+ pkg.mkdir(parents = True)
+ fa = types.ModuleType("flash_attn")
+ fa.__version__ = "2.8.3"
+ fa.__file__ = str(pkg / "__init__.py")
+ iface = types.ModuleType("flash_attn.flash_attn_interface")
+ iface.__file__ = str(pkg / "flash_attn_interface.py")
+ iface.USE_TRITON_ROCM = triton
+ for f in (fa.__file__, iface.__file__):
+ open(f, "w").write("# py\n")
+ if triton:
+ kdir = tmp_path / "site" / "triton_amd"
+ kdir.mkdir()
+ (kdir / "fwd_prefill.py").write_text("# kernel v1\n")
+ binary = kdir / "flash_attn_2.py"
+ else:
+ binary = tmp_path / "site" / "flash_attn_2_cuda.cpython-312-x86_64-linux-gnu.so"
+ binary.write_bytes(b"\x7fELF v1")
+ gpu = types.ModuleType("flash_attn_gpu")
+ gpu.__file__ = str(binary)
+ iface.flash_attn_gpu = gpu
+ monkeypatch.setitem(sys.modules, "flash_attn", fa)
+ monkeypatch.setitem(sys.modules, "flash_attn.flash_attn_interface", iface)
+ if triton:
+ pytest.importorskip("triton")
+ return kdir / "fwd_prefill.py"
+ monkeypatch.setitem(sys.modules, "flash_attn_2_cuda", gpu)
+ return binary
+
+
[email protected]("triton", [False, True])
[email protected]("how", ["content", "mtime"])
+def test_rebuilt_flash_attn_binary_changes_the_key(monkeypatch, tmp_path, triton, how):
+ pytest.importorskip("diffusers.models.attention_dispatch")
+ import os
+
+ kernel = _fake_build(monkeypatch, tmp_path, triton = triton)
+ key = _REAL_TOOLCHAIN_KEY("cuda:0")
+ assert key is not None and ("|triton" in key) is triton and _REAL_TOOLCHAIN_KEY("cuda:0") == key
+ st = os.stat(kernel)
+ if how == "content":
+ kernel.write_bytes(kernel.read_bytes() + b" rebuilt")
+ os.utime(kernel, ns = (st.st_atime_ns, st.st_mtime_ns))
+ else:
+ os.utime(kernel, ns = (st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != key
+
+
+def test_rebuilt_flash_attn_binary_forces_retiming(monkeypatch, tmp_path):
+ pytest.importorskip("diffusers.models.attention_dispatch")
+ import os
+
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ binary = _fake_build(
+ monkeypatch, tmp_path
+ ) # after _rocm, which installs a bare flash_attn module
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)
+ assert _REAL_TOOLCHAIN_KEY("cuda:0") is not None
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ _restart(monkeypatch)
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ assert len(timed) == 1
+ binary.write_bytes(b"\x7fELF v2 rebuilt")
+ st = os.stat(binary)
+ os.utime(binary, ns = (st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))
+ _restart(monkeypatch)
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ assert len(timed) == 2
+
+
+# Malformed cached records are a cache miss (re-timed), never a verdict, and never break the verdict log.
+
+_GOOD = {
+ "native_ms": 80.0,
+ "flash_ms": 70.0,
+ "rel_l1": 1e-3,
+ "max_rel": 2e-3,
+ "finite": True,
+ "tokens": 4096,
+}
+
+
+ "field, value",
+ [
+ ("native_ms", "80"),
+ ("flash_ms", "70"),
+ ("rel_l1", "1e-3"),
+ ("finite", "true"),
+ ("finite", 1),
+ ("tokens", "4096"),
+ ("tokens", 4096.5),
+ ("tokens", True),
+ ("native_ms", {"ms": 80}),
+ ("flash_ms", [70.0]),
+ ("rel_l1", None),
+ ("native_ms", float("nan")),
+ ("flash_ms", float("inf")),
+ ("rel_l1", float("nan")),
+ ("native_ms", -80.0),
+ ("flash_ms", 0.0),
+ ("rel_l1", -1e-3),
+ ("tokens", -4096),
+ ("admitted", "yes"),
+ ("geometry", {"hidden": "3072", "heads": 24, "head_dim": 128}),
+ ("geometry", [3072, 24, 128]),
+ ("native_ms", True),
+ ("drop", "flash_ms"),
+ ("drop", "max_rel"),
+ ("max_rel", "2e-3"),
+ ("max_rel", float("inf")),
+ ("max_rel", -1.0),
+ ("block_rel_l1", "x"),
+ ],
+)
+def test_malformed_cached_record_is_retimed(monkeypatch, tmp_path, field, value):
+ import json
+
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed, 90.0))
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ (path,) = _cache_files(tmp_path)
+ data = json.loads(path.read_text())
+ (slot,) = data["verdicts"]
+ rec = dict(_GOOD) # a stored win: a malformed copy of it must not admit flash
+ if field == "drop":
+ rec.pop(value)
+ else:
+ rec[field] = value
+ data["verdicts"][slot] = rec
+ path.write_text(json.dumps(data))
+ _restart(monkeypatch)
+ logs = []
+
+ class Log:
+ def info(self, *a):
+ logs.append(a[0] % a[1:])
+
+ assert (
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, logger = Log(), target = _target()
+ )
+ is None
+ )
+ assert len(timed) == 2 and any("probe" in line and "-> native" in line for line in logs)
+
+
+def test_non_finite_probe_verdict_is_stored_and_reused(monkeypatch, tmp_path):
+ # a flash output with NaN is a real (native) answer: stored, read back, never re-timed and never raising
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+
+ def nan_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ ):
+ timed.append(1)
+ return {**_GOOD, "finite": False, "rel_l1": float("nan"), "max_rel": float("nan")}
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", nan_time)
+ assert (
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ is None
+ )
+ _restart(monkeypatch)
+ assert (
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ is None
+ )
+ assert len(timed) == 1
+
+
+def test_implausible_probe_timing_is_native_and_not_stored(monkeypatch, tmp_path):
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(
+ att, "_time_rocm_flash_block", lambda *a, **k: {**_GOOD, "native_ms": 0.0, "flash_ms": 0.0}
+ )
+ assert (
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ is None
+ )
+ assert _cache_files(tmp_path) == []
+
+
+ "rec",
+ [
+ {},
+ {"flash_ms": "x", "native_ms": {"a": 1}, "rel_l1": [1], "tokens": "n", "admitted": "yes"},
+ {**_GOOD, "flash_ms": float("nan"), "rel_l1": float("inf"), "admitted": True},
+ ],
+)
+def test_verdict_log_never_raises(rec):
+ class Log:
+ def info(self, fmt, *a):
+ fmt % a
+
+ att._log_rocm_flash_verdict(Log(), "cuda:0", (3072, 24, 128), rec, "cached")
+ att._log_rocm_flash_verdict(Log(), "cuda:0", (), rec, "cached")
+
+
+# Accuracy is judged on the raw attention output: a wrong attention the residual + FFN would hide is still rejected.
+
+
+def test_wrong_attention_is_rejected_though_the_block_would_pass(monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ def off_by_20pct(q, k, v, **kw):
+ out = torch.nn.functional.scaled_dot_product_attention(
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
+ ).transpose(1, 2)
+ return out * 1.2
+
+ _diffusers_flash(monkeypatch, off_by_20pct)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
+ # inner == hidden as in a real DiT; at 1024 tokens the block output moves only ~1.7%
+ rec = att._time_rocm_flash_block("cpu", torch.bfloat16, 256, 4, 64, tokens = 1024)
+ # the old block-level check would have admitted it
+ assert rec["block_rel_l1"] <= att.ROCM_FLASH_BLOCK_TOL
+ assert rec["rel_l1"] > 0.15 and rec["max_rel"] > 0.1
+ assert att._rocm_flash_record_ok(rec) and not att._rocm_flash_wins(
+ {**rec, "native_ms": 80.0, "flash_ms": 40.0}
+ )
+
+
[email protected]("max_rel, wins", [(1e-2, True), (0.2, False)])
+def test_max_abs_error_alone_rejects(max_rel, wins):
+ assert att._rocm_flash_wins({**_GOOD, "max_rel": max_rel}) is wins
+
+
+# Lengths: calls are gated per power-of-two length bucket; an unseen bucket is timed on its first eager call and never
+# inherits the 4096 verdict.
+
+
+ "tokens, bucket",
+ [
+ (1, 256),
+ (300, 256),
+ (900, 512),
+ (4096, 4096),
+ (4608, 4096),
+ (8191, 4096),
+ (27280, 16384),
+ (50000, 32768),
+ (140000, 131072),
+ ],
+)
+def test_length_buckets(tokens, bucket):
+ assert att._rocm_flash_bucket(tokens) == bucket
+
+
+def _length_env(monkeypatch, verdicts):
+ """An auto load admitted at 4096; ``verdicts[tokens]`` is the flash_ms (native 80) the bucket probe measures."""
+ _rocm(monkeypatch)
+ guarded, ran = _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ kv_tokens = None,
+ ctx_dim = None,
+ batch = 1,
+ ):
+ size = tokens if kv_tokens is None else (tokens, kv_tokens)
+ if batch != 1:
+ size = (size, f"b{batch}")
+ if ctx_dim is not None:
+ size = (size, f"ctx{ctx_dim}")
+ timed.append(size)
+ return {
+ **_GOOD,
+ "flash_ms": verdicts[size],
+ "tokens": tokens,
+ **({} if kv_tokens is None else {"kv_tokens": kv_tokens}),
+ **({} if batch == 1 else {"batch": batch}),
+ }
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
+ dit = _Dit()
+ assert (
+ att.apply_attention_backend(
+ types.SimpleNamespace(transformer = dit), att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ == "flash"
+ )
+ monkeypatch.setattr(
+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d in ("cuda", "cpu") else d
+ )
+ return dit, guarded, ran, timed
+
+
+def test_unprobed_long_bucket_does_not_inherit_the_4096_verdict(monkeypatch):
+ dit, guarded, ran, timed = _length_env(
+ monkeypatch, {4096: 70.0, 16384: 90.0, (256, 1024): 70.0}
+ )
+ assert timed == [4096]
+ dit.run(guarded, [(4608, None), (27280, None), (20000, None), (16, 1200)])
+ assert timed == [4096, 16384, (256, 1024)]
+ assert ran == ["flash", "native", "native", "flash"]
+ assert att._ROCM_FLASH_ROUTED == {"slower or inexact at this size": 2}
+
+
+def test_50k_call_runs_flash_only_if_its_own_bucket_admitted_it(monkeypatch):
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 90.0, 65536: 60.0})
+ dit.run(guarded, [(50000, None), (70000, None), (140000, None)])
+ assert timed == [4096, 32768, 65536, 131072]
+ assert ran == ["native", "flash", "native"]
+
+
+def test_bucket_probe_error_stays_native_without_retrying_every_call(monkeypatch):
+ verdicts = {4096: 70.0}
+ dit, guarded, ran, timed = _length_env(monkeypatch, verdicts)
+ dit.run(guarded, [(40000, None), (40000, None)])
+ assert timed == [4096, 32768] and ran == ["native", "native"]
+ deferred = getattr(dit, att.ROCM_FLASH_DEFERRED_ATTR)
+ assert len(deferred) == 1 and next(iter(deferred)) not in att._ROCM_FLASH_BUCKETS
+ verdicts[32768] = 60.0
+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = dit)))
+ dit.run(guarded, [(40000, None)])
+ assert timed == [4096, 32768, 32768] and ran == ["native", "native", "flash"]
+
+
+# An in-forward probe sits on top of the live request's tensors; no room for it defers, never rejects.
+
+
+def test_bucket_probe_without_room_beside_the_forward_is_deferred_not_rejected(monkeypatch):
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 60.0})
+ needs = []
+ monkeypatch.setattr(
+ att, "_rocm_flash_probe_fits", lambda device, need: needs.append(need) or False
+ )
+ dit.run(guarded, [(40000, None), (40000, None)])
+ assert timed == [4096] and ran == ["native", "native"] and len(needs) == 1
+ assert att._ROCM_FLASH_BUCKETS == {} and len(getattr(dit, att.ROCM_FLASH_DEFERRED_ATTR)) == 1
+ assert att._ROCM_FLASH_ROUTED == {"timing deferred": 2}
+ assert not any(k[-1] == 32768 or 32768 in k for k in att.ROCM_FLASH_ADMISSIONS)
+ # Room again at the next generation: timed and admitted.
+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True)
+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = dit)))
+ dit.run(guarded, [(40000, None)])
+ assert timed == [4096, 32768] and ran[-1] == "flash"
+
+
+def test_deferral_belongs_to_its_dit_not_to_other_generations(monkeypatch):
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 60.0})
+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: False)
+ dit.run(guarded, [(40000, None)])
+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True)
+ other = _Dit()
+ att.apply_attention_backend(
+ types.SimpleNamespace(transformer = other), att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = other)))
+ dit.run(guarded, [(40000, None)])
+ assert timed == [4096] and ran == ["native", "native"]
+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = dit)))
+ dit.run(guarded, [(40000, None)])
+ assert timed == [4096, 32768] and ran[-1] == "flash"
+
+
+def test_a_moved_sdpa_baseline_drops_the_in_memory_verdicts(monkeypatch):
+ baseline = ["sdpafem,aotriton-exp1"]
+ monkeypatch.setattr(att, "_native_sdpa_baseline", lambda: baseline[0])
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 16384: 60.0})
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: "tc-" + baseline[0])
+ dit.run(guarded, [(20000, None), (20000, None)])
+ assert timed == [4096, 16384] and ran == ["flash", "flash"]
+ baseline[0] = "sdpa--m,aotriton-exp1"
+ dit.run(guarded, [(20000, None)])
+ assert timed == [4096, 16384, 16384] and len(att.ROCM_FLASH_ADMISSIONS) == 1
+ dit.run(guarded, [(20000, None)])
+ assert timed == [4096, 16384, 16384]
+
+
+def test_probe_bound_adds_math_sdpa_scores_when_fused_sdpa_is_off(monkeypatch):
+ torch = pytest.importorskip("torch")
+ g = (3072, 24, 128)
+ monkeypatch.setattr(att, "_native_sdpa_fused_serves", lambda *a: True)
+ fused = att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16)
+ monkeypatch.setattr(att, "_native_sdpa_fused_serves", lambda *a: False)
+ math = att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16)
+ assert math - fused == 2 * 24 * 16384 * 16384 * 4
+
+
+def test_fused_sdpa_eligibility_is_asked_for_the_probe_shape(monkeypatch):
+ torch = pytest.importorskip("torch")
+ seen = []
+
+ def can_flash(params, debug):
+ seen.append((tuple(params.query.shape), tuple(params.key.shape)))
+ return params.query.shape[2] < 32768
+
+ monkeypatch.setattr(torch.backends.cuda, "can_use_flash_attention", can_flash)
+ monkeypatch.setattr(torch.backends.cuda, "can_use_efficient_attention", lambda p, d: False)
+ monkeypatch.setattr(torch.backends.cuda, "flash_sdp_enabled", lambda: True)
+ g = (3072, 24, 128)
+ short = att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16, "cpu")
+ long = att._rocm_flash_probe_bytes(g, 32768, 512, 2, torch.bfloat16, "cpu")
+ assert seen == [
+ ((1, 24, 16384, 128), (1, 24, 16384, 128)),
+ ((2, 24, 32768, 128), (2, 24, 512, 128)),
+ ]
+ assert short < 4 * 2**30
+ assert long > 2 * 2 * 24 * 32768 * 512 * 4
+ monkeypatch.setattr(torch.backends.cuda, "flash_sdp_enabled", lambda: False)
+ assert (
+ att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16, "cpu") - short
+ == 2 * 24 * 16384**2 * 4
+ )
+
+
+def test_probe_tensors_are_drawn_on_the_device():
+ import inspect
+ src = inspect.getsource(att._time_rocm_flash_block)
+ assert ".to(device, dtype)" not in src and "torch.Generator(device = device)" in src
+
+
+def test_native_sdpa_baseline_keys_the_verdicts(monkeypatch, tmp_path):
+ pytest.importorskip("diffusers.models.attention_dispatch")
+ torch = pytest.importorskip("torch")
+ _fake_build(monkeypatch, tmp_path)
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)
+ flags = {"flash": True, "mem": True}
+ monkeypatch.setattr(torch.backends.cuda, "flash_sdp_enabled", lambda: flags["flash"])
+ monkeypatch.setattr(torch.backends.cuda, "mem_efficient_sdp_enabled", lambda: flags["mem"])
+ monkeypatch.setattr(torch.backends.cuda, "math_sdp_enabled", lambda: True)
+ monkeypatch.setenv("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")
+ fused = _REAL_TOOLCHAIN_KEY("cuda:0")
+ assert fused is not None and "|sdpafem,aotriton-exp1|" in fused
+ flags.update(flash = False, mem = False)
+ math_only = _REAL_TOOLCHAIN_KEY("cuda:0")
+ assert math_only != fused and "|sdpa--m,aotriton-exp1|" in math_only
+ flags.update(flash = True, mem = True)
+ monkeypatch.delenv("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL")
+ assert _REAL_TOOLCHAIN_KEY("cuda:0") not in (fused, math_only)
+
+
+def test_probe_byte_bound_covers_the_measured_peak(monkeypatch):
+ torch = pytest.importorskip("torch")
+ monkeypatch.setattr(att, "_native_sdpa_fused_serves", lambda *a: True)
+ for tokens, peak in ((1024, 0.31), (65536, 5.46)):
+ need = (
+ att._rocm_flash_probe_bytes((3072, 24, 128), tokens, tokens, 1, torch.bfloat16) / 2**30
+ )
+ assert peak <= need < 2 * peak
+
+
+def test_probe_fits_counts_cached_blocks_and_fails_closed(monkeypatch):
+ torch = pytest.importorskip("torch")
+ gib = 1 << 30
+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda d: (2 * gib, 80 * gib))
+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d: 10 * gib)
+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda d: 7 * gib)
+ assert _REAL_PROBE_FITS("cuda:0", 3 * gib)
+ assert not _REAL_PROBE_FITS("cuda:0", 4 * gib)
+
+ def boom(d):
+ raise RuntimeError("no device")
+
+ monkeypatch.setattr(torch.cuda, "mem_get_info", boom)
+ assert not _REAL_PROBE_FITS("cuda:0", 1)
+
+
+def test_unprobed_bucket_under_capture_runs_native_untimed(monkeypatch):
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 8192: 70.0})
+ monkeypatch.setattr(att, "_rocm_flash_capturing", lambda: True)
+ dit.run(guarded, [(9000, None)])
+ assert timed == [4096] and ran == ["native"]
+ monkeypatch.setattr(att, "_rocm_flash_capturing", lambda: False)
+ dit.run(guarded, [(9000, None)])
+ assert timed == [4096, 8192] and ran == ["native", "flash"]
+
+
+def test_bucket_verdicts_persist(monkeypatch, tmp_path):
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 16384: 60.0})
+ dit.run(guarded, [(20000, None)])
+ assert timed == [4096, 16384]
+ _restart(monkeypatch)
+ monkeypatch.setattr(att, "_ROCM_FLASH_BUCKETS", {})
+ dit2, guarded2, ran2, timed2 = _length_env(monkeypatch, {})
+ dit2.run(guarded2, [(20000, None)])
+ assert timed2 == [] and ran2 == ["flash"]
+
+
[email protected]("order", ["auto_first", "explicit_first"])
[email protected]("explicit", ["flash", "mode1"])
+def test_explicit_flash_is_never_length_gated(monkeypatch, order, explicit):
+ # an auto model and an explicitly selected one share (heads, head_dim): only the auto one's calls are gated
+ _rocm(monkeypatch)
+ guarded, ran = _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ ):
+ timed.append(tokens)
+ return {**_GOOD, "flash_ms": 70.0 if tokens == 4096 else 90.0, "tokens": tokens}
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
+ auto_dit, explicit_dit = _Dit(), _Dit()
+
+ def load_auto():
+ assert (
+ att.apply_attention_backend(
+ types.SimpleNamespace(transformer = auto_dit),
+ att.ROCM_FLASH_CANDIDATE,
+ target = _target(),
+ )
+ == "flash"
+ )
+
+ def load_explicit():
+ if explicit == "flash":
+ backend = att.apply_attention_backend(
+ types.SimpleNamespace(transformer = explicit_dit), "flash", target = _target()
+ )
+ else:
+ monkeypatch.setenv(att.ROCM_FLASH_ENV, "1")
+ backend = att.apply_attention_backend(
+ types.SimpleNamespace(transformer = explicit_dit),
+ att.ROCM_FLASH_CANDIDATE,
+ target = _target(),
+ )
+ monkeypatch.delenv(att.ROCM_FLASH_ENV)
+ assert backend == "flash"
+
+ for load in (load_auto, load_explicit) if order == "auto_first" else (load_explicit, load_auto):
+ load()
+ monkeypatch.setattr(
+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d in ("cuda", "cpu") else d
+ )
+ explicit_dit.run(guarded, [(40000, None)])
+ assert ran == ["flash"] and timed == [4096]
+ auto_dit.run(guarded, [(40000, None)])
+ assert ran == ["flash", "native"] and timed == [4096, 32768]
+ explicit_dit.run(guarded, [(40000, None)])
+ assert ran[-1] == "flash" and getattr(att._ROCM_FLASH_LOCAL, "gate", None) is None
+
+
+def test_reapplying_explicit_flash_disarms_the_gate(monkeypatch):
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 90.0})
+ assert (
+ att.apply_attention_backend(
+ types.SimpleNamespace(transformer = dit), "flash", target = _target()
+ )
+ == "flash"
+ )
+ dit.run(guarded, [(40000, None)])
+ assert ran == ["flash"] and timed == [4096]
+
+
+def test_failed_forward_does_not_leave_its_gate_active(monkeypatch):
+ torch = pytest.importorskip("torch")
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+
+ class Boom(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.attn = torch.nn.Module()
+ self.attn.heads = 2
+ self.attn.to_q = torch.nn.Linear(32, 32, bias = False)
+
+ def set_attention_backend(self, name):
+ pass
+
+ def forward(self, x):
+ raise RuntimeError("cancelled")
+
+ dit = Boom()
+ att.apply_attention_backend(
+ types.SimpleNamespace(transformer = dit), att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ with pytest.raises(RuntimeError):
+ dit(torch.zeros(1))
+ assert getattr(att._ROCM_FLASH_LOCAL, "gate", None) is None
+
+
+class _Dit:
+ """A DiT whose forward makes attention calls (masked or not) through the guarded FLASH backend."""
+
+ def __init__(self):
+ torch = pytest.importorskip("torch")
+ self.inner = torch.nn.Module()
+ self.backends = []
+ self.inner.heads = 2
+ self.inner.to_q = torch.nn.Linear(32, 32, bias = False)
+ self.hooks = {"pre": [], "post": []}
+
+ def modules(self):
+ return iter([self.inner])
+
+ def set_attention_backend(self, name):
+ self.backends.append(name)
+
+ def register_forward_pre_hook(self, fn):
+ self.hooks["pre"].append(fn)
+ return fn
+
+ def register_forward_hook(
+ self,
+ fn,
+ always_call = False,
+ ):
+ self.hooks["post"].append(fn)
+ return fn
+
+ def run(self, guarded, specs):
+ """One forward making a call per ``(q_tokens, kv_tokens or None)``, unmasked."""
+ import torch
+
+ for fn in self.hooks["pre"]:
+ fn(self, ())
+ for tokens, kv in specs:
+ q = torch.zeros(1, tokens, 2, 16, dtype = torch.bfloat16)
+ k = torch.zeros(1, kv or tokens, 2, 16, dtype = torch.bfloat16)
+ guarded(q, k, k, None)
+ for fn in self.hooks["post"]:
+ fn(self, (), None)
+
+ def __call__(self, guarded, masks):
+ import torch
+
+ for fn in self.hooks["pre"]:
+ fn(self, ())
+ q = torch.zeros(1, 4096, 2, 16, dtype = torch.bfloat16)
+ for masked in masks:
+ guarded(q, q, q, attn_mask = torch.ones(1, 1, 8, 8, dtype = torch.bool) if masked else None)
+ for fn in self.hooks["post"]:
+ fn(self, (), None)
+
+
+def _guarded_flash(monkeypatch):
+ ad = pytest.importorskip("diffusers.models.attention_dispatch")
+ backends = ad._AttentionBackendRegistry._backends
+ ran = []
+
+ def fake_flash(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ **kw,
+ ):
+ if attn_mask is not None:
+ raise ValueError("`attn_mask` is not supported for flash-attn 2.")
+ ran.append("flash")
+ return query
+
+ def fake_native(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ **kw,
+ ):
+ ran.append("native")
+ return query
+
+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, fake_flash)
+ monkeypatch.setitem(backends, ad.AttentionBackendName.NATIVE, fake_native)
+ assert att._install_rocm_flash_dispatch_guard() is True
+ return backends[ad.AttentionBackendName.FLASH], ran
+
+
+def _state(pipe, backend):
+ import dataclasses
+
+ @dataclasses.dataclass(frozen = True)
+ class State:
+ pipe: object
+ attention_backend: object
+ resolved: dict
+
+ resolved = {
+ "attention_backend": {"source": "auto", "value": backend or "native", "reason": "load"}
+ }
+ return State(pipe, backend, resolved)
+
+
+def _load(monkeypatch):
+ dit = _Dit()
+ pipe = types.SimpleNamespace(transformer = dit)
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
+ return dit, pipe, _state(pipe, "flash")
+
+
+ "masks, value, phrase",
+ [
+ (
+ [True, True],
+ "native",
+ "every attention call of the last generation ran native SDPA (attn_mask: 2)",
+ ),
+ ([False, False], "flash", "on every attention call"),
+ (
+ [True, False],
+ "flash",
+ "on 1 of 2 attention calls of the last generation; the rest ran native SDPA (attn_mask: 1)",
+ ),
+ ],
+)
+def test_badge_reports_the_routes_the_run_took(monkeypatch, masks, value, phrase):
+ _rocm(monkeypatch)
+ guarded, ran = _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+ dit, pipe, state = _load(monkeypatch)
+ att.begin_attention_run(state)
+ dit(guarded, masks)
+ att.report_attention_run(state)
+ assert ran == ["native" if m else "flash" for m in masks]
+ entry = state.resolved["attention_backend"]
+ assert entry["value"] == value and phrase in entry["reason"] and entry["source"] == "auto"
+ # report only: the engaged backend (recipes, calibration and compile keys) is untouched, nothing was switched
+ assert state.attention_backend == "flash" and dit.backends == ["flash"]
+ ran.clear()
+ dit(guarded, [False])
+ assert ran == ["flash"]
+
+
+def test_mixed_mask_model_keeps_flash_for_unmasked_calls_across_loads(monkeypatch, tmp_path):
+ # HunyuanVideo 1.5 style: padded batches mask, unpadded ones do not. A masked-only first run changes nothing for
+ # the next load, which serves its unmasked calls on flash.
+ _rocm(monkeypatch)
+ guarded, ran = _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))
+ dit, pipe, state = _load(monkeypatch)
+ dit(guarded, [True, True])
+ _restart(monkeypatch)
+ dit2, pipe2, state2 = _load(monkeypatch)
+ assert len(timed) == 1 and dit2.backends == ["flash"]
+ ran.clear()
+ dit2(guarded, [False, True])
+ assert ran == ["flash", "native"]
+ assert not list((tmp_path / "rocm_flash").glob("masked*"))
+
+
+def test_each_generation_reports_its_own_calls(monkeypatch):
+ # run 1 flash, run 2 masked only, run 3 flash: each badge describes its own run, never a lifetime mix
+ _rocm(monkeypatch)
+ guarded, ran = _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+ dit, pipe, state = _load(monkeypatch)
+ for masks, value in (([False, False], "flash"), ([True, True], "native"), ([False], "flash")):
+ att.begin_attention_run(state)
+ dit(guarded, masks)
+ att.report_attention_run(state)
+ assert state.resolved["attention_backend"]["value"] == value
+ assert state.attention_backend == "flash"
+
+
+def test_compiled_run_reports_routing_untracked(monkeypatch):
+ # a compiled / graphed run never re-enters the guard's Python: the badge says routing is not tracked
+ _rocm(monkeypatch)
+ _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+ dit, pipe, state = _load(monkeypatch)
+ att.begin_attention_run(state)
+ dit.run(lambda *a, **k: None, [])
+ att.report_attention_run(state)
+ entry = state.resolved["attention_backend"]
+ assert (
+ entry["value"] == "flash"
+ and "not tracked for compiled or graphed attention" in entry["reason"]
+ )
+
+
+def test_route_tallies_are_per_thread(monkeypatch):
+ # an image and a video forward on two threads at once: each DiT's tally gets only its own calls, and each thread's
+ # gate is its own
+ import threading
+
+ _rocm(monkeypatch)
+ guarded, ran = _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+ a, _, _ = _load(monkeypatch)
+ b = _Dit()
+ assert (
+ att.apply_attention_backend(types.SimpleNamespace(transformer = b), "flash", target = _target())
+ == "flash"
+ )
+ inside = threading.Barrier(2)
+ gates = {}
+
+ class Pausing:
+ def __init__(self, name):
+ self.name = name
+
+ def __call__(self, *args, **kwargs):
+ if self.name not in gates:
+ inside.wait(timeout = 10)
+ gates[self.name] = att._active_rocm_flash_gate()
+ return guarded(*args, **kwargs)
+
+ ta = threading.Thread(target = a, args = (Pausing("a"), [True, True, True]))
+ tb = threading.Thread(target = b, args = (Pausing("b"), [False]))
+ ta.start(), tb.start()
+ ta.join(10), tb.join(10)
+ tally_a = getattr(a, att.ROCM_FLASH_TALLY_ATTR)
+ tally_b = getattr(b, att.ROCM_FLASH_TALLY_ATTR)
+ assert tally_a == {"attn_mask": 3} and tally_b == {"flash": 1}
+ assert gates["a"] == getattr(a, att.ROCM_FLASH_GATE_ATTR) and gates["b"] is None
+
+
+# The activation memory guard must not trust the flash pick when calls can run native SDPA.
+
+
[email protected]("routed, quadratic", [(True, True), (False, False)])
+def test_memory_guard_budgets_quadratic_when_flash_can_hand_calls_to_native(
+ monkeypatch, routed, quadratic
+):
+ import core.inference.diffusion as diffusion
+
+ _rocm(monkeypatch)
+ _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+ monkeypatch.setattr(diffusion, "sdpa_math_only", lambda target: True)
+ import core.inference.diffusion_qwenimage21_math as qm
+
+ monkeypatch.setattr(qm, "bounded_math_attention", lambda pipe: False)
+ dit, pipe, state = _load(monkeypatch)
+ if not routed:
+ setattr(dit, att._ROCM_FLASH_HOOKS_ATTR, None)
+ assert diffusion._quadratic_attention(object(), state.attention_backend, pipe) is quadratic
+
+
+def test_diffusers_dispatch_change_retimes(monkeypatch, tmp_path):
+ ad = pytest.importorskip("diffusers.models.attention_dispatch")
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ binary = _fake_build(monkeypatch, tmp_path)
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)
+ backends = ad._AttentionBackendRegistry._backends
+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, ad._flash_attention)
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))
+ key = _REAL_TOOLCHAIN_KEY("cuda:0")
+ assert key is not None and "|diffusers" in key and _REAL_TOOLCHAIN_KEY("cuda:0") == key
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ _restart(monkeypatch)
+
+ def _flash_attention(
+ query,
+ key,
+ value,
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ scale = None,
+ ):
+ return query # a diffusers release whose flash wrapper changed
+
+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, _flash_attention)
+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != key
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ assert len(timed) == 2
+ # a Studio guard around the same function does not change it
+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, ad._flash_attention)
+ assert _REAL_GUARD() and _REAL_TOOLCHAIN_KEY("cuda:0") == key
+ del binary
+
+
[email protected]("module", ["core.inference.diffusion", "core.inference.video"])
+def test_both_generate_paths_mark_the_generation_boundary(module):
+ import ast
+ import importlib
+ import inspect
+
+ src = inspect.getsource(importlib.import_module(module))
+ tree = ast.parse(src)
+ gens = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "generate"]
+ called = {
+ getattr(c.func, "id", None) for g in gens for c in ast.walk(g) if isinstance(c, ast.Call)
+ }
+ assert gens and {"begin_attention_run", "report_attention_run"} <= called
+
+
+# Compiled execution (Studio compiles the denoiser's blocks on ROCm since its regional compile probe): an auto-gated
+# DiT's traced call takes its bucket's verdict at trace time, timing the bucket then if it is unknown, and the answer is
+# baked into the graph without a graph break. Real hooks, real torch.compile on CPU.
+
+
+class _CompiledDit(__import__("torch").nn.Module):
+ def __init__(self, guarded):
+ import torch
+
+ super().__init__()
+ self.attn = torch.nn.Module()
+ self.attn.heads = 2
+ self.attn.to_q = torch.nn.Linear(32, 32, bias = False)
+ self._guarded = guarded
+
+ def set_attention_backend(self, name):
+ pass
+
+ def forward(self, q):
+ return self._guarded(q, q, q, None)
+
+
+def _compiled_env(monkeypatch, auto, flash_ms):
+ torch = pytest.importorskip("torch")
+ _rocm(monkeypatch)
+ ad = pytest.importorskip("diffusers.models.attention_dispatch")
+ backends = ad._AttentionBackendRegistry._backends
+ monkeypatch.setitem(
+ backends,
+ ad.AttentionBackendName.FLASH,
+ lambda query, key, value, attn_mask = None, **k: query + 1,
+ )
+ monkeypatch.setitem(
+ backends,
+ ad.AttentionBackendName.NATIVE,
+ lambda query, key, value, attn_mask = None, **k: query - 1,
+ )
+ assert _REAL_GUARD()
+ guarded = backends[ad.AttentionBackendName.FLASH]
+ _apply_env(monkeypatch)
+ timed = []
+
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ **kw,
+ ):
+ timed.append(tokens)
+ return {**_GOOD, "flash_ms": flash_ms.get(tokens, 90.0), "tokens": tokens}
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
+ dit = _CompiledDit(guarded)
+ backend = att.ROCM_FLASH_CANDIDATE if auto else "flash"
+ assert (
+ att.apply_attention_backend(
+ types.SimpleNamespace(transformer = dit), backend, target = _target()
+ )
+ == "flash"
+ )
+ monkeypatch.setattr(
+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d in ("cuda", "cpu") else d
+ )
+ return torch, dit, timed
+
+
[email protected]("auto", [True, False])
+def test_compiled_auto_dit_takes_the_bucket_verdict_under_real_torch_compile(monkeypatch, auto):
+ # 4096 admitted at load; 8192 untimed until a traced call needs it, and it wins; 16384 loses
+ torch, dit, timed = _compiled_env(monkeypatch, auto, {4096: 70.0, 8192: 70.0})
+ q = torch.zeros(1, 4608, 2, 16, dtype = torch.bfloat16)
+ assert float(dit(q).flatten()[0]) == 1.0
+ torch._dynamo.reset()
+ counters = __import__("torch._dynamo.utils", fromlist = ["counters"]).counters
+ counters.clear()
+ compiled = torch.compile(dit, backend = "eager", fullgraph = True)
+ assert float(compiled(q).flatten()[0]) == 1.0 # traced: the known verdict, no graph break
+ q8 = torch.zeros(1, 9000, 2, 16, dtype = torch.bfloat16)
+ assert float(compiled(q8).flatten()[0]) == 1.0 # timed at trace time, then flash
+ q16 = torch.zeros(1, 17000, 2, 16, dtype = torch.bfloat16)
+ assert float(compiled(q16).flatten()[0]) == (-1.0 if auto else 1.0)
+ # explicit flash is never gated, so nothing past the load is timed for it
+ assert timed == ([4096, 8192, 16384] if auto else [])
+ assert not counters["graph_break"]
+ assert getattr(att._ROCM_FLASH_LOCAL, "gate", None) is None
+ torch._dynamo.reset()
+
+
+def test_compiled_dynamic_lengths_recompile_per_bucket_not_per_length(monkeypatch):
+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0, 8192: 90.0})
+ torch._dynamo.reset()
+ frames = []
+ compiled = torch.compile(
+ dit,
+ backend = lambda gm, inputs: frames.append(gm) or gm.forward,
+ dynamic = True,
+ fullgraph = True,
+ )
+ bf16 = torch.bfloat16
+ out = [
+ float(compiled(torch.zeros(1, n, 2, 16, dtype = bf16)).flatten()[0])
+ for n in (4352, 4608, 5000)
+ ]
+ assert out == [1.0, 1.0, 1.0] and len(frames) == 1
+ assert float(compiled(torch.zeros(1, 9000, 2, 16, dtype = bf16)).flatten()[0]) == -1.0
+ assert len(frames) == 2 and timed == [4096, 8192]
+ torch._dynamo.reset()
+
+
+def test_traced_call_with_no_room_for_its_probe_runs_native(monkeypatch):
+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0, 8192: 70.0})
+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: False)
+ torch._dynamo.reset()
+ compiled = torch.compile(dit, backend = "eager", fullgraph = True)
+ assert float(compiled(torch.zeros(1, 9000, 2, 16, dtype = torch.bfloat16)).flatten()[0]) == -1.0
+ assert (
+ timed == [4096]
+ and att._ROCM_FLASH_BUCKETS.get(("cuda:0", "torch.bfloat16", 32, 2, 16, 8192, 8192, 1))
+ is None
+ )
+ torch._dynamo.reset()
+
+
+# Cross-attention is timed as the rectangle that runs (long video query x short text key), never as a square.
+
+
+def test_cross_attention_is_not_decided_by_a_square_probe(monkeypatch):
+ dit, guarded, ran, timed = _length_env(
+ monkeypatch, {4096: 70.0, 16384: 60.0, (16384, 512): 90.0}
+ )
+ dit.run(guarded, [(27280, None), (27280, 512), (27280, 600)])
+ assert timed == [4096, 16384, (16384, 512)]
+ assert ran == ["flash", "native", "native"]
+ rec = att.ROCM_FLASH_ADMISSIONS[("cuda:0", "torch.bfloat16", (32, 2, 16), 16384, 512, 1)]
+ assert rec["kv_tokens"] == 512 and rec["admitted"] is False
+
+
+def test_rectangular_probe_runs_on_cpu(monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ def flash_attn_func(q, k, v, **kw):
+ assert q.shape[1] == 64 and k.shape[1] == 16
+ return torch.nn.functional.scaled_dot_product_attention(
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
+ ).transpose(1, 2)
+
+ _diffusers_flash(monkeypatch, flash_attn_func)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
+ rec = att._time_rocm_flash_block("cpu", torch.bfloat16, 64, 4, 16, tokens = 64, kv_tokens = 16)
+ assert rec["tokens"] == 64 and rec["kv_tokens"] == 16 and rec["rel_l1"] == 0.0
+ assert att._rocm_flash_record_ok(rec)
+
+
+def test_named_partition_is_in_run_accounting(monkeypatch):
+ from core.inference.video import _NamedDiTView
+
+ _rocm(monkeypatch)
+ guarded, ran = _guarded_flash(monkeypatch)
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))
+ ref = _Dit()
+ pipe = types.SimpleNamespace(transformer_ref = ref)
+ assert att._attention_dits(pipe) == [] # the stock enumeration cannot see it
+ view = _NamedDiTView(pipe, "transformer_ref")
+ assert att.apply_attention_backend(view, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
+ state = _state(pipe, "flash")
+ assert att.attention_may_run_native(pipe)
+ att.begin_attention_run(state)
+ ref(guarded, [True, True])
+ att.report_attention_run(state)
+ assert state.resolved["attention_backend"]["value"] == "native"
+ att.begin_attention_run(state)
+ assert getattr(ref, att.ROCM_FLASH_TALLY_ATTR) == {}
+
+
+def test_batch_two_is_not_decided_by_a_batch_one_verdict(monkeypatch):
+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, (4096, "b2"): 90.0})
+ import torch
+
+ def call(batch):
+ for fn in dit.hooks["pre"]:
+ fn(dit, ())
+ q = torch.zeros(batch, 4608, 2, 16, dtype = torch.bfloat16)
+ guarded(q, q, q, None)
+ for fn in dit.hooks["post"]:
+ fn(dit, (), None)
+
+ call(1)
+ call(2)
+ call(3)
+ assert timed == [4096, (4096, "b2")] and ran == ["flash", "native", "native"]
+ rec = att.ROCM_FLASH_ADMISSIONS[("cuda:0", "torch.bfloat16", (32, 2, 16), 4096, 4096, 2)]
+ assert rec["batch"] == 2 and rec["admitted"] is False
+
+
+def test_batched_probe_runs_on_cpu(monkeypatch):
+ torch = pytest.importorskip("torch")
+ shapes = []
+
+ def flash_attn_func(q, k, v, **kw):
+ shapes.append((tuple(q.shape), tuple(k.shape)))
+ return torch.nn.functional.scaled_dot_product_attention(
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)
+ ).transpose(1, 2)
+
+ _diffusers_flash(monkeypatch, flash_attn_func)
+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)
+ rec = att._time_rocm_flash_block(
+ "cpu", torch.bfloat16, 64, 4, 16, tokens = 64, kv_tokens = 16, ctx_dim = 48, batch = 2
+ )
+ assert shapes[0] == ((2, 64, 4, 16), (2, 16, 4, 16))
+ assert (
+ rec["batch"] == 2
+ and rec["kv_tokens"] == 16
+ and rec["rel_l1"] == 0.0
+ and att._rocm_flash_record_ok(rec)
+ )
+
+
+def test_context_width_is_part_of_the_geometry(monkeypatch):
+ torch = pytest.importorskip("torch")
+
+ class Cross(torch.nn.Module):
+ def __init__(self, kin):
+ super().__init__()
+ self.heads = 2
+ self.to_q = torch.nn.Linear(32, 32, bias = False)
+ self.to_k = torch.nn.Linear(kin, 32, bias = False)
+
+ dit = torch.nn.Module()
+ dit.self_attn, dit.cross = Cross(32), Cross(48)
+ original = att._attention_dits
+ try:
+ att._attention_dits = lambda p: [dit]
+ assert att._dit_attention_geometries(types.SimpleNamespace()) == (
+ (32, 2, 16),
+ (32, 2, 16, 48),
+ )
+ finally:
+ att._attention_dits = original
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ seen = []
+
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ kv_tokens = None,
+ ctx_dim = None,
+ batch = 1,
+ ):
+ seen.append(ctx_dim)
+ return {**_GOOD, "tokens": tokens}
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
+ assert att._rocm_flash_geometry_admitted("cuda:0", "torch.bfloat16", (32, 2, 16, 48))
+ assert att._rocm_flash_geometry_admitted("cuda:0", "torch.bfloat16", (32, 2, 16))
+ assert seen == [48, None]
+ rec = att.ROCM_FLASH_ADMISSIONS[("cuda:0", "torch.bfloat16", (32, 2, 16, 48), 4096, 4096, 1)]
+ assert rec["geometry"]["context"] == 48 and att._rocm_flash_record_ok(rec)
+
+
+# MiniMax-H3's resident INT8 load fuses to_q / to_k / to_v into to_qkv before the backend is applied.
+
+
+def test_fused_qkv_geometry_is_collected():
+ torch = pytest.importorskip("torch")
+
+ class Fused(torch.nn.Module):
+ def __init__(self, out):
+ super().__init__()
+ self.heads = 2
+ self.to_qkv = torch.nn.Linear(32, out, bias = False)
+
+ dit = torch.nn.Module()
+ dit.fused, dit.odd = Fused(96), Fused(95)
+ original = att._attention_dits
+ try:
+ att._attention_dits = lambda p: [dit]
+ assert att._dit_attention_geometries(types.SimpleNamespace()) == ((32, 2, 16),)
+ finally:
+ att._attention_dits = original
+
+
+# The key carries the ROCm runtime / kernel driver the process runs on, not only the wheel's build.
+
+
+def test_driver_change_retimes(monkeypatch, tmp_path):
+ pytest.importorskip("diffusers.models.attention_dispatch")
+ _rocm(monkeypatch)
+ _apply_env(monkeypatch)
+ _fake_build(monkeypatch, tmp_path)
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)
+ driver = ["driver:hipdriver70051234,amdgpu6.12.12"]
+ monkeypatch.setattr(att, "_rocm_driver_version", lambda: driver[0])
+ timed = []
+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))
+ key = _REAL_TOOLCHAIN_KEY("cuda:0")
+ assert key is not None and "|driver:hipdriver70051234,amdgpu6.12.12|" in key
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ _restart(monkeypatch)
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ assert len(timed) == 1
+ driver[0] = "driver:hipdriver70151234,amdgpu6.14.4"
+ _restart(monkeypatch)
+ att.apply_attention_backend(
+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()
+ )
+ assert len(timed) == 2
+
+
+def test_driver_version_reads_without_raising(monkeypatch):
+ monkeypatch.setattr(att, "_ROCM_DRIVER_VERSION", [])
+ v = att._rocm_driver_version()
+ assert v.startswith("driver:") and att._rocm_driver_version() is v
+
+
+def test_attention_status_reason():
+ assert att.attention_status_reason(None, "flash") == "verified ROCm flash attention"
+ assert att.attention_status_reason("auto", "flash") == "verified ROCm flash attention"
+ assert att.attention_status_reason(None, "_native_cudnn") == "cuDNN fused attention upgrade"
+ assert att.attention_status_reason(None, None) == "diffusers default"
+ assert att.attention_status_reason("flash", "flash") == "requested"
+ assert att.attention_status_reason("flash", None) == "diffusers default"
+
+
+# The in-memory admission key keeps geometry, query length, key length and batch in separate fields: a cross-attention
+# geometry whose context width equals a bucket edge must not alias a narrower geometry at rectangular lengths.
+
+
+def test_admission_key_does_not_alias_geometry_and_lengths(monkeypatch):
+ _apply_env(monkeypatch)
+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: None)
+ timed = []
+
+ def fake_time(
+ device,
+ dtype,
+ hidden,
+ heads,
+ head_dim,
+ tokens = 4096,
+ kv_tokens = None,
+ **kw,
+ ):
+ timed.append((hidden, heads, head_dim, tokens, kv_tokens, kw.get("ctx_dim")))
+ wins = kv_tokens is None
+ return {
+ "native_ms": 80.0,
+ "flash_ms": 60.0 if wins else 90.0,
+ "rel_l1": 1e-3,
+ "max_rel": 1e-3,
+ "finite": True,
+ "tokens": tokens,
+ **({} if kv_tokens is None else {"kv_tokens": kv_tokens}),
+ }
+
+ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)
+ assert (
+ att._rocm_flash_geometry_admitted("cuda:0", "bf16", (32, 2, 16, 256), tokens = 4096) is True
+ )
+ # (32, 2, 16) at 256 x 4096 flattens to the same tuple as (32, 2, 16, 256) at 4096: it must be timed on its own
+ assert (
+ att._rocm_flash_geometry_admitted("cuda:0", "bf16", (32, 2, 16), tokens = 256, kv_tokens = 4096)
+ is False
+ )
+ assert timed == [(32, 2, 16, 4096, None, 256), (32, 2, 16, 256, 4096, None)]
+ assert len(att.ROCM_FLASH_ADMISSIONS) == 2
+
+
+# A re-apply on the same pipe that no longer picks ROCm flash takes the tracking off with it: the badge and the memory
+# guard must not keep reading the pipe as flash.
+
+
+def test_reapply_to_native_removes_rocm_flash_tracking(monkeypatch):
+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0})
+ pipe = types.SimpleNamespace(transformer = dit)
+ att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target())
+ assert att.attention_may_run_native(pipe) and att.rocm_flash_ran(pipe) is not None
+ hooks = len(dit._forward_pre_hooks) + len(dit._forward_hooks)
+ assert att.apply_attention_backend(pipe, None, target = _target()) is None
+ assert not att.attention_may_run_native(pipe) and att.rocm_flash_ran(pipe) is None
+ assert len(dit._forward_pre_hooks) + len(dit._forward_hooks) == hooks - 2
+ entry = {"value": "native", "reason": "diffusers default"}
+ state = types.SimpleNamespace(pipe = pipe, resolved = {"attention_backend": entry})
+ att.begin_attention_run(state)
+ att.report_attention_run(state)
+ assert state.resolved["attention_backend"] == entry
+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"
+ assert att.attention_may_run_native(pipe)