Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: pick ROCm flash attention by a whole-block timing probe instead of a model table (#13792)
Daniel Han3c14a2b3359b
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex c8c35d6f365..7d1c35c10a7 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -250,10 +250,14 @@ from .diffusion_vae_fp16 import enable_fp16_vae_decodefrom .diffusion_vae_tiling import install as install_wide_vae_tilesfrom .diffusion_attention import (apply_attention_backend,+ attention_status_reason,auto_attention_reason,normalize_attention_backend,sdpa_math_only,sdpa_subquadratic_confirmed,+ attention_may_run_native,+ begin_attention_run,+ report_attention_run,select_attention_backend,_ensure_attention_backend_installed,)@@ -2295,9 +2299,14 @@ def _quadratic_attention(pipe: Any = None,) -> bool:"""Whether attention can only run on SDPA math (quadratic memory); False when unknown.- Any engaged non-native backend is a fused kernel, so the SDPA probe only speaks for native."""+ An engaged non-native backend is a fused kernel, so the SDPA probe only speaks for native, unless that backend can+ hand calls to native SDPA (ROCm flash: masked calls, lengths it does not serve): then budget as native."""if engaged_backend is not None and str(engaged_backend) != "native":- return False+ try:+ if pipe is None or not attention_may_run_native(pipe):+ return False+ except Exception: # noqa: BLE001+ return Falsetry:if not sdpa_math_only(target):return False@@ -9588,11 +9597,7 @@ class DiffusionBackend:"attention_backend": (attention_backend,attention_engaged or "native",- auto_attention_reason(attention_engaged)- if attention_engaged and attention_backend is None- else "diffusers default"- if attention_engaged is None- else "requested",+ attention_status_reason(attention_backend, attention_engaged),),"memory_mode": (memory_mode,@@ -11924,6 +11929,7 @@ class DiffusionBackend:# Publish an active (step 0) state before the slow pre-denoise setup so a reload mount probe does not# read idle.self._gen = _GenState(total_steps = steps)+ begin_attention_run(state)# Reset in the finally, so a failed or cancelled generation frees its reused outputs.static_skip_pipe = Nonerestore_scheduler: Optional[tuple[Any, Any]] = None@@ -12759,6 +12765,9 @@ class DiffusionBackend:compile_cache.save_async(state.compile_cache_ctx, logger = logger)except Exception: # noqa: BLE001 - cache persistence is best-effortpass+ # The status badge reports the attention routes this run took (report only: the recipe keeps the+ # engaged backend).+ report_attention_run(state)# Last word on cancellation, AFTER the post-denoise work: the event stays registered through the# compile-cache bookkeeping and the page still shows Stop for as long as progress reads active, so a Stop# landing there was answered cancelled = true and then contradicted by the image the route persisted.diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex 33d84bc5b54..ffc128b8850 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -24,7 +24,9 @@ Attention is bandwidth-bound, so a better kernel is a real win orthogonal to weisage - SageAttention 2 (INT8 QK); quantized, small quality cost. A pip SageAttention 2 when installed, else thekernels-community build from the Hugging Face kernels hub (``sage_hub``, Ampere / Ada / Hopper).xformers / aiter - memory-efficient (NVIDIA) / AITER (AMD ROCm).- flash on ROCm - only when the DAO-AILab ROCm build imports and passes an on-card check; ``auto`` stays native.+ flash on ROCm - only when the DAO-AILab ROCm build passes an on-card check; ``auto`` also needs a block timing win+ (``UNSLOTH_DIFFUSION_ROCM_FLASH``: auto | 0 | 1). The timing persists per toolchain and geometry under+ the Studio cache (``UNSLOTH_DIFFUSION_ROCM_FLASH_CACHE_DIR``), so a restart reads it instead of re-timing.Best-effort: an unavailable backend falls back to the diffusers default. torch/diffusers lazy."""@@ -445,8 +447,8 @@ def select_attention_backend(An explicit alias is honored (apply falls back if its kernel is unavailable). ``auto``upgrades to cuDNN on NVIDIA CUDA only when a speed profile is active (so ``off`` stays- bit-identical), and to verified ROCm flash for ``ROCM_AUTO_FLASH_FAMILIES`` on gfx11 unless speed is- explicitly ``off``; elsewhere returns None (native)."""+ bit-identical), and on ROCm with verified flash_attn to ``ROCM_FLASH_CANDIDATE`` (resolved by timing in+ ``apply_attention_backend``) unless speed is explicitly ``off``; elsewhere returns None (native)."""alias = normalize_attention_backend(requested)if alias != ATTN_AUTO:backend = _ALIASES[alias]@@ -472,10 +474,8 @@ def select_attention_backend(return backendif speed_active and _is_cuda_nvidia(target) and _cudnn_attention_supported():return "_native_cudnn"- if speed_active and ROCM_AUTO_FLASH and _is_cuda_rocm(target) and _rocm_flash_attn_runs(target):- return "flash"if (speed_active or speed_unset) and _rocm_auto_flash(target, family):- return "flash"+ return ROCM_FLASH_CANDIDATEreturn None@@ -490,16 +490,51 @@ def auto_attention_reason(engaged: Optional[str]) -> str:return f"{engaged} attention upgrade"+def attention_status_reason(requested: Optional[str], engaged: Optional[str]) -> str:+ """Status reason for the load's attention pick: the auto reason for None / "auto", else "requested"."""+ try:+ is_auto = normalize_attention_backend(requested) == ATTN_AUTO+ except ValueError:+ is_auto = False+ if is_auto or not engaged:+ return auto_attention_reason(engaged)+ return "requested"++def _is_cuda_rocm(target: Any) -> bool:return getattr(target, "device", None) == "cuda" and not _is_cuda_nvidia(target)-# ``auto`` -> verified ROCm flash under a speed profile. Off: gfx1151 gains over AOTriton SDPA were too small/narrow.-ROCM_AUTO_FLASH = False-# Also with speed unset (the UI default). gfx1151 klein 1024^2: 7.37-7.40 s vs SDPA 8.04-8.09 s, LPIPS 0.008.-# FLUX.1 measured LPIPS 0.048 vs SDPA (over the 0.02 bar): stays native.-ROCM_AUTO_FLASH_FAMILIES = frozenset({"flux.2-klein"})-ROCM_AUTO_FLASH_ARCHES = ("gfx11",)+ROCM_FLASH_CANDIDATE = "_rocm_flash_candidate"+ROCM_FLASH_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH" # auto (timed) | 0 (auto never picks flash) | 1 (verified is enough)+ROCM_FLASH_MIN_GAIN = 0.03+# Accuracy is judged on the raw attention output (flash vs native SDPA, same inputs), not the block: the residual and+# FFN would dilute a wrong attention. Rel L1 over the output, and the largest abs error over the largest abs output.+ROCM_FLASH_BLOCK_TOL = 2e-2 # attention output rel L1+ROCM_FLASH_MAX_TOL = 5e-2 # attention max abs error / max abs output+ROCM_FLASH_PROBE_TOKENS = 4096 # the load-time bucket: a 1024^2 image DiT sequence+# Calls are gated per (query, key) length bucket pair (powers of two from 256 up, open ended): a pair is timed at its+# lower edges on first use, so the 4096 verdict never decides a 30k-token video call, and a long-video x short-text+# cross-attention is timed as that rectangle, not as square self-attention.+ROCM_FLASH_MIN_BUCKET = 256+ROCM_FLASH_LONG_PROBE = (+ 16384 # from here the probe times fewer runs (one native SDPA call is seconds at 64k tokens)+)+ROCM_FLASH_ADMISSIONS: dict[tuple, dict] = {}+# Bump when the block probe changes what it measures: every stored verdict is then re-timed.+# 2: the flash arm runs through diffusers' dispatch (the generation path), not flash_attn_func directly.+# 3: accuracy from the raw attention output (rel L1 + max_rel), not the block output; verdicts per length bucket.+ROCM_FLASH_PROBE_VERSION = 3+ROCM_FLASH_CACHE_DIR_ENV = "UNSLOTH_DIFFUSION_ROCM_FLASH_CACHE_DIR"+++def _rocm_flash_mode() -> str:+ raw = os.environ.get(ROCM_FLASH_ENV, "auto").strip().lower()+ if raw in ("0", "false", "no", "off", "never"):+ return "0"+ if raw in ("1", "true", "yes", "on", "always"):+ return "1"+ return "auto"def _rocm_gfx_arch(target: Any) -> str:@@ -531,15 +566,889 @@ def _flash_attn_installed() -> bool:return False-def _rocm_auto_flash(target: Any, family: Any) -> bool:- if not _is_cuda_rocm(target):+def _rocm_auto_flash(target: Any, family: Any = None) -> bool:+ if not _is_cuda_rocm(target) or _rocm_flash_mode() == "0":return False- name = family if isinstance(family, str) else getattr(family, "name", None)- if name not in ROCM_AUTO_FLASH_FAMILIES:+ return _flash_attn_installed() and _rocm_flash_attn_runs(target)+++def _dit_attention_geometries(pipe: Any) -> tuple[tuple[int, ...], ...]:+ """Every distinct (hidden, heads, head_dim[, context width]) among the attention modules of every denoiser DiT, read off ``to_q``+ + ``heads`` (or ``num_heads``); flash is set on all of them, so all of them are timed."""+ found: set[tuple[int, int, int]] = set()+ for dit in _attention_dits(pipe):+ modules = getattr(dit, "modules", None)+ if not callable(modules):+ continue+ try:+ for module in modules():+ heads = getattr(module, "heads", None)+ if not isinstance(heads, int) or isinstance(heads, bool):+ heads = getattr(module, "num_heads", None)+ to_q = getattr(module, "to_q", None)+ fin = getattr(to_q, "in_features", None)+ fout = getattr(to_q, "out_features", None)+ if to_q is None:+ # Fused self-attention (MiniMax-H3 resident INT8 deletes to_q / to_k / to_v): Q is a third of to_qkv.+ to_qkv = getattr(module, "to_qkv", None)+ fin = getattr(to_qkv, "in_features", None)+ qkv_out = getattr(to_qkv, "out_features", None)+ fout = (+ qkv_out // 3+ if isinstance(qkv_out, int) and qkv_out > 0 and qkv_out % 3 == 0+ else None+ )+ if not all(+ isinstance(v, int) and not isinstance(v, bool) and v > 0+ for v in (heads, fin, fout)+ ):+ continue+ if fout % heads:+ continue+ # Cross-attention reads K / V from a context of another width (to_k.in_features): a 4th element.+ kin = getattr(getattr(module, "to_k", None), "in_features", None)+ if isinstance(kin, int) and not isinstance(kin, bool) and kin > 0 and kin != fin:+ found.add((fin, heads, fout // heads, kin))+ else:+ found.add((fin, heads, fout // heads))+ except Exception: # noqa: BLE001+ continue+ return tuple(sorted(found))+++def _time_rocm_flash_block(+ device: str,+ dtype: Any,+ hidden: int,+ heads: int,+ head_dim: int,+ tokens: int = ROCM_FLASH_PROBE_TOKENS,+ kv_tokens: Optional[int] = None,+ ctx_dim: Optional[int] = None,+ batch: int = 1,+) -> dict:+ """``kv_tokens`` / ``ctx_dim`` (default ``tokens`` / ``hidden``): a cross-attention block, queries from the image+ stream and keys / values from a ``kv_tokens``-long context ``ctx_dim`` wide. ``batch`` samples per call. Median ms of one synthetic DiT block with native SDPA vs flash_attn (timing only), plus the error of the raw+ flash attention output against native SDPA on the same q / k / v (accuracy).++ Both arms go through diffusers' ``dispatch_attention_fn`` exactly as generation does (flash behind Studio's mask+ guard), so a flash_attn build whose ``flash_attn_func`` rejects the arguments diffusers passes fails here."""+ import time++ import torch+ import torch.nn.functional as F+ from diffusers.models.attention_dispatch import AttentionBackendName, dispatch_attention_fn++ if not _install_rocm_flash_dispatch_guard():+ raise RuntimeError("diffusers flash backend registry not found")+ if dtype not in (torch.float16, torch.bfloat16):+ dtype = torch.bfloat16+ inner = heads * head_dim+ # Drawn on the device in the run dtype: a long bucket staged in host float32 would need GiBs of unbudgeted RAM.+ gen = torch.Generator(device = device).manual_seed(0)++ def rand(*shape):+ return torch.randn(shape, generator = gen, device = device, dtype = dtype)++ def w(rows: int, cols: int):+ return rand(rows, cols) * cols**-0.5++ kv = tokens if kv_tokens is None else int(kv_tokens)+ width = hidden if ctx_dim is None else int(ctx_dim)+ b = max(int(batch), 1)+ w_qkv, w_out, w_fc1, w_fc2 = (+ w(3 * inner, hidden),+ w(hidden, inner),+ w(4 * hidden, hidden),+ w(hidden, 4 * hidden),+ )+ x = rand(b, tokens, hidden)+ cross = kv != tokens or width != hidden+ ctx = rand(b, kv, width) if cross else None+ w_kv = w(2 * inner, width) if cross else None++ def project():+ xn = F.layer_norm(x, (hidden,))+ if ctx is None:+ h = F.linear(xn, w_qkv).view(b, tokens, 3, heads, head_dim)+ return h[:, :, 0], h[:, :, 1], h[:, :, 2]+ q = F.linear(xn, w_qkv[:inner]).view(b, tokens, heads, head_dim)+ h = F.linear(ctx, w_kv).view(b, kv, 2, heads, head_dim)+ return q, h[:, :, 0], h[:, :, 1]++ def via(backend):+ def attn(q, k, v):+ out = dispatch_attention_fn(+ q, k, v, attn_mask = None, dropout_p = 0.0, is_causal = False, backend = backend+ )+ return out[0] if isinstance(out, tuple) else out++ return attn++ native = via(AttentionBackendName.NATIVE)++ def block(attn):+ with torch.no_grad():+ a = attn(*project()).reshape(b, tokens, inner)+ x2 = x + F.linear(a, w_out)+ return x2 + F.linear(+ F.gelu(F.linear(F.layer_norm(x2, (hidden,)), w_fc1), approximate = "tanh"), w_fc2+ )++ arms = {"native": native, "flash": via(AttentionBackendName.FLASH)}+ with torch.no_grad():+ qkv = project()+ ref = native(*qkv).float()+ got = arms["flash"](*qkv).float()+ del qkv+ if tuple(got.shape) != tuple(ref.shape):+ raise RuntimeError(f"flash output shape {tuple(got.shape)} != {tuple(ref.shape)}")+ finite = bool(torch.isfinite(got).all())+ err = (got - ref).abs()+ rel = float(err.mean() / ref.abs().mean().clamp_min(1e-12))+ max_rel = float(err.max() / ref.abs().max().clamp_min(1e-12))+ del ref, got, err+ # diagnostic only: how much of the attention error survives the residual + FFN+ block_rel = None+ if tokens < ROCM_FLASH_LONG_PROBE:+ bref = block(native).float()+ bgot = block(arms["flash"]).float()+ block_rel = float((bgot - bref).abs().mean() / bref.abs().mean().clamp_min(1e-12))+ del bref, bgot+ times: dict[str, list] = {name: [] for name in arms}+ long = tokens >= ROCM_FLASH_LONG_PROBE+ for _ in range(1 if long else 2):+ for fn in arms.values():+ block(fn)+ torch.cuda.synchronize(device)+ for _ in range(3 if long else 5):+ for name, fn in arms.items():+ torch.cuda.synchronize(device)+ t0 = time.perf_counter()+ block(fn)+ torch.cuda.synchronize(device)+ times[name].append((time.perf_counter() - t0) * 1000.0)+ med = {name: sorted(v)[len(v) // 2] for name, v in times.items()}+ return {+ "native_ms": round(med["native"], 3),+ "flash_ms": round(med["flash"], 3),+ "rel_l1": rel,+ "max_rel": max_rel,+ "block_rel_l1": block_rel,+ "finite": finite,+ "tokens": tokens,+ **({} if kv == tokens else {"kv_tokens": kv}),+ **({} if b == 1 else {"batch": b}),+ }+++def _rocm_flash_block_admitted(+ pipe: Any,+ target: Any,+ logger: Any = None,+) -> bool:+ """Resolve ``ROCM_FLASH_CANDIDATE``: flash only if it runs and (mode auto) wins the block timing with matching output."""+ if target is None or not _rocm_auto_flash(target):return False- if not _rocm_gfx_arch(target).startswith(ROCM_AUTO_FLASH_ARCHES):+ mode = _rocm_flash_mode()+ if mode == "1":+ return _install_rocm_flash_dispatch_guard()+ geometries = _dit_attention_geometries(pipe)+ if not geometries:return False- return _flash_attn_installed() and _rocm_flash_attn_runs(target)+ device = _indexed_cuda_device(+ str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "")+ )+ dtype = getattr(target, "dtype", None)+ for geometry in geometries:+ if not _rocm_flash_geometry_admitted(device, dtype, geometry, logger):+ return False+ # diffusers' flash backend raises on attn_mask: masked calls (e.g. mixed-padding batches) must run native+ if not _install_rocm_flash_dispatch_guard():+ return False+ gate: dict[tuple, frozenset] = {}+ for geometry in geometries:+ shape = (device, str(dtype), geometry[1], geometry[2])+ gate[shape] = gate.get(shape, frozenset()) | {tuple(geometry)}+ for dit in _attention_dits(pipe):+ try:+ setattr(dit, ROCM_FLASH_GATE_ATTR, gate)+ except Exception: # noqa: BLE001+ pass+ return True+++# Length gating belongs to the DiT an auto load admitted, not to a shape: the gate {(device, dtype, heads, head_dim):+# hidden sizes} lives on that DiT and is active only while it runs (its forward hooks), so a model loaded with explicit+# flash or UNSLOTH_DIFFUSION_ROCM_FLASH=1 is never gated, whichever loaded first.+ROCM_FLASH_GATE_ATTR = "_unsloth_rocm_flash_gate"+# Per thread (concurrent image and video forwards never see each other's): ``gate`` and ``tally`` of the DiT forward+# running now, and ``probing`` while a timing probe runs. Dynamo traces threading.local reads.+_ROCM_FLASH_LOCAL = threading.local()+# (device, dtype, *geometry, query bucket, key bucket, batch bucket) -> flash serves it, filled on an eager call's+# first use (the probe or a stored verdict). In memory: a bucket the probe rejected stays native.+_ROCM_FLASH_BUCKETS: dict[tuple, bool] = {}+# On each hooked DiT: bucket keys whose probe could not run (no room beside the live forward, or it raised), native+# until that DiT's next generation retries them. Never a verdict, so a transient shortfall cannot disable a working+# kernel; per DiT, so another pipeline's generation boundary does not clear it.+ROCM_FLASH_DEFERRED_ATTR = "_unsloth_rocm_flash_deferred"+_ROCM_FLASH_GATE_LOCK = threading.Lock()+# The native SDPA baseline the in-memory verdicts were timed against; when it moves (a later load re-enables or disables+# the fused SDPA kernels), both in-memory caches are dropped and the disk cache, keyed on it, decides again.+_ROCM_FLASH_BASELINE: list = [None]+++def _sync_rocm_flash_baseline() -> None:+ try:+ current = _native_sdpa_baseline()+ except Exception: # noqa: BLE001+ current = None+ if current != _ROCM_FLASH_BASELINE[0]:+ ROCM_FLASH_ADMISSIONS.clear()+ _ROCM_FLASH_BUCKETS.clear()+ _ROCM_FLASH_BASELINE[0] = current+++def _rocm_flash_bucket(tokens: Any) -> int:+ """Largest power of two <= tokens, at least ROCM_FLASH_MIN_BUCKET, open ended."""+ bucket = ROCM_FLASH_MIN_BUCKET+ while tokens >= bucket * 2:+ bucket *= 2+ return bucket+++def _rocm_flash_batch_bucket(batch: int) -> int:+ """Largest power of two <= batch (1, 2, 4, ...), open ended."""+ bucket = 1+ while batch >= bucket * 2:+ bucket *= 2+ return bucket+++def _active_rocm_flash_gate() -> Optional[dict]:+ return getattr(_ROCM_FLASH_LOCAL, "gate", None)+++def _rocm_flash_capturing() -> bool:+ """A CUDA / HIP graph capture is under way (replays cannot re-route a call)."""+ try:+ import torch+ return bool(torch.cuda.is_current_stream_capturing())+ except Exception: # noqa: BLE001+ return False+++def _rocm_flash_length_reason(query: Any, key: Any) -> Optional[str]:+ """None when flash serves this call, else why not. Only calls made inside a DiT an auto load admitted are gated+ (explicit flash and ``UNSLOTH_DIFFUSION_ROCM_FLASH=1`` are not). A call is decided by its (batch, query length, key+ length) bucket, timed on first use. A traced (torch.compile) call is decided once, at trace time, by+ ``_rocm_flash_traced_reason``: its answer is baked into the graph, and the bucket edges compare symbolically, so a+ dynamic length guards on its bucket range, not on its value (a new bucket recompiles, a new length inside one+ does not). A graph capture runs native."""+ gate = _active_rocm_flash_gate()+ if not gate:+ return None+ traced = _compiling()+ if not traced and _rocm_flash_capturing():+ return "compiled or captured"+ if getattr(_ROCM_FLASH_LOCAL, "probing", False):+ return None+ try:+ batch, seq, heads, head_dim = query.shape+ kv = key.shape[1]+ raw_device, dtype_name = str(query.device), str(query.dtype)+ except Exception: # noqa: BLE001 - not an NHD tensor call: leave it to the kernel as before+ return None+ if traced:+ # Nothing here may graph-break: the device index, the verdict tables and any probe are read in plain Python+ # by the constant-marked helper, called with constants only (the gate's own key, never a traced size).+ match = None+ for shape, found in gate.items():+ if shape[1] == dtype_name and heads == shape[2] and head_dim == shape[3]:+ match = (shape, found)+ break+ if match is None:+ return None+ return _rocm_flash_traced_reason(+ raw_device,+ dtype_name,+ match[0][2],+ match[0][3],+ tuple(sorted(match[1])),+ _rocm_flash_bucket(seq),+ _rocm_flash_bucket(kv),+ _rocm_flash_batch_bucket(batch),+ )+ device = _indexed_cuda_device(raw_device)+ try:+ geometries = gate.get((device, dtype_name, int(heads), int(head_dim)))+ except Exception: # noqa: BLE001+ return None+ if not geometries:+ return None+ return _rocm_flash_bucket_reason(+ device,+ query.dtype,+ tuple(sorted(geometries)),+ _rocm_flash_bucket(int(seq)),+ _rocm_flash_bucket(int(kv)),+ _rocm_flash_batch_bucket(int(batch)),+ )+++def _rocm_flash_traced_reason(+ raw_device: str,+ dtype_name: str,+ heads: int,+ head_dim: int,+ geometries: tuple,+ q_bucket: int,+ kv_bucket: int,+ b_bucket: int,+) -> Optional[str]:+ """The traced call's route, run in plain Python while Dynamo traces (``torch.compiler.assume_constant_result``,+ marked when the guard installs): the same verdict as an eager call of this bucket, timing it now if unknown (no+ room beside the live forward -> native for this graph). Never raises: anything odd runs native."""+ try:+ import torch++ device = _indexed_cuda_device(raw_device)+ gate = _active_rocm_flash_gate()+ if gate and (device, dtype_name, heads, head_dim) not in gate:+ return "compiled or captured"+ dtype = getattr(torch, dtype_name.rpartition(".")[2], None)+ if not isinstance(dtype, torch.dtype):+ return "compiled or captured"+ return _rocm_flash_bucket_reason(device, dtype, geometries, q_bucket, kv_bucket, b_bucket)+ except Exception: # noqa: BLE001+ return "compiled or captured"+++def _rocm_flash_bucket_reason(+ device: str, dtype: Any, geometries: tuple, q_bucket: int, kv_bucket: int, b_bucket: int+) -> Optional[str]:+ _sync_rocm_flash_baseline()+ keys = [(device, str(dtype), *g, q_bucket, kv_bucket, b_bucket) for g in geometries]+ if all(_ROCM_FLASH_BUCKETS.get(k) is True for k in keys):+ return None+ if any(_ROCM_FLASH_BUCKETS.get(k) is False for k in keys):+ return "slower or inexact at this size"+ deferred = getattr(_ROCM_FLASH_LOCAL, "deferred", None)+ if not isinstance(deferred, set):+ deferred = set() # no owning DiT: nothing to remember the deferral on+ if any(k in deferred for k in keys):+ return "timing deferred"+ with _ROCM_FLASH_GATE_LOCK:+ verdict = True+ for k, g in zip(keys, geometries):+ ok = _ROCM_FLASH_BUCKETS.get(k)+ if ok is None:+ ok = _rocm_flash_geometry_admitted(+ device,+ dtype,+ g,+ tokens = q_bucket,+ batch = b_bucket,+ live = True,+ **({} if kv_bucket == q_bucket else {"kv_tokens": kv_bucket}),+ )+ if ok is None:+ deferred.add(k)+ return "timing deferred"+ _ROCM_FLASH_BUCKETS[k] = ok+ verdict = verdict and ok+ return None if verdict else "slower or inexact at this size"+++def _native_sdpa_fused_serves(+ device: Any, dtype: Any, heads: int, head_dim: int, tokens: int, kv_tokens: int, batch: int+) -> bool:+ """Whether native SDPA takes an enabled fused (flash / memory-efficient) kernel at exactly this shape, asked of+ torch's own dispatcher with zero-size-storage views; otherwise (or when it cannot say) it runs the math backend."""+ try:+ import torch+ from torch.backends.cuda import (+ SDPAParams,+ can_use_efficient_attention,+ can_use_flash_attention,+ )++ cuda = torch.backends.cuda++ def view(n):+ return torch.empty((1, 1, 1, head_dim), device = device, dtype = dtype).expand(+ batch, heads, n, head_dim+ )++ q, k = view(tokens), view(kv_tokens)+ try:+ params = SDPAParams(q, k, k, None, 0.0, False, False)+ except TypeError: # torch before enable_gqa+ params = SDPAParams(q, k, k, None, 0.0, False)+ return bool(+ (cuda.flash_sdp_enabled() and can_use_flash_attention(params, False))+ or (cuda.mem_efficient_sdp_enabled() and can_use_efficient_attention(params, False))+ )+ except Exception: # noqa: BLE001+ return False+++def _rocm_flash_probe_bytes(+ geometry: tuple,+ tokens: int,+ kv_tokens: int,+ batch: int,+ dtype: Any,+ device: Any = "cuda",+) -> int:+ """Upper bound on what ``_time_rocm_flash_block`` allocates: its weights, the block's widest activations and the+ fp32 accuracy outputs (measured on gfx1151: 5.46 GiB at Wan's 64k tokens, this gives 8.1 GiB). Unless a fused SDPA+ kernel is confirmed for this exact shape (``guard_rocm_fused_sdpa`` may have turned them off, or one may not serve+ this length / head size), the native arm runs math SDPA, which also holds fp32 scores and their softmax, batch x+ heads x queries x keys each."""+ hidden, heads, head_dim = (int(v) for v in geometry[:3])+ width = int(geometry[3]) if len(geometry) > 3 else hidden+ inner = heads * head_dim+ elem = getattr(dtype, "itemsize", 2) or 2+ weights = (4 * inner * hidden + 8 * hidden * hidden + 2 * inner * width) * elem+ per_query = (11 * hidden + 3 * inner) * elem + 2 * inner * 4+ per_key = (width + 2 * inner) * elem+ fused = _native_sdpa_fused_serves(device, dtype, heads, head_dim, tokens, kv_tokens, batch)+ scores = 0 if fused else 2 * batch * heads * tokens * kv_tokens * 4+ return weights + batch * (tokens * per_query + kv_tokens * per_key) + scores+++def _rocm_flash_probe_fits(device: str, need: int) -> bool:+ """Room for the probe beside the live forward: device free plus the allocator's cached-but-unused blocks, with+ 25% + 256 MiB to spare. Unknown memory state counts as no room."""+ try:+ import torch++ free, _total = torch.cuda.mem_get_info(device)+ cached = torch.cuda.memory_reserved(device) - torch.cuda.memory_allocated(device)+ return free + max(int(cached), 0) >= need * 1.25 + (256 << 20)+ except Exception: # noqa: BLE001+ return False+++def _rocm_flash_geometry_admitted(+ device: str,+ dtype: Any,+ geometry: tuple,+ logger: Any = None,+ tokens: int = ROCM_FLASH_PROBE_TOKENS,+ kv_tokens: Optional[int] = None,+ batch: int = 1,+ live: bool = False,+) -> Optional[bool]:+ """Whether flash is admitted for ``geometry`` (hidden, heads, head_dim[, context width]) at ``batch`` x ``tokens``+ queries x ``kv_tokens`` keys (default square, batch 1). None when the probe could not run (it raised, or ``live``,+ inside a running forward, and there is no room for it beside that forward's tensors): nothing is cached."""+ kv = tokens if kv_tokens is None else int(kv_tokens)+ # Geometry, lengths and batch as separate fields: a flat tuple aliases (h, H, d, ctx) at n with (h, H, d) at ctx x n.+ key = (device, str(dtype), tuple(geometry), tokens, kv, batch)+ _sync_rocm_flash_baseline()+ cached = ROCM_FLASH_ADMISSIONS.get(key)+ if cached is not None:+ return bool(cached.get("admitted"))+ toolchain = _rocm_flash_toolchain_key(device)+ path = _rocm_flash_cache_path(toolchain)+ size = (str(tokens) if kv == tokens else f"{tokens}x{kv}") + ("" if batch == 1 else f"b{batch}")+ slot = "|".join((str(dtype), *(str(v) for v in geometry), size))+ stored = _load_rocm_flash_verdicts(path, toolchain).get(slot)+ if (+ stored is not None+ and _rocm_flash_record_ok(stored)+ and stored["tokens"] == tokens+ and stored.get("kv_tokens", stored["tokens"]) == kv+ and stored.get("batch", 1) == batch+ ):+ rec = {**stored, "admitted": _rocm_flash_wins(stored), "cached": True}+ ROCM_FLASH_ADMISSIONS[key] = rec+ _log_rocm_flash_verdict(logger, device, geometry, rec, "cached")+ return rec["admitted"]+ if live:+ need = _rocm_flash_probe_bytes(geometry, tokens, kv, batch, dtype, device)+ if not _rocm_flash_probe_fits(device, need):+ _module_logger().info(+ "diffusion.attention: ROCm flash timing at %s tokens deferred, no room beside this forward "+ "(needs about %.2f GiB); native SDPA for now",+ size,+ need / 2**30,+ )+ return None+ _ROCM_FLASH_LOCAL.probing = True # the probe's own flash calls are neither gated nor counted+ try:+ shape = {} if kv == tokens else {"kv_tokens": kv}+ if len(geometry) > 3:+ shape["ctx_dim"] = geometry[3]+ if batch != 1:+ shape["batch"] = batch+ rec = _time_rocm_flash_block(device, dtype, *geometry[:3], tokens = tokens, **shape)+ except Exception as exc: # noqa: BLE001 - not cached: retried next load+ _module_logger().info(+ "diffusion.attention: ROCm flash timing probe could not run (%s); native SDPA", exc+ )+ return None+ finally:+ _ROCM_FLASH_LOCAL.probing = False+ try:+ import torch+ torch.cuda.empty_cache()+ except Exception: # noqa: BLE001+ pass+ rec = dict(rec) if isinstance(rec, dict) else {}+ rec["geometry"] = {"hidden": geometry[0], "heads": geometry[1], "head_dim": geometry[2]}+ if len(geometry) > 3:+ rec["geometry"]["context"] = geometry[3]+ valid = _rocm_flash_record_ok(rec)+ rec["admitted"] = valid and _rocm_flash_wins(rec)+ ROCM_FLASH_ADMISSIONS[key] = rec+ if valid: # an implausible timing is not stored, so it cannot pin a verdict+ _save_rocm_flash_verdict(path, toolchain, slot, rec)+ _log_rocm_flash_verdict(logger, device, geometry, rec, "probe")+ return rec["admitted"]+++def _real_number(value: Any) -> bool:+ return isinstance(value, (int, float)) and not isinstance(value, bool)+++def _rocm_flash_record_ok(rec: Any) -> bool:+ """Every field a verdict is derived from has the type and range the probe writes; anything else is a cache miss."""+ import math+ try:+ if not isinstance(rec, dict) or not isinstance(rec.get("finite"), bool):+ return False+ for name in ("native_ms", "flash_ms"):+ v = rec.get(name)+ if not _real_number(v) or not math.isfinite(v) or v <= 0:+ return False+ for name in ("tokens", "kv_tokens", "batch"):+ n = rec.get(name, 1)+ if not isinstance(n, int) or isinstance(n, bool) or n <= 0:+ return False+ rel = rec.get("rel_l1")+ if rec["finite"]:+ if not _real_number(rel) or not math.isfinite(rel) or rel < 0:+ return False+ elif rel is not None and not _real_number(rel): # a non-finite output may leave NaN here+ return False+ max_rel = rec.get("max_rel")+ if rec["finite"]:+ if not _real_number(max_rel) or not math.isfinite(max_rel) or max_rel < 0:+ return False+ elif max_rel is not None and not _real_number(max_rel):+ return False+ if rec.get("block_rel_l1") is not None and not _real_number(rec["block_rel_l1"]):+ return False+ if "admitted" in rec and not isinstance(rec["admitted"], bool):+ return False+ geometry = rec.get("geometry")+ if geometry is not None and not (+ isinstance(geometry, dict)+ and all(+ isinstance(geometry.get(k), int)+ and not isinstance(geometry.get(k), bool)+ and geometry.get(k) > 0+ for k in ("hidden", "heads", "head_dim")+ + (("context",) if "context" in geometry else ())+ )+ ):+ return False+ return True+ except Exception: # noqa: BLE001+ return False+++def _rocm_flash_wins(rec: dict) -> bool:+ """Re-derived from the stored numbers on a cache hit, so a threshold change applies without re-timing."""+ try:+ return bool(+ rec["finite"] is True+ and float(rec["rel_l1"]) <= ROCM_FLASH_BLOCK_TOL+ and float(rec["max_rel"]) <= ROCM_FLASH_MAX_TOL+ and float(rec["flash_ms"]) <= float(rec["native_ms"]) * (1.0 - ROCM_FLASH_MIN_GAIN)+ )+ except Exception: # noqa: BLE001 - malformed record: never flash+ return False+++def _log_rocm_flash_verdict(+ logger: Any, device: str, geometry: tuple, rec: dict, source: str+) -> None:+ """Never raises: the values are formatted defensively, whatever a record holds."""++ def num(value: Any, spec: str) -> str:+ try:+ return format(float(value), spec) if _real_number(value) else "?"+ except Exception: # noqa: BLE001+ return "?"++ try:+ (logger or _module_logger()).info(+ "diffusion.attention: ROCm flash block %s %s (hidden %s, heads %s, head_dim %s, %s tokens): flash %s ms "+ "vs native %s ms, attention rel L1 %s, max %s -> %s",+ source,+ device,+ *(str(g) for g in tuple(geometry)[:3]),+ num(rec.get("tokens"), ".0f"),+ num(rec.get("flash_ms"), ".2f"),+ num(rec.get("native_ms"), ".2f"),+ num(rec.get("rel_l1"), ".2e"),+ num(rec.get("max_rel"), ".2e"),+ "flash" if rec.get("admitted") is True else "native",+ )+ except Exception: # noqa: BLE001 - logging is diagnostic only+ pass+++_ROCM_DRIVER_VERSION: list = []+++def _rocm_driver_version() -> str:+ """The ROCm stack this process actually runs on, not the wheel's build (``torch.version.hip``): the HIP runtime's+ ``hipDriverGetVersion`` from the libamdhip64 this process loaded (HIP runtime API), plus the amdgpu kernel driver+ (``/sys/module/amdgpu/version`` for the DKMS driver, else the kernel release for the in-tree one). Read once per+ process; parts that cannot be read are left out."""+ if _ROCM_DRIVER_VERSION:+ return _ROCM_DRIVER_VERSION[0]+ import ctypes+ import platform++ parts = []+ try:+ lib = None+ if os.name == "nt":+ names = ["amdhip64_7.dll", "amdhip64_6.dll", "amdhip64.dll"]+ else:+ with open("/proc/self/maps", encoding = "utf-8", errors = "replace") as f:+ names = sorted(+ {line.split()[-1] for line in f if "libamdhip64" in line and "/" in line}+ )+ for name in names:+ try:+ lib = ctypes.CDLL(name)+ break+ except OSError:+ continue+ if lib is not None:+ version = ctypes.c_int(0)+ if lib.hipDriverGetVersion(ctypes.byref(version)) == 0:+ parts.append(f"hipdriver{version.value}")+ except Exception: # noqa: BLE001+ pass+ try:+ with open("/sys/module/amdgpu/version", encoding = "utf-8") as f:+ parts.append(f"amdgpu{f.read().strip()}")+ except OSError:+ try:+ parts.append(f"kernel{platform.release()}")+ except Exception: # noqa: BLE001+ pass+ _ROCM_DRIVER_VERSION.append("driver:" + (",".join(parts) or "?"))+ return _ROCM_DRIVER_VERSION[0]+++def _diffusers_flash_path_fingerprint() -> str:+ """diffusers version + a hash of the code generation runs flash through (``dispatch_attention_fn`` and the+ registered FLASH backend, unwrapped from Studio's guard), so a diffusers change that alters the call re-times."""+ import hashlib+ import inspect++ import diffusers+ from diffusers.models.attention_dispatch import (+ AttentionBackendName,+ _AttentionBackendRegistry,+ dispatch_attention_fn,+ )++ flash = _AttentionBackendRegistry._backends[AttentionBackendName.FLASH]+ while (+ getattr(flash, _SAGE_GUARD_ATTR, False) and getattr(flash, "__wrapped__", None) is not None+ ):+ flash = flash.__wrapped__+ parts = []+ for fn in (dispatch_attention_fn, flash):+ try:+ parts.append(inspect.getsource(fn))+ except (OSError, TypeError):+ code = getattr(fn, "__code__", None)+ parts.append(repr((code.co_code, code.co_consts)) if code is not None else repr(fn))+ digest = hashlib.sha1("\n".join(parts).encode("utf-8")).hexdigest()[:12]+ return f"diffusers{getattr(diffusers, '__version__', '?')}:{digest}"+++def _flash_attn_build_fingerprint() -> str:+ """size + mtime_ns of the flash_attn kernel files this process loaded: the CK extension (``flash_attn_2_cuda``) or,+ for the Triton AMD backend, the source files of the package ``flash_attn_interface.flash_attn_gpu`` came from. A+ rebuild in place (same version, same path) changes it. Stat only."""+ import importlib.util+ import sys++ files: set[str] = set()+ for name in ("flash_attn", "flash_attn.flash_attn_interface", "flash_attn_2_cuda"):+ f = getattr(sys.modules.get(name), "__file__", None)+ if isinstance(f, str) and f:+ files.add(f)+ gpu = getattr(sys.modules.get("flash_attn.flash_attn_interface"), "flash_attn_gpu", None)+ gpu_file = getattr(gpu, "__file__", None)+ if isinstance(gpu_file, str) and gpu_file:+ files.add(gpu_file)+ if gpu_file.endswith(".py"):+ folder = os.path.dirname(gpu_file)+ files.update(os.path.join(folder, n) for n in os.listdir(folder) if n.endswith(".py"))+ elif "flash_attn_2_cuda" not in sys.modules:+ try:+ spec = importlib.util.find_spec("flash_attn_2_cuda")+ if spec is not None and spec.origin:+ files.add(spec.origin)+ except Exception: # noqa: BLE001+ pass+ parts = []+ for f in sorted(files):+ try:+ st = os.stat(f)+ parts.append(f"{os.path.realpath(f)}:{st.st_size}:{st.st_mtime_ns}")+ except OSError:+ parts.append(f"{f}:missing")+ import hashlib++ return hashlib.sha1("\n".join(parts).encode("utf-8")).hexdigest()[:12]+++def _rocm_flash_toolchain_key(device: str) -> Optional[str]:+ """What the block timing depends on: card + gfx arch, torch, HIP, the flash_attn build (version, install path, CK or+ Triton backend) and the probe itself. None (in-memory only) when any part is unreadable."""+ try:+ import hashlib+ import importlib.metadata+ import inspect+ import sys++ import torch+ from utils.hardware.hardware import _props_gfx_arch++ index = torch.device(device).index+ index = torch.cuda.current_device() if index is None else index+ props = torch.cuda.get_device_properties(index)+ fa = sys.modules.get("flash_attn")+ if fa is None:+ import flash_attn as fa+ fa_version = getattr(fa, "__version__", None) or importlib.metadata.version("flash_attn")+ # flash_attn_interface reads exactly this to route to the Triton AMD kernels instead of the CK extension, and+ # falls back to Triton itself when the CK extension does not import; its USE_TRITON_ROCM is the outcome.+ triton_amd = getattr(+ sys.modules.get("flash_attn.flash_attn_interface"), "USE_TRITON_ROCM", None+ )+ if not isinstance(triton_amd, bool):+ triton_amd = os.environ.get("FLASH_ATTENTION_TRITON_AMD_ENABLE", "FALSE") == "TRUE"+ backend = "ck"+ if triton_amd:+ import triton+ backend = f"triton{triton.__version__}"+ src = inspect.getsource(_time_rocm_flash_block)+ return "|".join(+ (+ f"v{ROCM_FLASH_PROBE_VERSION}",+ str(props.name),+ _props_gfx_arch(props) or "gfx?",+ f"torch{torch.__version__}",+ f"hip{getattr(torch.version, 'hip', None)}",+ _rocm_driver_version(),+ _native_sdpa_baseline(),+ f"flash_attn{fa_version}",+ os.path.dirname(os.path.realpath(fa.__file__)),+ backend,+ f"build{_flash_attn_build_fingerprint()}",+ _diffusers_flash_path_fingerprint(),+ hashlib.sha1(src.encode("utf-8")).hexdigest()[:12],+ )+ )+ except Exception: # noqa: BLE001 - unidentifiable toolchain: never persisted+ return None+++def _native_sdpa_baseline() -> str:+ """The native arm a verdict was timed against: which SDPA backends are on (``guard_rocm_fused_sdpa``, the user, or+ ``UNSLOTH_ALLOW_ROCM_FUSED_SDPA`` move them) and the AOTriton experimental switch that decides whether they run."""+ import torch++ cuda = torch.backends.cuda+ flags = "".join(+ name if enabled() else "-"+ for name, enabled in (+ ("f", cuda.flash_sdp_enabled),+ ("e", cuda.mem_efficient_sdp_enabled),+ ("m", cuda.math_sdp_enabled),+ )+ )+ experimental = os.environ.get("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "").strip() or "unset"+ return f"sdpa{flags},aotriton-exp{experimental}"+++def _rocm_flash_cache_path(toolchain: Optional[str]) -> Optional[str]:+ if toolchain is None:+ return None+ import hashlib++ root = (os.environ.get(ROCM_FLASH_CACHE_DIR_ENV) or "").strip()+ if not root:+ try:+ from utils.paths.storage_roots import cache_root+ root = str(cache_root() / "diffusion_rocm_flash")+ except Exception: # noqa: BLE001 - outside Studio: in-memory only+ return None+ return os.path.join(+ root, "probe_" + hashlib.sha1(toolchain.encode("utf-8")).hexdigest()[:16] + ".json"+ )+++def _load_rocm_flash_verdicts(path: Optional[str], toolchain: Optional[str]) -> dict:+ import json+ if path is None or not os.path.isfile(path):+ return {}+ try:+ with open(path, encoding = "utf-8") as f:+ data = json.load(f)+ if not isinstance(data, dict) or data.get("key") != toolchain:+ return {}+ verdicts = data.get("verdicts")+ if not isinstance(verdicts, dict):+ return {}+ return {str(slot): rec for slot, rec in verdicts.items() if _rocm_flash_record_ok(rec)}+ except Exception: # noqa: BLE001 - a corrupt / unreadable file is re-timed+ return {}+++def _save_rocm_flash_verdict(+ path: Optional[str], toolchain: Optional[str], slot: str, rec: dict+) -> None:+ """Best effort, under a file lock: merge what another process stored since, then write a temp file and replace."""+ if path is None:+ return+ import contextlib+ import json++ tmp = f"{path}.{os.getpid()}.{threading.get_ident()}.tmp"+ try:+ os.makedirs(os.path.dirname(path), exist_ok = True)+ try:+ from filelock import FileLock+ lock: Any = FileLock(f"{path}.lock", timeout = 60)+ except ImportError:+ lock = contextlib.nullcontext()+ with lock:+ verdicts = _load_rocm_flash_verdicts(path, toolchain)+ verdicts[slot] = {k: v for k, v in rec.items() if k not in ("admitted", "cached")}+ with open(tmp, "w", encoding = "utf-8") as f:+ json.dump({"key": toolchain, "verdicts": verdicts}, f, indent = 1, sort_keys = True)+ os.replace(tmp, path)+ except Exception: # noqa: BLE001 - read-only cache / lock timeout: timed again next load+ try:+ os.remove(tmp)+ except OSError:+ pass# Max abs error of the probe's flash_attn output against an fp32 reference (bf16 eps is ~4e-3).@@ -868,11 +1777,13 @@ def _install_dispatch_guard(reroute_reason: Any,note: Any,make_op: Any = None,+ ran: Any = None,) -> bool:"""Wrap diffusers' registered ``backend`` so calls its kernel cannot take run native. Idempotent.False when the registry is not where expected: the caller then does not engage the backend. ``make_op(fn)``- optionally returns an opaque op for plain (no lse, no context-parallel) calls."""+ optionally returns an opaque op for plain (no lse, no context-parallel) calls. ``ran()`` is called for each call+ the kernel itself takes."""try:from diffusers.models.attention_dispatch import (AttentionBackendName,@@ -901,6 +1812,8 @@ def _install_dispatch_guard(bound.get("query"), bound.get("key"), bound.get("value"), bound.get("attn_mask"))if reason is None:+ if ran is not None:+ ran()extra = {k: v for k, v in kwargs.items() if k not in names}plain = (op is not None@@ -967,6 +1880,232 @@ def _note_fa4_reroute(reason: str) -> None:_FA4_OP_NAME = "unsloth_studio::flash_4_hub_attention_nhd"+# diffusers' flash (FA2) backend raises on any attn_mask; on ROCm those calls run native instead.+_ROCM_FLASH_ROUTED: dict[str, int] = {}+_ROCM_FLASH_ROUTED_LOGGED: set[str] = set()+++def _rocm_flash_reroute_reason(query: Any, key: Any, value: Any, attn_mask: Any) -> Optional[str]:+ if attn_mask is not None:+ return "attn_mask"+ return _rocm_flash_length_reason(query, key)+++def _install_rocm_flash_dispatch_guard() -> bool:+ try:+ import torch+ torch.compiler.assume_constant_result(_rocm_flash_traced_reason)+ except Exception: # noqa: BLE001 - torch without it: a traced call graph-breaks into the helper, still correct+ pass+ return _install_dispatch_guard(+ "flash",+ _rocm_flash_reroute_reason,+ lambda reason: (+ _note_rocm_flash_route(reason),+ _note_reroute(+ "ROCm flash attention", reason, _ROCM_FLASH_ROUTED, _ROCM_FLASH_ROUTED_LOGGED+ ),+ ),+ ran = _note_rocm_flash_ran,+ )+++# Per DiT, the routes its attention calls took in the current generation: {"flash": n, "<reroute reason>": m}. Report+# only: nothing reads it to choose a kernel, size memory or key a calibration.+ROCM_FLASH_TALLY_ATTR = "_unsloth_rocm_flash_tally"+_ROCM_FLASH_HOOKS_ATTR = "_unsloth_rocm_flash_hooks"+# On the pipe: every DiT the hooks went on, whatever attribute holds it (a MiniMax-H3 view hooks transformer_ref, which+# _attention_dits does not enumerate). Reset, report and the memory guard iterate this.+_ROCM_FLASH_DITS_ATTR = "_unsloth_rocm_flash_dits"+++def _rocm_flash_dits(pipe: Any) -> list:+ """The hooked DiTs of ``pipe`` (registry, then any hooked one _attention_dits still sees), each once."""+ found: list = []+ registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None) if pipe is not None else None+ for dit in list(registry or ()) + list(_attention_dits(pipe) if pipe is not None else ()):+ if getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None) and not any(dit is d for d in found):+ found.append(dit)+ return found+++def _compiling() -> bool:+ try:+ import torch+ return bool(torch.compiler.is_compiling())+ except Exception: # noqa: BLE001+ return False+++def _note_rocm_flash_route(route: str) -> None:+ """Count one eager call on the running DiT's tally (this thread's). Traced and replayed calls are not counted."""+ if _compiling() or getattr(_ROCM_FLASH_LOCAL, "probing", False):+ return+ tally = getattr(_ROCM_FLASH_LOCAL, "tally", None)+ if isinstance(tally, dict):+ tally[route] = tally.get(route, 0) + 1+++def _note_rocm_flash_ran() -> None:+ _note_rocm_flash_route("flash")+++def _track_rocm_flash_calls(pipe: Any) -> None:+ """Forward hooks on each DiT running ROCm flash: while it runs, this thread's gate (auto loads only) and tally are+ its own. Each hooked DiT is recorded on the pipe (``_rocm_flash_dits``). Idempotent."""+ for dit in _attention_dits(pipe):+ try:+ registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None)+ if not isinstance(registry, list):+ registry = []+ setattr(pipe, _ROCM_FLASH_DITS_ATTR, registry)+ if not any(dit is d for d in registry):+ registry.append(dit)+ except Exception: # noqa: BLE001+ pass+ register_pre = getattr(dit, "register_forward_pre_hook", None)+ register_post = getattr(dit, "register_forward_hook", None)+ if not callable(register_pre) or not callable(register_post):+ continue+ if getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None):+ continue++ # Also while Dynamo traces a compiled DiT (it replays these threading.local writes), so a traced call sees the+ # gate and runs native.+ def pre(+ _module,+ _args,+ dit = dit,+ ) -> None:+ outer = getattr(_ROCM_FLASH_LOCAL, "stack", None)+ if outer is None:+ outer = _ROCM_FLASH_LOCAL.stack = []+ outer.append(+ (+ getattr(_ROCM_FLASH_LOCAL, "gate", None),+ getattr(_ROCM_FLASH_LOCAL, "tally", None),+ getattr(_ROCM_FLASH_LOCAL, "deferred", None),+ )+ )+ _ROCM_FLASH_LOCAL.gate = getattr(dit, ROCM_FLASH_GATE_ATTR, None) or None+ tally = getattr(dit, ROCM_FLASH_TALLY_ATTR, None)+ _ROCM_FLASH_LOCAL.tally = tally if isinstance(tally, dict) else None+ deferred = getattr(dit, ROCM_FLASH_DEFERRED_ATTR, None)+ _ROCM_FLASH_LOCAL.deferred = deferred if isinstance(deferred, set) else None++ def post(_module, _args, _output) -> None:+ stack = getattr(_ROCM_FLASH_LOCAL, "stack", None)+ gate, tally, deferred = stack.pop() if stack else (None, None, None)+ _ROCM_FLASH_LOCAL.gate, _ROCM_FLASH_LOCAL.tally = gate, tally+ _ROCM_FLASH_LOCAL.deferred = deferred++ try:+ try: # also after a forward that raised, so a failed run never leaves its gate active+ post_handle = register_post(post, always_call = True)+ except TypeError:+ post_handle = register_post(post)+ handles = [register_pre(pre), post_handle]+ setattr(dit, _ROCM_FLASH_HOOKS_ATTR, handles)+ setattr(dit, ROCM_FLASH_TALLY_ATTR, {})+ setattr(dit, ROCM_FLASH_DEFERRED_ATTR, set())+ except Exception: # noqa: BLE001+ pass+++def _untrack_rocm_flash_calls(pipe: Any) -> None:+ """Undo ``_track_rocm_flash_calls`` for this pipe's DiTs once they no longer run ROCm flash (a re-apply picked+ another backend), so the badge and the memory guard stop treating them as flash. Only the DiTs this pipe (or view)+ owns: an MoE view leaves the other expert's hooks alone."""+ dits = _attention_dits(pipe) if pipe is not None else []+ for dit in dits:+ handles = getattr(dit, _ROCM_FLASH_HOOKS_ATTR, None)+ if not handles:+ continue+ for handle in handles:+ try:+ handle.remove()+ except Exception: # noqa: BLE001+ pass+ for attr in (_ROCM_FLASH_HOOKS_ATTR, ROCM_FLASH_TALLY_ATTR, ROCM_FLASH_DEFERRED_ATTR):+ try:+ setattr(dit, attr, None)+ except Exception: # noqa: BLE001+ pass+ registry = getattr(pipe, _ROCM_FLASH_DITS_ATTR, None) if pipe is not None else None+ if isinstance(registry, list) and dits:+ registry[:] = [d for d in registry if not any(d is x for x in dits)]+++def attention_may_run_native(pipe: Any) -> bool:+ """Whether some attention call of this pipe can run native SDPA although flash is engaged: ROCm flash routes masked+ calls (explicit or auto), and in auto lengths flash does not win (or could not time beside a live forward), to native. For+ memory budgets: conservative, never from what a past run reported."""+ return bool(_rocm_flash_dits(pipe))+++def rocm_flash_ran(pipe: Any) -> Optional[tuple[str, str]]:+ """``(value, reason)`` describing the current generation's routes for a ROCm flash pipe, or None when it is not one.+ A run with no counted call (compiled or graphed attention, which the guard cannot see) says so."""+ dits = _rocm_flash_dits(pipe)+ if not dits:+ return None+ total: dict[str, int] = {}+ for dit in dits:+ tally = getattr(dit, ROCM_FLASH_TALLY_ATTR, None)+ if isinstance(tally, dict):+ for name, count in tally.items():+ if isinstance(count, int):+ total[name] = total.get(name, 0) + count+ flash = total.pop("flash", 0)+ native = sum(total.values())+ if flash + native == 0:+ return "flash", (+ "ROCm flash_attn; per-call routing is not tracked for compiled or graphed attention (masked calls and "+ "lengths flash does not serve run native SDPA)"+ )+ why = ", ".join(f"{name}: {count}" for name, count in sorted(total.items()))+ if native == 0:+ return "flash", "ROCm flash_attn on every attention call of the last generation"+ if flash == 0:+ return (+ "native",+ f"ROCm flash_attn engaged, but every attention call of the last generation ran native SDPA ({why})",+ )+ return "flash", (+ f"ROCm flash_attn on {flash} of {flash + native} attention calls of the last generation; the rest ran native "+ f"SDPA ({why})"+ )+++def begin_attention_run(state: Any) -> None:+ """A generation boundary: the per-DiT route tallies restart, so the report after it describes this run alone, and+ this run's DiTs retry buckets whose probe could not run."""+ for dit in _rocm_flash_dits(getattr(state, "pipe", None)):+ if isinstance(getattr(dit, ROCM_FLASH_TALLY_ATTR, None), dict):+ try:+ setattr(dit, ROCM_FLASH_TALLY_ATTR, {})+ setattr(dit, ROCM_FLASH_DEFERRED_ATTR, set())+ except Exception: # noqa: BLE001+ pass+++def report_attention_run(state: Any) -> None:+ """After a generation: the ``resolved`` attention badge (value + reason) describes the routes it took. Report only:+ ``state.attention_backend`` stays the engaged backend, which recipes, calibration keys and compile caches key on."""+ try:+ ran = rocm_flash_ran(getattr(state, "pipe", None))+ resolved = getattr(state, "resolved", None)+ entry = resolved.get("attention_backend") if isinstance(resolved, dict) else None+ if ran is None or not isinstance(entry, dict):+ return+ value, reason = ran+ if entry.get("value") != value or entry.get("reason") != reason:+ entry = {**entry, "value": value, "reason": reason}+ object.__setattr__(state, "resolved", {**resolved, "attention_backend": entry})+ except Exception: # noqa: BLE001 - reporting only+ pass++def _install_fa4_dispatch_guard() -> bool:return _install_dispatch_guard("flash_4_hub",@@ -1998,6 +3137,13 @@ def apply_attention_backend(if target is not None:_configure_native_attention(pipe, target, logger)return None+ for dit in _attention_dits(pipe): # only an auto admission below re-arms per-length gating+ try:+ setattr(dit, ROCM_FLASH_GATE_ATTR, None)+ except Exception: # noqa: BLE001+ pass+ if backend == ROCM_FLASH_CANDIDATE:+ backend = "flash" if _rocm_flash_block_admitted(pipe, target, logger) else Noneif backend is not None:_ensure_attention_backend_installed(backend, logger)if backend == "sage":@@ -2017,6 +3163,19 @@ def apply_attention_backend(and not _rocm_flash_attn_runs(target)):backend = None+ if (+ backend == "flash"+ and target is not None+ and _is_cuda_rocm(target)+ and not _install_rocm_flash_dispatch_guard()+ ):+ # An explicit request too: without the guard a masked call would raise mid-generation, not run native.+ if logger is not None:+ logger.warning(+ "diffusion.attention: this diffusers build does not expose the flash backend the way Unsloth "+ "expects, so masked attention calls could not be routed around it; using the default backend"+ )+ backend = Noneif (backend == "_native_cudnn"and target is not None@@ -2054,6 +3213,10 @@ def apply_attention_backend(engaged = Falseif engaged:_tag_dits(pipe, backend)+ if backend == "flash" and target is not None and _is_cuda_rocm(target):+ _track_rocm_flash_calls(pipe)+ else:+ _untrack_rocm_flash_calls(pipe)# set_attention_backend also pins the backend process-wide. Each DiT's processors keep it locally, so reset# the global to native ONCE, else a later component inherits this kernel._reset_global_backend_to_native(logger)@@ -2063,6 +3226,7 @@ def apply_attention_backend(# No backend requested, or every set failed: pin native so a stale process-wide backend cannot leak in. One reset# covers every fresh DiT._tag_dits(pipe, None)+ _untrack_rocm_flash_calls(pipe)_restore_native_backend(setters[0], logger)# Native means torch's SDPA dispatch decides per call, and on a device with no fused kernel that decision is MATH.# Say so now; the flags this would otherwise be read off lie (#8225).diff --git a/studio/backend/core/inference/video.py b/studio/backend/core/inference/video.pyindex eb7f2d6182e..f97c417db6e 100644--- a/studio/backend/core/inference/video.py+++ b/studio/backend/core/inference/video.py@@ -52,8 +52,11 @@ from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_mfrom .diffusion_attention import (SDPA_MATH_ONLY_MESSAGE,apply_attention_backend,+ attention_status_reason,install_hunyuan_attention_trim,sdpa_math_only,+ begin_attention_run,+ report_attention_run,select_attention_backend,)from .diffusion_flow_shift import apply_comfy_flow_shift@@ -9453,7 +9456,7 @@ class VideoBackend:"attention_backend": (attention_backend,attention_engaged or "native",- "cuDNN fused attention on NVIDIA when a speed profile is active",+ attention_status_reason(attention_backend, attention_engaged),),"transformer_cache": (None if cache_auto else transformer_cache,@@ -10761,7 +10764,7 @@ class VideoBackend:"attention_backend": (attention_backend,attention_engaged or "native",- "cuDNN fused attention on NVIDIA when a speed profile is active",+ attention_status_reason(attention_backend, attention_engaged),),"transformer_cache": ((@@ -11446,6 +11449,7 @@ class VideoBackend:if _resolved_inputs is not None and _resolved_inputs.state is not state:raise RuntimeError(VIDEO_CANCELLED_MSG)self._active_generate_cancel = cancel+ begin_attention_run(state)# Bound below, once the request is resolved. None means the failure beat the resolution, and there is# nothing truthful to report.request_shape: Optional[dict[str, Any]] = None@@ -12234,6 +12238,8 @@ class VideoBackend:# A cancel during the blocking export/mux must still discard the clip; re-check before it is persisted.if cancel.is_set():raise RuntimeError(VIDEO_CANCELLED_MSG)+ # The status badge reports the attention routes this run took (report only, keys are unchanged).+ report_attention_run(state)if len(video_frames) and not fam.modular_workflow:from . import video_stream_residencydiff --git a/studio/backend/tests/test_diffusion_attention_rocm_flash.py b/studio/backend/tests/test_diffusion_attention_rocm_flash.pyindex 2c8e59475fa..22c87d263a3 100644--- a/studio/backend/tests/test_diffusion_attention_rocm_flash.py+++ b/studio/backend/tests/test_diffusion_attention_rocm_flash.py@@ -6,6 +6,8 @@ NVIDIA path unchanged; the CUDA flash-attn wheel is never pip installed onto ROCfrom __future__ import annotations+import importlib+import osimport subprocessimport sysimport types@@ -15,15 +17,31 @@ import pytestimport core.inference.diffusion_attention as attfrom core.inference.diffusion_attention import select_attention_backend+try: # before any test puts a bare flash_attn module in sys.modules (diffusers' import probes its spec)+ importlib.import_module("diffusers.models.attention_dispatch")+except Exception: # noqa: BLE001+ pass++_REAL_TOOLCHAIN_KEY = att._rocm_flash_toolchain_key+_REAL_PROBE_FITS = att._rocm_flash_probe_fits+_REAL_GUARD = att._install_rocm_flash_dispatch_guard+@pytest.fixture(autouse = True)-def _clean(monkeypatch):+def _clean(monkeypatch, tmp_path):monkeypatch.setattr(att, "_ROCM_FLASH_PROBE_CACHE", {}, raising = False)+ monkeypatch.setenv(att.ROCM_FLASH_CACHE_DIR_ENV, str(tmp_path / "rocm_flash"))+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: "tc-A", raising = False)monkeypatch.setattr(att, "_indexed_cuda_device", lambda d: "cuda:0" if d == "cuda" else d, raising = False)monkeypatch.setattr(att, "_cuda_capability", lambda: (9, 0))monkeypatch.setattr(att, "_INSTALL_ATTEMPTED", set())+ monkeypatch.setattr(att, "_ROCM_FLASH_LOCAL", __import__("threading").local(), raising = False)+ monkeypatch.setattr(att, "_ROCM_FLASH_BUCKETS", {}, raising = False)+ monkeypatch.setattr(att, "_ROCM_FLASH_BASELINE", [None], raising = False)+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True, raising = False)+ monkeypatch.setattr(att, "_ROCM_FLASH_ROUTED", {})def _target(device = "cuda", dtype = "torch.bfloat16"):@@ -96,11 +114,13 @@ def test_rocm_unaskable_check_is_native_and_not_cached(monkeypatch):assert att._ROCM_FLASH_PROBE_CACHE == {}-def test_rocm_auto_unchanged(monkeypatch):+def test_rocm_auto_is_a_timed_candidate_not_flash(monkeypatch):calls = _rocm(monkeypatch)- assert select_attention_backend(_target(), "auto", speed_active = True) is None+ assert (+ select_attention_backend(_target(), "auto", speed_active = True) == att.ROCM_FLASH_CANDIDATE+ )assert select_attention_backend(_target(), "auto", speed_active = False) is None- assert calls == []+ assert calls == [("cuda:0", "torch.bfloat16")]@pytest.mark.parametrize("alias", ["flash3", "flash4", "sage", "cudnn", "xformers"])@@ -242,54 +262,57 @@ def test_cudnn_head_dim_probe_still_answers_fp32_at_bf16(monkeypatch):assert seen == [torch.bfloat16]-# ``auto`` -> flash for FLUX.2-klein on ROCm gfx11 (measured on gfx1151); every other family and arch unchanged.--def _klein():return types.SimpleNamespace(name = "flux.2-klein")-def _arch(monkeypatch, arch = "gfx1151"):- monkeypatch.setattr(att, "_rocm_gfx_arch", lambda target: arch)---def test_rocm_auto_flash_for_klein_on_gfx11(monkeypatch):- calls = _rocm(monkeypatch)- _arch(monkeypatch)[email protected]("family", ["flux.2-klein", "flux.1", "z-image", "qwen-image", None])+def test_rocm_auto_candidate_for_every_family(monkeypatch, family):+ _rocm(monkeypatch)+ fam = None if family is None else types.SimpleNamespace(name = family)assert (- select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) == "flash"+ select_attention_backend(_target(), "auto", speed_active = True, family = fam)+ == att.ROCM_FLASH_CANDIDATE)assert (- select_attention_backend(_target(), None, speed_active = True, family = "flux.2-klein")- == "flash"+ select_attention_backend(_target(), None, speed_active = False, family = fam, speed_unset = True)+ == att.ROCM_FLASH_CANDIDATE)- # unset speed (ROCm resolves it to `off`): the default path, so flash too- assert (- select_attention_backend(- _target(), "auto", speed_active = False, family = _klein(), speed_unset = True- )- == "flash"+ assert select_attention_backend(_target(), "auto", speed_active = False, family = fam) is None+++def test_rocm_auto_without_flash_attn_is_silent(monkeypatch):+ calls = _rocm(monkeypatch, flash_attn = False)+ logged = []+ monkeypatch.setattr(+ att, "_module_logger", lambda: types.SimpleNamespace(warning = lambda *a: logged.append(a)))- # an explicit `off` stays native (the bit-identical reference path)- assert select_attention_backend(_target(), "auto", speed_active = False, family = _klein()) is None- assert calls == [("cuda:0", "torch.bfloat16")]+ assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None+ assert calls == [] and logged == [][email protected]("family", ["flux.1", "flux.2", "z-image", "qwen-image", "sdxl", None])-def test_rocm_auto_stays_native_for_unmeasured_families(monkeypatch, family):+def test_rocm_auto_failed_probe_stays_native(monkeypatch):+ _rocm(monkeypatch, probe = False)+ assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None+++def test_rocm_auto_explicit_native_wins(monkeypatch):calls = _rocm(monkeypatch)- _arch(monkeypatch)- fam = None if family is None else types.SimpleNamespace(name = family)- assert select_attention_backend(_target(), "auto", speed_active = True, family = fam) is None- assert (- select_attention_backend(- _target(), "auto", speed_active = False, family = fam, speed_unset = True+ for alias in ("native", "sdpa"):+ assert (+ select_attention_backend(_target(), alias, speed_active = True, family = _klein()) is None)- is None- )assert calls == []+def test_rocm_kill_switch_zero_keeps_auto_native(monkeypatch):+ calls = _rocm(monkeypatch)+ monkeypatch.setenv(att.ROCM_FLASH_ENV, "0")+ assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None+ assert calls == []+ assert select_attention_backend(_target(), "flash", speed_active = True) == "flash"++def test_rocm_gfx_arch_reads_the_selected_card(monkeypatch):import torch@@ -306,39 +329,173 @@ def test_rocm_gfx_arch_reads_the_selected_card(monkeypatch):assert asked == [1][email protected]("arch", ["gfx942", "gfx90a", "gfx1201", "gfx1030", ""])-def test_rocm_auto_klein_flash_gfx11_only(monkeypatch, arch):- calls = _rocm(monkeypatch)- _arch(monkeypatch, arch)- assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None- assert calls == []+def _pipe_with_attention(+ hidden = 3072,+ heads = 24,+ head_dim = 128,+ n = 3,+):+ set_to = []+ mods = [+ types.SimpleNamespace(+ heads = heads,+ to_q = types.SimpleNamespace(in_features = hidden, out_features = heads * head_dim),+ )+ for _ in range(n)+ ]+ dit = types.SimpleNamespace(set_attention_backend = set_to.append, modules = lambda: iter(mods))+ return types.SimpleNamespace(transformer = dit), set_to-def test_rocm_auto_klein_without_flash_attn_is_silent(monkeypatch, caplog):- calls = _rocm(monkeypatch, flash_attn = False)- _arch(monkeypatch)- logged = []+def _apply_env(monkeypatch):+ monkeypatch.setattr(att, "_active_attention_backend", lambda: att.ATTN_NATIVE)+ monkeypatch.setattr(att, "warn_if_sdpa_math_only", lambda *a, **k: False)+ monkeypatch.setattr(att, "_configure_native_attention", lambda *a, **k: None)+ monkeypatch.setattr(att, "ROCM_FLASH_ADMISSIONS", {})+ monkeypatch.setattr(att, "_install_rocm_flash_dispatch_guard", lambda: True, raising = False)+++ "flash_ms, rel, engaged",+ [(70.0, 1e-3, "flash"), (78.0, 1e-3, None), (90.0, 1e-3, None), (60.0, 0.5, None)],+)+def test_apply_resolves_candidate_by_block_timing(monkeypatch, flash_ms, rel, engaged):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ timed = []++ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ ):+ timed.append((device, hidden, heads, head_dim))+ return {+ "native_ms": 80.0,+ "flash_ms": flash_ms,+ "rel_l1": rel,+ "max_rel": rel,+ "finite": True,+ "tokens": 4096,+ }++ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)+ pipe, set_to = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged+ assert timed == [("cuda:0", 3072, 24, 128)]+ assert set_to == (["flash"] if engaged else [])+ pipe2, _ = _pipe_with_attention()+ att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target())+ assert len(timed) == 1+ rec = next(iter(att.ROCM_FLASH_ADMISSIONS.values()))+ assert rec["admitted"] is (engaged == "flash") and rec["geometry"] == {+ "hidden": 3072,+ "heads": 24,+ "head_dim": 128,+ }+++def test_apply_candidate_kill_switch_one_skips_timing(monkeypatch):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setenv(att.ROCM_FLASH_ENV, "1")monkeypatch.setattr(- att, "_module_logger", lambda: types.SimpleNamespace(warning = lambda *a: logged.append(a))+ att, "_time_rocm_flash_block", lambda *a, **k: pytest.fail("timed under =1"))- assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None- assert calls == [] and logged == []+ pipe, set_to = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"-def test_rocm_auto_klein_failed_probe_stays_native(monkeypatch):- _rocm(monkeypatch, probe = False)- _arch(monkeypatch)- assert select_attention_backend(_target(), "auto", speed_active = True, family = _klein()) is None+def test_apply_candidate_probe_error_is_native_and_retried(monkeypatch):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ n = []+ def boom(*a, **k):+ n.append(1)+ raise RuntimeError("out of memory")++ monkeypatch.setattr(att, "_time_rocm_flash_block", boom)+ pipe, set_to = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None+ assert len(n) == 2 and set_to == []+++def test_apply_candidate_without_geometry_is_native(monkeypatch):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(+ att, "_time_rocm_flash_block", lambda *a, **k: pytest.fail("no geometry, nothing to time")+ )+ set_to = []+ dit = types.SimpleNamespace(set_attention_backend = set_to.append, modules = lambda: iter(()))+ pipe = types.SimpleNamespace(transformer = dit)+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None+++def test_candidate_never_pip_installs(monkeypatch):+ assert att._ensure_backend_package(att.ROCM_FLASH_CANDIDATE) is None-def test_rocm_auto_klein_explicit_native_wins(monkeypatch):- calls = _rocm(monkeypatch)- _arch(monkeypatch)- for alias in ("native", "sdpa"):- assert (- select_attention_backend(_target(), alias, speed_active = True, family = _klein()) is None- )- assert calls == []++def _diffusers_flash(monkeypatch, flash_attn_func):+ """diffusers' real FLASH backend (unguarded, restored after the test) calling ``flash_attn_func``."""+ ad = pytest.importorskip("diffusers.models.attention_dispatch")+ backends = ad._AttentionBackendRegistry._backends+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, ad._flash_attention)+ monkeypatch.setattr(ad, "flash_attn_func", flash_attn_func)+ return ad+++def test_block_timer_runs_on_cpu_shapes(monkeypatch):+ torch = pytest.importorskip("torch")+ seen = []++ def flash_attn_func(+ q,+ k,+ v,+ dropout_p = 0.0,+ softmax_scale = None,+ causal = False,+ window_size = (-1, -1),+ return_attn_probs = False,+ ):+ seen.append((dropout_p, softmax_scale, causal, tuple(window_size), return_attn_probs))+ return torch.nn.functional.scaled_dot_product_attention(+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)+ ).transpose(1, 2)++ _diffusers_flash(monkeypatch, flash_attn_func)+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)+ att._ROCM_FLASH_LOCAL.tally = {}+ rec = att._time_rocm_flash_block("cpu", torch.bfloat16, 64, 4, 16, tokens = 32)+ assert rec["finite"] and rec["rel_l1"] == 0.0 and rec["native_ms"] > 0 and rec["flash_ms"] > 0+ assert len(seen) == 9 and set(seen) == {(0.0, None, False, (-1, -1), False)}+ assert att._ROCM_FLASH_LOCAL.tally == {"flash": 9}+++def test_flash_attn_rejecting_the_diffusers_arguments_is_not_admitted(monkeypatch, tmp_path):+ torch = pytest.importorskip("torch")+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_indexed_cuda_device", lambda d: "cpu")++ def old_api(q, k, v):+ return q++ _diffusers_flash(monkeypatch, old_api)+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)+ monkeypatch.setattr(torch.cuda, "empty_cache", lambda *a, **k: None)+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_TOKENS", 32)+ pipe, set_to = _pipe_with_attention(hidden = 64, heads = 4, head_dim = 16)+ with pytest.raises(TypeError):+ att._time_rocm_flash_block("cpu", torch.bfloat16, 64, 4, 16, tokens = 32)+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None+ assert set_to == [] and att.ROCM_FLASH_ADMISSIONS == {} and _cache_files(tmp_path) == []def test_nvidia_auto_klein_unchanged(monkeypatch):@@ -381,8 +538,1736 @@ def test_auto_attention_reason_names_the_engaged_kernel():def test_status_reason_is_derived_from_the_engaged_backend():- src = open(- att.__file__.replace("diffusion_attention.py", "diffusion.py"), encoding = "utf-8"- ).read()- assert src.count("auto_attention_reason(attention_engaged)") == 2- assert '"cuDNN fused attention upgrade"' not in src+ def read(name):+ return open(att.__file__.replace("diffusion_attention.py", name), encoding = "utf-8").read()++ image, video = read("diffusion.py"), read("video.py")+ assert image.count("attention_status_reason(attention_backend, attention_engaged)") == 1+ assert image.count("auto_attention_reason(attention_engaged)") == 1+ assert '"cuDNN fused attention upgrade"' not in image+ assert video.count("attention_status_reason(attention_backend, attention_engaged)") == 2+ assert "cuDNN fused attention on NVIDIA" not in video+++# Masked attention (HunyuanVideo 1.5 mixed padding) must never reach diffusers' flash backend, which raises on+# attn_mask; and every attention geometry that receives the backend must pass the timing, not only the most common one.+++def test_auto_flash_engages_only_with_the_mask_guard(monkeypatch):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(+ att,+ "_time_rocm_flash_block",+ lambda *a, **k: {+ "native_ms": 80.0,+ "flash_ms": 60.0,+ "rel_l1": 1e-3,+ "max_rel": 2e-3,+ "finite": True,+ "tokens": 4096,+ },+ )+ installed = []+ monkeypatch.setattr(+ att, "_install_rocm_flash_dispatch_guard", lambda: installed.append(1) or True+ )+ pipe, set_to = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"+ assert installed and set_to == ["flash"]+ # a diffusers build whose registry cannot be guarded: auto stays native rather than risk a masked call+ monkeypatch.setattr(att, "_install_rocm_flash_dispatch_guard", lambda: False)+ pipe2, set_to2 = _pipe_with_attention()+ assert att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) is None+ assert set_to2 == []+ pipe3, set_to3 = _pipe_with_attention()+ assert att.apply_attention_backend(pipe3, "flash", target = _target()) is None+ assert set_to3 == []+++def test_rocm_flash_guard_routes_masked_calls_to_native(monkeypatch):+ torch = pytest.importorskip("torch")+ ad = pytest.importorskip("diffusers.models.attention_dispatch")+ backends = ad._AttentionBackendRegistry._backends+ calls = []++ def fake_flash(+ query,+ key,+ value,+ attn_mask = None,+ **kw,+ ):+ if attn_mask is not None:+ raise ValueError("`attn_mask` is not supported for flash-attn 2.")+ calls.append("flash")+ return query++ def fake_native(+ query,+ key,+ value,+ attn_mask = None,+ **kw,+ ):+ calls.append("native")+ return query++ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, fake_flash)+ monkeypatch.setitem(backends, ad.AttentionBackendName.NATIVE, fake_native)+ assert att._install_rocm_flash_dispatch_guard() is True+ guarded = backends[ad.AttentionBackendName.FLASH]+ q = torch.zeros(1, 8, 2, 16, dtype = torch.bfloat16)+ mask = torch.ones(1, 1, 8, 8, dtype = torch.bool)+ guarded(q, q, q, attn_mask = mask)+ guarded(q, q, q, None)+ assert calls == ["native", "flash"]+ assert att._install_rocm_flash_dispatch_guard() is True+ assert backends[ad.AttentionBackendName.FLASH] is guarded+++def _two_geometry_pipe():+ set_to = []++ def dit(hidden, heads, head_dim, n):+ mods = [+ types.SimpleNamespace(+ heads = heads,+ to_q = types.SimpleNamespace(in_features = hidden, out_features = heads * head_dim),+ )+ for _ in range(n)+ ]+ return types.SimpleNamespace(+ set_attention_backend = set_to.append, modules = lambda: iter(mods)+ )++ first, second = dit(3072, 24, 128, 5), dit(1536, 12, 128, 1)+ monkey_dits = [first, second]+ return types.SimpleNamespace(transformer = first, transformer_2 = second), set_to, monkey_dits++[email protected]("minority_wins, engaged", [(True, "flash"), (False, None)])+def test_every_attention_geometry_is_timed(monkeypatch, minority_wins, engaged):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ pipe, set_to, dits = _two_geometry_pipe()+ monkeypatch.setattr(att, "_attention_dits", lambda p: dits)+ timed = []++ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ ):+ timed.append((hidden, heads, head_dim))+ flash = 60.0 if (hidden == 3072 or minority_wins) else 85.0+ return {+ "native_ms": 80.0,+ "flash_ms": flash,+ "rel_l1": 1e-3,+ "max_rel": 2e-3,+ "finite": True,+ "tokens": 4096,+ }++ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged+ assert (1536, 12, 128) in timed+ if engaged:+ assert sorted(timed) == [(1536, 12, 128), (3072, 24, 128)]+ assert set_to == ["flash", "flash"] and len(att.ROCM_FLASH_ADMISSIONS) == 2+ else:+ assert set_to == []+ n = len(timed)+ pipe2, set_to2, dits2 = _two_geometry_pipe()+ monkeypatch.setattr(att, "_attention_dits", lambda p: dits2)+ assert (+ att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) is None+ )+ assert len(timed) == n+++def test_num_heads_attention_modules_are_measured():+ # Krea 2's attention exposes num_heads, not heads: without the fallback auto could never time it.+ import types++ from torch import nn++ class Krea2LikeAttention(nn.Module):+ def __init__(self) -> None:+ super().__init__()+ self.num_heads = 24+ self.to_q = nn.Linear(3072, 3072, bias = False)++ dit = nn.Module()+ dit.attn = Krea2LikeAttention()+ pipe = types.SimpleNamespace()+ original = att._attention_dits+ try:+ att._attention_dits = lambda p: [dit]+ assert att._dit_attention_geometries(pipe) == ((3072, 24, 128),)+ finally:+ att._attention_dits = original+++def _timer(timed, flash_ms = 70.0):+ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ ):+ timed.append((hidden, heads, head_dim))+ return {+ "native_ms": 80.0,+ "flash_ms": flash_ms,+ "rel_l1": 1e-3,+ "max_rel": 2e-3,+ "finite": True,+ "tokens": 4096,+ }++ return fake_time+++def _restart(monkeypatch):+ """A new process: the in-memory verdicts are gone, the cache dir is not."""+ monkeypatch.setattr(att, "ROCM_FLASH_ADMISSIONS", {})+ monkeypatch.setattr(att, "_ROCM_FLASH_PROBE_CACHE", {})+++def _cache_files(tmp_path):+ root = tmp_path / "rocm_flash"+ return sorted(root.glob("probe_*.json")) if root.is_dir() else []++[email protected]("flash_ms, engaged", [(70.0, "flash"), (90.0, None)])+def test_block_verdict_persists_across_processes(monkeypatch, tmp_path, flash_ms, engaged):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed, flash_ms))+ pipe, _ = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged+ assert len(timed) == 1 and len(_cache_files(tmp_path)) == 1+ _restart(monkeypatch)+ monkeypatch.setattr(+ att,+ "_time_rocm_flash_block",+ lambda *a, **k: pytest.fail("re-timed despite a stored verdict"),+ )+ pipe2, set_to2 = _pipe_with_attention()+ assert att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged+ assert set_to2 == (["flash"] if engaged else [])+ rec = next(iter(att.ROCM_FLASH_ADMISSIONS.values()))+ assert rec["cached"] is True and rec["flash_ms"] == flash_ms++[email protected]("change", ["torch", "flash_attn", "probe_version", "geometry", "dtype"])+def test_block_verdict_key_mismatch_retimes(monkeypatch, tmp_path, change):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))+ toolchain = {"torch": "torch2.9", "flash_attn": "flash_attn2.8.3"}+ monkeypatch.setattr(+ att,+ "_rocm_flash_toolchain_key",+ lambda device: f"v{att.ROCM_FLASH_PROBE_VERSION}|{toolchain['torch']}|{toolchain['flash_attn']}",+ )+ pipe, _ = _pipe_with_attention()+ att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target())+ assert len(timed) == 1+ _restart(monkeypatch)+ target, geometry = _target(), {}+ if change == "torch":+ toolchain["torch"] = "torch2.10"+ elif change == "flash_attn":+ toolchain["flash_attn"] = "flash_attn2.8.4"+ elif change == "probe_version":+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_VERSION", att.ROCM_FLASH_PROBE_VERSION + 1)+ elif change == "geometry":+ geometry = {"hidden": 2048, "heads": 16, "head_dim": 128}+ else:+ target = _target(dtype = "torch.float16")+ pipe2, _ = _pipe_with_attention(**geometry)+ att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = target)+ assert len(timed) == 2+++ "payload", ["{not json", "", "[]", '{"key": "tc-A", "verdicts": {"x": 1}}']+)+def test_corrupt_cache_file_retimes(monkeypatch, tmp_path, payload):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))+ pipe, _ = _pipe_with_attention()+ att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target())+ (path,) = _cache_files(tmp_path)+ path.write_text(payload)+ _restart(monkeypatch)+ pipe2, _ = _pipe_with_attention()+ assert att.apply_attention_backend(pipe2, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"+ assert len(timed) == 2+ _restart(monkeypatch)+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ assert len(timed) == 2+++def test_probe_error_is_not_persisted(monkeypatch, tmp_path):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)++ def boom(*a, **k):+ raise RuntimeError("out of memory")++ monkeypatch.setattr(att, "_time_rocm_flash_block", boom)+ pipe, _ = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) is None+ assert _cache_files(tmp_path) == []+++def test_unidentifiable_toolchain_stays_in_memory(monkeypatch, tmp_path):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: None)+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))+ pipe, _ = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"+ assert _cache_files(tmp_path) == [] and not (tmp_path / "rocm_flash").exists()+++def test_unwritable_cache_dir_still_admits(monkeypatch, tmp_path):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ blocker = tmp_path / "file"+ blocker.write_text("")+ monkeypatch.setenv(att.ROCM_FLASH_CACHE_DIR_ENV, str(blocker / "sub"))+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))+ pipe, _ = _pipe_with_attention()+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"++[email protected]("mode, engaged", [("0", None), ("1", "flash")])+def test_forced_modes_neither_read_nor_write_the_cache(monkeypatch, tmp_path, mode, engaged):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ # a stored verdict that would say the opposite of the forced mode+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed, 70.0 if mode == "0" else 90.0))+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ (path,) = _cache_files(tmp_path)+ before = path.read_bytes()+ _restart(monkeypatch)+ monkeypatch.setenv(att.ROCM_FLASH_ENV, mode)+ monkeypatch.setattr(att, "_load_rocm_flash_verdicts", lambda *a: pytest.fail("cache read"))+ monkeypatch.setattr(att, "_save_rocm_flash_verdict", lambda *a: pytest.fail("cache write"))+ monkeypatch.setattr(att, "_time_rocm_flash_block", lambda *a, **k: pytest.fail("timed"))+ pipe, _ = _pipe_with_attention(hidden = 2048, heads = 16)+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == engaged+ assert path.read_bytes() == before and att.ROCM_FLASH_ADMISSIONS == {}+++def test_cache_dir_defaults_under_the_studio_cache(monkeypatch, tmp_path):+ monkeypatch.delenv(att.ROCM_FLASH_CACHE_DIR_ENV)+ import utils.paths.storage_roots as roots++ monkeypatch.setattr(roots, "cache_root", lambda: tmp_path / "cache")+ path = att._rocm_flash_cache_path("tc-A")+ assert path.startswith(str(tmp_path / "cache" / "diffusion_rocm_flash" / "probe_"))+ assert att._rocm_flash_cache_path("tc-B") != path and att._rocm_flash_cache_path(None) is None+++def test_toolchain_key_names_the_flash_attn_build(monkeypatch):+ pytest.importorskip("diffusers.models.attention_dispatch")+ import torch++ props = types.SimpleNamespace(name = "AMD Radeon 8060S", gcnArchName = "gfx1151:sramecc-:xnack-")+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda i: props)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(torch.version, "hip", "7.1.0", raising = False)+ fa = types.ModuleType("flash_attn")+ fa.__version__ = "2.8.3"+ fa.__file__ = "/x/site-packages/flash_attn/__init__.py"+ monkeypatch.setitem(sys.modules, "flash_attn", fa)+ # A real flash_attn build already imported (a ROCm host) would answer the backend through its USE_TRITON_ROCM.+ monkeypatch.delitem(sys.modules, "flash_attn.flash_attn_interface", raising = False)+ monkeypatch.delenv("FLASH_ATTENTION_TRITON_AMD_ENABLE", raising = False)+ ck = _REAL_TOOLCHAIN_KEY("cuda:0")+ assert ck is not None and ck.startswith(f"v{att.ROCM_FLASH_PROBE_VERSION}|")+ for part in (+ "AMD Radeon 8060S",+ "|gfx1151|",+ "|hip7.1.0|",+ "|flash_attn2.8.3|",+ os.sep + "flash_attn|ck|",+ ):+ assert part in ck, part+ fa.__version__ = "2.8.4"+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != ck+ fa.__version__ = "2.8.3"+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_VERSION", att.ROCM_FLASH_PROBE_VERSION + 1)+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != ck+ monkeypatch.setattr(att, "ROCM_FLASH_PROBE_VERSION", att.ROCM_FLASH_PROBE_VERSION - 1)+ assert _REAL_TOOLCHAIN_KEY("cuda:0") == ck+ pytest.importorskip("triton")+ monkeypatch.setenv("FLASH_ATTENTION_TRITON_AMD_ENABLE", "TRUE")+ assert "|triton" in _REAL_TOOLCHAIN_KEY("cuda:0")+++# A flash_attn rebuilt in place (same version, same path) must re-time: the key fingerprints the loaded kernel files.+++def _fake_build(+ monkeypatch,+ tmp_path,+ *,+ triton = False,+):+ import torch++ props = types.SimpleNamespace(name = "AMD Radeon 8060S", gcnArchName = "gfx1151")+ monkeypatch.setattr(torch.cuda, "get_device_properties", lambda i: props)+ monkeypatch.setattr(torch.cuda, "current_device", lambda: 0)+ monkeypatch.setattr(torch.version, "hip", "7.1.0", raising = False)+ pkg = tmp_path / "site" / "flash_attn"+ pkg.mkdir(parents = True)+ fa = types.ModuleType("flash_attn")+ fa.__version__ = "2.8.3"+ fa.__file__ = str(pkg / "__init__.py")+ iface = types.ModuleType("flash_attn.flash_attn_interface")+ iface.__file__ = str(pkg / "flash_attn_interface.py")+ iface.USE_TRITON_ROCM = triton+ for f in (fa.__file__, iface.__file__):+ open(f, "w").write("# py\n")+ if triton:+ kdir = tmp_path / "site" / "triton_amd"+ kdir.mkdir()+ (kdir / "fwd_prefill.py").write_text("# kernel v1\n")+ binary = kdir / "flash_attn_2.py"+ else:+ binary = tmp_path / "site" / "flash_attn_2_cuda.cpython-312-x86_64-linux-gnu.so"+ binary.write_bytes(b"\x7fELF v1")+ gpu = types.ModuleType("flash_attn_gpu")+ gpu.__file__ = str(binary)+ iface.flash_attn_gpu = gpu+ monkeypatch.setitem(sys.modules, "flash_attn", fa)+ monkeypatch.setitem(sys.modules, "flash_attn.flash_attn_interface", iface)+ if triton:+ pytest.importorskip("triton")+ return kdir / "fwd_prefill.py"+ monkeypatch.setitem(sys.modules, "flash_attn_2_cuda", gpu)+ return binary++[email protected]("triton", [False, True])[email protected]("how", ["content", "mtime"])+def test_rebuilt_flash_attn_binary_changes_the_key(monkeypatch, tmp_path, triton, how):+ pytest.importorskip("diffusers.models.attention_dispatch")+ import os++ kernel = _fake_build(monkeypatch, tmp_path, triton = triton)+ key = _REAL_TOOLCHAIN_KEY("cuda:0")+ assert key is not None and ("|triton" in key) is triton and _REAL_TOOLCHAIN_KEY("cuda:0") == key+ st = os.stat(kernel)+ if how == "content":+ kernel.write_bytes(kernel.read_bytes() + b" rebuilt")+ os.utime(kernel, ns = (st.st_atime_ns, st.st_mtime_ns))+ else:+ os.utime(kernel, ns = (st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != key+++def test_rebuilt_flash_attn_binary_forces_retiming(monkeypatch, tmp_path):+ pytest.importorskip("diffusers.models.attention_dispatch")+ import os++ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ binary = _fake_build(+ monkeypatch, tmp_path+ ) # after _rocm, which installs a bare flash_attn module+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)+ assert _REAL_TOOLCHAIN_KEY("cuda:0") is not None+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ _restart(monkeypatch)+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ assert len(timed) == 1+ binary.write_bytes(b"\x7fELF v2 rebuilt")+ st = os.stat(binary)+ os.utime(binary, ns = (st.st_atime_ns, st.st_mtime_ns + 1_000_000_000))+ _restart(monkeypatch)+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ assert len(timed) == 2+++# Malformed cached records are a cache miss (re-timed), never a verdict, and never break the verdict log.++_GOOD = {+ "native_ms": 80.0,+ "flash_ms": 70.0,+ "rel_l1": 1e-3,+ "max_rel": 2e-3,+ "finite": True,+ "tokens": 4096,+}+++ "field, value",+ [+ ("native_ms", "80"),+ ("flash_ms", "70"),+ ("rel_l1", "1e-3"),+ ("finite", "true"),+ ("finite", 1),+ ("tokens", "4096"),+ ("tokens", 4096.5),+ ("tokens", True),+ ("native_ms", {"ms": 80}),+ ("flash_ms", [70.0]),+ ("rel_l1", None),+ ("native_ms", float("nan")),+ ("flash_ms", float("inf")),+ ("rel_l1", float("nan")),+ ("native_ms", -80.0),+ ("flash_ms", 0.0),+ ("rel_l1", -1e-3),+ ("tokens", -4096),+ ("admitted", "yes"),+ ("geometry", {"hidden": "3072", "heads": 24, "head_dim": 128}),+ ("geometry", [3072, 24, 128]),+ ("native_ms", True),+ ("drop", "flash_ms"),+ ("drop", "max_rel"),+ ("max_rel", "2e-3"),+ ("max_rel", float("inf")),+ ("max_rel", -1.0),+ ("block_rel_l1", "x"),+ ],+)+def test_malformed_cached_record_is_retimed(monkeypatch, tmp_path, field, value):+ import json++ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed, 90.0))+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ (path,) = _cache_files(tmp_path)+ data = json.loads(path.read_text())+ (slot,) = data["verdicts"]+ rec = dict(_GOOD) # a stored win: a malformed copy of it must not admit flash+ if field == "drop":+ rec.pop(value)+ else:+ rec[field] = value+ data["verdicts"][slot] = rec+ path.write_text(json.dumps(data))+ _restart(monkeypatch)+ logs = []++ class Log:+ def info(self, *a):+ logs.append(a[0] % a[1:])++ assert (+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, logger = Log(), target = _target()+ )+ is None+ )+ assert len(timed) == 2 and any("probe" in line and "-> native" in line for line in logs)+++def test_non_finite_probe_verdict_is_stored_and_reused(monkeypatch, tmp_path):+ # a flash output with NaN is a real (native) answer: stored, read back, never re-timed and never raising+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ timed = []++ def nan_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ ):+ timed.append(1)+ return {**_GOOD, "finite": False, "rel_l1": float("nan"), "max_rel": float("nan")}++ monkeypatch.setattr(att, "_time_rocm_flash_block", nan_time)+ assert (+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ is None+ )+ _restart(monkeypatch)+ assert (+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ is None+ )+ assert len(timed) == 1+++def test_implausible_probe_timing_is_native_and_not_stored(monkeypatch, tmp_path):+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(+ att, "_time_rocm_flash_block", lambda *a, **k: {**_GOOD, "native_ms": 0.0, "flash_ms": 0.0}+ )+ assert (+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ is None+ )+ assert _cache_files(tmp_path) == []+++ "rec",+ [+ {},+ {"flash_ms": "x", "native_ms": {"a": 1}, "rel_l1": [1], "tokens": "n", "admitted": "yes"},+ {**_GOOD, "flash_ms": float("nan"), "rel_l1": float("inf"), "admitted": True},+ ],+)+def test_verdict_log_never_raises(rec):+ class Log:+ def info(self, fmt, *a):+ fmt % a++ att._log_rocm_flash_verdict(Log(), "cuda:0", (3072, 24, 128), rec, "cached")+ att._log_rocm_flash_verdict(Log(), "cuda:0", (), rec, "cached")+++# Accuracy is judged on the raw attention output: a wrong attention the residual + FFN would hide is still rejected.+++def test_wrong_attention_is_rejected_though_the_block_would_pass(monkeypatch):+ torch = pytest.importorskip("torch")++ def off_by_20pct(q, k, v, **kw):+ out = torch.nn.functional.scaled_dot_product_attention(+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)+ ).transpose(1, 2)+ return out * 1.2++ _diffusers_flash(monkeypatch, off_by_20pct)+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)+ # inner == hidden as in a real DiT; at 1024 tokens the block output moves only ~1.7%+ rec = att._time_rocm_flash_block("cpu", torch.bfloat16, 256, 4, 64, tokens = 1024)+ # the old block-level check would have admitted it+ assert rec["block_rel_l1"] <= att.ROCM_FLASH_BLOCK_TOL+ assert rec["rel_l1"] > 0.15 and rec["max_rel"] > 0.1+ assert att._rocm_flash_record_ok(rec) and not att._rocm_flash_wins(+ {**rec, "native_ms": 80.0, "flash_ms": 40.0}+ )++[email protected]("max_rel, wins", [(1e-2, True), (0.2, False)])+def test_max_abs_error_alone_rejects(max_rel, wins):+ assert att._rocm_flash_wins({**_GOOD, "max_rel": max_rel}) is wins+++# Lengths: calls are gated per power-of-two length bucket; an unseen bucket is timed on its first eager call and never+# inherits the 4096 verdict.+++ "tokens, bucket",+ [+ (1, 256),+ (300, 256),+ (900, 512),+ (4096, 4096),+ (4608, 4096),+ (8191, 4096),+ (27280, 16384),+ (50000, 32768),+ (140000, 131072),+ ],+)+def test_length_buckets(tokens, bucket):+ assert att._rocm_flash_bucket(tokens) == bucket+++def _length_env(monkeypatch, verdicts):+ """An auto load admitted at 4096; ``verdicts[tokens]`` is the flash_ms (native 80) the bucket probe measures."""+ _rocm(monkeypatch)+ guarded, ran = _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ timed = []++ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ kv_tokens = None,+ ctx_dim = None,+ batch = 1,+ ):+ size = tokens if kv_tokens is None else (tokens, kv_tokens)+ if batch != 1:+ size = (size, f"b{batch}")+ if ctx_dim is not None:+ size = (size, f"ctx{ctx_dim}")+ timed.append(size)+ return {+ **_GOOD,+ "flash_ms": verdicts[size],+ "tokens": tokens,+ **({} if kv_tokens is None else {"kv_tokens": kv_tokens}),+ **({} if batch == 1 else {"batch": batch}),+ }++ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)+ dit = _Dit()+ assert (+ att.apply_attention_backend(+ types.SimpleNamespace(transformer = dit), att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ == "flash"+ )+ monkeypatch.setattr(+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d in ("cuda", "cpu") else d+ )+ return dit, guarded, ran, timed+++def test_unprobed_long_bucket_does_not_inherit_the_4096_verdict(monkeypatch):+ dit, guarded, ran, timed = _length_env(+ monkeypatch, {4096: 70.0, 16384: 90.0, (256, 1024): 70.0}+ )+ assert timed == [4096]+ dit.run(guarded, [(4608, None), (27280, None), (20000, None), (16, 1200)])+ assert timed == [4096, 16384, (256, 1024)]+ assert ran == ["flash", "native", "native", "flash"]+ assert att._ROCM_FLASH_ROUTED == {"slower or inexact at this size": 2}+++def test_50k_call_runs_flash_only_if_its_own_bucket_admitted_it(monkeypatch):+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 90.0, 65536: 60.0})+ dit.run(guarded, [(50000, None), (70000, None), (140000, None)])+ assert timed == [4096, 32768, 65536, 131072]+ assert ran == ["native", "flash", "native"]+++def test_bucket_probe_error_stays_native_without_retrying_every_call(monkeypatch):+ verdicts = {4096: 70.0}+ dit, guarded, ran, timed = _length_env(monkeypatch, verdicts)+ dit.run(guarded, [(40000, None), (40000, None)])+ assert timed == [4096, 32768] and ran == ["native", "native"]+ deferred = getattr(dit, att.ROCM_FLASH_DEFERRED_ATTR)+ assert len(deferred) == 1 and next(iter(deferred)) not in att._ROCM_FLASH_BUCKETS+ verdicts[32768] = 60.0+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = dit)))+ dit.run(guarded, [(40000, None)])+ assert timed == [4096, 32768, 32768] and ran == ["native", "native", "flash"]+++# An in-forward probe sits on top of the live request's tensors; no room for it defers, never rejects.+++def test_bucket_probe_without_room_beside_the_forward_is_deferred_not_rejected(monkeypatch):+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 60.0})+ needs = []+ monkeypatch.setattr(+ att, "_rocm_flash_probe_fits", lambda device, need: needs.append(need) or False+ )+ dit.run(guarded, [(40000, None), (40000, None)])+ assert timed == [4096] and ran == ["native", "native"] and len(needs) == 1+ assert att._ROCM_FLASH_BUCKETS == {} and len(getattr(dit, att.ROCM_FLASH_DEFERRED_ATTR)) == 1+ assert att._ROCM_FLASH_ROUTED == {"timing deferred": 2}+ assert not any(k[-1] == 32768 or 32768 in k for k in att.ROCM_FLASH_ADMISSIONS)+ # Room again at the next generation: timed and admitted.+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True)+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = dit)))+ dit.run(guarded, [(40000, None)])+ assert timed == [4096, 32768] and ran[-1] == "flash"+++def test_deferral_belongs_to_its_dit_not_to_other_generations(monkeypatch):+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 60.0})+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: False)+ dit.run(guarded, [(40000, None)])+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: True)+ other = _Dit()+ att.apply_attention_backend(+ types.SimpleNamespace(transformer = other), att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = other)))+ dit.run(guarded, [(40000, None)])+ assert timed == [4096] and ran == ["native", "native"]+ att.begin_attention_run(types.SimpleNamespace(pipe = types.SimpleNamespace(transformer = dit)))+ dit.run(guarded, [(40000, None)])+ assert timed == [4096, 32768] and ran[-1] == "flash"+++def test_a_moved_sdpa_baseline_drops_the_in_memory_verdicts(monkeypatch):+ baseline = ["sdpafem,aotriton-exp1"]+ monkeypatch.setattr(att, "_native_sdpa_baseline", lambda: baseline[0])+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 16384: 60.0})+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: "tc-" + baseline[0])+ dit.run(guarded, [(20000, None), (20000, None)])+ assert timed == [4096, 16384] and ran == ["flash", "flash"]+ baseline[0] = "sdpa--m,aotriton-exp1"+ dit.run(guarded, [(20000, None)])+ assert timed == [4096, 16384, 16384] and len(att.ROCM_FLASH_ADMISSIONS) == 1+ dit.run(guarded, [(20000, None)])+ assert timed == [4096, 16384, 16384]+++def test_probe_bound_adds_math_sdpa_scores_when_fused_sdpa_is_off(monkeypatch):+ torch = pytest.importorskip("torch")+ g = (3072, 24, 128)+ monkeypatch.setattr(att, "_native_sdpa_fused_serves", lambda *a: True)+ fused = att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16)+ monkeypatch.setattr(att, "_native_sdpa_fused_serves", lambda *a: False)+ math = att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16)+ assert math - fused == 2 * 24 * 16384 * 16384 * 4+++def test_fused_sdpa_eligibility_is_asked_for_the_probe_shape(monkeypatch):+ torch = pytest.importorskip("torch")+ seen = []++ def can_flash(params, debug):+ seen.append((tuple(params.query.shape), tuple(params.key.shape)))+ return params.query.shape[2] < 32768++ monkeypatch.setattr(torch.backends.cuda, "can_use_flash_attention", can_flash)+ monkeypatch.setattr(torch.backends.cuda, "can_use_efficient_attention", lambda p, d: False)+ monkeypatch.setattr(torch.backends.cuda, "flash_sdp_enabled", lambda: True)+ g = (3072, 24, 128)+ short = att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16, "cpu")+ long = att._rocm_flash_probe_bytes(g, 32768, 512, 2, torch.bfloat16, "cpu")+ assert seen == [+ ((1, 24, 16384, 128), (1, 24, 16384, 128)),+ ((2, 24, 32768, 128), (2, 24, 512, 128)),+ ]+ assert short < 4 * 2**30+ assert long > 2 * 2 * 24 * 32768 * 512 * 4+ monkeypatch.setattr(torch.backends.cuda, "flash_sdp_enabled", lambda: False)+ assert (+ att._rocm_flash_probe_bytes(g, 16384, 16384, 1, torch.bfloat16, "cpu") - short+ == 2 * 24 * 16384**2 * 4+ )+++def test_probe_tensors_are_drawn_on_the_device():+ import inspect+ src = inspect.getsource(att._time_rocm_flash_block)+ assert ".to(device, dtype)" not in src and "torch.Generator(device = device)" in src+++def test_native_sdpa_baseline_keys_the_verdicts(monkeypatch, tmp_path):+ pytest.importorskip("diffusers.models.attention_dispatch")+ torch = pytest.importorskip("torch")+ _fake_build(monkeypatch, tmp_path)+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)+ flags = {"flash": True, "mem": True}+ monkeypatch.setattr(torch.backends.cuda, "flash_sdp_enabled", lambda: flags["flash"])+ monkeypatch.setattr(torch.backends.cuda, "mem_efficient_sdp_enabled", lambda: flags["mem"])+ monkeypatch.setattr(torch.backends.cuda, "math_sdp_enabled", lambda: True)+ monkeypatch.setenv("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL", "1")+ fused = _REAL_TOOLCHAIN_KEY("cuda:0")+ assert fused is not None and "|sdpafem,aotriton-exp1|" in fused+ flags.update(flash = False, mem = False)+ math_only = _REAL_TOOLCHAIN_KEY("cuda:0")+ assert math_only != fused and "|sdpa--m,aotriton-exp1|" in math_only+ flags.update(flash = True, mem = True)+ monkeypatch.delenv("TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL")+ assert _REAL_TOOLCHAIN_KEY("cuda:0") not in (fused, math_only)+++def test_probe_byte_bound_covers_the_measured_peak(monkeypatch):+ torch = pytest.importorskip("torch")+ monkeypatch.setattr(att, "_native_sdpa_fused_serves", lambda *a: True)+ for tokens, peak in ((1024, 0.31), (65536, 5.46)):+ need = (+ att._rocm_flash_probe_bytes((3072, 24, 128), tokens, tokens, 1, torch.bfloat16) / 2**30+ )+ assert peak <= need < 2 * peak+++def test_probe_fits_counts_cached_blocks_and_fails_closed(monkeypatch):+ torch = pytest.importorskip("torch")+ gib = 1 << 30+ monkeypatch.setattr(torch.cuda, "mem_get_info", lambda d: (2 * gib, 80 * gib))+ monkeypatch.setattr(torch.cuda, "memory_reserved", lambda d: 10 * gib)+ monkeypatch.setattr(torch.cuda, "memory_allocated", lambda d: 7 * gib)+ assert _REAL_PROBE_FITS("cuda:0", 3 * gib)+ assert not _REAL_PROBE_FITS("cuda:0", 4 * gib)++ def boom(d):+ raise RuntimeError("no device")++ monkeypatch.setattr(torch.cuda, "mem_get_info", boom)+ assert not _REAL_PROBE_FITS("cuda:0", 1)+++def test_unprobed_bucket_under_capture_runs_native_untimed(monkeypatch):+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 8192: 70.0})+ monkeypatch.setattr(att, "_rocm_flash_capturing", lambda: True)+ dit.run(guarded, [(9000, None)])+ assert timed == [4096] and ran == ["native"]+ monkeypatch.setattr(att, "_rocm_flash_capturing", lambda: False)+ dit.run(guarded, [(9000, None)])+ assert timed == [4096, 8192] and ran == ["native", "flash"]+++def test_bucket_verdicts_persist(monkeypatch, tmp_path):+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 16384: 60.0})+ dit.run(guarded, [(20000, None)])+ assert timed == [4096, 16384]+ _restart(monkeypatch)+ monkeypatch.setattr(att, "_ROCM_FLASH_BUCKETS", {})+ dit2, guarded2, ran2, timed2 = _length_env(monkeypatch, {})+ dit2.run(guarded2, [(20000, None)])+ assert timed2 == [] and ran2 == ["flash"]++[email protected]("order", ["auto_first", "explicit_first"])[email protected]("explicit", ["flash", "mode1"])+def test_explicit_flash_is_never_length_gated(monkeypatch, order, explicit):+ # an auto model and an explicitly selected one share (heads, head_dim): only the auto one's calls are gated+ _rocm(monkeypatch)+ guarded, ran = _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ timed = []++ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ ):+ timed.append(tokens)+ return {**_GOOD, "flash_ms": 70.0 if tokens == 4096 else 90.0, "tokens": tokens}++ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)+ auto_dit, explicit_dit = _Dit(), _Dit()++ def load_auto():+ assert (+ att.apply_attention_backend(+ types.SimpleNamespace(transformer = auto_dit),+ att.ROCM_FLASH_CANDIDATE,+ target = _target(),+ )+ == "flash"+ )++ def load_explicit():+ if explicit == "flash":+ backend = att.apply_attention_backend(+ types.SimpleNamespace(transformer = explicit_dit), "flash", target = _target()+ )+ else:+ monkeypatch.setenv(att.ROCM_FLASH_ENV, "1")+ backend = att.apply_attention_backend(+ types.SimpleNamespace(transformer = explicit_dit),+ att.ROCM_FLASH_CANDIDATE,+ target = _target(),+ )+ monkeypatch.delenv(att.ROCM_FLASH_ENV)+ assert backend == "flash"++ for load in (load_auto, load_explicit) if order == "auto_first" else (load_explicit, load_auto):+ load()+ monkeypatch.setattr(+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d in ("cuda", "cpu") else d+ )+ explicit_dit.run(guarded, [(40000, None)])+ assert ran == ["flash"] and timed == [4096]+ auto_dit.run(guarded, [(40000, None)])+ assert ran == ["flash", "native"] and timed == [4096, 32768]+ explicit_dit.run(guarded, [(40000, None)])+ assert ran[-1] == "flash" and getattr(att._ROCM_FLASH_LOCAL, "gate", None) is None+++def test_reapplying_explicit_flash_disarms_the_gate(monkeypatch):+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, 32768: 90.0})+ assert (+ att.apply_attention_backend(+ types.SimpleNamespace(transformer = dit), "flash", target = _target()+ )+ == "flash"+ )+ dit.run(guarded, [(40000, None)])+ assert ran == ["flash"] and timed == [4096]+++def test_failed_forward_does_not_leave_its_gate_active(monkeypatch):+ torch = pytest.importorskip("torch")+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))++ class Boom(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.attn = torch.nn.Module()+ self.attn.heads = 2+ self.attn.to_q = torch.nn.Linear(32, 32, bias = False)++ def set_attention_backend(self, name):+ pass++ def forward(self, x):+ raise RuntimeError("cancelled")++ dit = Boom()+ att.apply_attention_backend(+ types.SimpleNamespace(transformer = dit), att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ with pytest.raises(RuntimeError):+ dit(torch.zeros(1))+ assert getattr(att._ROCM_FLASH_LOCAL, "gate", None) is None+++class _Dit:+ """A DiT whose forward makes attention calls (masked or not) through the guarded FLASH backend."""++ def __init__(self):+ torch = pytest.importorskip("torch")+ self.inner = torch.nn.Module()+ self.backends = []+ self.inner.heads = 2+ self.inner.to_q = torch.nn.Linear(32, 32, bias = False)+ self.hooks = {"pre": [], "post": []}++ def modules(self):+ return iter([self.inner])++ def set_attention_backend(self, name):+ self.backends.append(name)++ def register_forward_pre_hook(self, fn):+ self.hooks["pre"].append(fn)+ return fn++ def register_forward_hook(+ self,+ fn,+ always_call = False,+ ):+ self.hooks["post"].append(fn)+ return fn++ def run(self, guarded, specs):+ """One forward making a call per ``(q_tokens, kv_tokens or None)``, unmasked."""+ import torch++ for fn in self.hooks["pre"]:+ fn(self, ())+ for tokens, kv in specs:+ q = torch.zeros(1, tokens, 2, 16, dtype = torch.bfloat16)+ k = torch.zeros(1, kv or tokens, 2, 16, dtype = torch.bfloat16)+ guarded(q, k, k, None)+ for fn in self.hooks["post"]:+ fn(self, (), None)++ def __call__(self, guarded, masks):+ import torch++ for fn in self.hooks["pre"]:+ fn(self, ())+ q = torch.zeros(1, 4096, 2, 16, dtype = torch.bfloat16)+ for masked in masks:+ guarded(q, q, q, attn_mask = torch.ones(1, 1, 8, 8, dtype = torch.bool) if masked else None)+ for fn in self.hooks["post"]:+ fn(self, (), None)+++def _guarded_flash(monkeypatch):+ ad = pytest.importorskip("diffusers.models.attention_dispatch")+ backends = ad._AttentionBackendRegistry._backends+ ran = []++ def fake_flash(+ query,+ key,+ value,+ attn_mask = None,+ **kw,+ ):+ if attn_mask is not None:+ raise ValueError("`attn_mask` is not supported for flash-attn 2.")+ ran.append("flash")+ return query++ def fake_native(+ query,+ key,+ value,+ attn_mask = None,+ **kw,+ ):+ ran.append("native")+ return query++ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, fake_flash)+ monkeypatch.setitem(backends, ad.AttentionBackendName.NATIVE, fake_native)+ assert att._install_rocm_flash_dispatch_guard() is True+ return backends[ad.AttentionBackendName.FLASH], ran+++def _state(pipe, backend):+ import dataclasses++ @dataclasses.dataclass(frozen = True)+ class State:+ pipe: object+ attention_backend: object+ resolved: dict++ resolved = {+ "attention_backend": {"source": "auto", "value": backend or "native", "reason": "load"}+ }+ return State(pipe, backend, resolved)+++def _load(monkeypatch):+ dit = _Dit()+ pipe = types.SimpleNamespace(transformer = dit)+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"+ return dit, pipe, _state(pipe, "flash")+++ "masks, value, phrase",+ [+ (+ [True, True],+ "native",+ "every attention call of the last generation ran native SDPA (attn_mask: 2)",+ ),+ ([False, False], "flash", "on every attention call"),+ (+ [True, False],+ "flash",+ "on 1 of 2 attention calls of the last generation; the rest ran native SDPA (attn_mask: 1)",+ ),+ ],+)+def test_badge_reports_the_routes_the_run_took(monkeypatch, masks, value, phrase):+ _rocm(monkeypatch)+ guarded, ran = _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))+ dit, pipe, state = _load(monkeypatch)+ att.begin_attention_run(state)+ dit(guarded, masks)+ att.report_attention_run(state)+ assert ran == ["native" if m else "flash" for m in masks]+ entry = state.resolved["attention_backend"]+ assert entry["value"] == value and phrase in entry["reason"] and entry["source"] == "auto"+ # report only: the engaged backend (recipes, calibration and compile keys) is untouched, nothing was switched+ assert state.attention_backend == "flash" and dit.backends == ["flash"]+ ran.clear()+ dit(guarded, [False])+ assert ran == ["flash"]+++def test_mixed_mask_model_keeps_flash_for_unmasked_calls_across_loads(monkeypatch, tmp_path):+ # HunyuanVideo 1.5 style: padded batches mask, unpadded ones do not. A masked-only first run changes nothing for+ # the next load, which serves its unmasked calls on flash.+ _rocm(monkeypatch)+ guarded, ran = _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))+ dit, pipe, state = _load(monkeypatch)+ dit(guarded, [True, True])+ _restart(monkeypatch)+ dit2, pipe2, state2 = _load(monkeypatch)+ assert len(timed) == 1 and dit2.backends == ["flash"]+ ran.clear()+ dit2(guarded, [False, True])+ assert ran == ["flash", "native"]+ assert not list((tmp_path / "rocm_flash").glob("masked*"))+++def test_each_generation_reports_its_own_calls(monkeypatch):+ # run 1 flash, run 2 masked only, run 3 flash: each badge describes its own run, never a lifetime mix+ _rocm(monkeypatch)+ guarded, ran = _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))+ dit, pipe, state = _load(monkeypatch)+ for masks, value in (([False, False], "flash"), ([True, True], "native"), ([False], "flash")):+ att.begin_attention_run(state)+ dit(guarded, masks)+ att.report_attention_run(state)+ assert state.resolved["attention_backend"]["value"] == value+ assert state.attention_backend == "flash"+++def test_compiled_run_reports_routing_untracked(monkeypatch):+ # a compiled / graphed run never re-enters the guard's Python: the badge says routing is not tracked+ _rocm(monkeypatch)+ _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))+ dit, pipe, state = _load(monkeypatch)+ att.begin_attention_run(state)+ dit.run(lambda *a, **k: None, [])+ att.report_attention_run(state)+ entry = state.resolved["attention_backend"]+ assert (+ entry["value"] == "flash"+ and "not tracked for compiled or graphed attention" in entry["reason"]+ )+++def test_route_tallies_are_per_thread(monkeypatch):+ # an image and a video forward on two threads at once: each DiT's tally gets only its own calls, and each thread's+ # gate is its own+ import threading++ _rocm(monkeypatch)+ guarded, ran = _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))+ a, _, _ = _load(monkeypatch)+ b = _Dit()+ assert (+ att.apply_attention_backend(types.SimpleNamespace(transformer = b), "flash", target = _target())+ == "flash"+ )+ inside = threading.Barrier(2)+ gates = {}++ class Pausing:+ def __init__(self, name):+ self.name = name++ def __call__(self, *args, **kwargs):+ if self.name not in gates:+ inside.wait(timeout = 10)+ gates[self.name] = att._active_rocm_flash_gate()+ return guarded(*args, **kwargs)++ ta = threading.Thread(target = a, args = (Pausing("a"), [True, True, True]))+ tb = threading.Thread(target = b, args = (Pausing("b"), [False]))+ ta.start(), tb.start()+ ta.join(10), tb.join(10)+ tally_a = getattr(a, att.ROCM_FLASH_TALLY_ATTR)+ tally_b = getattr(b, att.ROCM_FLASH_TALLY_ATTR)+ assert tally_a == {"attn_mask": 3} and tally_b == {"flash": 1}+ assert gates["a"] == getattr(a, att.ROCM_FLASH_GATE_ATTR) and gates["b"] is None+++# The activation memory guard must not trust the flash pick when calls can run native SDPA.++[email protected]("routed, quadratic", [(True, True), (False, False)])+def test_memory_guard_budgets_quadratic_when_flash_can_hand_calls_to_native(+ monkeypatch, routed, quadratic+):+ import core.inference.diffusion as diffusion++ _rocm(monkeypatch)+ _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))+ monkeypatch.setattr(diffusion, "sdpa_math_only", lambda target: True)+ import core.inference.diffusion_qwenimage21_math as qm++ monkeypatch.setattr(qm, "bounded_math_attention", lambda pipe: False)+ dit, pipe, state = _load(monkeypatch)+ if not routed:+ setattr(dit, att._ROCM_FLASH_HOOKS_ATTR, None)+ assert diffusion._quadratic_attention(object(), state.attention_backend, pipe) is quadratic+++def test_diffusers_dispatch_change_retimes(monkeypatch, tmp_path):+ ad = pytest.importorskip("diffusers.models.attention_dispatch")+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ binary = _fake_build(monkeypatch, tmp_path)+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)+ backends = ad._AttentionBackendRegistry._backends+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, ad._flash_attention)+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))+ key = _REAL_TOOLCHAIN_KEY("cuda:0")+ assert key is not None and "|diffusers" in key and _REAL_TOOLCHAIN_KEY("cuda:0") == key+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ _restart(monkeypatch)++ def _flash_attention(+ query,+ key,+ value,+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ scale = None,+ ):+ return query # a diffusers release whose flash wrapper changed++ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, _flash_attention)+ assert _REAL_TOOLCHAIN_KEY("cuda:0") != key+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ assert len(timed) == 2+ # a Studio guard around the same function does not change it+ monkeypatch.setitem(backends, ad.AttentionBackendName.FLASH, ad._flash_attention)+ assert _REAL_GUARD() and _REAL_TOOLCHAIN_KEY("cuda:0") == key+ del binary++[email protected]("module", ["core.inference.diffusion", "core.inference.video"])+def test_both_generate_paths_mark_the_generation_boundary(module):+ import ast+ import importlib+ import inspect++ src = inspect.getsource(importlib.import_module(module))+ tree = ast.parse(src)+ gens = [n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "generate"]+ called = {+ getattr(c.func, "id", None) for g in gens for c in ast.walk(g) if isinstance(c, ast.Call)+ }+ assert gens and {"begin_attention_run", "report_attention_run"} <= called+++# Compiled execution (Studio compiles the denoiser's blocks on ROCm since its regional compile probe): an auto-gated+# DiT's traced call takes its bucket's verdict at trace time, timing the bucket then if it is unknown, and the answer is+# baked into the graph without a graph break. Real hooks, real torch.compile on CPU.+++class _CompiledDit(__import__("torch").nn.Module):+ def __init__(self, guarded):+ import torch++ super().__init__()+ self.attn = torch.nn.Module()+ self.attn.heads = 2+ self.attn.to_q = torch.nn.Linear(32, 32, bias = False)+ self._guarded = guarded++ def set_attention_backend(self, name):+ pass++ def forward(self, q):+ return self._guarded(q, q, q, None)+++def _compiled_env(monkeypatch, auto, flash_ms):+ torch = pytest.importorskip("torch")+ _rocm(monkeypatch)+ ad = pytest.importorskip("diffusers.models.attention_dispatch")+ backends = ad._AttentionBackendRegistry._backends+ monkeypatch.setitem(+ backends,+ ad.AttentionBackendName.FLASH,+ lambda query, key, value, attn_mask = None, **k: query + 1,+ )+ monkeypatch.setitem(+ backends,+ ad.AttentionBackendName.NATIVE,+ lambda query, key, value, attn_mask = None, **k: query - 1,+ )+ assert _REAL_GUARD()+ guarded = backends[ad.AttentionBackendName.FLASH]+ _apply_env(monkeypatch)+ timed = []++ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ **kw,+ ):+ timed.append(tokens)+ return {**_GOOD, "flash_ms": flash_ms.get(tokens, 90.0), "tokens": tokens}++ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)+ dit = _CompiledDit(guarded)+ backend = att.ROCM_FLASH_CANDIDATE if auto else "flash"+ assert (+ att.apply_attention_backend(+ types.SimpleNamespace(transformer = dit), backend, target = _target()+ )+ == "flash"+ )+ monkeypatch.setattr(+ att, "_indexed_cuda_device", lambda d: "cuda:0" if d in ("cuda", "cpu") else d+ )+ return torch, dit, timed++[email protected]("auto", [True, False])+def test_compiled_auto_dit_takes_the_bucket_verdict_under_real_torch_compile(monkeypatch, auto):+ # 4096 admitted at load; 8192 untimed until a traced call needs it, and it wins; 16384 loses+ torch, dit, timed = _compiled_env(monkeypatch, auto, {4096: 70.0, 8192: 70.0})+ q = torch.zeros(1, 4608, 2, 16, dtype = torch.bfloat16)+ assert float(dit(q).flatten()[0]) == 1.0+ torch._dynamo.reset()+ counters = __import__("torch._dynamo.utils", fromlist = ["counters"]).counters+ counters.clear()+ compiled = torch.compile(dit, backend = "eager", fullgraph = True)+ assert float(compiled(q).flatten()[0]) == 1.0 # traced: the known verdict, no graph break+ q8 = torch.zeros(1, 9000, 2, 16, dtype = torch.bfloat16)+ assert float(compiled(q8).flatten()[0]) == 1.0 # timed at trace time, then flash+ q16 = torch.zeros(1, 17000, 2, 16, dtype = torch.bfloat16)+ assert float(compiled(q16).flatten()[0]) == (-1.0 if auto else 1.0)+ # explicit flash is never gated, so nothing past the load is timed for it+ assert timed == ([4096, 8192, 16384] if auto else [])+ assert not counters["graph_break"]+ assert getattr(att._ROCM_FLASH_LOCAL, "gate", None) is None+ torch._dynamo.reset()+++def test_compiled_dynamic_lengths_recompile_per_bucket_not_per_length(monkeypatch):+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0, 8192: 90.0})+ torch._dynamo.reset()+ frames = []+ compiled = torch.compile(+ dit,+ backend = lambda gm, inputs: frames.append(gm) or gm.forward,+ dynamic = True,+ fullgraph = True,+ )+ bf16 = torch.bfloat16+ out = [+ float(compiled(torch.zeros(1, n, 2, 16, dtype = bf16)).flatten()[0])+ for n in (4352, 4608, 5000)+ ]+ assert out == [1.0, 1.0, 1.0] and len(frames) == 1+ assert float(compiled(torch.zeros(1, 9000, 2, 16, dtype = bf16)).flatten()[0]) == -1.0+ assert len(frames) == 2 and timed == [4096, 8192]+ torch._dynamo.reset()+++def test_traced_call_with_no_room_for_its_probe_runs_native(monkeypatch):+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0, 8192: 70.0})+ monkeypatch.setattr(att, "_rocm_flash_probe_fits", lambda device, need: False)+ torch._dynamo.reset()+ compiled = torch.compile(dit, backend = "eager", fullgraph = True)+ assert float(compiled(torch.zeros(1, 9000, 2, 16, dtype = torch.bfloat16)).flatten()[0]) == -1.0+ assert (+ timed == [4096]+ and att._ROCM_FLASH_BUCKETS.get(("cuda:0", "torch.bfloat16", 32, 2, 16, 8192, 8192, 1))+ is None+ )+ torch._dynamo.reset()+++# Cross-attention is timed as the rectangle that runs (long video query x short text key), never as a square.+++def test_cross_attention_is_not_decided_by_a_square_probe(monkeypatch):+ dit, guarded, ran, timed = _length_env(+ monkeypatch, {4096: 70.0, 16384: 60.0, (16384, 512): 90.0}+ )+ dit.run(guarded, [(27280, None), (27280, 512), (27280, 600)])+ assert timed == [4096, 16384, (16384, 512)]+ assert ran == ["flash", "native", "native"]+ rec = att.ROCM_FLASH_ADMISSIONS[("cuda:0", "torch.bfloat16", (32, 2, 16), 16384, 512, 1)]+ assert rec["kv_tokens"] == 512 and rec["admitted"] is False+++def test_rectangular_probe_runs_on_cpu(monkeypatch):+ torch = pytest.importorskip("torch")++ def flash_attn_func(q, k, v, **kw):+ assert q.shape[1] == 64 and k.shape[1] == 16+ return torch.nn.functional.scaled_dot_product_attention(+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)+ ).transpose(1, 2)++ _diffusers_flash(monkeypatch, flash_attn_func)+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)+ rec = att._time_rocm_flash_block("cpu", torch.bfloat16, 64, 4, 16, tokens = 64, kv_tokens = 16)+ assert rec["tokens"] == 64 and rec["kv_tokens"] == 16 and rec["rel_l1"] == 0.0+ assert att._rocm_flash_record_ok(rec)+++def test_named_partition_is_in_run_accounting(monkeypatch):+ from core.inference.video import _NamedDiTView++ _rocm(monkeypatch)+ guarded, ran = _guarded_flash(monkeypatch)+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer([]))+ ref = _Dit()+ pipe = types.SimpleNamespace(transformer_ref = ref)+ assert att._attention_dits(pipe) == [] # the stock enumeration cannot see it+ view = _NamedDiTView(pipe, "transformer_ref")+ assert att.apply_attention_backend(view, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"+ state = _state(pipe, "flash")+ assert att.attention_may_run_native(pipe)+ att.begin_attention_run(state)+ ref(guarded, [True, True])+ att.report_attention_run(state)+ assert state.resolved["attention_backend"]["value"] == "native"+ att.begin_attention_run(state)+ assert getattr(ref, att.ROCM_FLASH_TALLY_ATTR) == {}+++def test_batch_two_is_not_decided_by_a_batch_one_verdict(monkeypatch):+ dit, guarded, ran, timed = _length_env(monkeypatch, {4096: 70.0, (4096, "b2"): 90.0})+ import torch++ def call(batch):+ for fn in dit.hooks["pre"]:+ fn(dit, ())+ q = torch.zeros(batch, 4608, 2, 16, dtype = torch.bfloat16)+ guarded(q, q, q, None)+ for fn in dit.hooks["post"]:+ fn(dit, (), None)++ call(1)+ call(2)+ call(3)+ assert timed == [4096, (4096, "b2")] and ran == ["flash", "native", "native"]+ rec = att.ROCM_FLASH_ADMISSIONS[("cuda:0", "torch.bfloat16", (32, 2, 16), 4096, 4096, 2)]+ assert rec["batch"] == 2 and rec["admitted"] is False+++def test_batched_probe_runs_on_cpu(monkeypatch):+ torch = pytest.importorskip("torch")+ shapes = []++ def flash_attn_func(q, k, v, **kw):+ shapes.append((tuple(q.shape), tuple(k.shape)))+ return torch.nn.functional.scaled_dot_product_attention(+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)+ ).transpose(1, 2)++ _diffusers_flash(monkeypatch, flash_attn_func)+ monkeypatch.setattr(torch.cuda, "synchronize", lambda *a, **k: None)+ rec = att._time_rocm_flash_block(+ "cpu", torch.bfloat16, 64, 4, 16, tokens = 64, kv_tokens = 16, ctx_dim = 48, batch = 2+ )+ assert shapes[0] == ((2, 64, 4, 16), (2, 16, 4, 16))+ assert (+ rec["batch"] == 2+ and rec["kv_tokens"] == 16+ and rec["rel_l1"] == 0.0+ and att._rocm_flash_record_ok(rec)+ )+++def test_context_width_is_part_of_the_geometry(monkeypatch):+ torch = pytest.importorskip("torch")++ class Cross(torch.nn.Module):+ def __init__(self, kin):+ super().__init__()+ self.heads = 2+ self.to_q = torch.nn.Linear(32, 32, bias = False)+ self.to_k = torch.nn.Linear(kin, 32, bias = False)++ dit = torch.nn.Module()+ dit.self_attn, dit.cross = Cross(32), Cross(48)+ original = att._attention_dits+ try:+ att._attention_dits = lambda p: [dit]+ assert att._dit_attention_geometries(types.SimpleNamespace()) == (+ (32, 2, 16),+ (32, 2, 16, 48),+ )+ finally:+ att._attention_dits = original+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ seen = []++ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ kv_tokens = None,+ ctx_dim = None,+ batch = 1,+ ):+ seen.append(ctx_dim)+ return {**_GOOD, "tokens": tokens}++ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)+ assert att._rocm_flash_geometry_admitted("cuda:0", "torch.bfloat16", (32, 2, 16, 48))+ assert att._rocm_flash_geometry_admitted("cuda:0", "torch.bfloat16", (32, 2, 16))+ assert seen == [48, None]+ rec = att.ROCM_FLASH_ADMISSIONS[("cuda:0", "torch.bfloat16", (32, 2, 16, 48), 4096, 4096, 1)]+ assert rec["geometry"]["context"] == 48 and att._rocm_flash_record_ok(rec)+++# MiniMax-H3's resident INT8 load fuses to_q / to_k / to_v into to_qkv before the backend is applied.+++def test_fused_qkv_geometry_is_collected():+ torch = pytest.importorskip("torch")++ class Fused(torch.nn.Module):+ def __init__(self, out):+ super().__init__()+ self.heads = 2+ self.to_qkv = torch.nn.Linear(32, out, bias = False)++ dit = torch.nn.Module()+ dit.fused, dit.odd = Fused(96), Fused(95)+ original = att._attention_dits+ try:+ att._attention_dits = lambda p: [dit]+ assert att._dit_attention_geometries(types.SimpleNamespace()) == ((32, 2, 16),)+ finally:+ att._attention_dits = original+++# The key carries the ROCm runtime / kernel driver the process runs on, not only the wheel's build.+++def test_driver_change_retimes(monkeypatch, tmp_path):+ pytest.importorskip("diffusers.models.attention_dispatch")+ _rocm(monkeypatch)+ _apply_env(monkeypatch)+ _fake_build(monkeypatch, tmp_path)+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", _REAL_TOOLCHAIN_KEY)+ driver = ["driver:hipdriver70051234,amdgpu6.12.12"]+ monkeypatch.setattr(att, "_rocm_driver_version", lambda: driver[0])+ timed = []+ monkeypatch.setattr(att, "_time_rocm_flash_block", _timer(timed))+ key = _REAL_TOOLCHAIN_KEY("cuda:0")+ assert key is not None and "|driver:hipdriver70051234,amdgpu6.12.12|" in key+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ _restart(monkeypatch)+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ assert len(timed) == 1+ driver[0] = "driver:hipdriver70151234,amdgpu6.14.4"+ _restart(monkeypatch)+ att.apply_attention_backend(+ _pipe_with_attention()[0], att.ROCM_FLASH_CANDIDATE, target = _target()+ )+ assert len(timed) == 2+++def test_driver_version_reads_without_raising(monkeypatch):+ monkeypatch.setattr(att, "_ROCM_DRIVER_VERSION", [])+ v = att._rocm_driver_version()+ assert v.startswith("driver:") and att._rocm_driver_version() is v+++def test_attention_status_reason():+ assert att.attention_status_reason(None, "flash") == "verified ROCm flash attention"+ assert att.attention_status_reason("auto", "flash") == "verified ROCm flash attention"+ assert att.attention_status_reason(None, "_native_cudnn") == "cuDNN fused attention upgrade"+ assert att.attention_status_reason(None, None) == "diffusers default"+ assert att.attention_status_reason("flash", "flash") == "requested"+ assert att.attention_status_reason("flash", None) == "diffusers default"+++# The in-memory admission key keeps geometry, query length, key length and batch in separate fields: a cross-attention+# geometry whose context width equals a bucket edge must not alias a narrower geometry at rectangular lengths.+++def test_admission_key_does_not_alias_geometry_and_lengths(monkeypatch):+ _apply_env(monkeypatch)+ monkeypatch.setattr(att, "_rocm_flash_toolchain_key", lambda device: None)+ timed = []++ def fake_time(+ device,+ dtype,+ hidden,+ heads,+ head_dim,+ tokens = 4096,+ kv_tokens = None,+ **kw,+ ):+ timed.append((hidden, heads, head_dim, tokens, kv_tokens, kw.get("ctx_dim")))+ wins = kv_tokens is None+ return {+ "native_ms": 80.0,+ "flash_ms": 60.0 if wins else 90.0,+ "rel_l1": 1e-3,+ "max_rel": 1e-3,+ "finite": True,+ "tokens": tokens,+ **({} if kv_tokens is None else {"kv_tokens": kv_tokens}),+ }++ monkeypatch.setattr(att, "_time_rocm_flash_block", fake_time)+ assert (+ att._rocm_flash_geometry_admitted("cuda:0", "bf16", (32, 2, 16, 256), tokens = 4096) is True+ )+ # (32, 2, 16) at 256 x 4096 flattens to the same tuple as (32, 2, 16, 256) at 4096: it must be timed on its own+ assert (+ att._rocm_flash_geometry_admitted("cuda:0", "bf16", (32, 2, 16), tokens = 256, kv_tokens = 4096)+ is False+ )+ assert timed == [(32, 2, 16, 4096, None, 256), (32, 2, 16, 256, 4096, None)]+ assert len(att.ROCM_FLASH_ADMISSIONS) == 2+++# A re-apply on the same pipe that no longer picks ROCm flash takes the tracking off with it: the badge and the memory+# guard must not keep reading the pipe as flash.+++def test_reapply_to_native_removes_rocm_flash_tracking(monkeypatch):+ torch, dit, timed = _compiled_env(monkeypatch, True, {4096: 70.0})+ pipe = types.SimpleNamespace(transformer = dit)+ att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target())+ assert att.attention_may_run_native(pipe) and att.rocm_flash_ran(pipe) is not None+ hooks = len(dit._forward_pre_hooks) + len(dit._forward_hooks)+ assert att.apply_attention_backend(pipe, None, target = _target()) is None+ assert not att.attention_may_run_native(pipe) and att.rocm_flash_ran(pipe) is None+ assert len(dit._forward_pre_hooks) + len(dit._forward_hooks) == hooks - 2+ entry = {"value": "native", "reason": "diffusers default"}+ state = types.SimpleNamespace(pipe = pipe, resolved = {"attention_backend": entry})+ att.begin_attention_run(state)+ att.report_attention_run(state)+ assert state.resolved["attention_backend"] == entry+ assert att.apply_attention_backend(pipe, att.ROCM_FLASH_CANDIDATE, target = _target()) == "flash"+ assert att.attention_may_run_native(pipe)