Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio FLUX.3: pass cuDNN attention the projections' strided Q/K/V instead of contiguous copies (#17180)

Daniel Handac1edbea5b6
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.py
index 8d72405295e..a39ab4f3f7e 100644
--- a/studio/backend/core/inference/diffusion_attention.py
+++ b/studio/backend/core/inference/diffusion_attention.py
@@ -3583,6 +3583,9 @@ def apply_attention_backend(
engaged = False
if engaged:
_tag_dits(pipe, backend)
+ if backend == "_native_cudnn" and target is not None and _is_cuda_nvidia(target):
+ from .diffusion_flux3_attn import install as install_flux3_strided
+ install_flux3_strided(pipe, backend, target, logger)
if backend == "flash" and target is not None and _is_cuda_rocm(target):
_track_rocm_flash_calls(pipe)
else:
diff --git a/studio/backend/core/inference/diffusion_flux3_attn.py b/studio/backend/core/inference/diffusion_flux3_attn.py
new file mode 100644
index 00000000000..d2ee940735c
--- /dev/null
+++ b/studio/backend/core/inference/diffusion_flux3_attn.py
@@ -0,0 +1,217 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""FLUX.3 cuDNN attention on strided [B, S, H, D] views, skipping diffusers' contiguous Q/K/V copies. Installed only
+when a load-time probe finds it bitwise equal to the contiguous call (UNSLOTH_DIFFUSION_FLUX3_STRIDED_ATTN=0 disables)."""
+
+from __future__ import annotations
+
+import os
+from functools import lru_cache
+from typing import Any, Optional
+
+from .rocm_bf16 import is_rocm_torch
+
+STRIDED_ATTN_ENV = "UNSLOTH_DIFFUSION_FLUX3_STRIDED_ATTN"
+_CUDNN = "_native_cudnn"
+_MODULE = "diffusers.models.transformers.transformer_flux3"
+# diffusion_autoopt.patches.source_digest of Flux3AttnProcessor.__call__ the subclass mirrors (d42ebf9c and 0.42.1).
+_PROCESSOR_DIGESTS = frozenset({"e3282b2e03be05d8"})
+# (device index, dtype, heads, head_dim) -> strided cuDNN output == contiguous cuDNN output, bitwise.
+_PROBED: dict[tuple, bool] = {}
+
+
+def enabled() -> bool:
+ return (os.environ.get(STRIDED_ATTN_ENV) or "1").strip().lower() not in (
+ "0",
+ "off",
+ "false",
+ "no",
+ )
+
+
+def _backend_value(backend: Any) -> Optional[str]:
+ if backend is None:
+ return None
+ return str(getattr(backend, "value", backend))
+
+
+@lru_cache(maxsize = None)
+def strided_processor_class() -> Any:
+ """Built once: dynamo guards on the class. Masks, context parallelism and non-cuDNN backends use the stock call."""
+ import torch.nn.functional as F
+ from diffusers.models.transformers import transformer_flux3 as tf3
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ class UnslothFlux3StridedAttnProcessor(tf3.Flux3AttnProcessor):
+ def __call__(
+ self,
+ attn: Any,
+ hidden_states: Any,
+ attention_mask: Any = None,
+ image_rotary_emb: Any = None,
+ ) -> Any:
+ if (
+ attention_mask is not None
+ or self._parallel_config is not None
+ or _backend_value(self._attention_backend) != _CUDNN
+ ):
+ return super().__call__(attn, hidden_states, attention_mask, image_rotary_emb)
+ query, key, value = tf3._get_qkv_projections(attn, hidden_states)
+
+ query = query.unflatten(-1, (attn.heads, -1))
+ key = key.unflatten(-1, (attn.heads, -1))
+ value = value.unflatten(-1, (attn.heads, -1))
+
+ query = attn.norm_q(query.float())
+ key = attn.norm_k(key.float())
+
+ if image_rotary_emb is not None:
+ # module attribute at call time: the fused RoPE install patches it there
+ query = tf3.apply_rotary_emb(query, image_rotary_emb, sequence_dim = 1)
+ key = tf3.apply_rotary_emb(key, image_rotary_emb, sequence_dim = 1)
+ query, key = query.to(value.dtype), key.to(value.dtype)
+
+ with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
+ out = F.scaled_dot_product_attention(
+ query.permute(0, 2, 1, 3),
+ key.permute(0, 2, 1, 3),
+ value.permute(0, 2, 1, 3),
+ attn_mask = None,
+ dropout_p = 0.0,
+ is_causal = False,
+ scale = None,
+ enable_gqa = False,
+ )
+ hidden_states = out.permute(0, 2, 1, 3).flatten(2, 3)
+ hidden_states = hidden_states.to(query.dtype)
+
+ hidden_states = attn.to_out[0](hidden_states)
+ hidden_states = attn.to_out[1](hidden_states)
+ return hidden_states
+
+ return UnslothFlux3StridedAttnProcessor
+
+
+def _attention_modules(transformer: Any) -> list:
+ try:
+ from diffusers.models.transformers.transformer_flux3 import Flux3Attention
+ except Exception: # noqa: BLE001
+ return []
+ modules = getattr(transformer, "modules", None)
+ if not callable(modules):
+ return []
+ return [m for m in modules() if isinstance(m, Flux3Attention)]
+
+
+def _device(target: Any) -> Any:
+ import torch
+
+ raw = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "cuda")
+ device = torch.device(raw)
+ if device.type == "cuda" and device.index is None:
+ device = torch.device("cuda", torch.cuda.current_device())
+ return device
+
+
+def strided_matches(device: Any, dtype: Any, heads: int, head_dim: int) -> bool:
+ """Cached per (card, dtype, heads, head_dim); False on any error."""
+ import torch
+ import torch.nn.functional as F
+ from torch.nn.attention import SDPBackend, sdpa_kernel
+
+ key = (device.index, dtype, int(heads), int(head_dim))
+ if key in _PROBED:
+ return _PROBED[key]
+ ok = False
+ try:
+ gen = torch.Generator(device = device).manual_seed(0)
+ q, k, v = (
+ torch.randn(
+ (2, 333, heads, head_dim), generator = gen, device = device, dtype = torch.float32
+ ).to(dtype)
+ for _ in range(3)
+ )
+ with torch.no_grad(), sdpa_kernel(SDPBackend.CUDNN_ATTENTION):
+ ref = F.scaled_dot_product_attention(
+ *(x.permute(0, 2, 1, 3).contiguous() for x in (q, k, v))
+ ).permute(0, 2, 1, 3)
+ got = F.scaled_dot_product_attention(
+ *(x.permute(0, 2, 1, 3) for x in (q, k, v))
+ ).permute(0, 2, 1, 3)
+ ok = bool(torch.equal(ref, got)) and bool(torch.isfinite(ref.float()).all())
+ except Exception: # noqa: BLE001 - an unprobeable card keeps diffusers' contiguous call
+ ok = False
+ _PROBED[key] = ok
+ return ok
+
+
+def install(
+ pipe: Any,
+ backend: Any,
+ target: Any,
+ logger: Any = None,
+) -> int:
+ """Returns the number of processors swapped. Idempotent; never raises."""
+ try:
+ return _install(pipe, backend, target, logger)
+ except Exception as exc: # noqa: BLE001 - the stock processors stay
+ if logger is not None:
+ logger.info("diffusion.flux3_attn: strided cuDNN attention skipped (%s)", exc)
+ return 0
+
+
+def _install(pipe: Any, backend: Any, target: Any, logger: Any) -> int:
+ if not enabled() or _backend_value(backend) != _CUDNN:
+ return 0
+ transformer = getattr(pipe, "transformer", None)
+ if transformer is None or type(transformer).__module__ != _MODULE:
+ return 0
+ import torch
+
+ if is_rocm_torch(torch) or not torch.cuda.is_available():
+ return 0
+ from .diffusion_autoopt.patches import source_digest
+ from diffusers.models.transformers import transformer_flux3 as tf3
+
+ if source_digest(tf3.Flux3AttnProcessor.__call__) not in _PROCESSOR_DIGESTS or not callable(
+ getattr(tf3, "_get_qkv_projections", None)
+ ):
+ if logger is not None:
+ logger.info(
+ "diffusion.flux3_attn: diffusers' FLUX.3 attention changed; stock processor kept"
+ )
+ return 0
+ device = _device(target)
+ if device.type != "cuda":
+ return 0
+ cls = strided_processor_class()
+ dtype = getattr(target, "dtype", None)
+ if dtype not in (torch.bfloat16, torch.float16):
+ # cuDNN attention runs bf16 / fp16 only; a quantized DiT computes in bf16
+ dtype = torch.bfloat16
+ swapped = 0
+ for attn in _attention_modules(transformer):
+ proc = getattr(attn, "processor", None)
+ if type(proc) is cls:
+ continue
+ if type(proc) is not tf3.Flux3AttnProcessor:
+ continue
+ if _backend_value(getattr(proc, "_attention_backend", None)) != _CUDNN:
+ continue
+ if not strided_matches(device, dtype, int(attn.heads), int(attn.head_dim)):
+ continue
+ new = cls()
+ new._attention_backend = proc._attention_backend
+ new._parallel_config = proc._parallel_config
+ attn.set_processor(new)
+ swapped += 1
+ if swapped and logger is not None:
+ logger.info(
+ "diffusion.flux3_attn: %d FLUX.3 attention calls pass cuDNN their strided views (no Q/K/V copies)",
+ swapped,
+ )
+ return swapped
+
+
+__all__ = ["STRIDED_ATTN_ENV", "enabled", "install", "strided_matches", "strided_processor_class"]
diff --git a/studio/backend/tests/test_diffusion_flux3_strided_attn.py b/studio/backend/tests/test_diffusion_flux3_strided_attn.py
new file mode 100644
index 00000000000..aa4326a01b1
--- /dev/null
+++ b/studio/backend/tests/test_diffusion_flux3_strided_attn.py
@@ -0,0 +1,185 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""CPU checks for diffusion_flux3_attn: cuDNN selection is a no-op, so this verifies the math, gates and trace, not cuDNN."""
+
+from __future__ import annotations
+
+import contextlib
+from types import SimpleNamespace
+
+import pytest
+
+torch = pytest.importorskip("torch")
+tf3 = pytest.importorskip("diffusers.models.transformers.transformer_flux3")
+
+from core.inference import diffusion_flux3_attn as fa # noqa: E402
+
+
[email protected](autouse = True)
+def _cpu_cudnn(monkeypatch):
+ monkeypatch.setattr(torch.nn.attention, "sdpa_kernel", lambda *a, **k: contextlib.nullcontext())
+ monkeypatch.delenv(fa.STRIDED_ATTN_ENV, raising = False)
+ fa.strided_processor_class.cache_clear()
+ fa._PROBED.clear()
+ yield
+ fa.strided_processor_class.cache_clear()
+ fa._PROBED.clear()
+
+
+def _tiny():
+ torch.manual_seed(0)
+ model = tf3.Flux3Transformer2DModel(
+ in_channels = 8,
+ cond_in_channels = 8,
+ num_layers = 1,
+ num_single_layers = 2,
+ attention_head_dim = 16,
+ num_attention_heads = 2,
+ joint_attention_dim = 24,
+ timestep_channels = 16,
+ axes_dims_rope = (4, 4, 4, 4),
+ )
+ return model.eval()
+
+
+def _attn(dtype = torch.float32):
+ torch.manual_seed(1)
+ attn = tf3.Flux3Attention(query_dim = 64, heads = 4, dim_head = 16).to(dtype).eval()
+ proc = attn.processor
+ proc._attention_backend = "_native_cudnn"
+ return attn, proc
+
+
+def _rope(n, d = 16):
+ g = torch.Generator().manual_seed(2)
+ ang = torch.rand(n, d // 2, generator = g) * 6.28
+ cos = torch.cos(ang).repeat_interleave(2, dim = -1)
+ sin = torch.sin(ang).repeat_interleave(2, dim = -1)
+ return cos, sin
+
+
[email protected]("dtype", [torch.float32, torch.bfloat16])
[email protected]("batch,seq", [(1, 40), (2, 33)])
+def test_strided_processor_matches_the_cudnn_dispatch_bitwise(dtype, batch, seq):
+ attn, stock = _attn(dtype)
+ x = torch.randn(batch, seq, 64, generator = torch.Generator().manual_seed(3)).to(dtype)
+ rope = _rope(seq)
+ with torch.no_grad():
+ want = stock(attn, x, None, rope)
+ new = fa.strided_processor_class()()
+ new._attention_backend = stock._attention_backend
+ got = new(attn, x, None, rope)
+ assert got.shape == want.shape and got.dtype == want.dtype
+ assert torch.equal(got, want)
+
+
+def test_masks_and_other_backends_take_the_stock_call(monkeypatch):
+ attn, stock = _attn()
+ new = fa.strided_processor_class()()
+ calls = []
+ real = tf3.Flux3AttnProcessor.__call__
+
+ def spy(self, *a, **k):
+ calls.append(1)
+ return real(self, *a, **k)
+
+ monkeypatch.setattr(tf3.Flux3AttnProcessor, "__call__", spy)
+ x = torch.randn(1, 12, 64)
+ with torch.no_grad():
+ new._attention_backend = "_native_cudnn"
+ new(attn, x, torch.ones(1, 12, dtype = torch.bool), _rope(12))
+ new._attention_backend = None
+ new(attn, x, None, _rope(12))
+ assert len(calls) == 2
+ new._attention_backend = "_native_cudnn"
+ new(attn, x, None, _rope(12))
+ assert len(calls) == 2 # the strided path did not call the stock processor
+
+
+def _cuda_target(monkeypatch, probe = True):
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(fa, "strided_matches", lambda *a, **k: probe)
+ return SimpleNamespace(device = "cuda", torch_device = "cuda:0", dtype = torch.bfloat16)
+
+
+def _pipe():
+ model = _tiny()
+ model.set_attention_backend("_native_cudnn")
+ return SimpleNamespace(transformer = model)
+
+
+def test_install_swaps_every_cudnn_processor_and_keeps_its_backend(monkeypatch):
+ pipe = _pipe()
+ target = _cuda_target(monkeypatch)
+ n = len(fa._attention_modules(pipe.transformer))
+ assert n > 0
+ assert fa.install(pipe, "_native_cudnn", target) == n
+ cls = fa.strided_processor_class()
+ for attn in fa._attention_modules(pipe.transformer):
+ assert type(attn.processor) is cls
+ assert fa._backend_value(attn.processor._attention_backend) == "_native_cudnn"
+ assert fa.install(pipe, "_native_cudnn", target) == 0 # idempotent
+
+
+def test_install_declines(monkeypatch):
+ target = _cuda_target(monkeypatch)
+ assert fa.install(_pipe(), "_native_flash", target) == 0
+ assert (
+ fa.install(SimpleNamespace(transformer = torch.nn.Linear(2, 2)), "_native_cudnn", target) == 0
+ )
+ assert (
+ fa.install(_pipe(), "_native_cudnn", SimpleNamespace(device = "cpu", torch_device = "cpu")) == 0
+ )
+ target = _cuda_target(monkeypatch, probe = False)
+ assert fa.install(_pipe(), "_native_cudnn", target) == 0
+ target = _cuda_target(monkeypatch)
+ monkeypatch.setattr(fa, "_PROCESSOR_DIGESTS", frozenset({"0000000000000000"}))
+ assert fa.install(_pipe(), "_native_cudnn", target) == 0
+ monkeypatch.setattr(fa, "_PROCESSOR_DIGESTS", frozenset({"e3282b2e03be05d8"}))
+ monkeypatch.setenv(fa.STRIDED_ATTN_ENV, "0")
+ assert fa.install(_pipe(), "_native_cudnn", target) == 0
+
+
+def test_processor_digest_is_the_pinned_diffusers_source():
+ from core.inference.diffusion_autoopt.patches import source_digest
+ assert source_digest(tf3.Flux3AttnProcessor.__call__) in fa._PROCESSOR_DIGESTS
+
+
+def test_whole_transformer_matches_and_compiles_without_breaks(monkeypatch):
+ from torch._dynamo.testing import CompileCounterWithBackend
+
+ pipe = _pipe()
+ g = torch.Generator().manual_seed(4)
+ kw = {
+ "hidden_states": torch.randn(1, 16, 8, generator = g),
+ "encoder_hidden_states": torch.randn(1, 5, 24, generator = g),
+ "timestep": torch.tensor([0.7]),
+ "img_ids": torch.randint(0, 8, (16, 4), generator = g).float(),
+ "txt_ids": torch.randint(0, 8, (5, 4), generator = g).float(),
+ "return_dict": False,
+ }
+ with torch.no_grad():
+ want = pipe.transformer(**kw)[0]
+ with monkeypatch.context() as m: # the fake CUDA flag only for the install, not for dynamo
+ assert fa.install(pipe, "_native_cudnn", _cuda_target(m)) > 0
+ got = pipe.transformer(**kw)[0]
+ assert torch.equal(got, want)
+ torch._dynamo.reset()
+ cc = CompileCounterWithBackend("inductor")
+ blocks = list(pipe.transformer.single_transformer_blocks)
+ eager = [b.forward for b in blocks]
+ for b in blocks:
+ b.forward = torch.compile(
+ type(b).forward.__get__(b), backend = cc, fullgraph = True, dynamic = True
+ )
+ try:
+ outs = [
+ pipe.transformer(**dict(kw, timestep = torch.tensor([t])))[0] for t in (0.7, 0.5, 0.3)
+ ]
+ finally:
+ for b, f in zip(blocks, eager):
+ b.forward = f
+ torch._dynamo.reset()
+ assert cc.frame_count == len(blocks) # fullgraph: no break; three steps, no recompile
+ assert torch.allclose(outs[0], want, atol = 1e-5, rtol = 1e-5)