Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio FLUX.3: pass cuDNN attention the projections' strided Q/K/V instead of contiguous copies (#17180)
Daniel Handac1edbea5b6
diff --git a/studio/backend/core/inference/diffusion_attention.py b/studio/backend/core/inference/diffusion_attention.pyindex 8d72405295e..a39ab4f3f7e 100644--- a/studio/backend/core/inference/diffusion_attention.py+++ b/studio/backend/core/inference/diffusion_attention.py@@ -3583,6 +3583,9 @@ def apply_attention_backend(engaged = Falseif engaged:_tag_dits(pipe, backend)+ if backend == "_native_cudnn" and target is not None and _is_cuda_nvidia(target):+ from .diffusion_flux3_attn import install as install_flux3_strided+ install_flux3_strided(pipe, backend, target, logger)if backend == "flash" and target is not None and _is_cuda_rocm(target):_track_rocm_flash_calls(pipe)else:diff --git a/studio/backend/core/inference/diffusion_flux3_attn.py b/studio/backend/core/inference/diffusion_flux3_attn.pynew file mode 100644index 00000000000..d2ee940735c--- /dev/null+++ b/studio/backend/core/inference/diffusion_flux3_attn.py@@ -0,0 +1,217 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""FLUX.3 cuDNN attention on strided [B, S, H, D] views, skipping diffusers' contiguous Q/K/V copies. Installed only+when a load-time probe finds it bitwise equal to the contiguous call (UNSLOTH_DIFFUSION_FLUX3_STRIDED_ATTN=0 disables)."""++from __future__ import annotations++import os+from functools import lru_cache+from typing import Any, Optional++from .rocm_bf16 import is_rocm_torch++STRIDED_ATTN_ENV = "UNSLOTH_DIFFUSION_FLUX3_STRIDED_ATTN"+_CUDNN = "_native_cudnn"+_MODULE = "diffusers.models.transformers.transformer_flux3"+# diffusion_autoopt.patches.source_digest of Flux3AttnProcessor.__call__ the subclass mirrors (d42ebf9c and 0.42.1).+_PROCESSOR_DIGESTS = frozenset({"e3282b2e03be05d8"})+# (device index, dtype, heads, head_dim) -> strided cuDNN output == contiguous cuDNN output, bitwise.+_PROBED: dict[tuple, bool] = {}+++def enabled() -> bool:+ return (os.environ.get(STRIDED_ATTN_ENV) or "1").strip().lower() not in (+ "0",+ "off",+ "false",+ "no",+ )+++def _backend_value(backend: Any) -> Optional[str]:+ if backend is None:+ return None+ return str(getattr(backend, "value", backend))+++@lru_cache(maxsize = None)+def strided_processor_class() -> Any:+ """Built once: dynamo guards on the class. Masks, context parallelism and non-cuDNN backends use the stock call."""+ import torch.nn.functional as F+ from diffusers.models.transformers import transformer_flux3 as tf3+ from torch.nn.attention import SDPBackend, sdpa_kernel++ class UnslothFlux3StridedAttnProcessor(tf3.Flux3AttnProcessor):+ def __call__(+ self,+ attn: Any,+ hidden_states: Any,+ attention_mask: Any = None,+ image_rotary_emb: Any = None,+ ) -> Any:+ if (+ attention_mask is not None+ or self._parallel_config is not None+ or _backend_value(self._attention_backend) != _CUDNN+ ):+ return super().__call__(attn, hidden_states, attention_mask, image_rotary_emb)+ query, key, value = tf3._get_qkv_projections(attn, hidden_states)++ query = query.unflatten(-1, (attn.heads, -1))+ key = key.unflatten(-1, (attn.heads, -1))+ value = value.unflatten(-1, (attn.heads, -1))++ query = attn.norm_q(query.float())+ key = attn.norm_k(key.float())++ if image_rotary_emb is not None:+ # module attribute at call time: the fused RoPE install patches it there+ query = tf3.apply_rotary_emb(query, image_rotary_emb, sequence_dim = 1)+ key = tf3.apply_rotary_emb(key, image_rotary_emb, sequence_dim = 1)+ query, key = query.to(value.dtype), key.to(value.dtype)++ with sdpa_kernel(SDPBackend.CUDNN_ATTENTION):+ out = F.scaled_dot_product_attention(+ query.permute(0, 2, 1, 3),+ key.permute(0, 2, 1, 3),+ value.permute(0, 2, 1, 3),+ attn_mask = None,+ dropout_p = 0.0,+ is_causal = False,+ scale = None,+ enable_gqa = False,+ )+ hidden_states = out.permute(0, 2, 1, 3).flatten(2, 3)+ hidden_states = hidden_states.to(query.dtype)++ hidden_states = attn.to_out[0](hidden_states)+ hidden_states = attn.to_out[1](hidden_states)+ return hidden_states++ return UnslothFlux3StridedAttnProcessor+++def _attention_modules(transformer: Any) -> list:+ try:+ from diffusers.models.transformers.transformer_flux3 import Flux3Attention+ except Exception: # noqa: BLE001+ return []+ modules = getattr(transformer, "modules", None)+ if not callable(modules):+ return []+ return [m for m in modules() if isinstance(m, Flux3Attention)]+++def _device(target: Any) -> Any:+ import torch++ raw = str(getattr(target, "torch_device", None) or getattr(target, "device", None) or "cuda")+ device = torch.device(raw)+ if device.type == "cuda" and device.index is None:+ device = torch.device("cuda", torch.cuda.current_device())+ return device+++def strided_matches(device: Any, dtype: Any, heads: int, head_dim: int) -> bool:+ """Cached per (card, dtype, heads, head_dim); False on any error."""+ import torch+ import torch.nn.functional as F+ from torch.nn.attention import SDPBackend, sdpa_kernel++ key = (device.index, dtype, int(heads), int(head_dim))+ if key in _PROBED:+ return _PROBED[key]+ ok = False+ try:+ gen = torch.Generator(device = device).manual_seed(0)+ q, k, v = (+ torch.randn(+ (2, 333, heads, head_dim), generator = gen, device = device, dtype = torch.float32+ ).to(dtype)+ for _ in range(3)+ )+ with torch.no_grad(), sdpa_kernel(SDPBackend.CUDNN_ATTENTION):+ ref = F.scaled_dot_product_attention(+ *(x.permute(0, 2, 1, 3).contiguous() for x in (q, k, v))+ ).permute(0, 2, 1, 3)+ got = F.scaled_dot_product_attention(+ *(x.permute(0, 2, 1, 3) for x in (q, k, v))+ ).permute(0, 2, 1, 3)+ ok = bool(torch.equal(ref, got)) and bool(torch.isfinite(ref.float()).all())+ except Exception: # noqa: BLE001 - an unprobeable card keeps diffusers' contiguous call+ ok = False+ _PROBED[key] = ok+ return ok+++def install(+ pipe: Any,+ backend: Any,+ target: Any,+ logger: Any = None,+) -> int:+ """Returns the number of processors swapped. Idempotent; never raises."""+ try:+ return _install(pipe, backend, target, logger)+ except Exception as exc: # noqa: BLE001 - the stock processors stay+ if logger is not None:+ logger.info("diffusion.flux3_attn: strided cuDNN attention skipped (%s)", exc)+ return 0+++def _install(pipe: Any, backend: Any, target: Any, logger: Any) -> int:+ if not enabled() or _backend_value(backend) != _CUDNN:+ return 0+ transformer = getattr(pipe, "transformer", None)+ if transformer is None or type(transformer).__module__ != _MODULE:+ return 0+ import torch++ if is_rocm_torch(torch) or not torch.cuda.is_available():+ return 0+ from .diffusion_autoopt.patches import source_digest+ from diffusers.models.transformers import transformer_flux3 as tf3++ if source_digest(tf3.Flux3AttnProcessor.__call__) not in _PROCESSOR_DIGESTS or not callable(+ getattr(tf3, "_get_qkv_projections", None)+ ):+ if logger is not None:+ logger.info(+ "diffusion.flux3_attn: diffusers' FLUX.3 attention changed; stock processor kept"+ )+ return 0+ device = _device(target)+ if device.type != "cuda":+ return 0+ cls = strided_processor_class()+ dtype = getattr(target, "dtype", None)+ if dtype not in (torch.bfloat16, torch.float16):+ # cuDNN attention runs bf16 / fp16 only; a quantized DiT computes in bf16+ dtype = torch.bfloat16+ swapped = 0+ for attn in _attention_modules(transformer):+ proc = getattr(attn, "processor", None)+ if type(proc) is cls:+ continue+ if type(proc) is not tf3.Flux3AttnProcessor:+ continue+ if _backend_value(getattr(proc, "_attention_backend", None)) != _CUDNN:+ continue+ if not strided_matches(device, dtype, int(attn.heads), int(attn.head_dim)):+ continue+ new = cls()+ new._attention_backend = proc._attention_backend+ new._parallel_config = proc._parallel_config+ attn.set_processor(new)+ swapped += 1+ if swapped and logger is not None:+ logger.info(+ "diffusion.flux3_attn: %d FLUX.3 attention calls pass cuDNN their strided views (no Q/K/V copies)",+ swapped,+ )+ return swapped+++__all__ = ["STRIDED_ATTN_ENV", "enabled", "install", "strided_matches", "strided_processor_class"]diff --git a/studio/backend/tests/test_diffusion_flux3_strided_attn.py b/studio/backend/tests/test_diffusion_flux3_strided_attn.pynew file mode 100644index 00000000000..aa4326a01b1--- /dev/null+++ b/studio/backend/tests/test_diffusion_flux3_strided_attn.py@@ -0,0 +1,185 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""CPU checks for diffusion_flux3_attn: cuDNN selection is a no-op, so this verifies the math, gates and trace, not cuDNN."""++from __future__ import annotations++import contextlib+from types import SimpleNamespace++import pytest++torch = pytest.importorskip("torch")+tf3 = pytest.importorskip("diffusers.models.transformers.transformer_flux3")++from core.inference import diffusion_flux3_attn as fa # noqa: E402++[email protected](autouse = True)+def _cpu_cudnn(monkeypatch):+ monkeypatch.setattr(torch.nn.attention, "sdpa_kernel", lambda *a, **k: contextlib.nullcontext())+ monkeypatch.delenv(fa.STRIDED_ATTN_ENV, raising = False)+ fa.strided_processor_class.cache_clear()+ fa._PROBED.clear()+ yield+ fa.strided_processor_class.cache_clear()+ fa._PROBED.clear()+++def _tiny():+ torch.manual_seed(0)+ model = tf3.Flux3Transformer2DModel(+ in_channels = 8,+ cond_in_channels = 8,+ num_layers = 1,+ num_single_layers = 2,+ attention_head_dim = 16,+ num_attention_heads = 2,+ joint_attention_dim = 24,+ timestep_channels = 16,+ axes_dims_rope = (4, 4, 4, 4),+ )+ return model.eval()+++def _attn(dtype = torch.float32):+ torch.manual_seed(1)+ attn = tf3.Flux3Attention(query_dim = 64, heads = 4, dim_head = 16).to(dtype).eval()+ proc = attn.processor+ proc._attention_backend = "_native_cudnn"+ return attn, proc+++def _rope(n, d = 16):+ g = torch.Generator().manual_seed(2)+ ang = torch.rand(n, d // 2, generator = g) * 6.28+ cos = torch.cos(ang).repeat_interleave(2, dim = -1)+ sin = torch.sin(ang).repeat_interleave(2, dim = -1)+ return cos, sin++[email protected]("dtype", [torch.float32, torch.bfloat16])[email protected]("batch,seq", [(1, 40), (2, 33)])+def test_strided_processor_matches_the_cudnn_dispatch_bitwise(dtype, batch, seq):+ attn, stock = _attn(dtype)+ x = torch.randn(batch, seq, 64, generator = torch.Generator().manual_seed(3)).to(dtype)+ rope = _rope(seq)+ with torch.no_grad():+ want = stock(attn, x, None, rope)+ new = fa.strided_processor_class()()+ new._attention_backend = stock._attention_backend+ got = new(attn, x, None, rope)+ assert got.shape == want.shape and got.dtype == want.dtype+ assert torch.equal(got, want)+++def test_masks_and_other_backends_take_the_stock_call(monkeypatch):+ attn, stock = _attn()+ new = fa.strided_processor_class()()+ calls = []+ real = tf3.Flux3AttnProcessor.__call__++ def spy(self, *a, **k):+ calls.append(1)+ return real(self, *a, **k)++ monkeypatch.setattr(tf3.Flux3AttnProcessor, "__call__", spy)+ x = torch.randn(1, 12, 64)+ with torch.no_grad():+ new._attention_backend = "_native_cudnn"+ new(attn, x, torch.ones(1, 12, dtype = torch.bool), _rope(12))+ new._attention_backend = None+ new(attn, x, None, _rope(12))+ assert len(calls) == 2+ new._attention_backend = "_native_cudnn"+ new(attn, x, None, _rope(12))+ assert len(calls) == 2 # the strided path did not call the stock processor+++def _cuda_target(monkeypatch, probe = True):+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(fa, "strided_matches", lambda *a, **k: probe)+ return SimpleNamespace(device = "cuda", torch_device = "cuda:0", dtype = torch.bfloat16)+++def _pipe():+ model = _tiny()+ model.set_attention_backend("_native_cudnn")+ return SimpleNamespace(transformer = model)+++def test_install_swaps_every_cudnn_processor_and_keeps_its_backend(monkeypatch):+ pipe = _pipe()+ target = _cuda_target(monkeypatch)+ n = len(fa._attention_modules(pipe.transformer))+ assert n > 0+ assert fa.install(pipe, "_native_cudnn", target) == n+ cls = fa.strided_processor_class()+ for attn in fa._attention_modules(pipe.transformer):+ assert type(attn.processor) is cls+ assert fa._backend_value(attn.processor._attention_backend) == "_native_cudnn"+ assert fa.install(pipe, "_native_cudnn", target) == 0 # idempotent+++def test_install_declines(monkeypatch):+ target = _cuda_target(monkeypatch)+ assert fa.install(_pipe(), "_native_flash", target) == 0+ assert (+ fa.install(SimpleNamespace(transformer = torch.nn.Linear(2, 2)), "_native_cudnn", target) == 0+ )+ assert (+ fa.install(_pipe(), "_native_cudnn", SimpleNamespace(device = "cpu", torch_device = "cpu")) == 0+ )+ target = _cuda_target(monkeypatch, probe = False)+ assert fa.install(_pipe(), "_native_cudnn", target) == 0+ target = _cuda_target(monkeypatch)+ monkeypatch.setattr(fa, "_PROCESSOR_DIGESTS", frozenset({"0000000000000000"}))+ assert fa.install(_pipe(), "_native_cudnn", target) == 0+ monkeypatch.setattr(fa, "_PROCESSOR_DIGESTS", frozenset({"e3282b2e03be05d8"}))+ monkeypatch.setenv(fa.STRIDED_ATTN_ENV, "0")+ assert fa.install(_pipe(), "_native_cudnn", target) == 0+++def test_processor_digest_is_the_pinned_diffusers_source():+ from core.inference.diffusion_autoopt.patches import source_digest+ assert source_digest(tf3.Flux3AttnProcessor.__call__) in fa._PROCESSOR_DIGESTS+++def test_whole_transformer_matches_and_compiles_without_breaks(monkeypatch):+ from torch._dynamo.testing import CompileCounterWithBackend++ pipe = _pipe()+ g = torch.Generator().manual_seed(4)+ kw = {+ "hidden_states": torch.randn(1, 16, 8, generator = g),+ "encoder_hidden_states": torch.randn(1, 5, 24, generator = g),+ "timestep": torch.tensor([0.7]),+ "img_ids": torch.randint(0, 8, (16, 4), generator = g).float(),+ "txt_ids": torch.randint(0, 8, (5, 4), generator = g).float(),+ "return_dict": False,+ }+ with torch.no_grad():+ want = pipe.transformer(**kw)[0]+ with monkeypatch.context() as m: # the fake CUDA flag only for the install, not for dynamo+ assert fa.install(pipe, "_native_cudnn", _cuda_target(m)) > 0+ got = pipe.transformer(**kw)[0]+ assert torch.equal(got, want)+ torch._dynamo.reset()+ cc = CompileCounterWithBackend("inductor")+ blocks = list(pipe.transformer.single_transformer_blocks)+ eager = [b.forward for b in blocks]+ for b in blocks:+ b.forward = torch.compile(+ type(b).forward.__get__(b), backend = cc, fullgraph = True, dynamic = True+ )+ try:+ outs = [+ pipe.transformer(**dict(kw, timestep = torch.tensor([t])))[0] for t in (0.7, 0.5, 0.3)+ ]+ finally:+ for b, f in zip(blocks, eager):+ b.forward = f+ torch._dynamo.reset()+ assert cc.frame_count == len(blocks) # fullgraph: no break; three steps, no recompile+ assert torch.allclose(outs[0], want, atol = 1e-5, rtol = 1e-5)