Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex attention: tuned backward tiles on A100 for head_dim 256 (Gemma 3 LoRA 1.9-2.7x faster) (#15193)

Daniel Han0705fa584791
diff --git a/tests/test_flex_tuned_backward_kernel_options.py b/tests/test_flex_tuned_backward_kernel_options.py
new file mode 100644
index 00000000000..3b7860adde1
--- /dev/null
+++ b/tests/test_flex_tuned_backward_kernel_options.py
@@ -0,0 +1,96 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""sm80 flex backward: tuned bwd_* kernel_options at 128 < head_dim <= 256."""
+
+import pytest
+
+import unsloth # noqa: F401 (must precede transformers)
+import unsloth.models._utils as u
+
+
[email protected]("head_dim", [129, 192, 256])
+def test_tuned_backward_options_in_range(head_dim):
+ opts = u._flex_tuned_bwd_kernel_options(head_dim, True)
+ assert opts == u._FLEX_TUNED_BWD_KERNEL_OPTIONS
+ # backward only: the forward template keeps Inductor's default
+ assert all(k.startswith("bwd_") for k in opts)
+ # a fresh dict every call, so a caller merging its own keys never mutates the constant
+ opts["bwd_num_warps"] = 99
+ assert u._FLEX_TUNED_BWD_KERNEL_OPTIONS["bwd_num_warps"] == 4
+
+
[email protected]("head_dim", [64, 128, 257, 512, None, "256"])
+def test_tuned_backward_options_out_of_range(head_dim):
+ assert u._flex_tuned_bwd_kernel_options(head_dim, True) is None
+
+
+def test_tuned_backward_options_need_a_measured_device():
+ assert u._flex_tuned_bwd_kernel_options(256, False) is None
+
+
+def test_tuned_backward_tiles_divide_the_default_sparse_block():
+ o = u._FLEX_TUNED_BWD_KERNEL_OPTIONS
+ for k in ("bwd_BLOCK_M1", "bwd_BLOCK_N1", "bwd_BLOCK_M2", "bwd_BLOCK_N2"):
+ assert 128 % o[k] == 0
+ assert o["bwd_BLOCK_N1"] % o["bwd_BLOCK_M1"] == 0
+ assert o["bwd_BLOCK_M2"] % o["bwd_BLOCK_N2"] == 0
+
+
[email protected]("value,expected", [("0", False)])
+def test_env_kill_switch(monkeypatch, value, expected):
+ monkeypatch.setenv(u._FLEX_TUNED_BWD_ENV_VAR, value)
+ assert u._flex_tuned_bwd_device_ok() is expected
+
+
+ "hip,torch_version,caps,env,expected",
+ [
+ (None, "2.14.0+cu130", [(8, 0)], "", True),
+ (None, "2.10.0+cu128", [(8, 0), (8, 0)], "", True),
+ (None, "2.14.0", [(8, 0), (8, 6)], "", False), # every visible device must be measured
+ (None, "2.14.0", [(8, 6)], "", False),
+ (None, "2.14.0", [(9, 0)], "", False),
+ (None, "2.14.0", [(10, 0)], "", False),
+ (None, "2.14.0", [(9, 0)], "1", True), # forced on any CUDA device
+ (None, "2.9.1", [(8, 0)], "", False), # no bwd_ keys before 2.10
+ (None, "2.9.1", [(8, 0)], "1", False),
+ ("7.2.0", "2.14.0", [(8, 0)], "", False), # ROCm never
+ ("7.2.0", "2.14.0", [(8, 0)], "1", False),
+ ],
+)
+def test_device_gate(monkeypatch, hip, torch_version, caps, env, expected):
+ import torch
+
+ monkeypatch.setattr(torch.version, "hip", hip, raising = False)
+ monkeypatch.setattr(torch, "__version__", torch_version)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: len(caps))
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = 0: caps[index])
+ monkeypatch.setenv(u._FLEX_TUNED_BWD_ENV_VAR, env)
+ assert u._flex_tuned_bwd_device_ok() is expected
+
+
+def test_wrapper_passes_tuned_backward_options(monkeypatch):
+ import torch
+
+ seen = {}
+
+ def fake_flex(module, query, key, value, *args, **kwargs):
+ seen.update(kwargs.get("kernel_options") or {})
+ return query
+
+ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: True)
+ wrapped = u._wrap_flex_attention_forward(fake_flex)
+
+ class M(torch.nn.Module):
+ is_causal = True
+
+ q = torch.randn(1, 2, 8, 256, requires_grad = True)
+ k = torch.randn(1, 2, 8, 256, requires_grad = True)
+ v = torch.randn(1, 2, 8, 256, requires_grad = True)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)
+ wrapped(M(), q, k, v, object()) # a non-None mask: stays on flex
+ for key_, val in u._FLEX_TUNED_BWD_KERNEL_OPTIONS.items():
+ assert seen[key_] == val
+ assert seen["FORCE_USE_FLEX_ATTENTION"] is True
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 65de0380a32..89d17810b29 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -736,6 +736,61 @@ def _flex_kernel_options_for_head_dim(head_dim):
return dict(_FLEX_LARGE_HEAD_DIM_KERNEL_OPTIONS)
+# Inductor's sm8x default flex BACKWARD config at 128 < head_dim <= 256 is (64, 64, 64, 64,
+# 2 stages, 4 warps). On an A100 (sm80) that config runs about 10x slower than a 32 x 32 tile with
+# one stage: Gemma 3 4B shapes (8 query / 4 kv heads, head_dim 256, bf16) fwd+bwd, causal
+# B8 x 1024: 32.4 ms default vs 3.2 ms; sliding 1024 window B2 x 4096: 53.3 vs 4.8 ms; causal
+# B2 x 4096: 115.3 vs 9.5 ms. In Gemma 3 4B LoRA training the flex backward was half of an A100
+# step. B200 (sm100, its own default) is faster on the default, so only the measured capability
+# gets the override. Backward only ("bwd_" keys): the forward template is unchanged, and so is the
+# math (tile sizes only reorder the float32 accumulation).
+_FLEX_TUNED_BWD_HEAD_DIM_RANGE = (128, 256) # (exclusive low, inclusive high)
+_FLEX_TUNED_BWD_CAPABILITIES = ((8, 0),)
+_FLEX_TUNED_BWD_KERNEL_OPTIONS = {
+ "bwd_BLOCK_M1": 32,
+ "bwd_BLOCK_N1": 32,
+ "bwd_BLOCK_M2": 32,
+ "bwd_BLOCK_N2": 32,
+ "bwd_num_stages": 1,
+ "bwd_num_warps": 4,
+}
+_FLEX_TUNED_BWD_ENV_VAR = "UNSLOTH_FLEX_TUNED_BACKWARD"
+# The flex templates read "bwd_"-prefixed kernel_options from torch 2.10 on; older releases have no
+# per-direction keys, so the override is never passed there.
+_FLEX_TUNED_BWD_MIN_TORCH = "2.10"
+
+
+def _flex_tuned_bwd_device_ok():
+ """True when every visible CUDA device has a capability the tuned backward was measured on.
+
+ Read once when the flex forward is wrapped (model load), never inside a compiled region.
+ UNSLOTH_FLEX_TUNED_BACKWARD=0 turns it off, =1 forces it on any CUDA device with torch >= 2.10."""
+ override = os.environ.get(_FLEX_TUNED_BWD_ENV_VAR, "").strip()
+ if override == "0":
+ return False
+ try:
+ if getattr(torch.version, "hip", None) or not torch.cuda.is_available():
+ return False
+ if Version(torch.__version__.split("+")[0]) < Version(_FLEX_TUNED_BWD_MIN_TORCH):
+ return False
+ if override == "1":
+ return True
+ return all(
+ tuple(torch.cuda.get_device_capability(index)) in _FLEX_TUNED_BWD_CAPABILITIES
+ for index in range(torch.cuda.device_count())
+ )
+ except Exception:
+ return False
+
+
+def _flex_tuned_bwd_kernel_options(head_dim, device_ok):
+ """bwd_* kernel_options for this head dim on a measured device, else None."""
+ low, high = _FLEX_TUNED_BWD_HEAD_DIM_RANGE
+ if not device_ok or not isinstance(head_dim, int) or not (low < head_dim <= high):
+ return None
+ return dict(_FLEX_TUNED_BWD_KERNEL_OPTIONS)
+
+
def _flex_call_needs_backward(query, key, value):
"""True when this flex call is recorded for autograd on an accelerator."""
try:
@@ -864,6 +919,8 @@ def _wrap_flex_attention_forward(flex_attention_forward):
if getattr(flex_attention_forward, "_unsloth_flex_kernel_options", False):
return flex_attention_forward
+ tuned_bwd_device_ok = _flex_tuned_bwd_device_ok()
+
@functools.wraps(flex_attention_forward)
def unsloth_flex_attention_forward(module, query, key, value, *args, **kwargs):
if _FLEX_MASKLESS_SDPA_ENABLED and _dropped_causal_mask(module, query, key, args, kwargs):
@@ -888,6 +945,14 @@ def _wrap_flex_attention_forward(flex_attention_forward):
)
except Exception:
kernel_options = None
+ if (
+ kernel_options is None
+ and tuned_bwd_device_ok
+ and hasattr(query, "dim")
+ and query.dim() == 4
+ and _flex_call_needs_backward(query, key, value)
+ ):
+ kernel_options = _flex_tuned_bwd_kernel_options(query.shape[-1], tuned_bwd_device_ok)
# Inductor picks flex_decoding for a static query length below 128. Its logsumexp gets a
# padded batch stride (comprehensive_padding) that the backward template ignores, so every
# batch row after the first reads a shifted LSE and the gradients blow up.