Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flex attention: tuned backward tiles on A100 for head_dim 256 (Gemma 3 LoRA 1.9-2.7x faster) (#15193)
Daniel Han0705fa584791
diff --git a/tests/test_flex_tuned_backward_kernel_options.py b/tests/test_flex_tuned_backward_kernel_options.pynew file mode 100644index 00000000000..3b7860adde1--- /dev/null+++ b/tests/test_flex_tuned_backward_kernel_options.py@@ -0,0 +1,96 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""sm80 flex backward: tuned bwd_* kernel_options at 128 < head_dim <= 256."""++import pytest++import unsloth # noqa: F401 (must precede transformers)+import unsloth.models._utils as u++[email protected]("head_dim", [129, 192, 256])+def test_tuned_backward_options_in_range(head_dim):+ opts = u._flex_tuned_bwd_kernel_options(head_dim, True)+ assert opts == u._FLEX_TUNED_BWD_KERNEL_OPTIONS+ # backward only: the forward template keeps Inductor's default+ assert all(k.startswith("bwd_") for k in opts)+ # a fresh dict every call, so a caller merging its own keys never mutates the constant+ opts["bwd_num_warps"] = 99+ assert u._FLEX_TUNED_BWD_KERNEL_OPTIONS["bwd_num_warps"] == 4++[email protected]("head_dim", [64, 128, 257, 512, None, "256"])+def test_tuned_backward_options_out_of_range(head_dim):+ assert u._flex_tuned_bwd_kernel_options(head_dim, True) is None+++def test_tuned_backward_options_need_a_measured_device():+ assert u._flex_tuned_bwd_kernel_options(256, False) is None+++def test_tuned_backward_tiles_divide_the_default_sparse_block():+ o = u._FLEX_TUNED_BWD_KERNEL_OPTIONS+ for k in ("bwd_BLOCK_M1", "bwd_BLOCK_N1", "bwd_BLOCK_M2", "bwd_BLOCK_N2"):+ assert 128 % o[k] == 0+ assert o["bwd_BLOCK_N1"] % o["bwd_BLOCK_M1"] == 0+ assert o["bwd_BLOCK_M2"] % o["bwd_BLOCK_N2"] == 0++[email protected]("value,expected", [("0", False)])+def test_env_kill_switch(monkeypatch, value, expected):+ monkeypatch.setenv(u._FLEX_TUNED_BWD_ENV_VAR, value)+ assert u._flex_tuned_bwd_device_ok() is expected+++ "hip,torch_version,caps,env,expected",+ [+ (None, "2.14.0+cu130", [(8, 0)], "", True),+ (None, "2.10.0+cu128", [(8, 0), (8, 0)], "", True),+ (None, "2.14.0", [(8, 0), (8, 6)], "", False), # every visible device must be measured+ (None, "2.14.0", [(8, 6)], "", False),+ (None, "2.14.0", [(9, 0)], "", False),+ (None, "2.14.0", [(10, 0)], "", False),+ (None, "2.14.0", [(9, 0)], "1", True), # forced on any CUDA device+ (None, "2.9.1", [(8, 0)], "", False), # no bwd_ keys before 2.10+ (None, "2.9.1", [(8, 0)], "1", False),+ ("7.2.0", "2.14.0", [(8, 0)], "", False), # ROCm never+ ("7.2.0", "2.14.0", [(8, 0)], "1", False),+ ],+)+def test_device_gate(monkeypatch, hip, torch_version, caps, env, expected):+ import torch++ monkeypatch.setattr(torch.version, "hip", hip, raising = False)+ monkeypatch.setattr(torch, "__version__", torch_version)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.cuda, "device_count", lambda: len(caps))+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda index = 0: caps[index])+ monkeypatch.setenv(u._FLEX_TUNED_BWD_ENV_VAR, env)+ assert u._flex_tuned_bwd_device_ok() is expected+++def test_wrapper_passes_tuned_backward_options(monkeypatch):+ import torch++ seen = {}++ def fake_flex(module, query, key, value, *args, **kwargs):+ seen.update(kwargs.get("kernel_options") or {})+ return query++ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: True)+ wrapped = u._wrap_flex_attention_forward(fake_flex)++ class M(torch.nn.Module):+ is_causal = True++ q = torch.randn(1, 2, 8, 256, requires_grad = True)+ k = torch.randn(1, 2, 8, 256, requires_grad = True)+ v = torch.randn(1, 2, 8, 256, requires_grad = True)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)+ wrapped(M(), q, k, v, object()) # a non-None mask: stays on flex+ for key_, val in u._FLEX_TUNED_BWD_KERNEL_OPTIONS.items():+ assert seen[key_] == val+ assert seen["FORCE_USE_FLEX_ATTENTION"] is Truediff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 65de0380a32..89d17810b29 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -736,6 +736,61 @@ def _flex_kernel_options_for_head_dim(head_dim):return dict(_FLEX_LARGE_HEAD_DIM_KERNEL_OPTIONS)+# Inductor's sm8x default flex BACKWARD config at 128 < head_dim <= 256 is (64, 64, 64, 64,+# 2 stages, 4 warps). On an A100 (sm80) that config runs about 10x slower than a 32 x 32 tile with+# one stage: Gemma 3 4B shapes (8 query / 4 kv heads, head_dim 256, bf16) fwd+bwd, causal+# B8 x 1024: 32.4 ms default vs 3.2 ms; sliding 1024 window B2 x 4096: 53.3 vs 4.8 ms; causal+# B2 x 4096: 115.3 vs 9.5 ms. In Gemma 3 4B LoRA training the flex backward was half of an A100+# step. B200 (sm100, its own default) is faster on the default, so only the measured capability+# gets the override. Backward only ("bwd_" keys): the forward template is unchanged, and so is the+# math (tile sizes only reorder the float32 accumulation).+_FLEX_TUNED_BWD_HEAD_DIM_RANGE = (128, 256) # (exclusive low, inclusive high)+_FLEX_TUNED_BWD_CAPABILITIES = ((8, 0),)+_FLEX_TUNED_BWD_KERNEL_OPTIONS = {+ "bwd_BLOCK_M1": 32,+ "bwd_BLOCK_N1": 32,+ "bwd_BLOCK_M2": 32,+ "bwd_BLOCK_N2": 32,+ "bwd_num_stages": 1,+ "bwd_num_warps": 4,+}+_FLEX_TUNED_BWD_ENV_VAR = "UNSLOTH_FLEX_TUNED_BACKWARD"+# The flex templates read "bwd_"-prefixed kernel_options from torch 2.10 on; older releases have no+# per-direction keys, so the override is never passed there.+_FLEX_TUNED_BWD_MIN_TORCH = "2.10"+++def _flex_tuned_bwd_device_ok():+ """True when every visible CUDA device has a capability the tuned backward was measured on.++ Read once when the flex forward is wrapped (model load), never inside a compiled region.+ UNSLOTH_FLEX_TUNED_BACKWARD=0 turns it off, =1 forces it on any CUDA device with torch >= 2.10."""+ override = os.environ.get(_FLEX_TUNED_BWD_ENV_VAR, "").strip()+ if override == "0":+ return False+ try:+ if getattr(torch.version, "hip", None) or not torch.cuda.is_available():+ return False+ if Version(torch.__version__.split("+")[0]) < Version(_FLEX_TUNED_BWD_MIN_TORCH):+ return False+ if override == "1":+ return True+ return all(+ tuple(torch.cuda.get_device_capability(index)) in _FLEX_TUNED_BWD_CAPABILITIES+ for index in range(torch.cuda.device_count())+ )+ except Exception:+ return False+++def _flex_tuned_bwd_kernel_options(head_dim, device_ok):+ """bwd_* kernel_options for this head dim on a measured device, else None."""+ low, high = _FLEX_TUNED_BWD_HEAD_DIM_RANGE+ if not device_ok or not isinstance(head_dim, int) or not (low < head_dim <= high):+ return None+ return dict(_FLEX_TUNED_BWD_KERNEL_OPTIONS)++def _flex_call_needs_backward(query, key, value):"""True when this flex call is recorded for autograd on an accelerator."""try:@@ -864,6 +919,8 @@ def _wrap_flex_attention_forward(flex_attention_forward):if getattr(flex_attention_forward, "_unsloth_flex_kernel_options", False):return flex_attention_forward+ tuned_bwd_device_ok = _flex_tuned_bwd_device_ok()+@functools.wraps(flex_attention_forward)def unsloth_flex_attention_forward(module, query, key, value, *args, **kwargs):if _FLEX_MASKLESS_SDPA_ENABLED and _dropped_causal_mask(module, query, key, args, kwargs):@@ -888,6 +945,14 @@ def _wrap_flex_attention_forward(flex_attention_forward):)except Exception:kernel_options = None+ if (+ kernel_options is None+ and tuned_bwd_device_ok+ and hasattr(query, "dim")+ and query.dim() == 4+ and _flex_call_needs_backward(query, key, value)+ ):+ kernel_options = _flex_tuned_bwd_kernel_options(query.shape[-1], tuned_bwd_device_ok)# Inductor picks flex_decoding for a static query length below 128. Its logsumexp gets a# padded batch stride (comprehensive_padding) that the backward template ignores, so every# batch row after the first reads a shifted LSE and the gradients blow up.