Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex tuned backward: leave caller tiles and small sparse blocks to Inductor (#15228)

Daniel Han99288b8d0dff
diff --git a/tests/test_flex_tuned_backward_kernel_options.py b/tests/test_flex_tuned_backward_kernel_options.py
index 3b7860adde1..bd75edd9d92 100644
--- a/tests/test_flex_tuned_backward_kernel_options.py
+++ b/tests/test_flex_tuned_backward_kernel_options.py
@@ -94,3 +94,118 @@ def test_wrapper_passes_tuned_backward_options(monkeypatch):
for key_, val in u._FLEX_TUNED_BWD_KERNEL_OPTIONS.items():
assert seen[key_] == val
assert seen["FORCE_USE_FLEX_ATTENTION"] is True
+
+
+def _wrapped_call(monkeypatch, mask, **kwargs):
+ import torch
+
+ seen = {}
+
+ def fake_flex(module, query, key, value, *args, **kw):
+ seen.update(kw.get("kernel_options") or {})
+ return query
+
+ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: True)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)
+ wrapped = u._wrap_flex_attention_forward(fake_flex)
+
+ class M(torch.nn.Module):
+ is_causal = True
+
+ q, k, v = (torch.randn(1, 2, 8, 256) for _ in range(3))
+ wrapped(M(), q, k, v, mask, **kwargs)
+ return seen
+
+
+class _Mask:
+ def __init__(self, block_size):
+ self.BLOCK_SIZE = block_size
+
+
+ "requested",
+ [{"bwd_BLOCK_M1": 64}, {"BLOCK_M": 64, "BLOCK_N": 64}, {"num_warps": 8}, {"bwd_num_stages": 2}],
+)
+def test_caller_tiles_are_not_mixed_with_tuned_ones(monkeypatch, requested):
+ # A partial caller set merged with ours could break N1 % M1 == 0 (Inductor rejects it).
+ seen = _wrapped_call(monkeypatch, object(), kernel_options = dict(requested))
+ for key_, val in requested.items():
+ assert seen[key_] == val
+ for key_ in u._FLEX_TUNED_BWD_KERNEL_OPTIONS:
+ if key_ not in requested:
+ assert key_ not in seen
+ assert seen["FORCE_USE_FLEX_ATTENTION"] is True
+
+
+def test_caller_non_tile_options_keep_tuned_backward(monkeypatch):
+ seen = _wrapped_call(monkeypatch, object(), kernel_options = {"PRESCALE_QK": True})
+ assert seen["PRESCALE_QK"] is True
+ for key_, val in u._FLEX_TUNED_BWD_KERNEL_OPTIONS.items():
+ assert seen[key_] == val
+
+
+ "block_size, tuned",
+ [
+ ((128, 128), True),
+ ((64, 128), True),
+ ((32, 32), True),
+ ((16, 16), False),
+ ((128, 16), False),
+ ],
+)
+def test_tuned_backward_only_when_tiles_divide_the_sparse_blocks(monkeypatch, block_size, tuned):
+ # Inductor shrinks its default tiles to fit 16-wide sparse blocks; fixed 32-wide ones cannot.
+ seen = _wrapped_call(monkeypatch, _Mask(block_size))
+ assert all((key_ in seen) == tuned for key_ in u._FLEX_TUNED_BWD_KERNEL_OPTIONS)
+ assert seen["FORCE_USE_FLEX_ATTENTION"] is True
+
+
+ not __import__("torch").cuda.is_available()
+ or __import__("torch").version.hip is not None
+ or tuple(int(x) for x in __import__("torch").__version__.split("+")[0].split(".")[:2])
+ < (2, 10),
+ reason = "needs NVIDIA CUDA and torch >= 2.10 (bwd_ kernel_options)",
+)
[email protected]("case", ["block16", "caller_m1_64"])
+def test_tuned_backward_compiles_for_small_blocks_and_partial_caller_tiles(monkeypatch, case):
+ # Real Inductor compile of the backward with the options the wrapper emits (tuned path forced).
+ import torch
+ from torch.nn.attention.flex_attention import create_block_mask, flex_attention
+
+ torch._dynamo.reset()
+ flex = torch.compile(flex_attention, fullgraph = True)
+
+ def real_flex(module, query, key, value, mask, **kw):
+ return flex(query, key, value, block_mask = mask, kernel_options = kw.get("kernel_options"))
+
+ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: True)
+ wrapped = u._wrap_flex_attention_forward(real_flex)
+
+ class M(torch.nn.Module):
+ is_causal = True
+
+ S = 128
+ block = 16 if case == "block16" else 128
+ mask = create_block_mask(
+ lambda b, h, q, kv: q >= kv, None, None, S, S, device = "cuda", BLOCK_SIZE = block
+ )
+ q, k, v = (
+ torch.randn(1, 2, S, 256, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ for _ in range(3)
+ )
+ if case == "block16":
+ # torch 2.10's Inductor cannot compile 16-wide sparse blocks at all (2.14 shrinks its tiles).
+ try:
+ flex(q, k, v, block_mask = mask).float().sum().backward()
+ except Exception as error:
+ pytest.skip(
+ f"this torch cannot compile 16-wide sparse blocks without Unsloth: {type(error).__name__}"
+ )
+ q.grad = k.grad = v.grad = None
+ torch._dynamo.reset()
+ extra = {"kernel_options": {"bwd_BLOCK_M1": 64}} if case == "caller_m1_64" else {}
+ out = wrapped(M(), q, k, v, mask, **extra)
+ out.float().sum().backward()
+ assert q.grad is not None and torch.isfinite(q.grad).all()
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index bd4881b82b8..1d69deb4187 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -763,7 +763,8 @@ _FLEX_TUNED_BWD_MIN_TORCH = "2.10"
def _flex_tuned_bwd_device_ok():
"""True when every visible CUDA device has a capability the tuned backward was measured on.
- Read once when the flex forward is wrapped (model load), never inside a compiled region.
+ Read once when the flex forward is wrapped, at `import unsloth` (set the env var before it), never
+ inside a compiled region.
UNSLOTH_FLEX_TUNED_BACKWARD=0 turns it off, =1 forces it on any CUDA device with torch >= 2.10."""
override = os.environ.get(_FLEX_TUNED_BWD_ENV_VAR, "").strip()
if override == "0":
@@ -791,6 +792,24 @@ def _flex_tuned_bwd_kernel_options(head_dim, device_ok):
return dict(_FLEX_TUNED_BWD_KERNEL_OPTIONS)
+def _flex_tuned_bwd_fits(args, kwargs):
+ """Whether the tuned backward tiles can join this call. Not when the caller set its own tiles
+ (a partial set merged with ours can break bwd_BLOCK_N1 % bwd_BLOCK_M1 == 0), nor when the block
+ mask's sparse blocks are not multiples of our tiles (Inductor's defaults shrink to fit those)."""
+ requested = kwargs.get("kernel_options") or {}
+ if any(k.startswith(("bwd_", "BLOCK_", "num_")) for k in requested):
+ return False
+ mask = args[0] if len(args) > 0 else kwargs.get("attention_mask", None)
+ block_size = getattr(mask, "BLOCK_SIZE", None)
+ if block_size is None:
+ return True
+ try:
+ tiles = [v for k, v in _FLEX_TUNED_BWD_KERNEL_OPTIONS.items() if "_BLOCK_" in k]
+ return all(int(size) % tile == 0 for size in block_size for tile in tiles)
+ except Exception:
+ return False
+
+
def _flex_call_needs_backward(query, key, value):
"""True when this flex call is recorded for autograd on an accelerator."""
try:
@@ -951,6 +970,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
and hasattr(query, "dim")
and query.dim() == 4
and _flex_call_needs_backward(query, key, value)
+ and _flex_tuned_bwd_fits(args, kwargs)
):
kernel_options = _flex_tuned_bwd_kernel_options(query.shape[-1], tuned_bwd_device_ok)
# Inductor picks flex_decoding for a static query length below 128. Its logsumexp gets a