Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex attention: small shared-memory tiles for short no-grad calls so one short prompt does not leave the process on eager attention (#16049)

Daniel Hana2f37e11fe29
diff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.py
index be66f8f402e..abd88d19a74 100644
--- a/tests/test_flex_small_smem_tiles.py
+++ b/tests/test_flex_small_smem_tiles.py
@@ -200,10 +200,55 @@ def test_wrapper_no_grad_prefill_gets_tiles_decode_does_not(monkeypatch):
monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
wrapped(_Causal(), *_qkv(128, grad = False), object())
assert seen.get("fwd_BLOCK_M") == 64
+ assert "FORCE_USE_FLEX_ATTENTION" not in seen # q_len >= 128 is the main kernel already
wrapped(_Causal(), *_qkv(8, grad = False), object())
assert seen == {} # flex_decoding keeps its own config
+def _gqa_qkv(
+ q_len,
+ q_heads,
+ kv_heads,
+ head_dim = 256,
+):
+ import torch
+ q = torch.randn(1, q_heads, q_len, head_dim)
+ return [q, torch.randn(1, kv_heads, q_len, head_dim), torch.randn(1, kv_heads, q_len, head_dim)]
+
+
+ "q_len,q_heads,kv_heads,tiled",
+ [
+ (1, 4, 1, False), # Gemma 3 1B decode step: 16 rows, flex_decoding fits
+ (16, 4, 1, False), # 64 rows still fit
+ (17, 4, 1, True), # 68 rows -> a 128-row tile: 114688 B
+ (28, 4, 1, True), # Gemma 3 1B / 270M chat prompt
+ (28, 8, 4, False), # Gemma 3 4B: 2 query heads per kv head, 56 rows
+ (40, 8, 4, True),
+ (127, 2, 2, True),
+ (28, 6, 2, False), # not a power of two: transformers repeats K / V, 28 rows
+ ],
+)
+def test_wrapper_short_prompt_whose_decode_tile_does_not_fit_gets_tiles(
+ monkeypatch, q_len, q_heads, kv_heads, tiled
+):
+ # A failed compile here pinned flex_attention to the eager fallback for the whole process.
+ wrapped, seen = _wrapped(monkeypatch)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ wrapped(_Causal(), *_gqa_qkv(q_len, q_heads, kv_heads), object())
+ if tiled:
+ assert {
+ k: seen.get(k) for k in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS
+ } == u._FLEX_SMALL_SMEM_KERNEL_OPTIONS
+ # On the main kernel: flex_decoding with these tiles is wrong on torch 2.10, fails on 2.11.
+ assert seen.get("FORCE_USE_FLEX_ATTENTION") is True
+ else:
+ assert seen == {}
+ wrapped, seen = _wrapped(monkeypatch, small_ok = False)
+ wrapped(_Causal(), *_gqa_qkv(q_len, q_heads, kv_heads), object())
+ assert seen == {} # other cards keep the defaults
+
+
def test_wrapper_untouched_off_class_and_outside_range(monkeypatch):
wrapped, seen = _wrapped(monkeypatch, small_ok = False)
monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)
@@ -275,3 +320,40 @@ def test_wrapper_training_calls_add_no_sequence_guard(monkeypatch):
return len(graphs)
assert graphs_for(True) == graphs_for(False)
+
+
+def test_wrapper_forced_short_prompt_drops_auto_backend(monkeypatch):
+ # torch 2.10+ rejects BACKEND with FORCE_USE_FLEX_ATTENTION and 2.9 has no BACKEND: AUTO is dropped.
+ wrapped, seen = _wrapped(monkeypatch)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = {"BACKEND": "AUTO"})
+ assert seen["FORCE_USE_FLEX_ATTENTION"] is True
+ assert "BACKEND" not in seen
+ assert seen["fwd_BLOCK_M"] == 64
+
+
+def test_wrapper_forced_short_prompt_triton_backend_keeps_tiles(monkeypatch):
+ wrapped, seen = _wrapped(monkeypatch)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = {"BACKEND": "TRITON"})
+ assert "FORCE_USE_FLEX_ATTENTION" not in seen
+ assert seen["BACKEND"] == "TRITON"
+ assert seen["fwd_BLOCK_M"] == 64
+
+
[email protected]("backend", ["TRITON_DECODE", "FLASH"])
+def test_wrapper_forced_short_prompt_keeps_explicit_backend_untouched(monkeypatch, backend):
+ # Our 64-row main-kernel tiles would make flex_decoding compute only 64 of 112 rows on 2.10.
+ wrapped, seen = _wrapped(monkeypatch)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = {"BACKEND": backend})
+ assert seen == {"BACKEND": backend}
+
+
+def test_wrapper_forced_short_prompt_keeps_caller_force_off_untouched(monkeypatch):
+ # The caller's FORCE off sends 112 rows to flex_decoding, which our main-kernel tiles do not cover.
+ wrapped, seen = _wrapped(monkeypatch)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ requested = {"FORCE_USE_FLEX_ATTENTION": False}
+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = dict(requested))
+ assert seen == requested
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index b8c677eae28..bfa1e58840a 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -985,6 +985,22 @@ _FLEX_SMALL_SMEM_KERNEL_OPTIONS = {
_FLEX_SMALL_SMEM_MIN_SHARED_MEMORY = 101376
_FLEX_SMALL_SMEM_CAPABILITY_MAJOR = 8
_FLEX_SMALL_SMEM_ENV_VAR = "UNSLOTH_FLEX_SMALL_SMEM_TILES"
+# flex_decoding (no-grad query below 128) holds next_pow2(q_len * query heads per kv head) query rows
+# in one tile. At head_dim 256 on these cards up to 64 rows fit (one-token decode: 16), 128 need
+# 114688 B. A short prompt's prefill (Gemma 3 1B: 28 tokens x 4 heads) then fails to compile,
+# unsloth_zoo's suppress_errors hides it, and Dynamo marks flex_attention as skipped for the rest
+# of the process, so every later call, long prompts and decode included, runs the eager fallback.
+_FLEX_SMALL_SMEM_DECODE_MAX_ROWS = 64
+
+
+def _flex_decode_rows(query, key):
+ """Query rows flex_decoding would put in one tile for this call (transformers repeats K / V
+ for a head count that is not a power of two, so each kv head then serves one query head)."""
+ q_heads, kv_heads = query.shape[1], key.shape[1]
+ group = 1
+ if kv_heads > 0 and q_heads % kv_heads == 0 and (q_heads & (q_heads - 1)) == 0:
+ group = q_heads // kv_heads
+ return query.shape[2] * group
def _flex_small_smem_device_ok():
@@ -1209,6 +1225,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
)
except Exception:
kernel_options = None
+ short_forced = False
# First: on these cards the default forward tile does not fit, so this full set beats a forced
# UNSLOTH_FLEX_TUNED_BACKWARD=1 (backward keys only).
if (
@@ -1216,20 +1233,31 @@ def _wrap_flex_attention_forward(flex_attention_forward):
and small_smem_device_ok
and hasattr(query, "dim")
and query.dim() == 4
- # Short no-grad queries go to flex_decoding, whose own tiles are left alone, unless the
- # caller (the block compile hook, also for no-grad checkpoint forwards) forces the main
- # kernel. Shape last, so training calls add no seq >= 128 guard.
+ # Short no-grad queries go to flex_decoding, whose own tiles are left alone while they
+ # fit, unless the caller (the block compile hook, also for no-grad checkpoint forwards)
+ # forces the main kernel. Shape last, so training calls add no seq >= 128 guard.
and (
_flex_call_needs_backward(query, key, value)
or _flex_main_kernel_forced(kwargs)
or decode_replaced
or query.shape[2] >= 128
+ or _flex_decode_rows(query, key) > _FLEX_SMALL_SMEM_DECODE_MAX_ROWS
)
and _flex_tuned_bwd_fits(
args, kwargs, _FLEX_SMALL_SMEM_KERNEL_OPTIONS, ("fwd_", "bwd_", "BLOCK_", "num_")
)
):
kernel_options = _flex_small_smem_kernel_options(query.shape[-1], small_smem_device_ok)
+ # Main kernel: flex_decoding with fwd_BLOCK_M 64 below its row count is wrong on 2.10 and
+ # fails to compile on 2.11. Shape last, as above (no training sequence guard).
+ if (
+ kernel_options is not None
+ and not _flex_call_needs_backward(query, key, value)
+ and not _flex_main_kernel_forced(kwargs)
+ and query.shape[2] < 128
+ ):
+ kernel_options["FORCE_USE_FLEX_ATTENTION"] = True
+ short_forced = True
if (
kernel_options is None
and tuned_bwd_device_ok
@@ -1255,7 +1283,23 @@ def _wrap_flex_attention_forward(flex_attention_forward):
requested = {k: v for k, v in requested.items() if k != "BACKEND"}
if kernel_options is not None:
# A caller that already asked for something keeps it: only fill the gaps.
+ ours_forced = (
+ kernel_options.get("FORCE_USE_FLEX_ATTENTION") is True
+ and "FORCE_USE_FLEX_ATTENTION" not in requested
+ )
+ if ours_forced and requested.get("BACKEND") == "AUTO":
+ requested = {k: v for k, v in requested.items() if k != "BACKEND"}
kernel_options.update(requested)
+ if short_forced and requested.get("FORCE_USE_FLEX_ATTENTION") is False:
+ # The caller turned the main kernel off: our main-kernel tiles do not cover flex_decoding.
+ kernel_options = dict(requested)
+ # torch 2.10+ rejects BACKEND with FORCE_USE_FLEX_ATTENTION: TRITON is the main kernel, so it
+ # keeps our tiles; any other backend keeps the caller's options, which our tiles do not fit.
+ elif ours_forced and "BACKEND" in kernel_options:
+ if kernel_options["BACKEND"] == "TRITON":
+ kernel_options.pop("FORCE_USE_FLEX_ATTENTION")
+ else:
+ kernel_options = dict(requested)
kwargs["kernel_options"] = kernel_options
return flex_attention_forward(module, query, key, value, *args, **kwargs)