Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flex attention: small shared-memory tiles for short no-grad calls so one short prompt does not leave the process on eager attention (#16049)
Daniel Hana2f37e11fe29
diff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.pyindex be66f8f402e..abd88d19a74 100644--- a/tests/test_flex_small_smem_tiles.py+++ b/tests/test_flex_small_smem_tiles.py@@ -200,10 +200,55 @@ def test_wrapper_no_grad_prefill_gets_tiles_decode_does_not(monkeypatch):monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)wrapped(_Causal(), *_qkv(128, grad = False), object())assert seen.get("fwd_BLOCK_M") == 64+ assert "FORCE_USE_FLEX_ATTENTION" not in seen # q_len >= 128 is the main kernel alreadywrapped(_Causal(), *_qkv(8, grad = False), object())assert seen == {} # flex_decoding keeps its own config+def _gqa_qkv(+ q_len,+ q_heads,+ kv_heads,+ head_dim = 256,+):+ import torch+ q = torch.randn(1, q_heads, q_len, head_dim)+ return [q, torch.randn(1, kv_heads, q_len, head_dim), torch.randn(1, kv_heads, q_len, head_dim)]+++ "q_len,q_heads,kv_heads,tiled",+ [+ (1, 4, 1, False), # Gemma 3 1B decode step: 16 rows, flex_decoding fits+ (16, 4, 1, False), # 64 rows still fit+ (17, 4, 1, True), # 68 rows -> a 128-row tile: 114688 B+ (28, 4, 1, True), # Gemma 3 1B / 270M chat prompt+ (28, 8, 4, False), # Gemma 3 4B: 2 query heads per kv head, 56 rows+ (40, 8, 4, True),+ (127, 2, 2, True),+ (28, 6, 2, False), # not a power of two: transformers repeats K / V, 28 rows+ ],+)+def test_wrapper_short_prompt_whose_decode_tile_does_not_fit_gets_tiles(+ monkeypatch, q_len, q_heads, kv_heads, tiled+):+ # A failed compile here pinned flex_attention to the eager fallback for the whole process.+ wrapped, seen = _wrapped(monkeypatch)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ wrapped(_Causal(), *_gqa_qkv(q_len, q_heads, kv_heads), object())+ if tiled:+ assert {+ k: seen.get(k) for k in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS+ } == u._FLEX_SMALL_SMEM_KERNEL_OPTIONS+ # On the main kernel: flex_decoding with these tiles is wrong on torch 2.10, fails on 2.11.+ assert seen.get("FORCE_USE_FLEX_ATTENTION") is True+ else:+ assert seen == {}+ wrapped, seen = _wrapped(monkeypatch, small_ok = False)+ wrapped(_Causal(), *_gqa_qkv(q_len, q_heads, kv_heads), object())+ assert seen == {} # other cards keep the defaults++def test_wrapper_untouched_off_class_and_outside_range(monkeypatch):wrapped, seen = _wrapped(monkeypatch, small_ok = False)monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)@@ -275,3 +320,40 @@ def test_wrapper_training_calls_add_no_sequence_guard(monkeypatch):return len(graphs)assert graphs_for(True) == graphs_for(False)+++def test_wrapper_forced_short_prompt_drops_auto_backend(monkeypatch):+ # torch 2.10+ rejects BACKEND with FORCE_USE_FLEX_ATTENTION and 2.9 has no BACKEND: AUTO is dropped.+ wrapped, seen = _wrapped(monkeypatch)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = {"BACKEND": "AUTO"})+ assert seen["FORCE_USE_FLEX_ATTENTION"] is True+ assert "BACKEND" not in seen+ assert seen["fwd_BLOCK_M"] == 64+++def test_wrapper_forced_short_prompt_triton_backend_keeps_tiles(monkeypatch):+ wrapped, seen = _wrapped(monkeypatch)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = {"BACKEND": "TRITON"})+ assert "FORCE_USE_FLEX_ATTENTION" not in seen+ assert seen["BACKEND"] == "TRITON"+ assert seen["fwd_BLOCK_M"] == 64++[email protected]("backend", ["TRITON_DECODE", "FLASH"])+def test_wrapper_forced_short_prompt_keeps_explicit_backend_untouched(monkeypatch, backend):+ # Our 64-row main-kernel tiles would make flex_decoding compute only 64 of 112 rows on 2.10.+ wrapped, seen = _wrapped(monkeypatch)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = {"BACKEND": backend})+ assert seen == {"BACKEND": backend}+++def test_wrapper_forced_short_prompt_keeps_caller_force_off_untouched(monkeypatch):+ # The caller's FORCE off sends 112 rows to flex_decoding, which our main-kernel tiles do not cover.+ wrapped, seen = _wrapped(monkeypatch)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ requested = {"FORCE_USE_FLEX_ATTENTION": False}+ wrapped(_Causal(), *_gqa_qkv(28, 4, 1), object(), kernel_options = dict(requested))+ assert seen == requesteddiff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex b8c677eae28..bfa1e58840a 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -985,6 +985,22 @@ _FLEX_SMALL_SMEM_KERNEL_OPTIONS = {_FLEX_SMALL_SMEM_MIN_SHARED_MEMORY = 101376_FLEX_SMALL_SMEM_CAPABILITY_MAJOR = 8_FLEX_SMALL_SMEM_ENV_VAR = "UNSLOTH_FLEX_SMALL_SMEM_TILES"+# flex_decoding (no-grad query below 128) holds next_pow2(q_len * query heads per kv head) query rows+# in one tile. At head_dim 256 on these cards up to 64 rows fit (one-token decode: 16), 128 need+# 114688 B. A short prompt's prefill (Gemma 3 1B: 28 tokens x 4 heads) then fails to compile,+# unsloth_zoo's suppress_errors hides it, and Dynamo marks flex_attention as skipped for the rest+# of the process, so every later call, long prompts and decode included, runs the eager fallback.+_FLEX_SMALL_SMEM_DECODE_MAX_ROWS = 64+++def _flex_decode_rows(query, key):+ """Query rows flex_decoding would put in one tile for this call (transformers repeats K / V+ for a head count that is not a power of two, so each kv head then serves one query head)."""+ q_heads, kv_heads = query.shape[1], key.shape[1]+ group = 1+ if kv_heads > 0 and q_heads % kv_heads == 0 and (q_heads & (q_heads - 1)) == 0:+ group = q_heads // kv_heads+ return query.shape[2] * groupdef _flex_small_smem_device_ok():@@ -1209,6 +1225,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):)except Exception:kernel_options = None+ short_forced = False# First: on these cards the default forward tile does not fit, so this full set beats a forced# UNSLOTH_FLEX_TUNED_BACKWARD=1 (backward keys only).if (@@ -1216,20 +1233,31 @@ def _wrap_flex_attention_forward(flex_attention_forward):and small_smem_device_okand hasattr(query, "dim")and query.dim() == 4- # Short no-grad queries go to flex_decoding, whose own tiles are left alone, unless the- # caller (the block compile hook, also for no-grad checkpoint forwards) forces the main- # kernel. Shape last, so training calls add no seq >= 128 guard.+ # Short no-grad queries go to flex_decoding, whose own tiles are left alone while they+ # fit, unless the caller (the block compile hook, also for no-grad checkpoint forwards)+ # forces the main kernel. Shape last, so training calls add no seq >= 128 guard.and (_flex_call_needs_backward(query, key, value)or _flex_main_kernel_forced(kwargs)or decode_replacedor query.shape[2] >= 128+ or _flex_decode_rows(query, key) > _FLEX_SMALL_SMEM_DECODE_MAX_ROWS)and _flex_tuned_bwd_fits(args, kwargs, _FLEX_SMALL_SMEM_KERNEL_OPTIONS, ("fwd_", "bwd_", "BLOCK_", "num_"))):kernel_options = _flex_small_smem_kernel_options(query.shape[-1], small_smem_device_ok)+ # Main kernel: flex_decoding with fwd_BLOCK_M 64 below its row count is wrong on 2.10 and+ # fails to compile on 2.11. Shape last, as above (no training sequence guard).+ if (+ kernel_options is not None+ and not _flex_call_needs_backward(query, key, value)+ and not _flex_main_kernel_forced(kwargs)+ and query.shape[2] < 128+ ):+ kernel_options["FORCE_USE_FLEX_ATTENTION"] = True+ short_forced = Trueif (kernel_options is Noneand tuned_bwd_device_ok@@ -1255,7 +1283,23 @@ def _wrap_flex_attention_forward(flex_attention_forward):requested = {k: v for k, v in requested.items() if k != "BACKEND"}if kernel_options is not None:# A caller that already asked for something keeps it: only fill the gaps.+ ours_forced = (+ kernel_options.get("FORCE_USE_FLEX_ATTENTION") is True+ and "FORCE_USE_FLEX_ATTENTION" not in requested+ )+ if ours_forced and requested.get("BACKEND") == "AUTO":+ requested = {k: v for k, v in requested.items() if k != "BACKEND"}kernel_options.update(requested)+ if short_forced and requested.get("FORCE_USE_FLEX_ATTENTION") is False:+ # The caller turned the main kernel off: our main-kernel tiles do not cover flex_decoding.+ kernel_options = dict(requested)+ # torch 2.10+ rejects BACKEND with FORCE_USE_FLEX_ATTENTION: TRITON is the main kernel, so it+ # keeps our tiles; any other backend keeps the caller's options, which our tiles do not fit.+ elif ours_forced and "BACKEND" in kernel_options:+ if kernel_options["BACKEND"] == "TRITON":+ kernel_options.pop("FORCE_USE_FLEX_ATTENTION")+ else:+ kernel_options = dict(requested)kwargs["kernel_options"] = kernel_optionsreturn flex_attention_forward(module, query, key, value, *args, **kwargs)