Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flex attention on torch < 2.14: fitting tiles for short decode calls so a batch-size change does not leave the process on eager attention (#16073)
Daniel Hand5d0200ec50e
diff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.pyindex abd88d19a74..7e2f7c07484 100644--- a/tests/test_flex_small_smem_tiles.py+++ b/tests/test_flex_small_smem_tiles.py@@ -139,6 +139,7 @@ def _wrapped(monkeypatch,small_ok = True,tuned_ok = False,+ short_tiles = False,):seen = {}@@ -149,6 +150,7 @@ def _wrapped(monkeypatch.setattr(u, "_flex_small_smem_device_ok", lambda: small_ok)monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: tuned_ok)+ monkeypatch.setattr(u, "_flex_small_smem_short_query_tiles", lambda: short_tiles)return u._wrap_flex_attention_forward(fake_flex), seen@@ -249,6 +251,87 @@ def test_wrapper_short_prompt_whose_decode_tile_does_not_fit_gets_tiles(assert seen == {} # other cards keep the defaults+ "version,expected",+ [+ ("2.10.0+cu128", True),+ ("2.11.0", True),+ ("2.13.1", True),+ ("2.14.0", False),+ ("2.14.1+cu130", False),+ ],+)+def test_short_query_tiles_torch_gate(monkeypatch, version, expected):+ monkeypatch.setattr(u.torch, "__version__", version)+ assert u._flex_small_smem_short_query_tiles() is expected+++ "q_len,q_heads,kv_heads,forced",+ [+ (1, 4, 1, False), # Gemma 3 1B decode step: tiles, flex_decoding stays eligible+ (4, 4, 1, False), # 16 rows: flex_decoding's own BLOCK_M would be 16+ (8, 4, 1, False), # 32 rows+ (16, 4, 1, False), # 64 rows+ (16, 8, 4, False), # Gemma 3 4B 16-token prompt, 32 rows+ (28, 8, 4, False), # Gemma 3 4B prompt, 56 rows+ (28, 6, 2, False), # repeated K / V, 28 rows+ (17, 4, 1, True), # 68 rows: main kernel, as in the default gate+ (127, 2, 2, True),+ ],+)+def test_wrapper_short_no_grad_queries_get_tiles_before_torch_2_14(+ monkeypatch, q_len, q_heads, kv_heads, forced+):+ # A batch size change makes the batch dim dynamic, which moves even one-token decode from+ # flex_decoding to the main kernel, whose default tile does not fit (torch 2.10 / 2.11).+ import torch++ qkv = [t.to(torch.bfloat16) for t in _gqa_qkv(q_len, q_heads, kv_heads)]+ wrapped, seen = _wrapped(monkeypatch, short_tiles = True)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ wrapped(_Causal(), *qkv, object())+ assert {+ k: seen.get(k) for k in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS+ } == u._FLEX_SMALL_SMEM_KERNEL_OPTIONS+ assert seen.get("FORCE_USE_FLEX_ATTENTION") is (True if forced else None)+ # flex_decoding derives SAFE_M_BOUNDARY from plain BLOCK_M, not fwd_BLOCK_M (torch < 2.14).+ assert seen.get("BLOCK_M") == (None if forced else 64)+ wrapped, seen = _wrapped(monkeypatch, small_ok = False, short_tiles = True)+ wrapped(_Causal(), *qkv, object())+ assert seen == {} # other cards keep the defaults+++ "head_dim,dtype,tiled",+ [+ (256, "bfloat16", True),+ (256, "float16", True),+ (256, "float32", False),+ (192, "bfloat16", False),+ ],+)+def test_short_query_tiles_only_where_measured(monkeypatch, head_dim, dtype, tiled):+ # fp32 needs 147456 B with these tiles on flex_decoding; head_dim 192 fails to build on 2.10.+ import torch++ wrapped, seen = _wrapped(monkeypatch, short_tiles = True)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ qkv = [t.to(getattr(torch, dtype)) for t in _gqa_qkv(1, 4, 1, head_dim = head_dim)]+ wrapped(_Causal(), *qkv, object())+ assert (seen.get("fwd_BLOCK_M") == 64) is tiled+ assert "FORCE_USE_FLEX_ATTENTION" not in seen+++def test_short_query_tiles_need_the_small_card(monkeypatch):+ calls = []+ monkeypatch.setattr(u, "_flex_small_smem_device_ok", lambda: False)+ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: False)+ monkeypatch.setattr(u, "_flex_small_smem_short_query_tiles", lambda: calls.append(1) or True)+ u._wrap_flex_attention_forward(lambda *a, **k: None)+ assert calls == []++def test_wrapper_untouched_off_class_and_outside_range(monkeypatch):wrapped, seen = _wrapped(monkeypatch, small_ok = False)monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex bfa1e58840a..0698721a41c 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -991,6 +991,14 @@ _FLEX_SMALL_SMEM_ENV_VAR = "UNSLOTH_FLEX_SMALL_SMEM_TILES"# unsloth_zoo's suppress_errors hides it, and Dynamo marks flex_attention as skipped for the rest# of the process, so every later call, long prompts and decode included, runs the eager fallback._FLEX_SMALL_SMEM_DECODE_MAX_ROWS = 64+# Before torch 2.14 a short no-grad query whose batch size changed since the first compile (batch 1,+# then 2) has a dynamic batch dim, which rules flex_decoding out even for one-token decode. The main+# kernel then takes its default head_dim 256 tile (151552 B), the compile fails, and the process+# goes eager as above. Short no-grad calls get the tiles there (flex_decoding takes them too while+# the batch is static); 2.14 compiles it unchanged. Measured at head_dim 256 in bf16 / fp16 only: in+# fp32 the tiles need 147456 B on flex_decoding, and at head_dim 192 they fail to build on 2.10.+_FLEX_SMALL_SMEM_DYNAMIC_DECODE_FIXED_TORCH = "2.14"+_FLEX_SMALL_SMEM_SHORT_QUERY_HEAD_DIM = 256def _flex_decode_rows(query, key):@@ -1031,6 +1039,25 @@ def _flex_small_smem_device_ok():return False+def _flex_small_smem_short_query_tiles():+ """Torch before 2.14: short no-grad queries on a small shared memory card always get the tiles,+ since a batch size change can move them from flex_decoding to the main kernel. Read at wrap time."""+ try:+ return Version(torch.__version__.split("+")[0]) < Version(+ _FLEX_SMALL_SMEM_DYNAMIC_DECODE_FIXED_TORCH+ )+ except Exception:+ return True+++def _flex_small_smem_short_query_fits(query):+ """A 16-bit head_dim 256 query, the case the short-query tiles were measured on."""+ return query.shape[-1] == _FLEX_SMALL_SMEM_SHORT_QUERY_HEAD_DIM and query.dtype in (+ torch.float16,+ torch.bfloat16,+ )++def _flex_small_smem_kernel_options(head_dim, device_ok):"""fwd_* / bwd_* kernel_options that fit an sm86 / sm89 card at this head dim, else None."""low, high = _FLEX_TUNED_BWD_HEAD_DIM_RANGE@@ -1186,6 +1213,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):tuned_bwd_device_ok = _flex_tuned_bwd_device_ok()small_smem_device_ok = _flex_small_smem_device_ok()+ small_smem_short_query_tiles = small_smem_device_ok and _flex_small_smem_short_query_tiles()# A failed flex_decoding compile makes Dynamo run flex eagerly for the rest of the process (full score matrix).decoding_unavailable = _flex_decoding_unavailable()@@ -1239,6 +1267,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):and (_flex_call_needs_backward(query, key, value)or _flex_main_kernel_forced(kwargs)+ or (small_smem_short_query_tiles and _flex_small_smem_short_query_fits(query))or decode_replacedor query.shape[2] >= 128or _flex_decode_rows(query, key) > _FLEX_SMALL_SMEM_DECODE_MAX_ROWS@@ -1248,16 +1277,21 @@ def _wrap_flex_attention_forward(flex_attention_forward):)):kernel_options = _flex_small_smem_kernel_options(query.shape[-1], small_smem_device_ok)- # Main kernel: flex_decoding with fwd_BLOCK_M 64 below its row count is wrong on 2.10 and- # fails to compile on 2.11. Shape last, as above (no training sequence guard).+ # Over 64 rows the main kernel: flex_decoding with fwd_BLOCK_M 64 below its row count is wrong+ # on 2.10 and fails to compile on 2.11. Shape last, as above (no training sequence guard).if (kernel_options is not Noneand not _flex_call_needs_backward(query, key, value)and not _flex_main_kernel_forced(kwargs)and query.shape[2] < 128):- kernel_options["FORCE_USE_FLEX_ATTENTION"] = True- short_forced = True+ if _flex_decode_rows(query, key) > _FLEX_SMALL_SMEM_DECODE_MAX_ROWS:+ kernel_options["FORCE_USE_FLEX_ATTENTION"] = True+ short_forced = True+ else:+ # Before 2.14 flex_decoding takes SAFE_M_BOUNDARY from plain BLOCK_M, so without it+ # the 64-row tile skips its row masks and overwrites other heads' rowmax / sumexp.+ kernel_options["BLOCK_M"] = kernel_options["fwd_BLOCK_M"]if (kernel_options is Noneand tuned_bwd_device_ok