Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex attention on torch < 2.14: fitting tiles for short decode calls so a batch-size change does not leave the process on eager attention (#16073)

Daniel Hand5d0200ec50e
diff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.py
index abd88d19a74..7e2f7c07484 100644
--- a/tests/test_flex_small_smem_tiles.py
+++ b/tests/test_flex_small_smem_tiles.py
@@ -139,6 +139,7 @@ def _wrapped(
monkeypatch,
small_ok = True,
tuned_ok = False,
+ short_tiles = False,
):
seen = {}
@@ -149,6 +150,7 @@ def _wrapped(
monkeypatch.setattr(u, "_flex_small_smem_device_ok", lambda: small_ok)
monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: tuned_ok)
+ monkeypatch.setattr(u, "_flex_small_smem_short_query_tiles", lambda: short_tiles)
return u._wrap_flex_attention_forward(fake_flex), seen
@@ -249,6 +251,87 @@ def test_wrapper_short_prompt_whose_decode_tile_does_not_fit_gets_tiles(
assert seen == {} # other cards keep the defaults
+ "version,expected",
+ [
+ ("2.10.0+cu128", True),
+ ("2.11.0", True),
+ ("2.13.1", True),
+ ("2.14.0", False),
+ ("2.14.1+cu130", False),
+ ],
+)
+def test_short_query_tiles_torch_gate(monkeypatch, version, expected):
+ monkeypatch.setattr(u.torch, "__version__", version)
+ assert u._flex_small_smem_short_query_tiles() is expected
+
+
+ "q_len,q_heads,kv_heads,forced",
+ [
+ (1, 4, 1, False), # Gemma 3 1B decode step: tiles, flex_decoding stays eligible
+ (4, 4, 1, False), # 16 rows: flex_decoding's own BLOCK_M would be 16
+ (8, 4, 1, False), # 32 rows
+ (16, 4, 1, False), # 64 rows
+ (16, 8, 4, False), # Gemma 3 4B 16-token prompt, 32 rows
+ (28, 8, 4, False), # Gemma 3 4B prompt, 56 rows
+ (28, 6, 2, False), # repeated K / V, 28 rows
+ (17, 4, 1, True), # 68 rows: main kernel, as in the default gate
+ (127, 2, 2, True),
+ ],
+)
+def test_wrapper_short_no_grad_queries_get_tiles_before_torch_2_14(
+ monkeypatch, q_len, q_heads, kv_heads, forced
+):
+ # A batch size change makes the batch dim dynamic, which moves even one-token decode from
+ # flex_decoding to the main kernel, whose default tile does not fit (torch 2.10 / 2.11).
+ import torch
+
+ qkv = [t.to(torch.bfloat16) for t in _gqa_qkv(q_len, q_heads, kv_heads)]
+ wrapped, seen = _wrapped(monkeypatch, short_tiles = True)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ wrapped(_Causal(), *qkv, object())
+ assert {
+ k: seen.get(k) for k in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS
+ } == u._FLEX_SMALL_SMEM_KERNEL_OPTIONS
+ assert seen.get("FORCE_USE_FLEX_ATTENTION") is (True if forced else None)
+ # flex_decoding derives SAFE_M_BOUNDARY from plain BLOCK_M, not fwd_BLOCK_M (torch < 2.14).
+ assert seen.get("BLOCK_M") == (None if forced else 64)
+ wrapped, seen = _wrapped(monkeypatch, small_ok = False, short_tiles = True)
+ wrapped(_Causal(), *qkv, object())
+ assert seen == {} # other cards keep the defaults
+
+
+ "head_dim,dtype,tiled",
+ [
+ (256, "bfloat16", True),
+ (256, "float16", True),
+ (256, "float32", False),
+ (192, "bfloat16", False),
+ ],
+)
+def test_short_query_tiles_only_where_measured(monkeypatch, head_dim, dtype, tiled):
+ # fp32 needs 147456 B with these tiles on flex_decoding; head_dim 192 fails to build on 2.10.
+ import torch
+
+ wrapped, seen = _wrapped(monkeypatch, short_tiles = True)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ qkv = [t.to(getattr(torch, dtype)) for t in _gqa_qkv(1, 4, 1, head_dim = head_dim)]
+ wrapped(_Causal(), *qkv, object())
+ assert (seen.get("fwd_BLOCK_M") == 64) is tiled
+ assert "FORCE_USE_FLEX_ATTENTION" not in seen
+
+
+def test_short_query_tiles_need_the_small_card(monkeypatch):
+ calls = []
+ monkeypatch.setattr(u, "_flex_small_smem_device_ok", lambda: False)
+ monkeypatch.setattr(u, "_flex_tuned_bwd_device_ok", lambda: False)
+ monkeypatch.setattr(u, "_flex_small_smem_short_query_tiles", lambda: calls.append(1) or True)
+ u._wrap_flex_attention_forward(lambda *a, **k: None)
+ assert calls == []
+
+
def test_wrapper_untouched_off_class_and_outside_range(monkeypatch):
wrapped, seen = _wrapped(monkeypatch, small_ok = False)
monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index bfa1e58840a..0698721a41c 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -991,6 +991,14 @@ _FLEX_SMALL_SMEM_ENV_VAR = "UNSLOTH_FLEX_SMALL_SMEM_TILES"
# unsloth_zoo's suppress_errors hides it, and Dynamo marks flex_attention as skipped for the rest
# of the process, so every later call, long prompts and decode included, runs the eager fallback.
_FLEX_SMALL_SMEM_DECODE_MAX_ROWS = 64
+# Before torch 2.14 a short no-grad query whose batch size changed since the first compile (batch 1,
+# then 2) has a dynamic batch dim, which rules flex_decoding out even for one-token decode. The main
+# kernel then takes its default head_dim 256 tile (151552 B), the compile fails, and the process
+# goes eager as above. Short no-grad calls get the tiles there (flex_decoding takes them too while
+# the batch is static); 2.14 compiles it unchanged. Measured at head_dim 256 in bf16 / fp16 only: in
+# fp32 the tiles need 147456 B on flex_decoding, and at head_dim 192 they fail to build on 2.10.
+_FLEX_SMALL_SMEM_DYNAMIC_DECODE_FIXED_TORCH = "2.14"
+_FLEX_SMALL_SMEM_SHORT_QUERY_HEAD_DIM = 256
def _flex_decode_rows(query, key):
@@ -1031,6 +1039,25 @@ def _flex_small_smem_device_ok():
return False
+def _flex_small_smem_short_query_tiles():
+ """Torch before 2.14: short no-grad queries on a small shared memory card always get the tiles,
+ since a batch size change can move them from flex_decoding to the main kernel. Read at wrap time."""
+ try:
+ return Version(torch.__version__.split("+")[0]) < Version(
+ _FLEX_SMALL_SMEM_DYNAMIC_DECODE_FIXED_TORCH
+ )
+ except Exception:
+ return True
+
+
+def _flex_small_smem_short_query_fits(query):
+ """A 16-bit head_dim 256 query, the case the short-query tiles were measured on."""
+ return query.shape[-1] == _FLEX_SMALL_SMEM_SHORT_QUERY_HEAD_DIM and query.dtype in (
+ torch.float16,
+ torch.bfloat16,
+ )
+
+
def _flex_small_smem_kernel_options(head_dim, device_ok):
"""fwd_* / bwd_* kernel_options that fit an sm86 / sm89 card at this head dim, else None."""
low, high = _FLEX_TUNED_BWD_HEAD_DIM_RANGE
@@ -1186,6 +1213,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
tuned_bwd_device_ok = _flex_tuned_bwd_device_ok()
small_smem_device_ok = _flex_small_smem_device_ok()
+ small_smem_short_query_tiles = small_smem_device_ok and _flex_small_smem_short_query_tiles()
# A failed flex_decoding compile makes Dynamo run flex eagerly for the rest of the process (full score matrix).
decoding_unavailable = _flex_decoding_unavailable()
@@ -1239,6 +1267,7 @@ def _wrap_flex_attention_forward(flex_attention_forward):
and (
_flex_call_needs_backward(query, key, value)
or _flex_main_kernel_forced(kwargs)
+ or (small_smem_short_query_tiles and _flex_small_smem_short_query_fits(query))
or decode_replaced
or query.shape[2] >= 128
or _flex_decode_rows(query, key) > _FLEX_SMALL_SMEM_DECODE_MAX_ROWS
@@ -1248,16 +1277,21 @@ def _wrap_flex_attention_forward(flex_attention_forward):
)
):
kernel_options = _flex_small_smem_kernel_options(query.shape[-1], small_smem_device_ok)
- # Main kernel: flex_decoding with fwd_BLOCK_M 64 below its row count is wrong on 2.10 and
- # fails to compile on 2.11. Shape last, as above (no training sequence guard).
+ # Over 64 rows the main kernel: flex_decoding with fwd_BLOCK_M 64 below its row count is wrong
+ # on 2.10 and fails to compile on 2.11. Shape last, as above (no training sequence guard).
if (
kernel_options is not None
and not _flex_call_needs_backward(query, key, value)
and not _flex_main_kernel_forced(kwargs)
and query.shape[2] < 128
):
- kernel_options["FORCE_USE_FLEX_ATTENTION"] = True
- short_forced = True
+ if _flex_decode_rows(query, key) > _FLEX_SMALL_SMEM_DECODE_MAX_ROWS:
+ kernel_options["FORCE_USE_FLEX_ATTENTION"] = True
+ short_forced = True
+ else:
+ # Before 2.14 flex_decoding takes SAFE_M_BOUNDARY from plain BLOCK_M, so without it
+ # the 64-row tile skips its row masks and overwrites other heads' rowmax / sumexp.
+ kernel_options["BLOCK_M"] = kernel_options["fwd_BLOCK_M"]
if (
kernel_options is None
and tuned_bwd_device_ok