Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex small-smem tiles: keep float32 Gemma 3 off flex on sm86 / sm89, tile forced no-grad calls (#15529)

Daniel Han67d3f8130937
diff --git a/tests/test_flex_preferred_shared_memory.py b/tests/test_flex_preferred_shared_memory.py
index c3789086e50..6613b4b734c 100644
--- a/tests/test_flex_preferred_shared_memory.py
+++ b/tests/test_flex_preferred_shared_memory.py
@@ -121,6 +121,7 @@ def test_l4_float32_gemma3_does_not_get_the_small_tiles(cards, monkeypatch):
monkeypatch.setattr(_utils.torch, "__version__", "2.10.0+cu130")
monkeypatch.delenv("UNSLOTH_FLEX_SMALL_SMEM_TILES", raising = False)
+ monkeypatch.delenv("UNSLOTH_FORCE_FLOAT32", raising = False)
cards(L4)
resolve = lambda dtype: _utils.resolve_attention_implementation(
SupportsFlex, _gemma3(), supports_sdpa = False, dtype = dtype
@@ -129,5 +130,9 @@ def test_l4_float32_gemma3_does_not_get_the_small_tiles(cards, monkeypatch):
assert resolve(torch.bfloat16) == "flex_attention"
assert resolve(torch.float16) == "flex_attention"
assert resolve(None) == "flex_attention"
+ # Forced float32 loads bf16 but runs Gemma 3 attention in float32.
+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")
+ assert resolve(torch.bfloat16) == "eager"
+ monkeypatch.delenv("UNSLOTH_FORCE_FLOAT32")
cards(A100)
assert resolve(torch.float32) == "flex_attention"
diff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.py
index 0cd4cd1ee89..f911d3c9bb2 100644
--- a/tests/test_flex_small_smem_tiles.py
+++ b/tests/test_flex_small_smem_tiles.py
@@ -224,3 +224,47 @@ def test_small_card_with_tuned_backward_forced_keeps_fitting_tiles(monkeypatch):
wrapped(_Causal(), *_qkv(256), object())
for k, v in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS.items():
assert seen[k] == v
+
+
[email protected]("requested", [{"FORCE_USE_FLEX_ATTENTION": True}, {"BACKEND": "TRITON"}])
+def test_wrapper_forced_main_kernel_no_grad_short_query_gets_tiles(monkeypatch, requested):
+ # Gradient checkpointing runs each block's first forward under no_grad, and the block compile
+ # hook forces the main kernel there, whose default head_dim 256 tiles do not fit (#15271).
+ wrapped, seen = _wrapped(monkeypatch)
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)
+ wrapped(_Causal(), *_qkv(64, grad = False), object(), kernel_options = dict(requested))
+ for k, v in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS.items():
+ assert seen[k] == v
+ for k, v in requested.items():
+ assert seen[k] == v
+ wrapped(_Causal(), *_qkv(64, grad = False), object(), kernel_options = {"BACKEND": "AUTO"})
+ assert "fwd_BLOCK_M" not in seen # flex_decoding may still run: its own tiles
+
+
+def test_wrapper_training_calls_add_no_sequence_guard(monkeypatch):
+ # The grad check must short-circuit before query.shape[2] >= 128, or a dynamic-shape block
+ # compiles once more when the sequence crosses 128.
+ import torch
+
+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)
+
+ def graphs_for(small_ok):
+ wrapped, _ = _wrapped(monkeypatch, small_ok = small_ok)
+ graphs = []
+
+ def backend(gm, example_inputs):
+ graphs.append(gm)
+ return gm.forward
+
+ torch._dynamo.reset()
+
+ @torch.compile(backend = backend, dynamic = True)
+ def block(q):
+ return wrapped(_Causal(), q, q, q, None, is_causal = False) * 2
+
+ for q_len in (256, 512, 64, 32):
+ block(torch.randn(1, 2, q_len, 256))
+ torch._dynamo.reset()
+ return len(graphs)
+
+ assert graphs_for(True) == graphs_for(False)
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index d99b7439f5d..67ae27ac0c8 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1033,6 +1033,14 @@ def _flex_call_needs_backward(query, key, value):
return False
+def _flex_main_kernel_forced(kwargs):
+ """True when the caller's kernel_options force the main flex kernel over flex_decoding."""
+ requested = kwargs.get("kernel_options") or {}
+ if not isinstance(requested, dict):
+ return False
+ return requested.get("FORCE_USE_FLEX_ATTENTION") is True or requested.get("BACKEND") == "TRITON"
+
+
# unsloth_zoo drops the causal mask of unpadded, cache-free batches; route that None to SDPA is_causal.
# Kill switch: UNSLOTH_FLEX_MASKLESS_SDPA=0.
_FLEX_MASKLESS_SDPA_ENABLED = os.environ.get("UNSLOTH_FLEX_MASKLESS_SDPA", "1") != "0"
@@ -1185,8 +1193,14 @@ def _wrap_flex_attention_forward(flex_attention_forward):
and small_smem_device_ok
and hasattr(query, "dim")
and query.dim() == 4
- # Short no-grad queries go to flex_decoding, whose own tiles are left alone.
- and (query.shape[2] >= 128 or _flex_call_needs_backward(query, key, value))
+ # Short no-grad queries go to flex_decoding, whose own tiles are left alone, unless the
+ # caller (the block compile hook, also for no-grad checkpoint forwards) forces the main
+ # kernel. Shape last, so training calls add no seq >= 128 guard.
+ and (
+ _flex_call_needs_backward(query, key, value)
+ or _flex_main_kernel_forced(kwargs)
+ or query.shape[2] >= 128
+ )
and _flex_tuned_bwd_fits(
args, kwargs, _FLEX_SMALL_SMEM_KERNEL_OPTIONS, ("fwd_", "bwd_", "BLOCK_", "num_")
)
@@ -1617,6 +1631,11 @@ def _enable_flex_attention_support(model_class, model_type = ""):
_FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY = 151552
+def _flex_attention_runs_float32(dtype):
+ """Attention runs in float32: a float32 load, or Gemma 3 / 4 under UNSLOTH_FORCE_FLOAT32."""
+ return dtype is torch.float32 or os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"
+
+
def _flex_preferred_fits(config, dtype = None):
"""False when a visible CUDA card lacks that shared memory; fails open on ROCm / no CUDA."""
head_dim = _text_attention_head_dim(config)
@@ -1627,9 +1646,9 @@ def _flex_preferred_fits(config, dtype = None):
try:
if getattr(torch.version, "hip", None) or not torch.cuda.is_available():
return True
- # sm86 / sm89, torch >= 2.10: small tiles fit 2-byte dtypes only; fp32 overflows smem.
+ # sm86 / sm89, torch >= 2.10: small tiles fit 2-byte dtypes only; fp32 (or forced fp32) overflows smem.
if _flex_small_smem_device_ok():
- return dtype is not torch.float32
+ return not _flex_attention_runs_float32(dtype)
return all(
torch.cuda.get_device_properties(index).shared_memory_per_block_optin
>= _FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY