Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Flex attention: keep float32 Gemma 3 off the sm86 / sm89 small tiles (#15401)

Daniel Han34969fde406e
diff --git a/tests/test_flex_preferred_shared_memory.py b/tests/test_flex_preferred_shared_memory.py
index 8f6b64d658b..c3789086e50 100644
--- a/tests/test_flex_preferred_shared_memory.py
+++ b/tests/test_flex_preferred_shared_memory.py
@@ -113,3 +113,21 @@ def test_rocm_cpu_and_a_failing_probe_keep_flex(cards, monkeypatch):
_utils.torch.cuda, "get_device_properties", side_effect = RuntimeError("probe")
):
assert _resolve(_gemma3()) == "flex_attention"
+
+
+def test_l4_float32_gemma3_does_not_get_the_small_tiles(cards, monkeypatch):
+ # fp32 small tiles overflow L4 smem ("No valid triton configs"), so fp32 keeps eager.
+ import torch
+
+ monkeypatch.setattr(_utils.torch, "__version__", "2.10.0+cu130")
+ monkeypatch.delenv("UNSLOTH_FLEX_SMALL_SMEM_TILES", raising = False)
+ cards(L4)
+ resolve = lambda dtype: _utils.resolve_attention_implementation(
+ SupportsFlex, _gemma3(), supports_sdpa = False, dtype = dtype
+ )
+ assert resolve(torch.float32) == "eager"
+ assert resolve(torch.bfloat16) == "flex_attention"
+ assert resolve(torch.float16) == "flex_attention"
+ assert resolve(None) == "flex_attention"
+ cards(A100)
+ assert resolve(torch.float32) == "flex_attention"
diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.py
index 59c2f3c22d2..6ae1a4d5a4d 100644
--- a/unsloth/models/_utils.py
+++ b/unsloth/models/_utils.py
@@ -1468,7 +1468,7 @@ def _enable_flex_attention_support(model_class, model_type = ""):
_FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY = 151552
-def _flex_preferred_fits(config):
+def _flex_preferred_fits(config, dtype = None):
"""False when a visible CUDA card lacks that shared memory; fails open on ROCm / no CUDA."""
head_dim = _text_attention_head_dim(config)
if head_dim is None or not (
@@ -1478,9 +1478,9 @@ def _flex_preferred_fits(config):
try:
if getattr(torch.version, "hip", None) or not torch.cuda.is_available():
return True
- # sm86 / sm89 with torch >= 2.10: the flex wrapper passes tiles that fit.
+ # sm86 / sm89, torch >= 2.10: small tiles fit 2-byte dtypes only; fp32 overflows smem.
if _flex_small_smem_device_ok():
- return True
+ return dtype is not torch.float32
return all(
torch.cuda.get_device_properties(index).shared_memory_per_block_optin
>= _FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY
@@ -1490,7 +1490,12 @@ def _flex_preferred_fits(config):
return True
-def _supports_flex_attention(model_class, config, model_type):
+def _supports_flex_attention(
+ model_class,
+ config,
+ model_type,
+ dtype = None,
+):
if os.environ.get("UNSLOTH_ENABLE_FLEX_ATTENTION", "1") == "0":
return False
if not getattr(model_class, "_supports_flex_attn", False):
@@ -1499,7 +1504,7 @@ def _supports_flex_attention(model_class, config, model_type):
return False
if not _flex_attention_gpu_is_supported():
return False
- if _config_prefers_flex_attention(config) and not _flex_preferred_fits(config):
+ if _config_prefers_flex_attention(config) and not _flex_preferred_fits(config, dtype):
return False
for attention_config in _iter_attention_configs(config):
attention_dropout = _config_get(attention_config, "attention_dropout", 0) or 0
@@ -2983,12 +2988,14 @@ def resolve_attention_implementation(
supports_flash_attention = _model_class_supports_flash_attention(
model_class
) and not _is_flash_excluded(model_type)
- supports_flex_attention = _supports_flex_attention(model_class, config, model_type)
+ supports_flex_attention = _supports_flex_attention(model_class, config, model_type, dtype)
# Before the ladder, since every branch below reads supports_flex_attention.
prefers_flex_for_head_dim = _prefers_flex_for_head_dim(config)
if prefers_flex_for_head_dim and not supports_flex_attention:
if _enable_flex_attention_support(model_class, model_type):
- supports_flex_attention = _supports_flex_attention(model_class, config, model_type)
+ supports_flex_attention = _supports_flex_attention(
+ model_class, config, model_type, dtype
+ )
# An explicit 1 forces flex over flash_attention_2 too, but only when flex can be scoped to the decoder
flex_forced_for_head_dim = (
_flex_large_head_dim_override() is True