Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flex attention: keep float32 Gemma 3 off the sm86 / sm89 small tiles (#15401)
Daniel Han34969fde406e
diff --git a/tests/test_flex_preferred_shared_memory.py b/tests/test_flex_preferred_shared_memory.pyindex 8f6b64d658b..c3789086e50 100644--- a/tests/test_flex_preferred_shared_memory.py+++ b/tests/test_flex_preferred_shared_memory.py@@ -113,3 +113,21 @@ def test_rocm_cpu_and_a_failing_probe_keep_flex(cards, monkeypatch):_utils.torch.cuda, "get_device_properties", side_effect = RuntimeError("probe")):assert _resolve(_gemma3()) == "flex_attention"+++def test_l4_float32_gemma3_does_not_get_the_small_tiles(cards, monkeypatch):+ # fp32 small tiles overflow L4 smem ("No valid triton configs"), so fp32 keeps eager.+ import torch++ monkeypatch.setattr(_utils.torch, "__version__", "2.10.0+cu130")+ monkeypatch.delenv("UNSLOTH_FLEX_SMALL_SMEM_TILES", raising = False)+ cards(L4)+ resolve = lambda dtype: _utils.resolve_attention_implementation(+ SupportsFlex, _gemma3(), supports_sdpa = False, dtype = dtype+ )+ assert resolve(torch.float32) == "eager"+ assert resolve(torch.bfloat16) == "flex_attention"+ assert resolve(torch.float16) == "flex_attention"+ assert resolve(None) == "flex_attention"+ cards(A100)+ assert resolve(torch.float32) == "flex_attention"diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex 59c2f3c22d2..6ae1a4d5a4d 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1468,7 +1468,7 @@ def _enable_flex_attention_support(model_class, model_type = ""):_FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY = 151552-def _flex_preferred_fits(config):+def _flex_preferred_fits(config, dtype = None):"""False when a visible CUDA card lacks that shared memory; fails open on ROCm / no CUDA."""head_dim = _text_attention_head_dim(config)if head_dim is None or not (@@ -1478,9 +1478,9 @@ def _flex_preferred_fits(config):try:if getattr(torch.version, "hip", None) or not torch.cuda.is_available():return True- # sm86 / sm89 with torch >= 2.10: the flex wrapper passes tiles that fit.+ # sm86 / sm89, torch >= 2.10: small tiles fit 2-byte dtypes only; fp32 overflows smem.if _flex_small_smem_device_ok():- return True+ return dtype is not torch.float32return all(torch.cuda.get_device_properties(index).shared_memory_per_block_optin>= _FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY@@ -1490,7 +1490,12 @@ def _flex_preferred_fits(config):return True-def _supports_flex_attention(model_class, config, model_type):+def _supports_flex_attention(+ model_class,+ config,+ model_type,+ dtype = None,+):if os.environ.get("UNSLOTH_ENABLE_FLEX_ATTENTION", "1") == "0":return Falseif not getattr(model_class, "_supports_flex_attn", False):@@ -1499,7 +1504,7 @@ def _supports_flex_attention(model_class, config, model_type):return Falseif not _flex_attention_gpu_is_supported():return False- if _config_prefers_flex_attention(config) and not _flex_preferred_fits(config):+ if _config_prefers_flex_attention(config) and not _flex_preferred_fits(config, dtype):return Falsefor attention_config in _iter_attention_configs(config):attention_dropout = _config_get(attention_config, "attention_dropout", 0) or 0@@ -2983,12 +2988,14 @@ def resolve_attention_implementation(supports_flash_attention = _model_class_supports_flash_attention(model_class) and not _is_flash_excluded(model_type)- supports_flex_attention = _supports_flex_attention(model_class, config, model_type)+ supports_flex_attention = _supports_flex_attention(model_class, config, model_type, dtype)# Before the ladder, since every branch below reads supports_flex_attention.prefers_flex_for_head_dim = _prefers_flex_for_head_dim(config)if prefers_flex_for_head_dim and not supports_flex_attention:if _enable_flex_attention_support(model_class, model_type):- supports_flex_attention = _supports_flex_attention(model_class, config, model_type)+ supports_flex_attention = _supports_flex_attention(+ model_class, config, model_type, dtype+ )# An explicit 1 forces flex over flash_attention_2 too, but only when flex can be scoped to the decoderflex_forced_for_head_dim = (_flex_large_head_dim_override() is True