Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Flex small-smem tiles: keep float32 Gemma 3 off flex on sm86 / sm89, tile forced no-grad calls (#15529)
Daniel Han67d3f8130937
diff --git a/tests/test_flex_preferred_shared_memory.py b/tests/test_flex_preferred_shared_memory.pyindex c3789086e50..6613b4b734c 100644--- a/tests/test_flex_preferred_shared_memory.py+++ b/tests/test_flex_preferred_shared_memory.py@@ -121,6 +121,7 @@ def test_l4_float32_gemma3_does_not_get_the_small_tiles(cards, monkeypatch):monkeypatch.setattr(_utils.torch, "__version__", "2.10.0+cu130")monkeypatch.delenv("UNSLOTH_FLEX_SMALL_SMEM_TILES", raising = False)+ monkeypatch.delenv("UNSLOTH_FORCE_FLOAT32", raising = False)cards(L4)resolve = lambda dtype: _utils.resolve_attention_implementation(SupportsFlex, _gemma3(), supports_sdpa = False, dtype = dtype@@ -129,5 +130,9 @@ def test_l4_float32_gemma3_does_not_get_the_small_tiles(cards, monkeypatch):assert resolve(torch.bfloat16) == "flex_attention"assert resolve(torch.float16) == "flex_attention"assert resolve(None) == "flex_attention"+ # Forced float32 loads bf16 but runs Gemma 3 attention in float32.+ monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")+ assert resolve(torch.bfloat16) == "eager"+ monkeypatch.delenv("UNSLOTH_FORCE_FLOAT32")cards(A100)assert resolve(torch.float32) == "flex_attention"diff --git a/tests/test_flex_small_smem_tiles.py b/tests/test_flex_small_smem_tiles.pyindex 0cd4cd1ee89..f911d3c9bb2 100644--- a/tests/test_flex_small_smem_tiles.py+++ b/tests/test_flex_small_smem_tiles.py@@ -224,3 +224,47 @@ def test_small_card_with_tuned_backward_forced_keeps_fitting_tiles(monkeypatch):wrapped(_Causal(), *_qkv(256), object())for k, v in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS.items():assert seen[k] == v++[email protected]("requested", [{"FORCE_USE_FLEX_ATTENTION": True}, {"BACKEND": "TRITON"}])+def test_wrapper_forced_main_kernel_no_grad_short_query_gets_tiles(monkeypatch, requested):+ # Gradient checkpointing runs each block's first forward under no_grad, and the block compile+ # hook forces the main kernel there, whose default head_dim 256 tiles do not fit (#15271).+ wrapped, seen = _wrapped(monkeypatch)+ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: False)+ wrapped(_Causal(), *_qkv(64, grad = False), object(), kernel_options = dict(requested))+ for k, v in u._FLEX_SMALL_SMEM_KERNEL_OPTIONS.items():+ assert seen[k] == v+ for k, v in requested.items():+ assert seen[k] == v+ wrapped(_Causal(), *_qkv(64, grad = False), object(), kernel_options = {"BACKEND": "AUTO"})+ assert "fwd_BLOCK_M" not in seen # flex_decoding may still run: its own tiles+++def test_wrapper_training_calls_add_no_sequence_guard(monkeypatch):+ # The grad check must short-circuit before query.shape[2] >= 128, or a dynamic-shape block+ # compiles once more when the sequence crosses 128.+ import torch++ monkeypatch.setattr(u, "_flex_call_needs_backward", lambda *a: True)++ def graphs_for(small_ok):+ wrapped, _ = _wrapped(monkeypatch, small_ok = small_ok)+ graphs = []++ def backend(gm, example_inputs):+ graphs.append(gm)+ return gm.forward++ torch._dynamo.reset()++ @torch.compile(backend = backend, dynamic = True)+ def block(q):+ return wrapped(_Causal(), q, q, q, None, is_causal = False) * 2++ for q_len in (256, 512, 64, 32):+ block(torch.randn(1, 2, q_len, 256))+ torch._dynamo.reset()+ return len(graphs)++ assert graphs_for(True) == graphs_for(False)diff --git a/unsloth/models/_utils.py b/unsloth/models/_utils.pyindex d99b7439f5d..67ae27ac0c8 100644--- a/unsloth/models/_utils.py+++ b/unsloth/models/_utils.py@@ -1033,6 +1033,14 @@ def _flex_call_needs_backward(query, key, value):return False+def _flex_main_kernel_forced(kwargs):+ """True when the caller's kernel_options force the main flex kernel over flex_decoding."""+ requested = kwargs.get("kernel_options") or {}+ if not isinstance(requested, dict):+ return False+ return requested.get("FORCE_USE_FLEX_ATTENTION") is True or requested.get("BACKEND") == "TRITON"++# unsloth_zoo drops the causal mask of unpadded, cache-free batches; route that None to SDPA is_causal.# Kill switch: UNSLOTH_FLEX_MASKLESS_SDPA=0._FLEX_MASKLESS_SDPA_ENABLED = os.environ.get("UNSLOTH_FLEX_MASKLESS_SDPA", "1") != "0"@@ -1185,8 +1193,14 @@ def _wrap_flex_attention_forward(flex_attention_forward):and small_smem_device_okand hasattr(query, "dim")and query.dim() == 4- # Short no-grad queries go to flex_decoding, whose own tiles are left alone.- and (query.shape[2] >= 128 or _flex_call_needs_backward(query, key, value))+ # Short no-grad queries go to flex_decoding, whose own tiles are left alone, unless the+ # caller (the block compile hook, also for no-grad checkpoint forwards) forces the main+ # kernel. Shape last, so training calls add no seq >= 128 guard.+ and (+ _flex_call_needs_backward(query, key, value)+ or _flex_main_kernel_forced(kwargs)+ or query.shape[2] >= 128+ )and _flex_tuned_bwd_fits(args, kwargs, _FLEX_SMALL_SMEM_KERNEL_OPTIONS, ("fwd_", "bwd_", "BLOCK_", "num_"))@@ -1617,6 +1631,11 @@ def _enable_flex_attention_support(model_class, model_type = ""):_FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY = 151552+def _flex_attention_runs_float32(dtype):+ """Attention runs in float32: a float32 load, or Gemma 3 / 4 under UNSLOTH_FORCE_FLOAT32."""+ return dtype is torch.float32 or os.environ.get("UNSLOTH_FORCE_FLOAT32", "0") == "1"++def _flex_preferred_fits(config, dtype = None):"""False when a visible CUDA card lacks that shared memory; fails open on ROCm / no CUDA."""head_dim = _text_attention_head_dim(config)@@ -1627,9 +1646,9 @@ def _flex_preferred_fits(config, dtype = None):try:if getattr(torch.version, "hip", None) or not torch.cuda.is_available():return True- # sm86 / sm89, torch >= 2.10: small tiles fit 2-byte dtypes only; fp32 overflows smem.+ # sm86 / sm89, torch >= 2.10: small tiles fit 2-byte dtypes only; fp32 (or forced fp32) overflows smem.if _flex_small_smem_device_ok():- return dtype is not torch.float32+ return not _flex_attention_runs_float32(dtype)return all(torch.cuda.get_device_properties(index).shared_memory_per_block_optin>= _FLEX_TEMPLATE_HEAD_DIM_256_SHARED_MEMORY