Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Large head attention follow-ups: compiled kill switch, 4-bit shard self check, forced float32 skip (#14188)

Daniel Han73d2598a2442
diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.py
index ed624ad6044..9c162e5f1d2 100644
--- a/tests/test_large_head_attention.py
+++ b/tests/test_large_head_attention.py
@@ -537,6 +537,103 @@ def test_install_skipped_when_every_self_check_fails(monkeypatch):
ALL_ATTENTION_FUNCTIONS["sdpa"] = saved
+def test_enabled_reads_the_same_value_compiled_and_eager(monkeypatch):
+ # An lru_cache wrapper is traced through by dynamo (with a warning), so a compiled caller
+ # re-read the environment while eager callers kept the cached value.
+ import sys
+
+ # dynamo resolves globals through sys.modules; this test loads the file standalone.
+ monkeypatch.setitem(sys.modules, LH.__name__, LH)
+ monkeypatch.setenv("UNSLOTH_HD512_ATTENTION", "1")
+ LH._enabled.cache_clear()
+ try:
+ assert LH._enabled() is True
+ monkeypatch.setenv("UNSLOTH_HD512_ATTENTION", "0")
+
+ def f(x):
+ return x + 1 if LH._enabled() else x - 1
+
+ x = torch.zeros(3)
+ torch._dynamo.reset()
+ compiled = torch.compile(f, backend = "eager", fullgraph = True)(x)
+ assert torch.equal(compiled, f(x))
+ assert torch.equal(compiled, x + 1)
+ finally:
+ monkeypatch.delenv("UNSLOTH_HD512_ATTENTION")
+ LH._enabled.cache_clear()
+ torch._dynamo.reset()
+
+
+def _install_with_params(monkeypatch, params, **attrs):
+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+
+ model = torch.nn.Linear(4, 4)
+ model.head_dim = 512
+ for name, value in attrs.items():
+ setattr(model, name, value)
+ monkeypatch.setattr(model, "parameters", lambda: iter(params))
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(torch.version, "hip", None)
+ monkeypatch.setattr(LH, "_PATCHED", [False])
+ checked = []
+ monkeypatch.setattr(
+ LH, "_self_check", lambda device, dtype: checked.append((device, dtype)) or True
+ )
+ saved = ALL_ATTENTION_FUNCTIONS["sdpa"]
+ try:
+ installed = LH.patch_large_head_attention(model)
+ finally:
+ ALL_ATTENTION_FUNCTIONS["sdpa"] = saved
+ return installed, checked
+
+
+class _FakeParam:
+ is_cuda = True
+ device = torch.device("cuda", 0)
+
+ def __init__(
+ self,
+ dtype,
+ quant_dtype = None,
+ ):
+ self.dtype = dtype
+ if quant_dtype is not None:
+ self.quant_state = type("QuantState", (), {"dtype": quant_dtype})()
+
+
+def test_patch_self_checks_the_compute_dtype_of_4bit_only_shards(monkeypatch):
+ # A GPU holding only bnb 4-bit weights (uint8 storage), e.g. embeddings offloaded to CPU, must
+ # still get its load-time check, else compiled blocks never use the kernel there.
+ cpu = type(
+ "CpuParam", (), {"is_cuda": False, "device": torch.device("cpu"), "dtype": torch.bfloat16}
+ )()
+ installed, checked = _install_with_params(
+ monkeypatch, [cpu, _FakeParam(torch.uint8, torch.bfloat16)]
+ )
+ assert installed is True
+ assert checked == [(torch.device("cuda", 0), torch.bfloat16)]
+
+
+def test_patch_self_checks_the_compute_dtype_of_float_packed_4bit(monkeypatch):
+ # bnb_4bit_quant_storage = float16 with bfloat16 compute: the storage dtype is a half type, but
+ # attention runs in bfloat16, so that is the verdict compiled blocks look up.
+ installed, checked = _install_with_params(
+ monkeypatch, [_FakeParam(torch.float16, torch.bfloat16)]
+ )
+ assert installed is True
+ assert checked == [(torch.device("cuda", 0), torch.bfloat16)]
+
+
+def test_patch_skips_the_fp16_self_check_under_forced_float32(monkeypatch):
+ # zoo's forced float32 Gemma-4 attention runs float32 SDPA directly, so the router never sees it.
+ installed, checked = _install_with_params(
+ monkeypatch, [_FakeParam(torch.float16)], _unsloth_forced_float32 = True
+ )
+ assert installed is False and checked == []
+ installed, checked = _install_with_params(monkeypatch, [_FakeParam(torch.float16)])
+ assert installed is True and checked == [(torch.device("cuda", 0), torch.float16)]
+
+
@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")
def test_router_uses_the_kernel_for_masks_made_under_inference_mode():
# generate() / eval build their masks under inference_mode. Sending those to SDPA repeated the
diff --git a/tests/test_norm_backward_grad_alias.py b/tests/test_norm_backward_grad_alias.py
index bb930800e0d..275dc1e495f 100644
--- a/tests/test_norm_backward_grad_alias.py
+++ b/tests/test_norm_backward_grad_alias.py
@@ -60,11 +60,20 @@ def _check(norm, reference, dtype):
@pytest.mark.parametrize("save_output", [False, True])
@pytest.mark.parametrize("train_weight", [False, True])
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
-def test_rms_layernorm_backward_leaves_shared_grad_alone(gemma, save_output, train_weight, dtype):
+def test_rms_layernorm_backward_leaves_shared_grad_alone(
+ gemma, save_output, train_weight, dtype, monkeypatch
+):
+ from unsloth.kernels import rms_layernorm
from unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm
+ # save_output only engages under UNSLOTH_RMSNORM_SAVE_OUTPUT, read once at import.
+ monkeypatch.setattr(rms_layernorm, "_SAVE_OUTPUT", save_output)
W = (1.0 + 0.1 * torch.randn(256, device = "cuda")).to(dtype).requires_grad_(train_weight)
norm = lambda x: Fast_RMS_Layernorm.apply(x, W, 1e-6, gemma, save_output)
+ probe = norm(torch.randn(3, 5, 256, device = "cuda", dtype = dtype).requires_grad_())
+ # The backward from Y only runs for a frozen W.
+ from_y = save_output and not train_weight
+ assert getattr(probe, "_unsloth_rmsnorm_saved", False) is from_y
_check(norm, lambda x: _rms_fp64(x, W, gemma), dtype)
diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.py
index e0e9673a0ea..c51814a552f 100644
--- a/unsloth/kernels/large_head_attention.py
+++ b/unsloth/kernels/large_head_attention.py
@@ -8,7 +8,6 @@
# non-decreasing down the rows (causal, packed, right padding, sliding window).
# UNSLOTH_HD512_ATTENTION=0 leaves SDPA untouched.
-import functools
import os
import sys
@@ -707,9 +706,18 @@ def large_head_attention(
return out.transpose(1, 2)
[email protected]_cache(maxsize = 1)
+# Read once, like a cache, but dynamo reads (and guards on) this list instead of tracing through an
+# lru_cache wrapper, which warns and re-reads the environment, so compiled code agrees with eager.
+_ENABLED = [None]
+
+
def _enabled():
- return os.environ.get("UNSLOTH_HD512_ATTENTION", "1") != "0"
+ if _ENABLED[0] is None:
+ _ENABLED[0] = os.environ.get("UNSLOTH_HD512_ATTENTION", "1") != "0"
+ return _ENABLED[0]
+
+
+_enabled.cache_clear = lambda: _ENABLED.__setitem__(0, None)
def _shapes_qualify(query, key, value):
@@ -933,13 +941,22 @@ def patch_large_head_attention(model = None):
head_dims = (getattr(m, "head_dim", 0) for m in model.modules())
if not any(isinstance(d, int) and d in _SUPPORTED_HEAD_DIMS for d in head_dims):
return False
+ # Forced float32 (zoo's Gemma-4 patch) runs attention in float32 on raw SDPA, never through
+ # this router: installing would only self check and autotune fp16 at load for nothing.
+ if getattr(model, "_unsloth_forced_float32", False) is True:
+ return False
# Compiled graphs only read the cached verdict, so check before any tracing.
try:
- targets = {
- (p.device, p.dtype)
- for p in model.parameters()
- if p.is_cuda and p.dtype in (torch.float16, torch.bfloat16)
- }
+ targets = set()
+ for p in model.parameters():
+ if not p.is_cuda:
+ continue
+ # bnb 4-bit weights are packed into uint8 or a float quant storage dtype that can
+ # differ from the compute dtype, which is on the quant state.
+ quant_state = getattr(p, "quant_state", None)
+ dtype = getattr(quant_state, "dtype", None) if quant_state is not None else p.dtype
+ if dtype in (torch.float16, torch.bfloat16):
+ targets.add((p.device, dtype))
except Exception:
targets = set()
passed = [_self_check(device, dtype) for device, dtype in targets]