Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Large head attention follow-ups: compiled kill switch, 4-bit shard self check, forced float32 skip (#14188)
Daniel Han73d2598a2442
diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.pyindex ed624ad6044..9c162e5f1d2 100644--- a/tests/test_large_head_attention.py+++ b/tests/test_large_head_attention.py@@ -537,6 +537,103 @@ def test_install_skipped_when_every_self_check_fails(monkeypatch):ALL_ATTENTION_FUNCTIONS["sdpa"] = saved+def test_enabled_reads_the_same_value_compiled_and_eager(monkeypatch):+ # An lru_cache wrapper is traced through by dynamo (with a warning), so a compiled caller+ # re-read the environment while eager callers kept the cached value.+ import sys++ # dynamo resolves globals through sys.modules; this test loads the file standalone.+ monkeypatch.setitem(sys.modules, LH.__name__, LH)+ monkeypatch.setenv("UNSLOTH_HD512_ATTENTION", "1")+ LH._enabled.cache_clear()+ try:+ assert LH._enabled() is True+ monkeypatch.setenv("UNSLOTH_HD512_ATTENTION", "0")++ def f(x):+ return x + 1 if LH._enabled() else x - 1++ x = torch.zeros(3)+ torch._dynamo.reset()+ compiled = torch.compile(f, backend = "eager", fullgraph = True)(x)+ assert torch.equal(compiled, f(x))+ assert torch.equal(compiled, x + 1)+ finally:+ monkeypatch.delenv("UNSLOTH_HD512_ATTENTION")+ LH._enabled.cache_clear()+ torch._dynamo.reset()+++def _install_with_params(monkeypatch, params, **attrs):+ from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS++ model = torch.nn.Linear(4, 4)+ model.head_dim = 512+ for name, value in attrs.items():+ setattr(model, name, value)+ monkeypatch.setattr(model, "parameters", lambda: iter(params))+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(torch.version, "hip", None)+ monkeypatch.setattr(LH, "_PATCHED", [False])+ checked = []+ monkeypatch.setattr(+ LH, "_self_check", lambda device, dtype: checked.append((device, dtype)) or True+ )+ saved = ALL_ATTENTION_FUNCTIONS["sdpa"]+ try:+ installed = LH.patch_large_head_attention(model)+ finally:+ ALL_ATTENTION_FUNCTIONS["sdpa"] = saved+ return installed, checked+++class _FakeParam:+ is_cuda = True+ device = torch.device("cuda", 0)++ def __init__(+ self,+ dtype,+ quant_dtype = None,+ ):+ self.dtype = dtype+ if quant_dtype is not None:+ self.quant_state = type("QuantState", (), {"dtype": quant_dtype})()+++def test_patch_self_checks_the_compute_dtype_of_4bit_only_shards(monkeypatch):+ # A GPU holding only bnb 4-bit weights (uint8 storage), e.g. embeddings offloaded to CPU, must+ # still get its load-time check, else compiled blocks never use the kernel there.+ cpu = type(+ "CpuParam", (), {"is_cuda": False, "device": torch.device("cpu"), "dtype": torch.bfloat16}+ )()+ installed, checked = _install_with_params(+ monkeypatch, [cpu, _FakeParam(torch.uint8, torch.bfloat16)]+ )+ assert installed is True+ assert checked == [(torch.device("cuda", 0), torch.bfloat16)]+++def test_patch_self_checks_the_compute_dtype_of_float_packed_4bit(monkeypatch):+ # bnb_4bit_quant_storage = float16 with bfloat16 compute: the storage dtype is a half type, but+ # attention runs in bfloat16, so that is the verdict compiled blocks look up.+ installed, checked = _install_with_params(+ monkeypatch, [_FakeParam(torch.float16, torch.bfloat16)]+ )+ assert installed is True+ assert checked == [(torch.device("cuda", 0), torch.bfloat16)]+++def test_patch_skips_the_fp16_self_check_under_forced_float32(monkeypatch):+ # zoo's forced float32 Gemma-4 attention runs float32 SDPA directly, so the router never sees it.+ installed, checked = _install_with_params(+ monkeypatch, [_FakeParam(torch.float16)], _unsloth_forced_float32 = True+ )+ assert installed is False and checked == []+ installed, checked = _install_with_params(monkeypatch, [_FakeParam(torch.float16)])+ assert installed is True and checked == [(torch.device("cuda", 0), torch.float16)]++@pytest.mark.skipif(not _SM80, reason = "the router engages on sm80+ only")def test_router_uses_the_kernel_for_masks_made_under_inference_mode():# generate() / eval build their masks under inference_mode. Sending those to SDPA repeated thediff --git a/tests/test_norm_backward_grad_alias.py b/tests/test_norm_backward_grad_alias.pyindex bb930800e0d..275dc1e495f 100644--- a/tests/test_norm_backward_grad_alias.py+++ b/tests/test_norm_backward_grad_alias.py@@ -60,11 +60,20 @@ def _check(norm, reference, dtype):@pytest.mark.parametrize("save_output", [False, True])@pytest.mark.parametrize("train_weight", [False, True])@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])-def test_rms_layernorm_backward_leaves_shared_grad_alone(gemma, save_output, train_weight, dtype):+def test_rms_layernorm_backward_leaves_shared_grad_alone(+ gemma, save_output, train_weight, dtype, monkeypatch+):+ from unsloth.kernels import rms_layernormfrom unsloth.kernels.rms_layernorm import Fast_RMS_Layernorm+ # save_output only engages under UNSLOTH_RMSNORM_SAVE_OUTPUT, read once at import.+ monkeypatch.setattr(rms_layernorm, "_SAVE_OUTPUT", save_output)W = (1.0 + 0.1 * torch.randn(256, device = "cuda")).to(dtype).requires_grad_(train_weight)norm = lambda x: Fast_RMS_Layernorm.apply(x, W, 1e-6, gemma, save_output)+ probe = norm(torch.randn(3, 5, 256, device = "cuda", dtype = dtype).requires_grad_())+ # The backward from Y only runs for a frozen W.+ from_y = save_output and not train_weight+ assert getattr(probe, "_unsloth_rmsnorm_saved", False) is from_y_check(norm, lambda x: _rms_fp64(x, W, gemma), dtype)diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.pyindex e0e9673a0ea..c51814a552f 100644--- a/unsloth/kernels/large_head_attention.py+++ b/unsloth/kernels/large_head_attention.py@@ -8,7 +8,6 @@# non-decreasing down the rows (causal, packed, right padding, sliding window).# UNSLOTH_HD512_ATTENTION=0 leaves SDPA untouched.-import functoolsimport osimport sys@@ -707,9 +706,18 @@ def large_head_attention(return out.transpose(1, 2)[email protected]_cache(maxsize = 1)+# Read once, like a cache, but dynamo reads (and guards on) this list instead of tracing through an+# lru_cache wrapper, which warns and re-reads the environment, so compiled code agrees with eager.+_ENABLED = [None]++def _enabled():- return os.environ.get("UNSLOTH_HD512_ATTENTION", "1") != "0"+ if _ENABLED[0] is None:+ _ENABLED[0] = os.environ.get("UNSLOTH_HD512_ATTENTION", "1") != "0"+ return _ENABLED[0]+++_enabled.cache_clear = lambda: _ENABLED.__setitem__(0, None)def _shapes_qualify(query, key, value):@@ -933,13 +941,22 @@ def patch_large_head_attention(model = None):head_dims = (getattr(m, "head_dim", 0) for m in model.modules())if not any(isinstance(d, int) and d in _SUPPORTED_HEAD_DIMS for d in head_dims):return False+ # Forced float32 (zoo's Gemma-4 patch) runs attention in float32 on raw SDPA, never through+ # this router: installing would only self check and autotune fp16 at load for nothing.+ if getattr(model, "_unsloth_forced_float32", False) is True:+ return False# Compiled graphs only read the cached verdict, so check before any tracing.try:- targets = {- (p.device, p.dtype)- for p in model.parameters()- if p.is_cuda and p.dtype in (torch.float16, torch.bfloat16)- }+ targets = set()+ for p in model.parameters():+ if not p.is_cuda:+ continue+ # bnb 4-bit weights are packed into uint8 or a float quant storage dtype that can+ # differ from the compute dtype, which is on the quant state.+ quant_state = getattr(p, "quant_state", None)+ dtype = getattr(quant_state, "dtype", None) if quant_state is not None else p.dtype+ if dtype in (torch.float16, torch.bfloat16):+ targets.add((p.device, dtype))except Exception:targets = set()passed = [_self_check(device, dtype) for device, dtype in targets]