Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Gemma2: keep softcapping attention under int32 indexing and fall back to eager if compile fails (#12154)
Daniel Han9c74453fdc3f
diff --git a/tests/test_gemma2_softcap_int32_split.py b/tests/test_gemma2_softcap_int32_split.pynew file mode 100644index 00000000000..ce4e4cf05fe--- /dev/null+++ b/tests/test_gemma2_softcap_int32_split.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Gemma2 softcapping attention stays under 2**31 score elements per compiled call.++Inductor indexes the [bsz, heads, q, q] score tensor in int32. Past 2**31 elements the last+batch rows wrap and come back NaN (torch 2.9: gemma-2-9b, a left padded batch of 8 at 4305 tokens).+"""++import types++import pytest++import unsloth # noqa: F401 (must precede transformers)+import unsloth.kernels.flex_attention as fk+from real_accelerator import has_real_cuda++torch = pytest.importorskip("torch")+++def _spy(monkeypatch):+ calls = []++ def fake(Q, K, V, causal_mask, self, bsz, q_len):+ calls.append((bsz, Q.shape[0], tuple(causal_mask.shape)))+ return (+ Q[:, 0]+ + causal_mask[..., :q_len, :q_len]+ .sum(-1)+ .reshape(bsz if causal_mask.dim() == 4 else 1, -1)[..., None]+ )++ monkeypatch.setattr(fk, "_compiled_slow_attention_softcapping", fake)+ return calls+++def _inputs(bsz, q_len, mask_dims):+ Q = torch.arange(bsz, dtype = torch.float32)[:, None, None, None].expand(bsz, 1, q_len, 2).clone()+ mask = torch.arange(bsz * q_len * q_len, dtype = torch.float32).reshape(bsz, 1, q_len, q_len)+ return Q, (mask if mask_dims == 4 else mask[0, 0])++[email protected]("mask_dims", [4, 2])+def test_batch_split_when_scores_pass_int32(monkeypatch, mask_dims):+ calls = _spy(monkeypatch)+ q_len = 8+ # 2**25 heads x 64 = 2**31 score elements per row, so every row is its own call.+ config = types.SimpleNamespace(num_attention_heads = 2**25)+ Q, mask = _inputs(3, q_len, mask_dims)+ out = fk.slow_attention_softcapping(+ Q, Q, Q, mask, types.SimpleNamespace(config = config), 3, q_len+ )+ assert [c[:2] for c in calls] == [(1, 1)] * 3+ assert all(c[2] == ((1, 1, q_len, q_len) if mask_dims == 4 else (q_len, q_len)) for c in calls)+ reference = fk._compiled_slow_attention_softcapping(+ Q, Q, Q, mask, types.SimpleNamespace(config = config), 3, q_len+ )+ torch.testing.assert_close(out, reference)+++def test_small_batches_take_one_call(monkeypatch):+ calls = _spy(monkeypatch)+ config = types.SimpleNamespace(num_attention_heads = 16)+ Q, mask = _inputs(4, 8, 4)+ fk.slow_attention_softcapping(Q, Q, Q, mask, types.SimpleNamespace(config = config), 4, 8)+ assert [c[:2] for c in calls] == [(4, 4)]+++ not has_real_cuda(), reason = "runs the compiled kernel past 2**31 score elements on CUDA"+)+def test_compiled_kernel_past_int32_matches_eager():+ if torch.cuda.mem_get_info()[0] < 40 * 2**30:+ pytest.skip("needs about 40 GB free for the 2.4e9 element score tensor")+ bsz, heads, kv_heads, q_len, head_dim = 8, 16, 8, 4305, 16+ config = types.SimpleNamespace(+ num_attention_heads = heads,+ num_key_value_heads = kv_heads,+ query_pre_attn_scalar = 256,+ attn_logit_softcapping = 50.0,+ )+ layer = types.SimpleNamespace(+ config = config, head_dim = head_dim, num_key_value_groups = heads // kv_heads+ )+ g = torch.Generator(device = "cuda").manual_seed(0)+ Q = torch.randn(bsz, heads, q_len, head_dim, device = "cuda", dtype = torch.bfloat16, generator = g)+ K = torch.randn(+ bsz, kv_heads, q_len, head_dim, device = "cuda", dtype = torch.bfloat16, generator = g+ )+ V = torch.randn(+ bsz, kv_heads, q_len, head_dim, device = "cuda", dtype = torch.bfloat16, generator = g+ )+ keep = torch.ones(bsz, q_len, dtype = torch.bool, device = "cuda")+ keep[-1, :3950] = False # the last row is mostly left padding, as in the failing batch+ i = torch.arange(q_len, device = "cuda")+ allowed = (i[None, :] <= i[:, None])[None, None] & keep[:, None, None, :]+ mask = torch.zeros(bsz, 1, q_len, q_len, device = "cuda", dtype = torch.bfloat16)+ mask.masked_fill_(~allowed, torch.finfo(torch.bfloat16).min)+ out = fk.slow_attention_softcapping(Q, K, V, mask, layer, bsz, q_len)+ assert torch.isfinite(out).all()+ eager = fk._compiled_slow_attention_softcapping._torchdynamo_orig_callable+ ref = eager(Q[-1:], K[-1:], V[-1:], mask[-1:], layer, 1, q_len)+ torch.testing.assert_close(out[-1:], ref, atol = 2e-2, rtol = 2e-2)+++def _raising_compiled(monkeypatch, error):+ calls = {"compiled": 0, "eager": 0}++ def compiled(*a):+ calls["compiled"] += 1+ raise error++ def eager(Q, K, V, causal_mask, self, bsz, q_len):+ calls["eager"] += 1+ return Q[:, 0]++ compiled._torchdynamo_orig_callable = eager+ monkeypatch.setattr(fk, "_compiled_slow_attention_softcapping", compiled)+ monkeypatch.setattr(fk, "_SOFTCAP_EAGER", {})+ return calls+++def test_compile_failure_falls_back_to_eager_once(monkeypatch):+ calls = _raising_compiled(monkeypatch, RuntimeError("inductor::_alloc_from_pool ..."))+ config = types.SimpleNamespace(num_attention_heads = 16)+ Q, mask = _inputs(2, 8, 4)+ layer = types.SimpleNamespace(config = config)+ for _ in range(3):+ out = fk.slow_attention_softcapping(Q, Q, Q, mask, layer, 2, 8)+ assert out.shape == (2, 8, 2)+ assert calls == {"compiled": 1, "eager": 3}+++def test_out_of_memory_is_not_swallowed(monkeypatch):+ calls = _raising_compiled(monkeypatch, torch.OutOfMemoryError("out of memory"))+ Q, mask = _inputs(2, 8, 4)+ with pytest.raises(torch.OutOfMemoryError):+ fk.slow_attention_softcapping(+ Q,+ Q,+ Q,+ mask,+ types.SimpleNamespace(config = types.SimpleNamespace(num_attention_heads = 16)),+ 2,+ 8,+ )+ assert calls["eager"] == 0diff --git a/unsloth/kernels/flex_attention.py b/unsloth/kernels/flex_attention.pyindex a1fc03a02a3..0384ec55512 100644--- a/unsloth/kernels/flex_attention.py+++ b/unsloth/kernels/flex_attention.py@@ -37,7 +37,7 @@ except:if not HAS_FLEX_ATTENTION:# Logit softcapping@torch.compile(fullgraph = True, dynamic = True, options = torch_compile_options)- def slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):+ def _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):n_heads = self.config.num_attention_headshead_dim = self.head_dimn_kv_heads = self.config.num_key_value_heads@@ -66,6 +66,46 @@ if not HAS_FLEX_ATTENTION:A = A.reshape(bsz, q_len, n_heads * head_dim)return A+ _SOFTCAP_EAGER = {}++ def _softcapping_attention(Q, K, V, causal_mask, self, bsz, q_len):+ key = Q.device.type+ if not _SOFTCAP_EAGER.get(key):+ try:+ return _compiled_slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len)+ except torch.OutOfMemoryError:+ raise+ except Exception as error:+ # ROCm torch 2.11 inductor rejects this graph (inductor::_alloc_from_pool aliasing) on gfx1151.+ logger.warning_once(+ f"Unsloth: compiled Gemma2 softcapping attention failed on {key}, using eager: {error}"+ )+ _SOFTCAP_EAGER[key] = True+ return _compiled_slow_attention_softcapping._torchdynamo_orig_callable(+ Q, K, V, causal_mask, self, bsz, q_len+ )++ def slow_attention_softcapping(Q, K, V, causal_mask, self, bsz, q_len):+ # Inductor indexes the scores in int32; past 2**31 elements the last rows come back NaN.+ rows = max(1, (2**31 - 1) // (self.config.num_attention_heads * q_len * q_len))+ if bsz <= rows:+ return _softcapping_attention(Q, K, V, causal_mask, self, bsz, q_len)+ per_row_mask = causal_mask.dim() == 4 and causal_mask.shape[0] == bsz+ return torch.cat(+ [+ _softcapping_attention(+ Q[i : i + rows],+ K[i : i + rows],+ V[i : i + rows],+ causal_mask[i : i + rows] if per_row_mask else causal_mask,+ self,+ min(rows, bsz - i),+ q_len,+ )+ for i in range(0, bsz, rows)+ ]+ )+create_flex_attention_causal_mask = Nonecreate_flex_attention_sliding_window_mask = Noneelse: