Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Large head attention: keep CUDA graph capture off the mask scan and bound int32 row offsets (#14790)
Daniel Hane79ad3e734b7
diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.pyindex 9c162e5f1d2..e90195fed87 100644--- a/tests/test_large_head_attention.py+++ b/tests/test_large_head_attention.py@@ -653,3 +653,141 @@ def test_router_uses_the_kernel_for_masks_made_under_inference_mode():assert not isinstance(out, str) and len(calls) == 0assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 2torch.testing.assert_close(out, reference)+++class _FakeCudaOperand:+ """A meta tensor that reports CUDA, so `_shapes_qualify` can be checked without a GPU."""++ def __init__(+ self,+ shape,+ stride,+ dtype = torch.float16,+ ):+ self._t = torch.empty_strided(shape, stride, device = "meta", dtype = dtype)+ self.shape, self.dtype, self.is_cuda, self.device = self._t.shape, dtype, True, "cuda:0"++ def dim(self):+ return self._t.dim()++ def numel(self):+ return self._t.numel()++ def stride(self, dim = None):+ return self._t.stride() if dim is None else self._t.stride(dim)+++def test_shapes_qualify_bounds_the_reachable_row_offset():+ # 2,560 elements, but row 4 starts 2**31 elements in: int32 row offsets wrap in the kernel.+ dense = _FakeCudaOperand((1, 1, 5, 512), (2560, 2560, 512, 1))+ wide = _FakeCudaOperand((1, 1, 5, 512), (0, 0, 1 << 29, 1))+ assert LH._shapes_qualify(dense, dense, dense)+ assert not LH._shapes_qualify(wide, dense, dense)+ assert not LH._shapes_qualify(dense, wide, dense)+ assert not LH._shapes_qualify(dense, dense, wide)+ # Batch and head offsets are int64 in the kernels, so a wide batch stride still qualifies.+ batch = _FakeCudaOperand((2, 1, 5, 512), (1 << 31, 2560, 512, 1))+ assert LH._shapes_qualify(batch, batch, batch)+ # The last row's last element at exactly 2**31 - 1 is the largest offset that still fits.+ edge = _FakeCudaOperand((1, 1, 2, 512), (0, 0, LH._INT32_MAX - 511, 1))+ dense_k = _FakeCudaOperand((1, 1, 2, 512), (1024, 1024, 512, 1))+ assert LH._shapes_qualify(edge, dense_k, dense_k)+ over = _FakeCudaOperand((1, 1, 2, 512), (0, 0, LH._INT32_MAX - 510, 1))+ assert not LH._shapes_qualify(over, dense_k, dense_k)+++def _wide_row_stride(shape, dtype, seed):+ # The storage runs to the last row's offset: 2**29 elements per row, over 4 GiB for five rows.+ need = ((shape[-2] - 1) * (1 << 29) + shape[-1]) * torch.empty((), dtype = dtype).element_size()+ if torch.cuda.mem_get_info()[0] < need + (1 << 30):+ pytest.skip(reason = f"needs {need / 2**30:.1f} GiB free for a 2**29 element row stride")+ gen = torch.Generator(device = "cuda").manual_seed(seed)+ x = torch.empty_strided(shape, (0, 0, 1 << 29, 1), device = "cuda", dtype = dtype)+ x.copy_(torch.randn(shape, device = "cuda", dtype = dtype, generator = gen))+ return x++[email protected](not _SM80, reason = "the router engages on sm80+ only")+def test_router_sends_a_wide_row_stride_query_to_sdpa():+ dtype = _cuda_dtype()+ S = 5+ q = _wide_row_stride((1, 1, S, 512), dtype, 0)+ k = torch.randn(1, 1, S, 512, device = "cuda", dtype = dtype)+ v = torch.randn_like(k)++ def original(module, query, key, value, attention_mask, **kwargs):+ out = torch.nn.functional.scaled_dot_product_attention(query, key, value, is_causal = True)+ return out.transpose(1, 2).contiguous(), None++ router = LH._make_router(original)+ before = LH.LARGE_HEAD_ENGAGED["calls"]+ out, _ = router(_Module(), q, k, v, None)+ torch.cuda.synchronize()+ assert LH.LARGE_HEAD_ENGAGED["calls"] == before+ ref = _reference(q.float(), k.float(), v.float(), _causal(S).cuda())+ assert torch.isfinite(out).all()+ assert (out.float() - ref).abs().max() <= 0.03 * ref.abs().max() + 0.02+ del q+ torch.cuda.empty_cache()++[email protected](not has_real_cuda(), reason = "needs CUDA")+def test_backward_copies_a_wide_row_stride_grad():+ dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16+ S = 5+ torch.manual_seed(1)+ q, k, v = [torch.randn(1, 1, S, 512, device = "cuda", dtype = dtype) for _ in range(3)]+ lo, hi = LH._dense_intervals(1, S, True, q.device)+ ours_in = [x.clone().requires_grad_() for x in (q, k, v)]+ out = LH.large_head_attention(*ours_in, lo, hi)+ grad = _wide_row_stride((1, 1, S, 512), dtype, 2) # [B, H, S, D], as the kernel reads dO+ out.backward(grad.transpose(1, 2))+ torch.cuda.synchronize()+ ref_in = [x.detach().float().requires_grad_() for x in (q, k, v)]+ _reference(*ref_in, _causal(S).cuda()).backward(grad.float().transpose(1, 2))+ for a, b in zip([x.grad for x in ours_in], [x.grad for x in ref_in]):+ assert torch.isfinite(a).all()+ assert (a.float() - b).abs().max() <= 0.03 * b.abs().max() + 0.02+ del grad+ torch.cuda.empty_cache()++[email protected](not _SM80, reason = "the router engages on sm80+ only")+def test_router_captures_into_a_cuda_graph_with_an_inference_mask():+ # The mask scan syncs the host ("operation not permitted when stream is capturing"), and a graph+ # would replay one scan's verdict for whatever the mask buffer holds later, so capture uses SDPA.+ def original(module, query, key, value, attention_mask, **kwargs):+ key, value = key.repeat_interleave(2, 1), value.repeat_interleave(2, 1)+ out = torch.nn.functional.scaled_dot_product_attention(+ query, key, value, attn_mask = attention_mask+ )+ return out.transpose(1, 2).contiguous(), None++ router = LH._make_router(original)+ device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()+ assert LH._self_check(device, dtype) is True+ with torch.inference_mode():+ q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)+ kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)+ mask = _causal(24).cuda()[None, None]+ eager_masked = router(_Module(), q, kv, kv, mask)[0]+ eager_dense = router(_Module(), q, kv, kv, None)[0]+ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ for _ in range(3):+ router(_Module(), q, kv, kv, mask)+ router(_Module(), q, kv, kv, None)+ stream.synchronize()+ before = LH.LARGE_HEAD_ENGAGED["calls"]+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph, stream = stream):+ masked = router(_Module(), q, kv, kv, mask)[0]+ dense = router(_Module(), q, kv, kv, None)[0]+ # The unmasked call still runs the kernel inside the graph; the masked one runs SDPA.+ assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1+ graph.replay()+ torch.cuda.synchronize()+ assert LH._SELF_CHECK[(str(device), dtype)] is True+ torch.testing.assert_close(masked, eager_masked, atol = 2e-2, rtol = 2e-2)+ torch.testing.assert_close(dense, eager_dense)diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.pyindex c51814a552f..95b7eb32d2f 100644--- a/unsloth/kernels/large_head_attention.py+++ b/unsloth/kernels/large_head_attention.py@@ -430,7 +430,7 @@ def _bwd_impl(do, q, k, v, o, lse, lo, hi, scale):B, H, S, D = q.shapeHKV = k.shape[1]G = H // HKV- if do.stride(-1) != 1:+ if do.stride(-1) != 1 or _row_span(do) > _INT32_MAX:do = do.contiguous()dq = torch.empty_like(q)dk = torch.empty_like(k)@@ -720,6 +720,19 @@ def _enabled():_enabled.cache_clear = lambda: _ENABLED.__setitem__(0, None)+def _row_span(x):+ """Largest element offset the kernels compute in int32 for a [B, H, S, D] operand.++ Batch and head offsets are int64 in every kernel; the row and column offsets+ (offs_m * stride_m + offs_d) are int32, so they are what must stay below 2**31. A strided+ view can have a small numel and still reach far past 2**31 elements along S. An operand+ whose last stride is not 1 is copied contiguous before launch, so its numel bounds it.+ """+ if x.stride(-1) != 1:+ return x.numel()+ return (x.shape[-2] - 1) * x.stride(-2) + (x.shape[-1] - 1)++def _shapes_qualify(query, key, value):if query.dim() != 4 or key.dim() != 4 or value.dim() != 4:return False@@ -737,7 +750,9 @@ def _shapes_qualify(query, key, value):return Falseif key.dtype != query.dtype or value.dtype != query.dtype:return False- return max(query.numel(), key.numel(), value.numel()) <= _INT32_MAX+ if max(query.numel(), key.numel(), value.numel()) > _INT32_MAX:+ return False+ return max(_row_span(query), _row_span(key), _row_span(value)) <= _INT32_MAX_SELF_CHECK = {}@@ -844,6 +859,14 @@ if hasattr(torch.compiler, "assume_constant_result"):_math_sdpa_requested = torch.compiler.assume_constant_result(_math_sdpa_requested)+def _is_capturing(device):+ try:+ with torch.cuda.device(device):+ return torch.cuda.is_current_stream_capturing()+ except Exception:+ return False++def _make_router(original):def _sdpa_large_head(module,@@ -875,15 +898,23 @@ def _make_router(original):B, S = query.shape[0], query.shape[2]intervals = Nonecompiling = torch.compiler.is_compiling()+ # A host sync (mask scan, first self check) is illegal while a CUDA graph is captured,+ # and a captured graph would replay one mask's verdict for whatever the buffer holds+ # later. So capture only reads cached self checks and sends masked calls to SDPA.+ capturing = not compiling and _is_capturing(query.device)if attention_mask is None:causal = is_causal if is_causal is not None else getattr(module, "is_causal", True)intervals = _dense_intervals(B, S, bool(causal), query.device)- elif not compiling and _SELF_CHECK.get((str(query.device), query.dtype), True):+ elif (+ not compiling+ and not capturing+ and _SELF_CHECK.get((str(query.device), query.dtype), True)+ ):# Reading a mask's structure needs a host sync; compiled graphs keep SDPA. Inference# masks cannot cache the verdict, so they are rescanned per layer: one sync each, only# at prefill (decode never qualifies), cheaper than SDPA's repeated KV heads and mask.intervals = intervals_from_mask(attention_mask, B, S)- if compiling:+ if compiling or capturing:checked = _SELF_CHECK.get((str(query.device), query.dtype), False)elif intervals is not None:checked = _self_check(query.device, query.dtype)