Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Large head attention: keep CUDA graph capture off the mask scan and bound int32 row offsets (#14790)

Daniel Hane79ad3e734b7
diff --git a/tests/test_large_head_attention.py b/tests/test_large_head_attention.py
index 9c162e5f1d2..e90195fed87 100644
--- a/tests/test_large_head_attention.py
+++ b/tests/test_large_head_attention.py
@@ -653,3 +653,141 @@ def test_router_uses_the_kernel_for_masks_made_under_inference_mode():
assert not isinstance(out, str) and len(calls) == 0
assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 2
torch.testing.assert_close(out, reference)
+
+
+class _FakeCudaOperand:
+ """A meta tensor that reports CUDA, so `_shapes_qualify` can be checked without a GPU."""
+
+ def __init__(
+ self,
+ shape,
+ stride,
+ dtype = torch.float16,
+ ):
+ self._t = torch.empty_strided(shape, stride, device = "meta", dtype = dtype)
+ self.shape, self.dtype, self.is_cuda, self.device = self._t.shape, dtype, True, "cuda:0"
+
+ def dim(self):
+ return self._t.dim()
+
+ def numel(self):
+ return self._t.numel()
+
+ def stride(self, dim = None):
+ return self._t.stride() if dim is None else self._t.stride(dim)
+
+
+def test_shapes_qualify_bounds_the_reachable_row_offset():
+ # 2,560 elements, but row 4 starts 2**31 elements in: int32 row offsets wrap in the kernel.
+ dense = _FakeCudaOperand((1, 1, 5, 512), (2560, 2560, 512, 1))
+ wide = _FakeCudaOperand((1, 1, 5, 512), (0, 0, 1 << 29, 1))
+ assert LH._shapes_qualify(dense, dense, dense)
+ assert not LH._shapes_qualify(wide, dense, dense)
+ assert not LH._shapes_qualify(dense, wide, dense)
+ assert not LH._shapes_qualify(dense, dense, wide)
+ # Batch and head offsets are int64 in the kernels, so a wide batch stride still qualifies.
+ batch = _FakeCudaOperand((2, 1, 5, 512), (1 << 31, 2560, 512, 1))
+ assert LH._shapes_qualify(batch, batch, batch)
+ # The last row's last element at exactly 2**31 - 1 is the largest offset that still fits.
+ edge = _FakeCudaOperand((1, 1, 2, 512), (0, 0, LH._INT32_MAX - 511, 1))
+ dense_k = _FakeCudaOperand((1, 1, 2, 512), (1024, 1024, 512, 1))
+ assert LH._shapes_qualify(edge, dense_k, dense_k)
+ over = _FakeCudaOperand((1, 1, 2, 512), (0, 0, LH._INT32_MAX - 510, 1))
+ assert not LH._shapes_qualify(over, dense_k, dense_k)
+
+
+def _wide_row_stride(shape, dtype, seed):
+ # The storage runs to the last row's offset: 2**29 elements per row, over 4 GiB for five rows.
+ need = ((shape[-2] - 1) * (1 << 29) + shape[-1]) * torch.empty((), dtype = dtype).element_size()
+ if torch.cuda.mem_get_info()[0] < need + (1 << 30):
+ pytest.skip(reason = f"needs {need / 2**30:.1f} GiB free for a 2**29 element row stride")
+ gen = torch.Generator(device = "cuda").manual_seed(seed)
+ x = torch.empty_strided(shape, (0, 0, 1 << 29, 1), device = "cuda", dtype = dtype)
+ x.copy_(torch.randn(shape, device = "cuda", dtype = dtype, generator = gen))
+ return x
+
+
[email protected](not _SM80, reason = "the router engages on sm80+ only")
+def test_router_sends_a_wide_row_stride_query_to_sdpa():
+ dtype = _cuda_dtype()
+ S = 5
+ q = _wide_row_stride((1, 1, S, 512), dtype, 0)
+ k = torch.randn(1, 1, S, 512, device = "cuda", dtype = dtype)
+ v = torch.randn_like(k)
+
+ def original(module, query, key, value, attention_mask, **kwargs):
+ out = torch.nn.functional.scaled_dot_product_attention(query, key, value, is_causal = True)
+ return out.transpose(1, 2).contiguous(), None
+
+ router = LH._make_router(original)
+ before = LH.LARGE_HEAD_ENGAGED["calls"]
+ out, _ = router(_Module(), q, k, v, None)
+ torch.cuda.synchronize()
+ assert LH.LARGE_HEAD_ENGAGED["calls"] == before
+ ref = _reference(q.float(), k.float(), v.float(), _causal(S).cuda())
+ assert torch.isfinite(out).all()
+ assert (out.float() - ref).abs().max() <= 0.03 * ref.abs().max() + 0.02
+ del q
+ torch.cuda.empty_cache()
+
+
[email protected](not has_real_cuda(), reason = "needs CUDA")
+def test_backward_copies_a_wide_row_stride_grad():
+ dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] >= 8 else torch.float16
+ S = 5
+ torch.manual_seed(1)
+ q, k, v = [torch.randn(1, 1, S, 512, device = "cuda", dtype = dtype) for _ in range(3)]
+ lo, hi = LH._dense_intervals(1, S, True, q.device)
+ ours_in = [x.clone().requires_grad_() for x in (q, k, v)]
+ out = LH.large_head_attention(*ours_in, lo, hi)
+ grad = _wide_row_stride((1, 1, S, 512), dtype, 2) # [B, H, S, D], as the kernel reads dO
+ out.backward(grad.transpose(1, 2))
+ torch.cuda.synchronize()
+ ref_in = [x.detach().float().requires_grad_() for x in (q, k, v)]
+ _reference(*ref_in, _causal(S).cuda()).backward(grad.float().transpose(1, 2))
+ for a, b in zip([x.grad for x in ours_in], [x.grad for x in ref_in]):
+ assert torch.isfinite(a).all()
+ assert (a.float() - b).abs().max() <= 0.03 * b.abs().max() + 0.02
+ del grad
+ torch.cuda.empty_cache()
+
+
[email protected](not _SM80, reason = "the router engages on sm80+ only")
+def test_router_captures_into_a_cuda_graph_with_an_inference_mask():
+ # The mask scan syncs the host ("operation not permitted when stream is capturing"), and a graph
+ # would replay one scan's verdict for whatever the mask buffer holds later, so capture uses SDPA.
+ def original(module, query, key, value, attention_mask, **kwargs):
+ key, value = key.repeat_interleave(2, 1), value.repeat_interleave(2, 1)
+ out = torch.nn.functional.scaled_dot_product_attention(
+ query, key, value, attn_mask = attention_mask
+ )
+ return out.transpose(1, 2).contiguous(), None
+
+ router = LH._make_router(original)
+ device, dtype = torch.device("cuda", torch.cuda.current_device()), _cuda_dtype()
+ assert LH._self_check(device, dtype) is True
+ with torch.inference_mode():
+ q = torch.randn(1, 4, 24, 512, device = "cuda", dtype = dtype)
+ kv = torch.randn(1, 2, 24, 512, device = "cuda", dtype = dtype)
+ mask = _causal(24).cuda()[None, None]
+ eager_masked = router(_Module(), q, kv, kv, mask)[0]
+ eager_dense = router(_Module(), q, kv, kv, None)[0]
+ stream = torch.cuda.Stream()
+ stream.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(stream):
+ for _ in range(3):
+ router(_Module(), q, kv, kv, mask)
+ router(_Module(), q, kv, kv, None)
+ stream.synchronize()
+ before = LH.LARGE_HEAD_ENGAGED["calls"]
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph, stream = stream):
+ masked = router(_Module(), q, kv, kv, mask)[0]
+ dense = router(_Module(), q, kv, kv, None)[0]
+ # The unmasked call still runs the kernel inside the graph; the masked one runs SDPA.
+ assert LH.LARGE_HEAD_ENGAGED["calls"] == before + 1
+ graph.replay()
+ torch.cuda.synchronize()
+ assert LH._SELF_CHECK[(str(device), dtype)] is True
+ torch.testing.assert_close(masked, eager_masked, atol = 2e-2, rtol = 2e-2)
+ torch.testing.assert_close(dense, eager_dense)
diff --git a/unsloth/kernels/large_head_attention.py b/unsloth/kernels/large_head_attention.py
index c51814a552f..95b7eb32d2f 100644
--- a/unsloth/kernels/large_head_attention.py
+++ b/unsloth/kernels/large_head_attention.py
@@ -430,7 +430,7 @@ def _bwd_impl(do, q, k, v, o, lse, lo, hi, scale):
B, H, S, D = q.shape
HKV = k.shape[1]
G = H // HKV
- if do.stride(-1) != 1:
+ if do.stride(-1) != 1 or _row_span(do) > _INT32_MAX:
do = do.contiguous()
dq = torch.empty_like(q)
dk = torch.empty_like(k)
@@ -720,6 +720,19 @@ def _enabled():
_enabled.cache_clear = lambda: _ENABLED.__setitem__(0, None)
+def _row_span(x):
+ """Largest element offset the kernels compute in int32 for a [B, H, S, D] operand.
+
+ Batch and head offsets are int64 in every kernel; the row and column offsets
+ (offs_m * stride_m + offs_d) are int32, so they are what must stay below 2**31. A strided
+ view can have a small numel and still reach far past 2**31 elements along S. An operand
+ whose last stride is not 1 is copied contiguous before launch, so its numel bounds it.
+ """
+ if x.stride(-1) != 1:
+ return x.numel()
+ return (x.shape[-2] - 1) * x.stride(-2) + (x.shape[-1] - 1)
+
+
def _shapes_qualify(query, key, value):
if query.dim() != 4 or key.dim() != 4 or value.dim() != 4:
return False
@@ -737,7 +750,9 @@ def _shapes_qualify(query, key, value):
return False
if key.dtype != query.dtype or value.dtype != query.dtype:
return False
- return max(query.numel(), key.numel(), value.numel()) <= _INT32_MAX
+ if max(query.numel(), key.numel(), value.numel()) > _INT32_MAX:
+ return False
+ return max(_row_span(query), _row_span(key), _row_span(value)) <= _INT32_MAX
_SELF_CHECK = {}
@@ -844,6 +859,14 @@ if hasattr(torch.compiler, "assume_constant_result"):
_math_sdpa_requested = torch.compiler.assume_constant_result(_math_sdpa_requested)
+def _is_capturing(device):
+ try:
+ with torch.cuda.device(device):
+ return torch.cuda.is_current_stream_capturing()
+ except Exception:
+ return False
+
+
def _make_router(original):
def _sdpa_large_head(
module,
@@ -875,15 +898,23 @@ def _make_router(original):
B, S = query.shape[0], query.shape[2]
intervals = None
compiling = torch.compiler.is_compiling()
+ # A host sync (mask scan, first self check) is illegal while a CUDA graph is captured,
+ # and a captured graph would replay one mask's verdict for whatever the buffer holds
+ # later. So capture only reads cached self checks and sends masked calls to SDPA.
+ capturing = not compiling and _is_capturing(query.device)
if attention_mask is None:
causal = is_causal if is_causal is not None else getattr(module, "is_causal", True)
intervals = _dense_intervals(B, S, bool(causal), query.device)
- elif not compiling and _SELF_CHECK.get((str(query.device), query.dtype), True):
+ elif (
+ not compiling
+ and not capturing
+ and _SELF_CHECK.get((str(query.device), query.dtype), True)
+ ):
# Reading a mask's structure needs a host sync; compiled graphs keep SDPA. Inference
# masks cannot cache the verdict, so they are rescanned per layer: one sync each, only
# at prefill (decode never qualifies), cheaper than SDPA's repeated KV heads and mask.
intervals = intervals_from_mask(attention_mask, B, S)
- if compiling:
+ if compiling or capturing:
checked = _SELF_CHECK.get((str(query.device), query.dtype), False)
elif intervals is not None:
checked = _self_check(query.device, query.dtype)