Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fix illegal memory access in block-FP8 LoRA kernels on long batches (#13141)

Daniel Hana0cd6acddecb
diff --git a/tests/test_fp8_int64_offsets.py b/tests/test_fp8_int64_offsets.py
new file mode 100644
index 00000000000..b2025efc686
--- /dev/null
+++ b/tests/test_fp8_int64_offsets.py
@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Block-FP8 kernels past 2**31 elements hit an illegal memory access from int32 offsets (#3921).
+import os
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+cuda_available = torch.cuda.is_available()
+
+pytestmark = pytest.mark.skipif(
+ not (cuda_available and torch.cuda.get_device_capability() >= (8, 9)),
+ reason = "FP8 GEMMs need CUDA sm89+",
+)
+
+
[email protected](scope = "module")
+def F():
+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")
+ from unsloth.kernels import fp8
+ return fp8
+
+
+def _need_free_gb(gb):
+ if torch.cuda.mem_get_info()[0] < gb * 1024**3:
+ pytest.skip(f"needs {gb} GB free GPU memory")
+
+
+def test_act_quant_past_int32(F):
+ _need_free_gb(16)
+ M, K = (2**31) // 4096 + 64, 4096
+ X = torch.randn(M, K, device = "cuda", dtype = torch.bfloat16)
+ q, s = F.act_quant(X, 128)
+ tail = slice(M - 64, M)
+ q_ref, s_ref = F.act_quant(X[tail].contiguous(), 128)
+ torch.cuda.synchronize()
+ assert torch.equal(s[tail], s_ref)
+ assert torch.equal(q[tail].view(torch.uint8), q_ref.view(torch.uint8))
+
+
+def test_block_matmul_output_past_int32(F):
+ _need_free_gb(16)
+ N, K = 32768, 256
+ M = (2**31) // N + 256
+ w = (torch.randn(N, K, device = "cuda") * 0.05).to(torch.float8_e4m3fn)
+ w_s = torch.rand(N // 128, K // 128, device = "cuda") * 0.01 + 0.001
+ X = torch.randn(M, K, device = "cuda", dtype = torch.bfloat16)
+ q, s = F.act_quant(X, 128)
+ out = F.w8a8_block_fp8_matmul_triton(q, w, s, w_s, [128, 128], output_dtype = torch.bfloat16)
+ rows = slice(M - 512, M) # straddles element 2**31
+ ref = F.w8a8_block_fp8_matmul_triton(
+ q[rows].contiguous(), w, s[rows].contiguous(), w_s, [128, 128], output_dtype = torch.bfloat16
+ )
+ torch.cuda.synchronize()
+ assert torch.equal(out[rows], ref)
+
+
+def test_torchao_route_hands_large_launches_to_int64_kernel(F, monkeypatch):
+ calls = []
+ monkeypatch.setattr(F, "w8a8_block_fp8_matmul_triton", lambda *a, **k: calls.append("triton"))
+ monkeypatch.setattr(
+ F, "torchao_blockwise_gemm", lambda *a, **k: calls.append("torchao") or a[0]
+ )
+ meta = lambda *shape: torch.empty(*shape, device = "meta", dtype = torch.float8_e4m3fn)
+ w_s = torch.empty(216, 40, device = "meta")
+ F.torchao_block_matmul(
+ meta(4096, 5120), meta(27648, 5120), torch.empty(4096, 40, device = "meta"), w_s, [128, 128]
+ )
+ F.torchao_block_matmul(
+ meta(81920, 5120), meta(27648, 5120), torch.empty(81920, 40, device = "meta"), w_s, [128, 128]
+ )
+ assert calls == ["torchao", "triton"]
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index e2689a549e1..fb216b54b81 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -18,6 +18,8 @@ import triton.language as tl
from torch.nn import functional as F
import math
from unsloth_zoo.utils import Version
+
+from .indexing import long_indexing
from unsloth_zoo.log import logger
from unsloth_zoo.temporary_patches.common import torch_compile
@@ -214,9 +216,18 @@ def weight_dequant(
# Copied from huggingface.co/deepseek-ai/DeepSeek-V3 inference/kernel.py
@triton.jit
-def act_quant_kernel(x_ptr, y_ptr, s_ptr, BLOCK_SIZE: tl.constexpr):
+def act_quant_kernel(
+ x_ptr,
+ y_ptr,
+ s_ptr,
+ BLOCK_SIZE: tl.constexpr,
+ LONG_INDEXING: tl.constexpr = False,
+):
pid = tl.program_id(axis = 0)
- offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+ if LONG_INDEXING:
+ offs = pid.to(tl.int64) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE).to(tl.int64)
+ else:
+ offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
x = tl.load(x_ptr + offs).to(tl.float32)
s = tl.max(tl.abs(x)) / 448.0
# All-zero row: keep scale at 1 so LoRA's zero dY does not become NaN (a deviation from the original
@@ -246,7 +257,7 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, tor
return (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)
with _fp8_triton_device_context(x):
- act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size)
+ act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size, LONG_INDEXING = long_indexing(x))
return y, s
@@ -278,6 +289,7 @@ def _w8a8_block_fp8_matmul(
BLOCK_SIZE_N: tl.constexpr,
BLOCK_SIZE_K: tl.constexpr,
GROUP_SIZE_M: tl.constexpr,
+ LONG_INDEXING: tl.constexpr = False,
):
"""Triton-accelerated function used to perform linear operations (dot
product) on input tensors `A` and `B` with block-wise quantization, and
@@ -297,6 +309,9 @@ def _w8a8_block_fp8_matmul(
offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % M
offs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % N
offs_k = tl.arange(0, BLOCK_SIZE_K)
+ if LONG_INDEXING:
+ offs_am = offs_am.to(tl.int64)
+ offs_bn = offs_bn.to(tl.int64)
a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
@@ -327,6 +342,9 @@ def _w8a8_block_fp8_matmul(
offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
+ if LONG_INDEXING:
+ offs_cm = offs_cm.to(tl.int64)
+ offs_cn = offs_cn.to(tl.int64)
c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
tl.store(c_ptrs, c, mask = c_mask)
@@ -399,6 +417,7 @@ def w8a8_block_fp8_matmul_triton(
BLOCK_SIZE_N = BLOCK_SIZE_N,
BLOCK_SIZE_K = BLOCK_SIZE_K,
GROUP_SIZE_M = 8,
+ LONG_INDEXING = long_indexing(A, B, C),
# Default 4 warps starve 128x128 tiles (8 is 1.3-2.1x faster from M >= 128); smaller decode tiles gain nothing.
num_warps = 8 if BLOCK_SIZE_M == 128 else 4,
)
@@ -419,6 +438,13 @@ def torchao_block_matmul(
block_size: list[int],
output_dtype: torch.dtype = torch.bfloat16,
) -> torch.Tensor:
+ # torchao's kernel indexes in int32 too, so route launches past 2**31 elements to the int64 one.
+ if long_indexing(act_q, weight_q) or (
+ act_q.numel() // act_q.shape[-1] * weight_q.shape[0] >= 2**31
+ ):
+ return w8a8_block_fp8_matmul_triton(
+ act_q, weight_q, act_scale, weight_scale, block_size, output_dtype = output_dtype
+ )
with _fp8_triton_device_context(act_q):
out = torchao_blockwise_gemm(
act_q.contiguous(),
diff --git a/unsloth/kernels/indexing.py b/unsloth/kernels/indexing.py
new file mode 100644
index 00000000000..57101287cbb
--- /dev/null
+++ b/unsloth/kernels/indexing.py
@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+
+__all__ = ["long_indexing"]
+
+try:
+ from math import sumprod as _sumprod
+except ImportError: # Python < 3.12
+
+ def _sumprod(a, b):
+ return sum(x * y for x, y in zip(a, b))
+
+
+def long_indexing(*tensors, block = 0):
+ # numel() is not enough for strided views (transposed Q / K): their offsets reach past it.
+ for t in tensors:
+ if t.is_contiguous():
+ extent = t.numel()
+ else:
+ stride = t.stride()
+ extent = _sumprod(t.shape, stride) - sum(stride) + 1
+ if extent + block > 2**31:
+ return True
+ return False
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index fcfe4ad736d..0ad6e33fdaa 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -28,6 +28,7 @@ from ..device_type import (
ALLOW_PREQUANTIZED_MODELS,
)
from ..bnb_availability import native_kernels_ready
+from .indexing import long_indexing
from .fp8 import weight_dequant, fp8_linear, can_use_fp8_rowwise_gemv, fp8_rowwise_gemv
from .nvfp4 import NVFP4QuantState, nvfp4_dequantize, nvfp4_linear
@@ -119,27 +120,6 @@ def is_rdna():
)
-try:
- from math import sumprod as _sumprod
-except ImportError: # Python < 3.12
-
- def _sumprod(a, b):
- return sum(x * y for x, y in zip(a, b))
-
-
-def long_indexing(*tensors, block = 0):
- # numel() is not enough for strided views (transposed Q / K): their offsets reach past it.
- for t in tensors:
- if t.is_contiguous():
- extent = t.numel()
- else:
- stride = t.stride()
- extent = _sumprod(t.shape, stride) - sum(stride) + 1
- if extent + block > 2**31:
- return True
- return False
-
-
def calculate_settings(
n: int,
) -> (