Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix illegal memory access in block-FP8 LoRA kernels on long batches (#13141)
Daniel Hana0cd6acddecb
diff --git a/tests/test_fp8_int64_offsets.py b/tests/test_fp8_int64_offsets.pynew file mode 100644index 00000000000..b2025efc686--- /dev/null+++ b/tests/test_fp8_int64_offsets.py@@ -0,0 +1,72 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Block-FP8 kernels past 2**31 elements hit an illegal memory access from int32 offsets (#3921).+import os++import pytest++torch = pytest.importorskip("torch")++cuda_available = torch.cuda.is_available()++pytestmark = pytest.mark.skipif(+ not (cuda_available and torch.cuda.get_device_capability() >= (8, 9)),+ reason = "FP8 GEMMs need CUDA sm89+",+)++[email protected](scope = "module")+def F():+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ from unsloth.kernels import fp8+ return fp8+++def _need_free_gb(gb):+ if torch.cuda.mem_get_info()[0] < gb * 1024**3:+ pytest.skip(f"needs {gb} GB free GPU memory")+++def test_act_quant_past_int32(F):+ _need_free_gb(16)+ M, K = (2**31) // 4096 + 64, 4096+ X = torch.randn(M, K, device = "cuda", dtype = torch.bfloat16)+ q, s = F.act_quant(X, 128)+ tail = slice(M - 64, M)+ q_ref, s_ref = F.act_quant(X[tail].contiguous(), 128)+ torch.cuda.synchronize()+ assert torch.equal(s[tail], s_ref)+ assert torch.equal(q[tail].view(torch.uint8), q_ref.view(torch.uint8))+++def test_block_matmul_output_past_int32(F):+ _need_free_gb(16)+ N, K = 32768, 256+ M = (2**31) // N + 256+ w = (torch.randn(N, K, device = "cuda") * 0.05).to(torch.float8_e4m3fn)+ w_s = torch.rand(N // 128, K // 128, device = "cuda") * 0.01 + 0.001+ X = torch.randn(M, K, device = "cuda", dtype = torch.bfloat16)+ q, s = F.act_quant(X, 128)+ out = F.w8a8_block_fp8_matmul_triton(q, w, s, w_s, [128, 128], output_dtype = torch.bfloat16)+ rows = slice(M - 512, M) # straddles element 2**31+ ref = F.w8a8_block_fp8_matmul_triton(+ q[rows].contiguous(), w, s[rows].contiguous(), w_s, [128, 128], output_dtype = torch.bfloat16+ )+ torch.cuda.synchronize()+ assert torch.equal(out[rows], ref)+++def test_torchao_route_hands_large_launches_to_int64_kernel(F, monkeypatch):+ calls = []+ monkeypatch.setattr(F, "w8a8_block_fp8_matmul_triton", lambda *a, **k: calls.append("triton"))+ monkeypatch.setattr(+ F, "torchao_blockwise_gemm", lambda *a, **k: calls.append("torchao") or a[0]+ )+ meta = lambda *shape: torch.empty(*shape, device = "meta", dtype = torch.float8_e4m3fn)+ w_s = torch.empty(216, 40, device = "meta")+ F.torchao_block_matmul(+ meta(4096, 5120), meta(27648, 5120), torch.empty(4096, 40, device = "meta"), w_s, [128, 128]+ )+ F.torchao_block_matmul(+ meta(81920, 5120), meta(27648, 5120), torch.empty(81920, 40, device = "meta"), w_s, [128, 128]+ )+ assert calls == ["torchao", "triton"]diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex e2689a549e1..fb216b54b81 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -18,6 +18,8 @@ import triton.language as tlfrom torch.nn import functional as Fimport mathfrom unsloth_zoo.utils import Version++from .indexing import long_indexingfrom unsloth_zoo.log import loggerfrom unsloth_zoo.temporary_patches.common import torch_compile@@ -214,9 +216,18 @@ def weight_dequant(# Copied from huggingface.co/deepseek-ai/DeepSeek-V3 inference/kernel.py@triton.jit-def act_quant_kernel(x_ptr, y_ptr, s_ptr, BLOCK_SIZE: tl.constexpr):+def act_quant_kernel(+ x_ptr,+ y_ptr,+ s_ptr,+ BLOCK_SIZE: tl.constexpr,+ LONG_INDEXING: tl.constexpr = False,+):pid = tl.program_id(axis = 0)- offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)+ if LONG_INDEXING:+ offs = pid.to(tl.int64) * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE).to(tl.int64)+ else:+ offs = pid * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)x = tl.load(x_ptr + offs).to(tl.float32)s = tl.max(tl.abs(x)) / 448.0# All-zero row: keep scale at 1 so LoRA's zero dY does not become NaN (a deviation from the original@@ -246,7 +257,7 @@ def act_quant(x: torch.Tensor, block_size: int = 128) -> tuple[torch.Tensor, torreturn (triton.cdiv(x.numel(), meta["BLOCK_SIZE"]),)with _fp8_triton_device_context(x):- act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size)+ act_quant_kernel[grid](x, y, s, BLOCK_SIZE = block_size, LONG_INDEXING = long_indexing(x))return y, s@@ -278,6 +289,7 @@ def _w8a8_block_fp8_matmul(BLOCK_SIZE_N: tl.constexpr,BLOCK_SIZE_K: tl.constexpr,GROUP_SIZE_M: tl.constexpr,+ LONG_INDEXING: tl.constexpr = False,):"""Triton-accelerated function used to perform linear operations (dotproduct) on input tensors `A` and `B` with block-wise quantization, and@@ -297,6 +309,9 @@ def _w8a8_block_fp8_matmul(offs_am = (pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)) % Moffs_bn = (pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)) % Noffs_k = tl.arange(0, BLOCK_SIZE_K)+ if LONG_INDEXING:+ offs_am = offs_am.to(tl.int64)+ offs_bn = offs_bn.to(tl.int64)a_ptrs = A + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)b_ptrs = B + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)@@ -327,6 +342,9 @@ def _w8a8_block_fp8_matmul(offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)+ if LONG_INDEXING:+ offs_cm = offs_cm.to(tl.int64)+ offs_cn = offs_cn.to(tl.int64)c_ptrs = C + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)tl.store(c_ptrs, c, mask = c_mask)@@ -399,6 +417,7 @@ def w8a8_block_fp8_matmul_triton(BLOCK_SIZE_N = BLOCK_SIZE_N,BLOCK_SIZE_K = BLOCK_SIZE_K,GROUP_SIZE_M = 8,+ LONG_INDEXING = long_indexing(A, B, C),# Default 4 warps starve 128x128 tiles (8 is 1.3-2.1x faster from M >= 128); smaller decode tiles gain nothing.num_warps = 8 if BLOCK_SIZE_M == 128 else 4,)@@ -419,6 +438,13 @@ def torchao_block_matmul(block_size: list[int],output_dtype: torch.dtype = torch.bfloat16,) -> torch.Tensor:+ # torchao's kernel indexes in int32 too, so route launches past 2**31 elements to the int64 one.+ if long_indexing(act_q, weight_q) or (+ act_q.numel() // act_q.shape[-1] * weight_q.shape[0] >= 2**31+ ):+ return w8a8_block_fp8_matmul_triton(+ act_q, weight_q, act_scale, weight_scale, block_size, output_dtype = output_dtype+ )with _fp8_triton_device_context(act_q):out = torchao_blockwise_gemm(act_q.contiguous(),diff --git a/unsloth/kernels/indexing.py b/unsloth/kernels/indexing.pynew file mode 100644index 00000000000..57101287cbb--- /dev/null+++ b/unsloth/kernels/indexing.py@@ -0,0 +1,24 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.++__all__ = ["long_indexing"]++try:+ from math import sumprod as _sumprod+except ImportError: # Python < 3.12++ def _sumprod(a, b):+ return sum(x * y for x, y in zip(a, b))+++def long_indexing(*tensors, block = 0):+ # numel() is not enough for strided views (transposed Q / K): their offsets reach past it.+ for t in tensors:+ if t.is_contiguous():+ extent = t.numel()+ else:+ stride = t.stride()+ extent = _sumprod(t.shape, stride) - sum(stride) + 1+ if extent + block > 2**31:+ return True+ return Falsediff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex fcfe4ad736d..0ad6e33fdaa 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -28,6 +28,7 @@ from ..device_type import (ALLOW_PREQUANTIZED_MODELS,)from ..bnb_availability import native_kernels_ready+from .indexing import long_indexingfrom .fp8 import weight_dequant, fp8_linear, can_use_fp8_rowwise_gemv, fp8_rowwise_gemvfrom .nvfp4 import NVFP4QuantState, nvfp4_dequantize, nvfp4_linear@@ -119,27 +120,6 @@ def is_rdna():)-try:- from math import sumprod as _sumprod-except ImportError: # Python < 3.12-- def _sumprod(a, b):- return sum(x * y for x, y in zip(a, b))---def long_indexing(*tensors, block = 0):- # numel() is not enough for strided views (transposed Q / K): their offsets reach past it.- for t in tensors:- if t.is_contiguous():- extent = t.numel()- else:- stride = t.stride()- extent = _sumprod(t.shape, stride) - sum(stride) + 1- if extent + block > 2**31:- return True- return False--def calculate_settings(n: int,) -> (