Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 per-tensor / rowwise dequant: one Triton pass over the e4m3fn bytes (#14886)
Daniel Haneffa4c381af7
diff --git a/tests/test_fp8_scale_dequant_one_pass.py b/tests/test_fp8_scale_dequant_one_pass.pynew file mode 100644index 00000000000..36012b88c47--- /dev/null+++ b/tests/test_fp8_scale_dequant_one_pass.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""Per-tensor / rowwise e4m3fn dequant on CUDA runs as one Triton pass with the torch expression's bits."""++import os++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("triton")++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.version.hip is not None, reason = "NVIDIA CUDA only"+)++[email protected](scope = "module")+def fp8():+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ from unsloth.kernels import fp8+ return fp8+++def F(fp8, monkeypatch):+ # Small test matrices: take the kernel at any size.+ monkeypatch.setattr(fp8, "_SCALE_DEQUANT_MIN_NUMEL", 0)+ return fp8+++def _reference(x, s, dtype):+ # main before the one-pass kernel: fp16 multiplies by the fp32 scale, other dtypes natively.+ if dtype == torch.float16:+ return (x.to(torch.float32) * s.to(torch.float32)).to(dtype)+ return x.to(dtype) * s.to(dtype)+++def _weight(m, n, scale_dtype = torch.float32, seed = 0):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ w = torch.randn(m, n, device = "cuda", generator = g)+ w = w * torch.logspace(-6, 2, m, device = "cuda")[:, None]+ s = (w.abs().amax(1, keepdim = True) / 448).to(scale_dtype)+ q = (w / s.float()).to(torch.float8_e4m3fn)+ # Every byte, including NaN, both zeros and subnormals, in the first rows.+ codes = torch.arange(256, device = "cuda", dtype = torch.int32).to(torch.uint8)+ q.view(torch.uint8)[: (256 + n - 1) // n].view(-1)[:256] = codes+ return q, s+++def _assert_same_bits(out, ref):+ # Bit patterns, so signed zeros and subnormals count; NaN payloads may differ.+ nan = ref.isnan()+ assert nan.any() and torch.equal(out.isnan(), nan)+ bits = torch.int32 if ref.dtype == torch.float32 else torch.int16+ assert torch.equal(out.view(bits)[~nan], ref.view(bits)[~nan])+++def _count_launches(F, monkeypatch):+ calls = []+ real = F._scale_dequant_one_pass+ monkeypatch.setattr(F, "_scale_dequant_one_pass", lambda *a: calls.append(1) or real(*a))+ return calls++[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])[email protected]("scale_dtype", [torch.float32, torch.bfloat16])[email protected]("layout", ["row", "row_t", "tensor", "tensor_t", "view_t"])+def test_one_pass_matches_torch_expression(F, monkeypatch, dtype, scale_dtype, layout):+ q, s = _weight(96, 160, scale_dtype)+ if layout == "row":+ x, xs, ref = q, s, _reference(q, s, dtype)+ elif layout == "row_t":+ # weight_dequant(W.t(), s): scale per column of the view.+ x, xs, ref = q.t(), s, _reference(q, s, dtype).t()+ elif layout == "tensor":+ t = s[17:18]+ x, xs, ref = q, t, _reference(q, t.view(1, 1), dtype)+ elif layout == "tensor_t":+ t = s[17:18].view(1)+ x, xs, ref = q.t(), t, _reference(q.t(), t.view(1, 1), dtype)+ else:+ # Transposed view whose rows carry the scales.+ qt = q.t().contiguous().t()+ x, xs, ref = qt, s, _reference(qt, s, dtype)+ calls = _count_launches(F, monkeypatch)+ out = F.weight_dequant(x, xs, dtype)+ assert calls, "one-pass kernel not used"+ assert out.dtype == dtype and out.shape == ref.shape and out.stride() == ref.stride()+ _assert_same_bits(out, ref)+++def test_fp16_extreme_scales_round_once(F, monkeypatch):+ calls = _count_launches(F, monkeypatch)+ q = torch.full((4, 8), 448.0, device = "cuda").to(torch.float8_e4m3fn)+ for scale in (1e-8, 1.234567e-4, 1e5):+ s = torch.full((4, 1), scale, device = "cuda")+ out = F.weight_dequant(q, s, torch.float16)+ assert torch.equal(out, (q.double() * s.double()).half())+ assert len(calls) == 3+++def test_other_inputs_keep_the_torch_path(F, monkeypatch):+ calls = _count_launches(F, monkeypatch)+ q, s = _weight(32, 48)+ F.weight_dequant(q, s, torch.float64)+ F.weight_dequant(q.to(torch.float8_e5m2), s, torch.bfloat16)+ F.weight_dequant(q, s.double(), torch.bfloat16)+ F.weight_dequant(q, s.clone().requires_grad_(), torch.bfloat16)+ assert not calls+++def test_trainable_fp8_input_keeps_autograd(F, monkeypatch):+ calls = _count_launches(F, monkeypatch)+ q, s = _weight(32, 48)+ x = q.detach().requires_grad_()+ out = F.weight_dequant(x, s, torch.bfloat16)+ assert out.requires_grad and not calls+++def test_strided_transposed_view_keeps_the_torch_path(F, monkeypatch):+ calls = _count_launches(F, monkeypatch)+ q, s = _weight(96, 160)+ # 48 rows of a transposed (160, 96) weight: the transpose of q[:, 95:143] is not dense.+ x = q.t()[95:143]+ assert F._is_transposed_view(x) and not x.t().is_contiguous()+ sx = torch.rand(48, 1, device = "cuda")+ _assert_same_bits(F._scale_mul(x, sx, torch.float16), _reference(x, sx, torch.float16))+ assert not calls+++def test_compiled_dequant_keeps_the_fused_expression(F, monkeypatch):+ calls = _count_launches(F, monkeypatch)+ q, s = _weight(64, 128)+ torch._dynamo.reset()+ fn = torch.compile(lambda x, y: F.weight_dequant(x, y, torch.float16), fullgraph = True)+ _assert_same_bits(fn(q, s), _reference(q, s, torch.float16))+ assert not calls+++def test_small_weights_keep_the_torch_path(fp8, monkeypatch):+ calls = _count_launches(fp8, monkeypatch)+ q, s = _weight(2048, 2048)+ fp8.weight_dequant(q, s, torch.bfloat16)+ assert not calls+ q, s = _weight(2049, 2048)+ _assert_same_bits(fp8.weight_dequant(q, s, torch.bfloat16), _reference(q, s, torch.bfloat16))+ assert len(calls) == 1diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 2725793758a..7a6b40838df 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -190,6 +190,8 @@ def _scale_mul(x, s, dtype):# fp16 out= makes TensorIterator compute in fp32 per element without fp32 temporaries. Dynamo# refuses a strided out=; under compile Inductor fuses the fp32 expression instead (same bits).# Other dtypes keep their dtype-native expression.+ if _scale_dequant_one_pass_ok(x, s, dtype):+ return _scale_dequant_one_pass(x, s, dtype)if dtype != torch.float16:return x.to(dtype) * s.to(dtype)if torch.compiler.is_compiling():@@ -198,6 +200,56 @@ def _scale_mul(x, s, dtype):return torch.mul(x16, s.to(torch.float32), out = torch.empty_like(x16))+# At or below 4M elements the launch dominates on A100 and newer, and torch's two passes are no slower.+_SCALE_DEQUANT_MIN_NUMEL = 1 << 22+++def _scale_dequant_one_pass_ok(x, s, dtype):+ # Eager NVIDIA e4m3fn only; compiled regions already fuse the torch expression.+ return (+ x.numel() > _SCALE_DEQUANT_MIN_NUMEL+ and x.is_cuda+ and torch.version.hip is None+ and x.dtype == torch.float8_e4m3fn+ and dtype in (torch.float16, torch.bfloat16, torch.float32)+ and x.dim() == 2+ and s.device == x.device+ and s.dtype in (torch.float32, torch.bfloat16, torch.float16)+ and not (torch.is_grad_enabled() and (x.requires_grad or s.requires_grad))+ # The kernel indexes dense rows, so a transposed view must be a dense transpose.+ and (x.is_contiguous() or (_is_transposed_view(x) and x.t().is_contiguous()))+ and not torch.compiler.is_compiling()+ )+++def _scale_dequant_one_pass(x, s, dtype):+ transposed = _is_transposed_view(x)+ if transposed:+ x = x.t()+ m, n = x.shape+ out = torch.empty((m, n), dtype = dtype, device = x.device)+ if out.numel() == 0:+ return out.t() if transposed else out+ xb = x.view(torch.uint8)+ BLOCK_M, BLOCK_N, num_warps = _SCALE_DEQUANT_TILE+ with _fp8_triton_device_context(x):+ _e4m3fn_scale_dequant_kernel[(triton.cdiv(m, BLOCK_M), triton.cdiv(n, BLOCK_N))](+ xb,+ s,+ out,+ m,+ n,+ 0 if s.numel() == 1 else s.stride(0),+ BLOCK_M = BLOCK_M,+ BLOCK_N = BLOCK_N,+ SCALE_ON_COLS = transposed,+ ROUND_SCALE = dtype != torch.float16,+ LONG_INDEXING = long_indexing(xb, out, block = BLOCK_M * n),+ num_warps = num_warps,+ )+ return out.t() if transposed else out++def weight_dequant(x: torch.Tensor,s: torch.Tensor,@@ -1254,6 +1306,46 @@ def _e4m3fn_bytes_dequant_kernel(_BYTES_DEQUANT_TILE = (8, 512, 4)+def _e4m3fn_scale_dequant_kernel(+ x_ptr,+ s_ptr,+ y_ptr,+ M,+ N,+ stride_s,+ BLOCK_M: tl.constexpr,+ BLOCK_N: tl.constexpr,+ SCALE_ON_COLS: tl.constexpr,+ ROUND_SCALE: tl.constexpr,+ LONG_INDEXING: tl.constexpr,+):+ # Scale per tensor (stride_s 0), row or column. ROUND_SCALE mirrors x.to(dtype) * s.to(dtype) (fp16 keeps+ # the fp32 scale); e4m3 times a bf16 scale is exact in fp32, so one rounding gives torch's bits.+ pid_m = tl.program_id(0)+ pid_n = tl.program_id(1)+ if LONG_INDEXING:+ pid_m = pid_m.to(tl.int64)+ pid_n = pid_n.to(tl.int64)+ r = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)+ c = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)+ mask = (r[:, None] < M) & (c[None, :] < N)+ offs = r[:, None] * N + c[None, :]+ x = _e4m3fn_byte_to_float(tl.load(x_ptr + offs, mask = mask, other = 0).to(tl.int32))+ if SCALE_ON_COLS:+ s = tl.load(s_ptr + c * stride_s, mask = c < N, other = 0.0)+ else:+ s = tl.load(s_ptr + r * stride_s, mask = r < M, other = 0.0)+ if ROUND_SCALE:+ s = s.to(y_ptr.dtype.element_ty)+ s = s.to(tl.float32)+ y = x * (s[None, :] if SCALE_ON_COLS else s[:, None])+ tl.store(y_ptr + offs, y.to(y_ptr.dtype.element_ty), mask = mask)+++_SCALE_DEQUANT_TILE = (16, 256, 4)++@_opaque_under_compile("fp8_e4m3fn_bytes_dequant",lambda weight, weight_scale, block_m, block_n, dtype: torch.empty(