Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 per-tensor / rowwise dequant: one Triton pass over the e4m3fn bytes (#14886)

Daniel Haneffa4c381af7
diff --git a/tests/test_fp8_scale_dequant_one_pass.py b/tests/test_fp8_scale_dequant_one_pass.py
new file mode 100644
index 00000000000..36012b88c47
--- /dev/null
+++ b/tests/test_fp8_scale_dequant_one_pass.py
@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""Per-tensor / rowwise e4m3fn dequant on CUDA runs as one Triton pass with the torch expression's bits."""
+
+import os
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("triton")
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.version.hip is not None, reason = "NVIDIA CUDA only"
+)
+
+
[email protected](scope = "module")
+def fp8():
+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")
+ from unsloth.kernels import fp8
+ return fp8
+
+
+def F(fp8, monkeypatch):
+ # Small test matrices: take the kernel at any size.
+ monkeypatch.setattr(fp8, "_SCALE_DEQUANT_MIN_NUMEL", 0)
+ return fp8
+
+
+def _reference(x, s, dtype):
+ # main before the one-pass kernel: fp16 multiplies by the fp32 scale, other dtypes natively.
+ if dtype == torch.float16:
+ return (x.to(torch.float32) * s.to(torch.float32)).to(dtype)
+ return x.to(dtype) * s.to(dtype)
+
+
+def _weight(m, n, scale_dtype = torch.float32, seed = 0):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ w = torch.randn(m, n, device = "cuda", generator = g)
+ w = w * torch.logspace(-6, 2, m, device = "cuda")[:, None]
+ s = (w.abs().amax(1, keepdim = True) / 448).to(scale_dtype)
+ q = (w / s.float()).to(torch.float8_e4m3fn)
+ # Every byte, including NaN, both zeros and subnormals, in the first rows.
+ codes = torch.arange(256, device = "cuda", dtype = torch.int32).to(torch.uint8)
+ q.view(torch.uint8)[: (256 + n - 1) // n].view(-1)[:256] = codes
+ return q, s
+
+
+def _assert_same_bits(out, ref):
+ # Bit patterns, so signed zeros and subnormals count; NaN payloads may differ.
+ nan = ref.isnan()
+ assert nan.any() and torch.equal(out.isnan(), nan)
+ bits = torch.int32 if ref.dtype == torch.float32 else torch.int16
+ assert torch.equal(out.view(bits)[~nan], ref.view(bits)[~nan])
+
+
+def _count_launches(F, monkeypatch):
+ calls = []
+ real = F._scale_dequant_one_pass
+ monkeypatch.setattr(F, "_scale_dequant_one_pass", lambda *a: calls.append(1) or real(*a))
+ return calls
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])
[email protected]("scale_dtype", [torch.float32, torch.bfloat16])
[email protected]("layout", ["row", "row_t", "tensor", "tensor_t", "view_t"])
+def test_one_pass_matches_torch_expression(F, monkeypatch, dtype, scale_dtype, layout):
+ q, s = _weight(96, 160, scale_dtype)
+ if layout == "row":
+ x, xs, ref = q, s, _reference(q, s, dtype)
+ elif layout == "row_t":
+ # weight_dequant(W.t(), s): scale per column of the view.
+ x, xs, ref = q.t(), s, _reference(q, s, dtype).t()
+ elif layout == "tensor":
+ t = s[17:18]
+ x, xs, ref = q, t, _reference(q, t.view(1, 1), dtype)
+ elif layout == "tensor_t":
+ t = s[17:18].view(1)
+ x, xs, ref = q.t(), t, _reference(q.t(), t.view(1, 1), dtype)
+ else:
+ # Transposed view whose rows carry the scales.
+ qt = q.t().contiguous().t()
+ x, xs, ref = qt, s, _reference(qt, s, dtype)
+ calls = _count_launches(F, monkeypatch)
+ out = F.weight_dequant(x, xs, dtype)
+ assert calls, "one-pass kernel not used"
+ assert out.dtype == dtype and out.shape == ref.shape and out.stride() == ref.stride()
+ _assert_same_bits(out, ref)
+
+
+def test_fp16_extreme_scales_round_once(F, monkeypatch):
+ calls = _count_launches(F, monkeypatch)
+ q = torch.full((4, 8), 448.0, device = "cuda").to(torch.float8_e4m3fn)
+ for scale in (1e-8, 1.234567e-4, 1e5):
+ s = torch.full((4, 1), scale, device = "cuda")
+ out = F.weight_dequant(q, s, torch.float16)
+ assert torch.equal(out, (q.double() * s.double()).half())
+ assert len(calls) == 3
+
+
+def test_other_inputs_keep_the_torch_path(F, monkeypatch):
+ calls = _count_launches(F, monkeypatch)
+ q, s = _weight(32, 48)
+ F.weight_dequant(q, s, torch.float64)
+ F.weight_dequant(q.to(torch.float8_e5m2), s, torch.bfloat16)
+ F.weight_dequant(q, s.double(), torch.bfloat16)
+ F.weight_dequant(q, s.clone().requires_grad_(), torch.bfloat16)
+ assert not calls
+
+
+def test_trainable_fp8_input_keeps_autograd(F, monkeypatch):
+ calls = _count_launches(F, monkeypatch)
+ q, s = _weight(32, 48)
+ x = q.detach().requires_grad_()
+ out = F.weight_dequant(x, s, torch.bfloat16)
+ assert out.requires_grad and not calls
+
+
+def test_strided_transposed_view_keeps_the_torch_path(F, monkeypatch):
+ calls = _count_launches(F, monkeypatch)
+ q, s = _weight(96, 160)
+ # 48 rows of a transposed (160, 96) weight: the transpose of q[:, 95:143] is not dense.
+ x = q.t()[95:143]
+ assert F._is_transposed_view(x) and not x.t().is_contiguous()
+ sx = torch.rand(48, 1, device = "cuda")
+ _assert_same_bits(F._scale_mul(x, sx, torch.float16), _reference(x, sx, torch.float16))
+ assert not calls
+
+
+def test_compiled_dequant_keeps_the_fused_expression(F, monkeypatch):
+ calls = _count_launches(F, monkeypatch)
+ q, s = _weight(64, 128)
+ torch._dynamo.reset()
+ fn = torch.compile(lambda x, y: F.weight_dequant(x, y, torch.float16), fullgraph = True)
+ _assert_same_bits(fn(q, s), _reference(q, s, torch.float16))
+ assert not calls
+
+
+def test_small_weights_keep_the_torch_path(fp8, monkeypatch):
+ calls = _count_launches(fp8, monkeypatch)
+ q, s = _weight(2048, 2048)
+ fp8.weight_dequant(q, s, torch.bfloat16)
+ assert not calls
+ q, s = _weight(2049, 2048)
+ _assert_same_bits(fp8.weight_dequant(q, s, torch.bfloat16), _reference(q, s, torch.bfloat16))
+ assert len(calls) == 1
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 2725793758a..7a6b40838df 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -190,6 +190,8 @@ def _scale_mul(x, s, dtype):
# fp16 out= makes TensorIterator compute in fp32 per element without fp32 temporaries. Dynamo
# refuses a strided out=; under compile Inductor fuses the fp32 expression instead (same bits).
# Other dtypes keep their dtype-native expression.
+ if _scale_dequant_one_pass_ok(x, s, dtype):
+ return _scale_dequant_one_pass(x, s, dtype)
if dtype != torch.float16:
return x.to(dtype) * s.to(dtype)
if torch.compiler.is_compiling():
@@ -198,6 +200,56 @@ def _scale_mul(x, s, dtype):
return torch.mul(x16, s.to(torch.float32), out = torch.empty_like(x16))
+# At or below 4M elements the launch dominates on A100 and newer, and torch's two passes are no slower.
+_SCALE_DEQUANT_MIN_NUMEL = 1 << 22
+
+
+def _scale_dequant_one_pass_ok(x, s, dtype):
+ # Eager NVIDIA e4m3fn only; compiled regions already fuse the torch expression.
+ return (
+ x.numel() > _SCALE_DEQUANT_MIN_NUMEL
+ and x.is_cuda
+ and torch.version.hip is None
+ and x.dtype == torch.float8_e4m3fn
+ and dtype in (torch.float16, torch.bfloat16, torch.float32)
+ and x.dim() == 2
+ and s.device == x.device
+ and s.dtype in (torch.float32, torch.bfloat16, torch.float16)
+ and not (torch.is_grad_enabled() and (x.requires_grad or s.requires_grad))
+ # The kernel indexes dense rows, so a transposed view must be a dense transpose.
+ and (x.is_contiguous() or (_is_transposed_view(x) and x.t().is_contiguous()))
+ and not torch.compiler.is_compiling()
+ )
+
+
+def _scale_dequant_one_pass(x, s, dtype):
+ transposed = _is_transposed_view(x)
+ if transposed:
+ x = x.t()
+ m, n = x.shape
+ out = torch.empty((m, n), dtype = dtype, device = x.device)
+ if out.numel() == 0:
+ return out.t() if transposed else out
+ xb = x.view(torch.uint8)
+ BLOCK_M, BLOCK_N, num_warps = _SCALE_DEQUANT_TILE
+ with _fp8_triton_device_context(x):
+ _e4m3fn_scale_dequant_kernel[(triton.cdiv(m, BLOCK_M), triton.cdiv(n, BLOCK_N))](
+ xb,
+ s,
+ out,
+ m,
+ n,
+ 0 if s.numel() == 1 else s.stride(0),
+ BLOCK_M = BLOCK_M,
+ BLOCK_N = BLOCK_N,
+ SCALE_ON_COLS = transposed,
+ ROUND_SCALE = dtype != torch.float16,
+ LONG_INDEXING = long_indexing(xb, out, block = BLOCK_M * n),
+ num_warps = num_warps,
+ )
+ return out.t() if transposed else out
+
+
def weight_dequant(
x: torch.Tensor,
s: torch.Tensor,
@@ -1254,6 +1306,46 @@ def _e4m3fn_bytes_dequant_kernel(
_BYTES_DEQUANT_TILE = (8, 512, 4)
+def _e4m3fn_scale_dequant_kernel(
+ x_ptr,
+ s_ptr,
+ y_ptr,
+ M,
+ N,
+ stride_s,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ SCALE_ON_COLS: tl.constexpr,
+ ROUND_SCALE: tl.constexpr,
+ LONG_INDEXING: tl.constexpr,
+):
+ # Scale per tensor (stride_s 0), row or column. ROUND_SCALE mirrors x.to(dtype) * s.to(dtype) (fp16 keeps
+ # the fp32 scale); e4m3 times a bf16 scale is exact in fp32, so one rounding gives torch's bits.
+ pid_m = tl.program_id(0)
+ pid_n = tl.program_id(1)
+ if LONG_INDEXING:
+ pid_m = pid_m.to(tl.int64)
+ pid_n = pid_n.to(tl.int64)
+ r = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
+ c = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ mask = (r[:, None] < M) & (c[None, :] < N)
+ offs = r[:, None] * N + c[None, :]
+ x = _e4m3fn_byte_to_float(tl.load(x_ptr + offs, mask = mask, other = 0).to(tl.int32))
+ if SCALE_ON_COLS:
+ s = tl.load(s_ptr + c * stride_s, mask = c < N, other = 0.0)
+ else:
+ s = tl.load(s_ptr + r * stride_s, mask = r < M, other = 0.0)
+ if ROUND_SCALE:
+ s = s.to(y_ptr.dtype.element_ty)
+ s = s.to(tl.float32)
+ y = x * (s[None, :] if SCALE_ON_COLS else s[:, None])
+ tl.store(y_ptr + offs, y.to(y_ptr.dtype.element_ty), mask = mask)
+
+
+_SCALE_DEQUANT_TILE = (16, 256, 4)
+
+
@_opaque_under_compile(
"fp8_e4m3fn_bytes_dequant",
lambda weight, weight_scale, block_m, block_n, dtype: torch.empty(