Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 block dequant below sm89: decode e4m3fn bytes in one Triton pass (#14408)

Daniel Han98010f6eee0e
diff --git a/tests/test_fp8_pre_sm89_byte_dequant.py b/tests/test_fp8_pre_sm89_byte_dequant.py
new file mode 100644
index 00000000000..38bf0cc2727
--- /dev/null
+++ b/tests/test_fp8_pre_sm89_byte_dequant.py
@@ -0,0 +1,248 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+"""Pre-sm89 block FP8 dequant decodes e4m3fn bytes in triton; it must match the torch fallback bit for bit."""
+
+import types
+
+import pytest
+import torch
+
+from unsloth.kernels import fp8
+
+CUDA = torch.cuda.is_available()
+needs_cuda = pytest.mark.skipif(not CUDA, reason = "needs CUDA")
+
+
+def pre_sm89(monkeypatch):
+ # Real pre-sm89 cards take this path anyway; newer ones are told they are an A100.
+ fp8._fp8_device_lacks_kernel.cache_clear()
+ if CUDA and torch.version.hip is None and torch.cuda.get_device_capability() >= (8, 9):
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (8, 0))
+ yield
+ fp8._fp8_device_lacks_kernel.cache_clear()
+
+
+def _weight(
+ m,
+ n,
+ device,
+ seed = 0,
+):
+ g = torch.Generator(device = "cpu").manual_seed(seed)
+ # Every byte value, incl. subnormals, -0.0 and both NaNs, then a realistic spread.
+ raw = torch.randint(0, 256, (m, n), generator = g, dtype = torch.uint8)
+ raw.view(-1)[: min(256, m * n)] = torch.arange(min(256, m * n), dtype = torch.uint8)
+ return raw.view(torch.float8_e4m3fn).to(device)
+
+
+def _scale(
+ m,
+ n,
+ block,
+ dtype,
+ device,
+ seed = 1,
+):
+ g = torch.Generator(device = "cpu").manual_seed(seed)
+ shape = (-(-m // block[0]), -(-n // block[1]))
+ s = torch.rand(shape, generator = g, dtype = torch.float32) * 1e-2 + 1e-4
+ return s.to(dtype).to(device)
+
+
+def _same_bits(a, b):
+ assert a.shape == b.shape and a.dtype == b.dtype
+ nan = torch.isnan(a)
+ assert torch.equal(nan, torch.isnan(b))
+ # NaN payloads may differ between the two casts; everything else (incl. -0.0) must match exactly.
+ ai = a.masked_fill(nan, 0).contiguous().view(-1).view(torch.uint8)
+ bi = b.masked_fill(nan, 0).contiguous().view(-1).view(torch.uint8)
+ assert torch.equal(ai, bi)
+
+
+SHAPES = [
+ (128, 128),
+ (256, 384),
+ (384, 256),
+ (128, 384),
+ (1, 128),
+ (130, 200),
+ (576, 7168 // 4),
+ (300, 129),
+ (17, 3),
+]
+
+
+@needs_cuda
[email protected]("shape", SHAPES)
[email protected]("block", [[128, 128], [64, 128], [128, 64]])
[email protected]("out_dtype", [torch.float16, torch.bfloat16, torch.float32])
+def test_bytes_kernel_matches_torch_fallback(pre_sm89, shape, block, out_dtype):
+ m, n = shape
+ w = _weight(m, n, "cuda")
+ s = _scale(m, n, block, torch.float32, "cuda")
+ ref = fp8._torch_blockwise_dequant(w, s, block, out_dtype)
+ out = fp8._blockwise_weight_dequant_any_shape(w, s, block, out_dtype)
+ _same_bits(out, ref)
+
+
+@needs_cuda
[email protected]("scale_dtype", [torch.float32, torch.bfloat16, torch.float16, "e8m0"])
+def test_scale_dtypes(pre_sm89, scale_dtype):
+ m, n, block = 384, 256, [128, 128]
+ w = _weight(m, n, "cuda")
+ if scale_dtype == "e8m0":
+ if not hasattr(torch, "float8_e8m0fnu"):
+ pytest.skip("no float8_e8m0fnu in this torch")
+ s = (2.0 ** torch.randint(-12, 2, (3, 2))).to("cuda").to(torch.float8_e8m0fnu)
+ ref_scale = s.to(torch.float32)
+ else:
+ s = _scale(m, n, block, scale_dtype, "cuda")
+ ref_scale = s
+ ref = fp8._torch_blockwise_dequant(w, ref_scale, block, torch.bfloat16)
+ _same_bits(fp8._blockwise_weight_dequant_any_shape(w, s, block, torch.bfloat16), ref)
+
+
+@needs_cuda
[email protected]("shape", [(256, 384), (130, 200), (384, 384)])
+def test_transposed_and_strided(pre_sm89, shape):
+ m, n = shape
+ block = [128, 128]
+ w = _weight(m, n, "cuda")
+ s = _scale(m, n, block, torch.float32, "cuda")
+ ref = fp8._torch_blockwise_dequant(w, s, block, torch.float16)
+ # weight_dequant on a transposed view (fast_lora backward's W.t()).
+ _same_bits(fp8.weight_dequant(w.t(), s, torch.float16), ref.t())
+ # Transposed weight with transposed scale grid (FP8_fbgemm_block_linear's re-orientation).
+ wt, st = w.t(), s.t()
+ _same_bits(
+ fp8._blockwise_weight_dequant_any_shape(wt, st, block, torch.float16),
+ fp8._torch_blockwise_dequant(wt, st, block, torch.float16),
+ )
+ wide = _weight(m, n + 128, "cuda")[:, :n]
+ _same_bits(
+ fp8._blockwise_weight_dequant_any_shape(wide, s, block, torch.bfloat16),
+ fp8._torch_blockwise_dequant(wide, s, block, torch.bfloat16),
+ )
+
+
+@needs_cuda
[email protected]("shape", [(14336, 4096), (4096, 14336)])
+def test_real_shapes_and_peak_memory(pre_sm89, shape):
+ m, n = shape
+ w = (torch.randn(m, n, device = "cuda") * 0.3).to(torch.float8_e4m3fn)
+ s = _scale(m, n, [128, 128], torch.float32, "cuda")
+ ref = fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16)
+ del_ref_bytes = ref.numel() * ref.element_size()
+ torch.cuda.synchronize()
+ torch.cuda.reset_peak_memory_stats()
+ before = torch.cuda.memory_allocated()
+ out = fp8._blockwise_weight_dequant_any_shape(w, s, [128, 128], torch.bfloat16)
+ torch.cuda.synchronize()
+ peak = torch.cuda.max_memory_allocated() - before
+ _same_bits(out, ref)
+ # Only the output is allocated (the chunked fallback adds float32 scratch on top).
+ assert peak <= del_ref_bytes + (1 << 20), peak
+
+
+@needs_cuda
+def test_routes_to_bytes_kernel_only_on_nvidia_pre_sm89(monkeypatch):
+ calls = []
+ orig = fp8._e4m3fn_bytes_dequant
+ monkeypatch.setattr(
+ fp8, "_e4m3fn_bytes_dequant", lambda *a, **k: calls.append(1) or orig(*a, **k)
+ )
+ monkeypatch.setattr(
+ fp8,
+ "weight_dequant_block",
+ lambda x, s, block_size = 128, dtype = torch.bfloat16: calls.append(2)
+ or torch.empty(x.shape, dtype = dtype, device = x.device),
+ )
+ w = _weight(256, 256, "cuda")
+ s = _scale(256, 256, [128, 128], torch.float32, "cuda")
+
+ def route(
+ capability,
+ hip = None,
+ weight = w,
+ ):
+ calls.clear()
+ with monkeypatch.context() as mp:
+ mp.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)
+ mp.setattr(torch.version, "hip", hip)
+ if hip is not None:
+ props = types.SimpleNamespace(gcnArchName = "gfx90a")
+ mp.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)
+ mp.setattr(fp8.triton, "__version__", "3.2.0")
+ fp8._fp8_device_lacks_kernel.cache_clear()
+ fp8._blockwise_weight_dequant_any_shape(weight, s, [128, 128], torch.bfloat16)
+ fp8._fp8_device_lacks_kernel.cache_clear()
+ return {1: "bytes", 2: "triton_fp8"}.get(calls[0], "?") if calls else "torch"
+
+ assert route((7, 5)) == "bytes"
+ assert route((8, 0)) == "bytes"
+ assert route((8, 6)) == "bytes"
+ # sm89+ keeps the native fp8 kernel; ragged shapes there keep the torch loop as before.
+ assert route((8, 9)) == "triton_fp8"
+ assert route((9, 0)) == "triton_fp8"
+ assert route((9, 0), weight = _weight(130, 256, "cuda")) == "torch"
+ # ROCm without fp8e4nv keeps the torch loop (unverified on AMD hardware).
+ assert route((9, 4), hip = "6.4.0") == "torch"
+ # e5m2 weights are compiled natively before sm89 and never reach the byte decoder.
+ assert route((8, 0), weight = w.float().to(torch.float8_e5m2)) == "triton_fp8"
+
+
+@needs_cuda
+def test_fp8_block_linear_forward_backward_unchanged(pre_sm89):
+ torch.manual_seed(0)
+ m, k = 384, 256
+ weight = (torch.randn(m, k, device = "cuda") * 0.3).to(torch.float8_e4m3fn)
+ scale = torch.rand(3, 2, device = "cuda") + 0.5
+ X = (torch.randn(8, k, device = "cuda", dtype = torch.float16)).requires_grad_(True)
+ out = fp8.FP8BlockQuantLinear.apply(X, weight, scale)
+ out.float().pow(2).sum().backward()
+ W = fp8._torch_blockwise_dequant(weight, scale, [128, 128], torch.float16)
+ Xr = X.detach().clone().requires_grad_(True)
+ ref = Xr @ W.T
+ ref.float().pow(2).sum().backward()
+ assert torch.equal(out, ref)
+ assert torch.equal(X.grad, Xr.grad)
+
+
+@needs_cuda
+def test_under_torch_compile(pre_sm89):
+ w = _weight(256, 384, "cuda")
+ s = _scale(256, 384, [128, 128], torch.float32, "cuda")
+ f = torch.compile(
+ lambda w, s: fp8._e4m3fn_bytes_dequant(w, s, 128, 128, torch.bfloat16), fullgraph = True
+ )
+ _same_bits(f(w, s), fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16))
+
+
+def test_cpu_keeps_torch_path(monkeypatch):
+ monkeypatch.setattr(
+ fp8,
+ "_e4m3fn_bytes_dequant",
+ lambda *a, **k: (_ for _ in ()).throw(AssertionError("byte kernel on CPU")),
+ )
+ w = _weight(130, 200, "cpu")
+ s = _scale(130, 200, [128, 128], torch.float32, "cpu")
+ out = fp8._blockwise_weight_dequant_any_shape(w, s, [128, 128], torch.bfloat16)
+ _same_bits(out, fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16))
+
+
+@needs_cuda
+def test_past_int32_offsets(pre_sm89):
+ # 2**31 + 128 * 512 elements: int32 offsets would wrap; spot-check the last rows against the torch loop.
+ m, n = (2**31) // 4096 + 128, 4096
+ if torch.cuda.mem_get_info()[0] < 8 * 1024**3:
+ pytest.skip("needs 8 GB free GPU memory")
+ w = (
+ torch.empty(m, n, dtype = torch.uint8, device = "cuda")
+ .random_(0, 126)
+ .view(torch.float8_e4m3fn)
+ )
+ s = _scale(m, n, [128, 128], torch.float32, "cuda")
+ out = fp8._blockwise_weight_dequant_any_shape(w, s, [128, 128], torch.float16)
+ tail = slice(m - 256, m)
+ _same_bits(out[tail], fp8._torch_blockwise_dequant(w[tail], s[-2:], [128, 128], torch.float16))
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index b71263da86b..fda34d8d430 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -648,6 +648,97 @@ def _torch_blockwise_dequant(weight, weight_scale, block_size, out_dtype):
return out
+def _e4m3fn_byte_to_float(b):
+ # No fp8e4nv before sm_89: decode E4M3FN bytes exactly (bias 7, exp 0 subnormal, 0x7F/0xFF NaN).
+ exp = (b >> 3) & 15
+ man = b & 7
+ normal = (((exp + 120) << 23) | (man << 20)).to(tl.float32, bitcast = True)
+ value = tl.where(exp == 0, man.to(tl.float32) * 0.001953125, normal)
+ value = tl.where((b & 127) == 127, float("nan"), value)
+ # Sign as a bit, not a negation: -x can lower to 0 - x, which turns code 0x80 (-0.0) into +0.0.
+ return (value.to(tl.int32, bitcast = True) | ((b & 128) << 24)).to(tl.float32, bitcast = True)
+
+
+def _e4m3fn_bytes_dequant_kernel(
+ x_ptr,
+ s_ptr,
+ y_ptr,
+ M,
+ N,
+ stride_xm,
+ stride_xn,
+ stride_sm,
+ stride_sn,
+ BLOCK_M_Q: tl.constexpr,
+ BLOCK_N_Q: tl.constexpr,
+ BLOCK_R: tl.constexpr,
+ BLOCK_C: tl.constexpr,
+ LONG_INDEXING: tl.constexpr,
+):
+ pid_r = tl.program_id(0)
+ pid_c = tl.program_id(1)
+ if LONG_INDEXING:
+ pid_r = pid_r.to(tl.int64)
+ pid_c = pid_c.to(tl.int64)
+ r = pid_r * BLOCK_R + tl.arange(0, BLOCK_R)
+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)
+ mask = (r[:, None] < M) & (c[None, :] < N)
+ b = tl.load(x_ptr + r[:, None] * stride_xm + c[None, :] * stride_xn, mask = mask, other = 0)
+ # Per-element scale gather: a tile spans several quant blocks (also covers ragged / rectangular ones).
+ s = tl.load(
+ s_ptr + (r // BLOCK_M_Q)[:, None] * stride_sm + (c // BLOCK_N_Q)[None, :] * stride_sn,
+ mask = mask,
+ other = 0.0,
+ ).to(tl.float32)
+ # One fp32 multiply then one round to the output dtype, as the torch fallback does.
+ y = _e4m3fn_byte_to_float(b.to(tl.int32)) * s
+ tl.store(y_ptr + r[:, None] * N + c[None, :], y.to(y_ptr.dtype.element_ty), mask = mask)
+
+
+# (rows, cols, num_warps) per program, from a tile sweep on T4 and A100.
+_BYTES_DEQUANT_TILE = (8, 512, 4)
+
+
+@_opaque_under_compile(
+ "fp8_e4m3fn_bytes_dequant",
+ lambda weight, weight_scale, block_m, block_n, dtype: torch.empty(
+ weight.shape, dtype = dtype, device = weight.device
+ ),
+)
+def _e4m3fn_bytes_dequant(
+ weight: torch.Tensor, weight_scale: torch.Tensor, block_m: int, block_n: int, dtype: torch.dtype
+) -> torch.Tensor:
+ # Raw bytes, as triton has no fp8e4nv below sm89; bit-identical to _torch_blockwise_dequant.
+ m, n = weight.shape
+ out = torch.empty((m, n), dtype = dtype, device = weight.device)
+ if out.numel() == 0:
+ return out
+ x = weight.view(torch.uint8)
+ BLOCK_R, BLOCK_C, num_warps = _BYTES_DEQUANT_TILE
+ grid = (triton.cdiv(m, BLOCK_R), triton.cdiv(n, BLOCK_C))
+ with _fp8_triton_device_context(weight):
+ _e4m3fn_bytes_dequant_kernel[grid](
+ x,
+ weight_scale,
+ out,
+ m,
+ n,
+ x.stride(0),
+ x.stride(1),
+ weight_scale.stride(0),
+ weight_scale.stride(1),
+ BLOCK_M_Q = block_m,
+ BLOCK_N_Q = block_n,
+ BLOCK_R = BLOCK_R,
+ BLOCK_C = BLOCK_C,
+ LONG_INDEXING = long_indexing(x, weight_scale, out),
+ num_warps = num_warps,
+ )
+ return out
+
+
def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dtype):
"""Blockwise fp8 weight dequant for any shape: triton when the weight tiles
evenly into block_size and this GPU can compile its fp8 dtype, else a
@@ -670,6 +761,11 @@ def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dt
):
# Uneven tiling, rectangular blocks, or an uncompilable fp8 dtype: the triton kernel uses one
# BLOCK_SIZE for both axes, so it mis-indexes when block_size[0] != [1].
+ if kernel_fp8_unsupported and torch.version.hip is None and weight_scale.ndim == 2:
+ # NVIDIA below sm89: one triton pass over the e4m3fn bytes, bit-identical to the torch loop.
+ return _e4m3fn_bytes_dequant(
+ weight, weight_scale, int(block_size[0]), int(block_size[1]), out_dtype
+ )
return _torch_blockwise_dequant(weight, weight_scale, block_size, out_dtype)
# Even tiling with square blocks: pass the real block size, since weight_dequant would silently
# default to 128 and dequantize wrongly.