Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 block dequant below sm89: decode e4m3fn bytes in one Triton pass (#14408)
Daniel Han98010f6eee0e
diff --git a/tests/test_fp8_pre_sm89_byte_dequant.py b/tests/test_fp8_pre_sm89_byte_dequant.pynew file mode 100644index 00000000000..38bf0cc2727--- /dev/null+++ b/tests/test_fp8_pre_sm89_byte_dequant.py@@ -0,0 +1,248 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.+"""Pre-sm89 block FP8 dequant decodes e4m3fn bytes in triton; it must match the torch fallback bit for bit."""++import types++import pytest+import torch++from unsloth.kernels import fp8++CUDA = torch.cuda.is_available()+needs_cuda = pytest.mark.skipif(not CUDA, reason = "needs CUDA")+++def pre_sm89(monkeypatch):+ # Real pre-sm89 cards take this path anyway; newer ones are told they are an A100.+ fp8._fp8_device_lacks_kernel.cache_clear()+ if CUDA and torch.version.hip is None and torch.cuda.get_device_capability() >= (8, 9):+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (8, 0))+ yield+ fp8._fp8_device_lacks_kernel.cache_clear()+++def _weight(+ m,+ n,+ device,+ seed = 0,+):+ g = torch.Generator(device = "cpu").manual_seed(seed)+ # Every byte value, incl. subnormals, -0.0 and both NaNs, then a realistic spread.+ raw = torch.randint(0, 256, (m, n), generator = g, dtype = torch.uint8)+ raw.view(-1)[: min(256, m * n)] = torch.arange(min(256, m * n), dtype = torch.uint8)+ return raw.view(torch.float8_e4m3fn).to(device)+++def _scale(+ m,+ n,+ block,+ dtype,+ device,+ seed = 1,+):+ g = torch.Generator(device = "cpu").manual_seed(seed)+ shape = (-(-m // block[0]), -(-n // block[1]))+ s = torch.rand(shape, generator = g, dtype = torch.float32) * 1e-2 + 1e-4+ return s.to(dtype).to(device)+++def _same_bits(a, b):+ assert a.shape == b.shape and a.dtype == b.dtype+ nan = torch.isnan(a)+ assert torch.equal(nan, torch.isnan(b))+ # NaN payloads may differ between the two casts; everything else (incl. -0.0) must match exactly.+ ai = a.masked_fill(nan, 0).contiguous().view(-1).view(torch.uint8)+ bi = b.masked_fill(nan, 0).contiguous().view(-1).view(torch.uint8)+ assert torch.equal(ai, bi)+++SHAPES = [+ (128, 128),+ (256, 384),+ (384, 256),+ (128, 384),+ (1, 128),+ (130, 200),+ (576, 7168 // 4),+ (300, 129),+ (17, 3),+]+++@needs_cuda[email protected]("shape", SHAPES)[email protected]("block", [[128, 128], [64, 128], [128, 64]])[email protected]("out_dtype", [torch.float16, torch.bfloat16, torch.float32])+def test_bytes_kernel_matches_torch_fallback(pre_sm89, shape, block, out_dtype):+ m, n = shape+ w = _weight(m, n, "cuda")+ s = _scale(m, n, block, torch.float32, "cuda")+ ref = fp8._torch_blockwise_dequant(w, s, block, out_dtype)+ out = fp8._blockwise_weight_dequant_any_shape(w, s, block, out_dtype)+ _same_bits(out, ref)+++@needs_cuda[email protected]("scale_dtype", [torch.float32, torch.bfloat16, torch.float16, "e8m0"])+def test_scale_dtypes(pre_sm89, scale_dtype):+ m, n, block = 384, 256, [128, 128]+ w = _weight(m, n, "cuda")+ if scale_dtype == "e8m0":+ if not hasattr(torch, "float8_e8m0fnu"):+ pytest.skip("no float8_e8m0fnu in this torch")+ s = (2.0 ** torch.randint(-12, 2, (3, 2))).to("cuda").to(torch.float8_e8m0fnu)+ ref_scale = s.to(torch.float32)+ else:+ s = _scale(m, n, block, scale_dtype, "cuda")+ ref_scale = s+ ref = fp8._torch_blockwise_dequant(w, ref_scale, block, torch.bfloat16)+ _same_bits(fp8._blockwise_weight_dequant_any_shape(w, s, block, torch.bfloat16), ref)+++@needs_cuda[email protected]("shape", [(256, 384), (130, 200), (384, 384)])+def test_transposed_and_strided(pre_sm89, shape):+ m, n = shape+ block = [128, 128]+ w = _weight(m, n, "cuda")+ s = _scale(m, n, block, torch.float32, "cuda")+ ref = fp8._torch_blockwise_dequant(w, s, block, torch.float16)+ # weight_dequant on a transposed view (fast_lora backward's W.t()).+ _same_bits(fp8.weight_dequant(w.t(), s, torch.float16), ref.t())+ # Transposed weight with transposed scale grid (FP8_fbgemm_block_linear's re-orientation).+ wt, st = w.t(), s.t()+ _same_bits(+ fp8._blockwise_weight_dequant_any_shape(wt, st, block, torch.float16),+ fp8._torch_blockwise_dequant(wt, st, block, torch.float16),+ )+ wide = _weight(m, n + 128, "cuda")[:, :n]+ _same_bits(+ fp8._blockwise_weight_dequant_any_shape(wide, s, block, torch.bfloat16),+ fp8._torch_blockwise_dequant(wide, s, block, torch.bfloat16),+ )+++@needs_cuda[email protected]("shape", [(14336, 4096), (4096, 14336)])+def test_real_shapes_and_peak_memory(pre_sm89, shape):+ m, n = shape+ w = (torch.randn(m, n, device = "cuda") * 0.3).to(torch.float8_e4m3fn)+ s = _scale(m, n, [128, 128], torch.float32, "cuda")+ ref = fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16)+ del_ref_bytes = ref.numel() * ref.element_size()+ torch.cuda.synchronize()+ torch.cuda.reset_peak_memory_stats()+ before = torch.cuda.memory_allocated()+ out = fp8._blockwise_weight_dequant_any_shape(w, s, [128, 128], torch.bfloat16)+ torch.cuda.synchronize()+ peak = torch.cuda.max_memory_allocated() - before+ _same_bits(out, ref)+ # Only the output is allocated (the chunked fallback adds float32 scratch on top).+ assert peak <= del_ref_bytes + (1 << 20), peak+++@needs_cuda+def test_routes_to_bytes_kernel_only_on_nvidia_pre_sm89(monkeypatch):+ calls = []+ orig = fp8._e4m3fn_bytes_dequant+ monkeypatch.setattr(+ fp8, "_e4m3fn_bytes_dequant", lambda *a, **k: calls.append(1) or orig(*a, **k)+ )+ monkeypatch.setattr(+ fp8,+ "weight_dequant_block",+ lambda x, s, block_size = 128, dtype = torch.bfloat16: calls.append(2)+ or torch.empty(x.shape, dtype = dtype, device = x.device),+ )+ w = _weight(256, 256, "cuda")+ s = _scale(256, 256, [128, 128], torch.float32, "cuda")++ def route(+ capability,+ hip = None,+ weight = w,+ ):+ calls.clear()+ with monkeypatch.context() as mp:+ mp.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)+ mp.setattr(torch.version, "hip", hip)+ if hip is not None:+ props = types.SimpleNamespace(gcnArchName = "gfx90a")+ mp.setattr(torch.cuda, "get_device_properties", lambda *a, **k: props)+ mp.setattr(fp8.triton, "__version__", "3.2.0")+ fp8._fp8_device_lacks_kernel.cache_clear()+ fp8._blockwise_weight_dequant_any_shape(weight, s, [128, 128], torch.bfloat16)+ fp8._fp8_device_lacks_kernel.cache_clear()+ return {1: "bytes", 2: "triton_fp8"}.get(calls[0], "?") if calls else "torch"++ assert route((7, 5)) == "bytes"+ assert route((8, 0)) == "bytes"+ assert route((8, 6)) == "bytes"+ # sm89+ keeps the native fp8 kernel; ragged shapes there keep the torch loop as before.+ assert route((8, 9)) == "triton_fp8"+ assert route((9, 0)) == "triton_fp8"+ assert route((9, 0), weight = _weight(130, 256, "cuda")) == "torch"+ # ROCm without fp8e4nv keeps the torch loop (unverified on AMD hardware).+ assert route((9, 4), hip = "6.4.0") == "torch"+ # e5m2 weights are compiled natively before sm89 and never reach the byte decoder.+ assert route((8, 0), weight = w.float().to(torch.float8_e5m2)) == "triton_fp8"+++@needs_cuda+def test_fp8_block_linear_forward_backward_unchanged(pre_sm89):+ torch.manual_seed(0)+ m, k = 384, 256+ weight = (torch.randn(m, k, device = "cuda") * 0.3).to(torch.float8_e4m3fn)+ scale = torch.rand(3, 2, device = "cuda") + 0.5+ X = (torch.randn(8, k, device = "cuda", dtype = torch.float16)).requires_grad_(True)+ out = fp8.FP8BlockQuantLinear.apply(X, weight, scale)+ out.float().pow(2).sum().backward()+ W = fp8._torch_blockwise_dequant(weight, scale, [128, 128], torch.float16)+ Xr = X.detach().clone().requires_grad_(True)+ ref = Xr @ W.T+ ref.float().pow(2).sum().backward()+ assert torch.equal(out, ref)+ assert torch.equal(X.grad, Xr.grad)+++@needs_cuda+def test_under_torch_compile(pre_sm89):+ w = _weight(256, 384, "cuda")+ s = _scale(256, 384, [128, 128], torch.float32, "cuda")+ f = torch.compile(+ lambda w, s: fp8._e4m3fn_bytes_dequant(w, s, 128, 128, torch.bfloat16), fullgraph = True+ )+ _same_bits(f(w, s), fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16))+++def test_cpu_keeps_torch_path(monkeypatch):+ monkeypatch.setattr(+ fp8,+ "_e4m3fn_bytes_dequant",+ lambda *a, **k: (_ for _ in ()).throw(AssertionError("byte kernel on CPU")),+ )+ w = _weight(130, 200, "cpu")+ s = _scale(130, 200, [128, 128], torch.float32, "cpu")+ out = fp8._blockwise_weight_dequant_any_shape(w, s, [128, 128], torch.bfloat16)+ _same_bits(out, fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16))+++@needs_cuda+def test_past_int32_offsets(pre_sm89):+ # 2**31 + 128 * 512 elements: int32 offsets would wrap; spot-check the last rows against the torch loop.+ m, n = (2**31) // 4096 + 128, 4096+ if torch.cuda.mem_get_info()[0] < 8 * 1024**3:+ pytest.skip("needs 8 GB free GPU memory")+ w = (+ torch.empty(m, n, dtype = torch.uint8, device = "cuda")+ .random_(0, 126)+ .view(torch.float8_e4m3fn)+ )+ s = _scale(m, n, [128, 128], torch.float32, "cuda")+ out = fp8._blockwise_weight_dequant_any_shape(w, s, [128, 128], torch.float16)+ tail = slice(m - 256, m)+ _same_bits(out[tail], fp8._torch_blockwise_dequant(w[tail], s[-2:], [128, 128], torch.float16))diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex b71263da86b..fda34d8d430 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -648,6 +648,97 @@ def _torch_blockwise_dequant(weight, weight_scale, block_size, out_dtype):return out+def _e4m3fn_byte_to_float(b):+ # No fp8e4nv before sm_89: decode E4M3FN bytes exactly (bias 7, exp 0 subnormal, 0x7F/0xFF NaN).+ exp = (b >> 3) & 15+ man = b & 7+ normal = (((exp + 120) << 23) | (man << 20)).to(tl.float32, bitcast = True)+ value = tl.where(exp == 0, man.to(tl.float32) * 0.001953125, normal)+ value = tl.where((b & 127) == 127, float("nan"), value)+ # Sign as a bit, not a negation: -x can lower to 0 - x, which turns code 0x80 (-0.0) into +0.0.+ return (value.to(tl.int32, bitcast = True) | ((b & 128) << 24)).to(tl.float32, bitcast = True)+++def _e4m3fn_bytes_dequant_kernel(+ x_ptr,+ s_ptr,+ y_ptr,+ M,+ N,+ stride_xm,+ stride_xn,+ stride_sm,+ stride_sn,+ BLOCK_M_Q: tl.constexpr,+ BLOCK_N_Q: tl.constexpr,+ BLOCK_R: tl.constexpr,+ BLOCK_C: tl.constexpr,+ LONG_INDEXING: tl.constexpr,+):+ pid_r = tl.program_id(0)+ pid_c = tl.program_id(1)+ if LONG_INDEXING:+ pid_r = pid_r.to(tl.int64)+ pid_c = pid_c.to(tl.int64)+ r = pid_r * BLOCK_R + tl.arange(0, BLOCK_R)+ c = pid_c * BLOCK_C + tl.arange(0, BLOCK_C)+ mask = (r[:, None] < M) & (c[None, :] < N)+ b = tl.load(x_ptr + r[:, None] * stride_xm + c[None, :] * stride_xn, mask = mask, other = 0)+ # Per-element scale gather: a tile spans several quant blocks (also covers ragged / rectangular ones).+ s = tl.load(+ s_ptr + (r // BLOCK_M_Q)[:, None] * stride_sm + (c // BLOCK_N_Q)[None, :] * stride_sn,+ mask = mask,+ other = 0.0,+ ).to(tl.float32)+ # One fp32 multiply then one round to the output dtype, as the torch fallback does.+ y = _e4m3fn_byte_to_float(b.to(tl.int32)) * s+ tl.store(y_ptr + r[:, None] * N + c[None, :], y.to(y_ptr.dtype.element_ty), mask = mask)+++# (rows, cols, num_warps) per program, from a tile sweep on T4 and A100.+_BYTES_DEQUANT_TILE = (8, 512, 4)+++@_opaque_under_compile(+ "fp8_e4m3fn_bytes_dequant",+ lambda weight, weight_scale, block_m, block_n, dtype: torch.empty(+ weight.shape, dtype = dtype, device = weight.device+ ),+)+def _e4m3fn_bytes_dequant(+ weight: torch.Tensor, weight_scale: torch.Tensor, block_m: int, block_n: int, dtype: torch.dtype+) -> torch.Tensor:+ # Raw bytes, as triton has no fp8e4nv below sm89; bit-identical to _torch_blockwise_dequant.+ m, n = weight.shape+ out = torch.empty((m, n), dtype = dtype, device = weight.device)+ if out.numel() == 0:+ return out+ x = weight.view(torch.uint8)+ BLOCK_R, BLOCK_C, num_warps = _BYTES_DEQUANT_TILE+ grid = (triton.cdiv(m, BLOCK_R), triton.cdiv(n, BLOCK_C))+ with _fp8_triton_device_context(weight):+ _e4m3fn_bytes_dequant_kernel[grid](+ x,+ weight_scale,+ out,+ m,+ n,+ x.stride(0),+ x.stride(1),+ weight_scale.stride(0),+ weight_scale.stride(1),+ BLOCK_M_Q = block_m,+ BLOCK_N_Q = block_n,+ BLOCK_R = BLOCK_R,+ BLOCK_C = BLOCK_C,+ LONG_INDEXING = long_indexing(x, weight_scale, out),+ num_warps = num_warps,+ )+ return out++def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dtype):"""Blockwise fp8 weight dequant for any shape: triton when the weight tilesevenly into block_size and this GPU can compile its fp8 dtype, else a@@ -670,6 +761,11 @@ def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dt):# Uneven tiling, rectangular blocks, or an uncompilable fp8 dtype: the triton kernel uses one# BLOCK_SIZE for both axes, so it mis-indexes when block_size[0] != [1].+ if kernel_fp8_unsupported and torch.version.hip is None and weight_scale.ndim == 2:+ # NVIDIA below sm89: one triton pass over the e4m3fn bytes, bit-identical to the torch loop.+ return _e4m3fn_bytes_dequant(+ weight, weight_scale, int(block_size[0]), int(block_size[1]), out_dtype+ )return _torch_blockwise_dequant(weight, weight_scale, block_size, out_dtype)# Even tiling with square blocks: pass the real block size, since weight_dequant would silently# default to 128 and dequantize wrongly.