Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 block decode: fused act_quant and narrow-N GEMM for 1 to 16 tokens (#14792)
Daniel Hanf232b9af2da7
diff --git a/tests/test_fp8_block_small_m.py b/tests/test_fp8_block_small_m.pynew file mode 100644index 00000000000..ba4ce2da119--- /dev/null+++ b/tests/test_fp8_block_small_m.py@@ -0,0 +1,160 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Decode-sized block FP8: act_quant fused into a narrow-N w8a8 GEMM (fp8_block_small_m_linear)."""++import pytest+import torch++if not torch.cuda.is_available():+ pytest.skip("needs CUDA", allow_module_level = True)+if torch.cuda.get_device_capability() < (8, 9):+ pytest.skip("fp8e4nv Triton kernels need sm89+", allow_module_level = True)++import unsloth # noqa: F401+import unsloth.kernels.fp8 as f++pytestmark = pytest.mark.skipif(+ f.fp8_block_matmul is not f.w8a8_block_fp8_matmul_triton,+ reason = "only the Triton block matmul route is replaced",+)+++def _block_weight(N, K, scale_dtype = torch.float32):+ W = torch.randn(N, K, device = "cuda") * 0.02+ s = W.abs().view(N // 128, 128, K // 128, 128).amax((1, 3)) / 448.0+ q = (W.view(N // 128, 128, K // 128, 128) / s[:, None, :, None]).view(N, K).to(torch.float8_e4m3fn)+ return q, s.contiguous().to(scale_dtype)+++def _main_path(X, q, s, monkeypatch):+ monkeypatch.setattr(f, "_FP8_BLOCK_SMALL_M", False)+ out = f.fp8_linear(X, q, s)+ monkeypatch.setattr(f, "_FP8_BLOCK_SMALL_M", True)+ return out+++def _oracle(X, q, s):+ M, K = X.numel() // X.shape[-1], X.shape[-1]+ N = q.shape[0]+ xq, xs = f.act_quant(X, 128)+ x64 = (xq.double().view(M, K // 128, 128) * xs.double().view(M, K // 128, 1)).view(M, K)+ w64 = (q.double().view(N // 128, 128, K // 128, 128) * s.double()[:, None, :, None]).view(N, K)+ return (x64 @ w64.t()).view(*X.shape[:-1], N)+++def _oracle_abs(X, q, s):+ M, K = X.numel() // X.shape[-1], X.shape[-1]+ N = q.shape[0]+ xq, xs = f.act_quant(X, 128)+ x64 = (xq.double().view(M, K // 128, 128) * xs.double().view(M, K // 128, 1)).view(M, K)+ w64 = (q.double().view(N // 128, 128, K // 128, 128) * s.double()[:, None, :, None]).view(N, K)+ return (x64.abs() @ w64.abs().t()).view(*X.shape[:-1], N)++[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("scale_dtype", [torch.float32, torch.bfloat16])[email protected]("N,K", [(4096, 4096), (1024, 4096), (6144, 2048), (2048, 6144)])[email protected]("M", [1, 3, 8, 16])+def test_small_m_matches_main(monkeypatch, dtype, scale_dtype, N, K, M):+ torch.manual_seed(0)+ q, s = _block_weight(N, K, scale_dtype)+ X = torch.randn(1, M, K, device = "cuda", dtype = dtype)+ X[0, 0, :128] = 0 # an all-zero block keeps scale 1, as act_quant does+ assert f.can_use_fp8_block_small_m(X, q, s, [128, 128])+ main = _main_path(X, q, s, monkeypatch)+ calls = []+ orig = f.fp8_block_small_m_linear+ monkeypatch.setattr(f, "fp8_block_small_m_linear", lambda *a: calls.append(1) or orig(*a))+ new = f.fp8_linear(X, q, s)+ assert calls == [1]+ ref = _oracle(X, q, s)+ split_k = f._small_m_config(M, N, K, X.device)[1]+ if split_k == 1:+ # Same quantized operands and the same accumulation order as main.+ assert torch.equal(new, main)+ else:+ # Only the fp32 sum order over K splits differs: bound is one output rounding plus fp32 sum error.+ mag = torch.maximum(new.double().abs(), main.double().abs())+ k_blocks = K // 128+ bound = torch.finfo(dtype).eps * mag + 2 * k_blocks * torch.finfo(torch.float32).eps * _oracle_abs(X, q, s)+ assert ((new.double() - main.double()).abs() <= bound + 1e-30).all()+ err_new = (new.double() - ref).abs().mean()+ err_main = (main.double() - ref).abs().mean()+ assert err_new <= err_main * 1.05+ assert torch.isfinite(new).all()+++def test_split_k_is_deterministic(monkeypatch):+ torch.manual_seed(0)+ q, s = _block_weight(1024, 4096)+ X = torch.randn(1, 4, 4096, device = "cuda", dtype = torch.bfloat16)+ monkeypatch.setattr(f, "_small_m_config", lambda *a: (32, 8, 4, 3))+ first = f.fp8_block_small_m_linear(X, q, s)+ for _ in range(20):+ assert torch.equal(f.fp8_block_small_m_linear(X, q, s), first)+++def test_larger_m_and_kill_switch_keep_main_path(monkeypatch):+ q, s = _block_weight(4096, 4096)+ for M in (17, 64):+ X = torch.randn(1, M, 4096, device = "cuda", dtype = torch.bfloat16)+ assert not f.can_use_fp8_block_small_m(X, q, s, [128, 128])+ X = torch.randn(1, 1, 4096, device = "cuda", dtype = torch.bfloat16)+ monkeypatch.setattr(f, "_FP8_BLOCK_SMALL_M", False)+ assert not f.can_use_fp8_block_small_m(X, q, s, [128, 128])+++def test_unsupported_layouts_fall_back():+ q, s = _block_weight(4096, 4096)+ X = torch.randn(1, 1, 4096, device = "cuda", dtype = torch.bfloat16)+ assert not f.can_use_fp8_block_small_m(X.float(), q, s, [128, 128])+ assert not f.can_use_fp8_block_small_m(X, q, s, [64, 128])+ assert not f.can_use_fp8_block_small_m(X, q, s[:, :-1], [128, 128])+ assert not f.can_use_fp8_block_small_m(X, q.to(torch.float8_e5m2), s, [128, 128])+++def test_pre_sm89_keeps_dequant_route(monkeypatch):+ # Below sm89 FP8BlockQuantLinear returns the dequant + matmul result before the small-M branch.+ q, s = _block_weight(1024, 4096)+ X = torch.randn(1, 2, 4096, device = "cuda", dtype = torch.bfloat16)+ monkeypatch.setattr(f, "_fp8_device_lacks_kernel", lambda device: True)+ monkeypatch.setattr(f, "fp8_block_small_m_linear", lambda *a: pytest.fail("small-M kernel used below sm89"))+ out = f.fp8_linear(X, q, s)+ ref = X @ f._blockwise_weight_dequant_any_shape(q, s, [128, 128], X.dtype).t()+ assert torch.equal(out, ref)+++def test_grad_path_still_uses_autograd_function():+ q, s = _block_weight(1024, 4096)+ X = torch.randn(1, 2, 4096, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ out = f.fp8_linear(X, q, s)+ assert out.grad_fn is not None+ out.float().sum().backward()+ ref = f._blockwise_weight_dequant_any_shape(q, s, [128, 128], X.dtype).float().sum(0)+ torch.testing.assert_close(X.grad[0, 0].float(), ref, rtol = 1e-2, atol = 1e-2)+++def test_compiled_path_adds_no_graph_or_break():+ # Compiled regions keep main's route: no break, and no recompile when M crosses the small-M bound.+ from torch._dynamo.utils import counters++ q, s = _block_weight(1024, 4096)+ torch._dynamo.reset()+ counters.clear()+ fn = torch.compile(lambda x: f.fp8_linear(x, q, s), dynamic = True)+ with torch.no_grad():+ for M in (1, 2, 16, 17, 64, 1):+ X = torch.randn(1, M, 4096, device = "cuda", dtype = torch.bfloat16)+ torch.testing.assert_close(fn(X), _main_path_eager(X, q, s), rtol = 0, atol = 0)+ assert sum(counters["graph_break"].values()) == 0+ assert counters["stats"]["unique_graphs"] <= 2+++def _main_path_eager(X, q, s):+ old = f._FP8_BLOCK_SMALL_M+ f._FP8_BLOCK_SMALL_M = False+ try:+ return f.fp8_linear(X, q, s)+ finally:+ f._FP8_BLOCK_SMALL_M = olddiff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 88e82acabdf..9282233d61c 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -441,6 +441,171 @@ def w8a8_block_fp8_matmul_triton(return C+# Decode-sized M: act_quant fused into a narrow-N w8a8 GEMM; the 128 x 128 tile above runs only N / 128 programs.[email protected](do_not_specialize = ["M"])+def _w8a8_block_fp8_small_m(+ X,+ B,+ C,+ Bs,+ P,+ L,+ M,+ N,+ K,+ stride_xm,+ stride_bn,+ stride_cm,+ stride_Bs_n,+ stride_Bs_k,+ KB_PER_SPLIT,+ GROUP_N: tl.constexpr,+ BLOCK_SIZE_M: tl.constexpr,+ BLOCK_SIZE_N: tl.constexpr,+ BLOCK_SIZE_K: tl.constexpr,+ SPLIT_K: tl.constexpr,+):+ pid_n = tl.program_id(axis = 0)+ pid_k = tl.program_id(axis = 1)+ offs_m = tl.arange(0, BLOCK_SIZE_M)+ offs_n = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)+ offs_k = tl.arange(0, BLOCK_SIZE_K)+ m_mask = offs_m < M+ n_mask = offs_n < N+ x_ptrs = X + offs_m[:, None] * stride_xm + offs_k[None, :]+ b_ptrs = B + offs_k[:, None] + offs_n[None, :] * stride_bn+ bs_ptrs = Bs + (offs_n // GROUP_N) * stride_Bs_n+ kb_start = pid_k * KB_PER_SPLIT+ kb_end = tl.minimum(kb_start + KB_PER_SPLIT, tl.cdiv(K, BLOCK_SIZE_K))++ accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype = tl.float32)+ for kb in range(kb_start, kb_end):+ k0 = kb * BLOCK_SIZE_K+ # Same quantization as act_quant_kernel: per row and 128 block, scale max|x| / 448, 1 when all zero.+ x = tl.load(x_ptrs + k0, mask = m_mask[:, None], other = 0.0).to(tl.float32)+ a_s = tl.max(tl.abs(x), axis = 1) / 448.0+ a_s = tl.where(a_s == 0, 1.0, a_s)+ a = (x / a_s[:, None]).to(B.dtype.element_ty)+ b = tl.load(b_ptrs + k0, mask = n_mask[None, :], other = 0.0)+ b_s = tl.load(bs_ptrs + kb * stride_Bs_k, mask = n_mask, other = 0.0)+ accumulator += tl.dot(a, b) * a_s[:, None] * b_s[None, :]++ c_ptrs = C + offs_m[:, None] * stride_cm + offs_n[None, :]+ c_mask = m_mask[:, None] & n_mask[None, :]+ if SPLIT_K == 1:+ tl.store(c_ptrs, accumulator.to(C.dtype.element_ty), mask = c_mask)+ else:+ # Deterministic split K: every split stores its partial, the last to arrive sums them in split order.+ p_ptrs = P + offs_m[:, None] * N + offs_n[None, :]+ tl.store(p_ptrs + pid_k * M * N, accumulator, mask = c_mask)+ tl.debug_barrier()+ arrived = tl.atomic_add(L + pid_n, 1, sem = "acq_rel")+ if arrived == SPLIT_K - 1:+ total = tl.load(p_ptrs, mask = c_mask, other = 0.0, cache_modifier = ".cg")+ for s in range(1, SPLIT_K):+ total += tl.load(p_ptrs + s * M * N, mask = c_mask, other = 0.0, cache_modifier = ".cg")+ tl.store(c_ptrs, total.to(C.dtype.element_ty), mask = c_mask)+++_FP8_BLOCK_SMALL_M = os.environ.get("UNSLOTH_FP8_BLOCK_SMALL_M", "1") != "0"+_SMALL_M_MAX = 16+++# Fitted on full config sweeps (L4, RTX PRO 6000; checked on B200): the widest N tile that still gives+# 2 tiles per SM, then split K while the grid stays within 4 programs per SM and each split keeps 8+ K blocks.+_SMALL_M_STAGES = {16: 5, 32: 4, 64: 3, 128: 4}+++def _small_m_config(M, N, K, device):+ """(BLOCK_SIZE_N, SPLIT_K, num_warps, num_stages) for the fused small-M kernel; depends on N, K only."""+ sms = _sm_count(device)+ block_n = 16+ for candidate in (128, 64, 32):+ if triton.cdiv(N, candidate) >= 2 * sms:+ block_n = candidate+ break+ tiles = triton.cdiv(N, block_n)+ k_blocks = triton.cdiv(K, 128)+ split_k = 1+ while tiles * split_k * 2 <= 4 * sms and k_blocks // (split_k * 2) >= 8:+ split_k *= 2+ return block_n, split_k, 4, _SMALL_M_STAGES[block_n]++[email protected]_cache(maxsize = None)+def _sm_count(device):+ return torch.cuda.get_device_properties(device).multi_processor_count+++def can_use_fp8_block_small_m(X, weight, weight_scale, block_size):+ # Eager only: under torch.compile the M bound below would guard on the symbolic M and recompile.+ if not _FP8_BLOCK_SMALL_M or fp8_block_matmul is not w8a8_block_fp8_matmul_triton or torch.compiler.is_compiling():+ return False+ if not X.is_cuda or X.dtype not in (torch.bfloat16, torch.float16):+ return False+ if list(block_size) != [128, 128] or weight.dtype != torch.float8_e4m3fn:+ return False+ K = X.shape[-1]+ M = X.numel() // K if K else 0+ N = weight.shape[0]+ if not (1 <= M <= _SMALL_M_MAX) or weight.shape[1] != K or K % 128 or N % 16:+ return False+ # Checkpoints store weight_scale_inv in bf16 too; the kernel promotes it exactly as the w8a8 one does.+ if weight_scale.dtype not in (torch.float32, torch.bfloat16, torch.float16) or tuple(weight_scale.shape) != (triton.cdiv(N, 128), K // 128):+ return False+ return not long_indexing(X, weight)+++@_opaque_under_compile(+ "fp8_block_small_m_linear",+ lambda X, weight, weight_scale: X.new_empty(X.shape[:-1] + (weight.shape[0],)),+)+def fp8_block_small_m_linear(X, weight, weight_scale):+ """act_quant(X) then the w8a8 block matmul in one launch, for M <= _SMALL_M_MAX rows."""+ if not X.is_contiguous():+ X = X.contiguous()+ N, K = weight.shape+ M = X.numel() // K+ out = X.new_empty(X.shape[:-1] + (N,))+ block_n, split_k, warps, stages = _small_m_config(M, N, K, X.device)+ k_blocks = triton.cdiv(K, 128)+ kb_per_split = triton.cdiv(k_blocks, split_k)+ split_k = triton.cdiv(k_blocks, kb_per_split)+ tiles = triton.cdiv(N, block_n)+ if split_k > 1:+ partial = torch.empty((split_k, M, N), device = X.device, dtype = torch.float32)+ arrivals = torch.zeros(tiles, device = X.device, dtype = torch.int32)+ else:+ partial = arrivals = out+ with _fp8_triton_device_context(X, static_device_count = True):+ _w8a8_block_fp8_small_m[(tiles, split_k)](+ X,+ weight,+ out,+ weight_scale,+ partial,+ arrivals,+ M,+ N,+ K,+ K,+ weight.stride(0),+ N,+ weight_scale.stride(0),+ weight_scale.stride(1),+ kb_per_split,+ GROUP_N = 128,+ BLOCK_SIZE_M = 16,+ BLOCK_SIZE_N = block_n,+ BLOCK_SIZE_K = 128,+ SPLIT_K = split_k,+ num_warps = warps,+ num_stages = stages,+ enable_fp_fusion = True,+ )+ return out++@_opaque_under_compile("fp8_block_matmul_torchao",lambda A, B, As, Bs, block_size, output_dtype = torch.bfloat16: A.new_empty(@@ -970,15 +1135,18 @@ class FP8BlockQuantLinear(torch.autograd.Function):output = torch_matmul(X, W_deq.T)return output if output.dtype == X.dtype else output.to(X.dtype)- qinput, scale = act_quant(X, block_size[1])- output = fp8_block_matmul(- qinput,- weight,- scale,- weight_scale,- block_size,- output_dtype = X.dtype,- )+ if can_use_fp8_block_small_m(X, weight, weight_scale, block_size):+ output = fp8_block_small_m_linear(X, weight, weight_scale)+ else:+ qinput, scale = act_quant(X, block_size[1])+ output = fp8_block_matmul(+ qinput,+ weight,+ scale,+ weight_scale,+ block_size,+ output_dtype = X.dtype,+ )ctx.weight = weightctx.weight_scale = original_weight_scalectx.block_size = block_size@@ -998,9 +1166,16 @@ class FP8BlockQuantLinear(torch.autograd.Function):# Not torch.compiled: after a model's second weight shape, automatic dynamic shapes recompile the# user Triton kernel into a version 40-80x slower than eager (the only work here is that kernel).def fp8_torch_block_quant_forward(X, weight, weight_scale):+ if not (torch.is_grad_enabled() and X.requires_grad):+ # Nothing to differentiate (decode, no_grad): skip the autograd.Function's per-call host cost.+ return FP8BlockQuantLinear.forward(_NO_CTX(), X, weight, weight_scale)return FP8BlockQuantLinear.apply(X, weight, weight_scale)+class _NO_CTX:+ pass++def _quantize_fp8_per_row(x, scale_ub = None):# Same as FBGEMM quantize_fp8_per_row: scale = max(min(row absmax, scale_ub) / 448, 1 / (448 * 512)), saturating cast.row_max = x.abs().amax(dim = -1, keepdim = True).float()