Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fall back from FBGEMM for rowwise FP8 on GPUs it has no kernel for (RTX PRO 6000 / 5090) (#12098)
Daniel Han5d6ecc420484
diff --git a/tests/test_fp8_rowwise_backend.py b/tests/test_fp8_rowwise_backend.pynew file mode 100644index 00000000000..917897bd733--- /dev/null+++ b/tests/test_fp8_rowwise_backend.py@@ -0,0 +1,93 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Rowwise FP8 (FbgemmFp8Linear) must run where FBGEMM has no kernel (sm120 raises "cutlass cannot initialize").+import os++import pytest++torch = pytest.importorskip("torch")++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),+ reason = "FP8 GEMMs need CUDA sm89+",+)++[email protected](scope = "module")+def F():+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ from unsloth.kernels import fp8+ return fp8+++def _rowwise_weight(N, K):+ w = torch.randn(N, K, device = "cuda") * 0.02+ scale = (w.abs().amax(1, keepdim = True) / 448).float()+ return (w / scale).to(torch.float8_e4m3fn), scale+++def _backends(F):+ device = torch.device("cuda", torch.cuda.current_device())+ ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)+ out = ["dequant"]+ if F._rowwise_gemm_works(+ lambda x: torch._scaled_mm(+ x, x.t(), scale_a = ones((128, 1)), scale_b = ones((1, 128)), out_dtype = torch.bfloat16+ ),+ device,+ ):+ out.append("scaled_mm")+ if F._fp8_rowwise_backend(device) == "fbgemm":+ out.append("fbgemm")+ return out+++def test_backend_is_one_that_runs_here(F):+ backend = F._fp8_rowwise_backend(torch.device("cuda", torch.cuda.current_device()))+ assert backend in ("fbgemm", "scaled_mm", "dequant")+ X = torch.randn(64, 256, device = "cuda", dtype = torch.bfloat16)+ w, s = _rowwise_weight(512, 256)+ y = F.fp8_linear(X, w, s)+ ref = X.float() @ (w.float() * s).t()+ assert torch.isfinite(y).all()+ assert ((y.float() - ref).norm() / ref.norm()) < 0.05++[email protected]("backend", ["dequant", "scaled_mm", "fbgemm"])+def test_every_backend_matches_reference(F, backend):+ if backend not in _backends(F):+ pytest.skip(f"{backend} is not usable on this GPU")+ torch.manual_seed(0)+ w, s = _rowwise_weight(768, 512)+ X = torch.randn(2, 48, 512, device = "cuda", dtype = torch.bfloat16, requires_grad = True)+ # FbgemmFp8Linear stores its bias in float32.+ bias = torch.randn(768, device = "cuda", dtype = torch.float32)+ y = F.FbgemmFp8Linear_matmul.apply(X, w, s, bias, backend)+ W = w.float() * s+ ref = X.detach().float() @ W.t() + bias.float()+ assert y.shape == (2, 48, 768) and y.dtype == torch.bfloat16+ assert ((y.float() - ref).norm() / ref.norm()) < 0.05+ y.float().sum().backward()+ ref_dx = torch.ones(2, 48, 768, device = "cuda") @ W+ assert ((X.grad.float() - ref_dx).norm() / ref_dx.norm()) < 0.01+ # Decode-sized calls take the dequant branch on the scaled_mm backend.+ assert (+ F.FbgemmFp8Linear_matmul.apply(X[:1, :1].detach(), w, s, bias, backend).dtype+ == torch.bfloat16+ )+++def test_quantize_matches_fbgemm(F):+ if not hasattr(torch.ops.fbgemm, "quantize_fp8_per_row"):+ pytest.skip("FBGEMM not installed")+ torch.manual_seed(0)+ x = torch.randn(256, 1024, device = "cuda", dtype = torch.bfloat16)+ x[3] *= 1000+ x[5] = 0+ for scale_ub in (None, torch.tensor([30.0], device = "cuda")):+ q_ref, s_ref = torch.ops.fbgemm.quantize_fp8_per_row(x, scale_ub = scale_ub)+ q, s = F._quantize_fp8_per_row(x, scale_ub)+ # FBGEMM's fast-math division can land 1 ulp lower, flipping exact rounding ties by one FP8 step.+ torch.testing.assert_close(s.view(-1), s_ref, rtol = 1e-6, atol = 0)+ assert (q.float() != q_ref.float()).float().mean() < 1e-3+ steps = (q.float() - q_ref.float()).abs() / q_ref.float().abs().clamp(min = 2**-9)+ assert steps.max() <= 0.125diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex e881cc1e6d5..7805ee58d0e 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -652,6 +652,54 @@ def fp8_torch_block_quant_forward(X, weight, weight_scale):return FP8BlockQuantLinear.apply(X, weight, weight_scale)+def _quantize_fp8_per_row(x, scale_ub = None):+ # Same as FBGEMM quantize_fp8_per_row: scale = max(min(row absmax, scale_ub) / 448, 1 / (448 * 512)), saturating cast.+ row_max = x.abs().amax(dim = -1, keepdim = True).float()+ if scale_ub is not None:+ row_max = torch.minimum(row_max, scale_ub.float())+ scale = torch.clamp(row_max / 448.0, min = 1.0 / (448.0 * 512.0))+ return (x.float() / scale).clamp(-448.0, 448.0).to(torch.float8_e4m3fn), scale+++def _rowwise_gemm_works(gemm, device):+ x = torch.ones(128, 128, dtype = torch.float8_e4m3fn, device = device)+ try:+ return bool((gemm(x) == 128).all())+ except Exception:+ return False++[email protected]_cache(maxsize = None)+def _probe_fp8_rowwise_backend(device):+ """Fastest rowwise FP8 GEMM that runs correctly on this device: FBGEMM ships no sm120 kernels ("cutlass+ cannot initialize" on RTX PRO 6000 / 5090), where torch._scaled_mm keeps FBGEMM's numerics; else dequant."""+ ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)+ if _has_fbgemm_rowwise():+ from unsloth.import_fixes import suppress_cuda_printf+ with suppress_cuda_printf():+ if _rowwise_gemm_works(+ lambda x: torch.ops.fbgemm.f8f8bf16_rowwise(x, x, ones(128), ones(128)), device+ ):+ return "fbgemm"+ if _rowwise_gemm_works(+ lambda x: torch._scaled_mm(+ x, x.t(), scale_a = ones((128, 1)), scale_b = ones((1, 128)), out_dtype = torch.bfloat16+ ),+ device,+ ):+ return "scaled_mm"+ return "dequant"+++def _fp8_rowwise_backend(device):+ return _probe_fp8_rowwise_backend(device)+++if hasattr(torch._dynamo, "assume_constant_result"):+ # Compiled callers run the probe once at trace time and bake the answer in; an lru_cache wrapper would be traced.+ _fp8_rowwise_backend = torch._dynamo.assume_constant_result(_fp8_rowwise_backend)++class FbgemmFp8Linear_matmul(torch.autograd.Function):@staticmethoddef forward(@@ -660,13 +708,15 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):weight,weight_scale,bias = None,+ backend = None,):+ if backend is None:+ backend = _fp8_rowwise_backend(x.device)+ rowwise = weight.shape[0] == weight_scale.shape[0] and not _is_transposed_view(weight)if (- weight.shape[0] == weight_scale.shape[0]+ backend == "fbgemm"+ and rowwiseand (weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0)- and not _is_transposed_view(weight)- and _has_fbgemm_rowwise()- and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)):# The kernel needs weight dims divisible by 8 (else "cutlass cannot implement"), and padding plus# f8f8bf16 is slower than dequant plus bf16 matmul.@@ -694,11 +744,41 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):output = output.to(x.device, x.dtype)output = output.reshape(output_shape)del x_quantized, x_scale+ elif (+ backend == "scaled_mm"+ and rowwise+ and (weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0)+ and x.shape[-1] == weight.shape[1]+ # Decode-sized calls: compiled dequant + GEMV beats quantizing the activation.+ and x.numel() > 64 * x.shape[-1]+ ):+ output_shape = (*x.shape[:-1], -1)+ x_quantized, x_scale = _quantize_fp8_per_row(+ x.reshape(-1, x.shape[-1]), getattr(weight, "input_scale_ub", None)+ )+ if not weight.is_contiguous():+ weight = weight.contiguous()+ output = torch._scaled_mm(+ x_quantized,+ weight.t(),+ scale_a = x_scale,+ scale_b = weight_scale.to(torch.float32).view(1, -1),+ out_dtype = x.dtype,+ use_fast_accum = True,+ )+ output = output + bias if bias is not None else output+ # FbgemmFp8Linear keeps its bias in float32; only cast when it promoted (a no-op .to() aliases).+ if output.dtype != x.dtype:+ output = output.to(x.dtype)+ output = output.reshape(output_shape)+ del x_quantized, x_scaleelif weight_scale.shape[0] in (weight.shape[0], weight.shape[1]):- # Transposed, non-divisible-by-8 (Qwen 2.5 VL 7B 3420x1280), no FBGEMM, or pre-sm89: dequant.+ # Transposed, non-divisible-by-8 (Qwen 2.5 VL 7B 3420x1280), or no working FP8 GEMM here: dequant.W_deq = weight_dequant(weight, weight_scale, x.dtype).Toutput = torch_matmul(x, W_deq)output = output + bias if bias is not None else output+ if output.dtype != x.dtype:+ output = output.to(x.dtype)del W_deqelse:raise ValueError(@@ -724,7 +804,9 @@ def fbgemm_fp8_linear(weight_scale,bias = None,):- return FbgemmFp8Linear_matmul.apply(X, weight, weight_scale, bias)+ return FbgemmFp8Linear_matmul.apply(+ X, weight, weight_scale, bias, _fp8_rowwise_backend(X.device)+ )class FP8_fbgemm_block_linear(torch.autograd.Function):