Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Fall back from FBGEMM for rowwise FP8 on GPUs it has no kernel for (RTX PRO 6000 / 5090) (#12098)

Daniel Han5d6ecc420484
diff --git a/tests/test_fp8_rowwise_backend.py b/tests/test_fp8_rowwise_backend.py
new file mode 100644
index 00000000000..917897bd733
--- /dev/null
+++ b/tests/test_fp8_rowwise_backend.py
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Rowwise FP8 (FbgemmFp8Linear) must run where FBGEMM has no kernel (sm120 raises "cutlass cannot initialize").
+import os
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
+ reason = "FP8 GEMMs need CUDA sm89+",
+)
+
+
[email protected](scope = "module")
+def F():
+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")
+ from unsloth.kernels import fp8
+ return fp8
+
+
+def _rowwise_weight(N, K):
+ w = torch.randn(N, K, device = "cuda") * 0.02
+ scale = (w.abs().amax(1, keepdim = True) / 448).float()
+ return (w / scale).to(torch.float8_e4m3fn), scale
+
+
+def _backends(F):
+ device = torch.device("cuda", torch.cuda.current_device())
+ ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)
+ out = ["dequant"]
+ if F._rowwise_gemm_works(
+ lambda x: torch._scaled_mm(
+ x, x.t(), scale_a = ones((128, 1)), scale_b = ones((1, 128)), out_dtype = torch.bfloat16
+ ),
+ device,
+ ):
+ out.append("scaled_mm")
+ if F._fp8_rowwise_backend(device) == "fbgemm":
+ out.append("fbgemm")
+ return out
+
+
+def test_backend_is_one_that_runs_here(F):
+ backend = F._fp8_rowwise_backend(torch.device("cuda", torch.cuda.current_device()))
+ assert backend in ("fbgemm", "scaled_mm", "dequant")
+ X = torch.randn(64, 256, device = "cuda", dtype = torch.bfloat16)
+ w, s = _rowwise_weight(512, 256)
+ y = F.fp8_linear(X, w, s)
+ ref = X.float() @ (w.float() * s).t()
+ assert torch.isfinite(y).all()
+ assert ((y.float() - ref).norm() / ref.norm()) < 0.05
+
+
[email protected]("backend", ["dequant", "scaled_mm", "fbgemm"])
+def test_every_backend_matches_reference(F, backend):
+ if backend not in _backends(F):
+ pytest.skip(f"{backend} is not usable on this GPU")
+ torch.manual_seed(0)
+ w, s = _rowwise_weight(768, 512)
+ X = torch.randn(2, 48, 512, device = "cuda", dtype = torch.bfloat16, requires_grad = True)
+ # FbgemmFp8Linear stores its bias in float32.
+ bias = torch.randn(768, device = "cuda", dtype = torch.float32)
+ y = F.FbgemmFp8Linear_matmul.apply(X, w, s, bias, backend)
+ W = w.float() * s
+ ref = X.detach().float() @ W.t() + bias.float()
+ assert y.shape == (2, 48, 768) and y.dtype == torch.bfloat16
+ assert ((y.float() - ref).norm() / ref.norm()) < 0.05
+ y.float().sum().backward()
+ ref_dx = torch.ones(2, 48, 768, device = "cuda") @ W
+ assert ((X.grad.float() - ref_dx).norm() / ref_dx.norm()) < 0.01
+ # Decode-sized calls take the dequant branch on the scaled_mm backend.
+ assert (
+ F.FbgemmFp8Linear_matmul.apply(X[:1, :1].detach(), w, s, bias, backend).dtype
+ == torch.bfloat16
+ )
+
+
+def test_quantize_matches_fbgemm(F):
+ if not hasattr(torch.ops.fbgemm, "quantize_fp8_per_row"):
+ pytest.skip("FBGEMM not installed")
+ torch.manual_seed(0)
+ x = torch.randn(256, 1024, device = "cuda", dtype = torch.bfloat16)
+ x[3] *= 1000
+ x[5] = 0
+ for scale_ub in (None, torch.tensor([30.0], device = "cuda")):
+ q_ref, s_ref = torch.ops.fbgemm.quantize_fp8_per_row(x, scale_ub = scale_ub)
+ q, s = F._quantize_fp8_per_row(x, scale_ub)
+ # FBGEMM's fast-math division can land 1 ulp lower, flipping exact rounding ties by one FP8 step.
+ torch.testing.assert_close(s.view(-1), s_ref, rtol = 1e-6, atol = 0)
+ assert (q.float() != q_ref.float()).float().mean() < 1e-3
+ steps = (q.float() - q_ref.float()).abs() / q_ref.float().abs().clamp(min = 2**-9)
+ assert steps.max() <= 0.125
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index e881cc1e6d5..7805ee58d0e 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -652,6 +652,54 @@ def fp8_torch_block_quant_forward(X, weight, weight_scale):
return FP8BlockQuantLinear.apply(X, weight, weight_scale)
+def _quantize_fp8_per_row(x, scale_ub = None):
+ # Same as FBGEMM quantize_fp8_per_row: scale = max(min(row absmax, scale_ub) / 448, 1 / (448 * 512)), saturating cast.
+ row_max = x.abs().amax(dim = -1, keepdim = True).float()
+ if scale_ub is not None:
+ row_max = torch.minimum(row_max, scale_ub.float())
+ scale = torch.clamp(row_max / 448.0, min = 1.0 / (448.0 * 512.0))
+ return (x.float() / scale).clamp(-448.0, 448.0).to(torch.float8_e4m3fn), scale
+
+
+def _rowwise_gemm_works(gemm, device):
+ x = torch.ones(128, 128, dtype = torch.float8_e4m3fn, device = device)
+ try:
+ return bool((gemm(x) == 128).all())
+ except Exception:
+ return False
+
+
[email protected]_cache(maxsize = None)
+def _probe_fp8_rowwise_backend(device):
+ """Fastest rowwise FP8 GEMM that runs correctly on this device: FBGEMM ships no sm120 kernels ("cutlass
+ cannot initialize" on RTX PRO 6000 / 5090), where torch._scaled_mm keeps FBGEMM's numerics; else dequant."""
+ ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)
+ if _has_fbgemm_rowwise():
+ from unsloth.import_fixes import suppress_cuda_printf
+ with suppress_cuda_printf():
+ if _rowwise_gemm_works(
+ lambda x: torch.ops.fbgemm.f8f8bf16_rowwise(x, x, ones(128), ones(128)), device
+ ):
+ return "fbgemm"
+ if _rowwise_gemm_works(
+ lambda x: torch._scaled_mm(
+ x, x.t(), scale_a = ones((128, 1)), scale_b = ones((1, 128)), out_dtype = torch.bfloat16
+ ),
+ device,
+ ):
+ return "scaled_mm"
+ return "dequant"
+
+
+def _fp8_rowwise_backend(device):
+ return _probe_fp8_rowwise_backend(device)
+
+
+if hasattr(torch._dynamo, "assume_constant_result"):
+ # Compiled callers run the probe once at trace time and bake the answer in; an lru_cache wrapper would be traced.
+ _fp8_rowwise_backend = torch._dynamo.assume_constant_result(_fp8_rowwise_backend)
+
+
class FbgemmFp8Linear_matmul(torch.autograd.Function):
@staticmethod
def forward(
@@ -660,13 +708,15 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
weight,
weight_scale,
bias = None,
+ backend = None,
):
+ if backend is None:
+ backend = _fp8_rowwise_backend(x.device)
+ rowwise = weight.shape[0] == weight_scale.shape[0] and not _is_transposed_view(weight)
if (
- weight.shape[0] == weight_scale.shape[0]
+ backend == "fbgemm"
+ and rowwise
and (weight.shape[0] % 8 == 0 and weight.shape[1] % 8 == 0)
- and not _is_transposed_view(weight)
- and _has_fbgemm_rowwise()
- and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)
):
# The kernel needs weight dims divisible by 8 (else "cutlass cannot implement"), and padding plus
# f8f8bf16 is slower than dequant plus bf16 matmul.
@@ -694,11 +744,41 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
output = output.to(x.device, x.dtype)
output = output.reshape(output_shape)
del x_quantized, x_scale
+ elif (
+ backend == "scaled_mm"
+ and rowwise
+ and (weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0)
+ and x.shape[-1] == weight.shape[1]
+ # Decode-sized calls: compiled dequant + GEMV beats quantizing the activation.
+ and x.numel() > 64 * x.shape[-1]
+ ):
+ output_shape = (*x.shape[:-1], -1)
+ x_quantized, x_scale = _quantize_fp8_per_row(
+ x.reshape(-1, x.shape[-1]), getattr(weight, "input_scale_ub", None)
+ )
+ if not weight.is_contiguous():
+ weight = weight.contiguous()
+ output = torch._scaled_mm(
+ x_quantized,
+ weight.t(),
+ scale_a = x_scale,
+ scale_b = weight_scale.to(torch.float32).view(1, -1),
+ out_dtype = x.dtype,
+ use_fast_accum = True,
+ )
+ output = output + bias if bias is not None else output
+ # FbgemmFp8Linear keeps its bias in float32; only cast when it promoted (a no-op .to() aliases).
+ if output.dtype != x.dtype:
+ output = output.to(x.dtype)
+ output = output.reshape(output_shape)
+ del x_quantized, x_scale
elif weight_scale.shape[0] in (weight.shape[0], weight.shape[1]):
- # Transposed, non-divisible-by-8 (Qwen 2.5 VL 7B 3420x1280), no FBGEMM, or pre-sm89: dequant.
+ # Transposed, non-divisible-by-8 (Qwen 2.5 VL 7B 3420x1280), or no working FP8 GEMM here: dequant.
W_deq = weight_dequant(weight, weight_scale, x.dtype).T
output = torch_matmul(x, W_deq)
output = output + bias if bias is not None else output
+ if output.dtype != x.dtype:
+ output = output.to(x.dtype)
del W_deq
else:
raise ValueError(
@@ -724,7 +804,9 @@ def fbgemm_fp8_linear(
weight_scale,
bias = None,
):
- return FbgemmFp8Linear_matmul.apply(X, weight, weight_scale, bias)
+ return FbgemmFp8Linear_matmul.apply(
+ X, weight, weight_scale, bias, _fp8_rowwise_backend(X.device)
+ )
class FP8_fbgemm_block_linear(torch.autograd.Function):