Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8: keep rowwise _scaled_mm for float32 where cuBLAS supports it (torch < 2.14) (#16393)

Daniel Han3d99bf6c5b25
diff --git a/tests/test_fp8_dequant_max_rows.py b/tests/test_fp8_dequant_max_rows.py
index 43739592e9f..5943e3c3438 100644
--- a/tests/test_fp8_dequant_max_rows.py
+++ b/tests/test_fp8_dequant_max_rows.py
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
-"""scaled_mm rowwise FP8 backend: 17-64 rows go to torch._scaled_mm only when the 16-bit weight spills L2 (CPU)."""
-
import pytest
import torch
from torch._subclasses.fake_tensor import FakeTensorMode
@@ -16,7 +14,6 @@ LARGE = (14336, 4096) # Llama-3.1-8B MLP: 112 MiB in bf16
@pytest.fixture
def route(monkeypatch):
- # NVIDIA routing; on ROCm the direct path is off and every call keeps the compiled path.
monkeypatch.setattr(torch.version, "hip", None, raising = False)
monkeypatch.setattr(F8, "_fp8_rowwise_backend", lambda device: "scaled_mm")
monkeypatch.setattr(F8, "_FP8_ROWWISE_DIRECT", True)
@@ -60,7 +57,6 @@ def test_row_bounds_are_unchanged(route):
def test_autocast_in_another_dtype_keeps_the_compiled_path(
route, monkeypatch, autocast_dtype, expected
):
- # The direct path would round the dequantized weight into X's dtype before autocast's GEMM.
monkeypatch.setattr(
torch, "is_autocast_enabled", lambda device_type = None: autocast_dtype is not None
)
@@ -97,8 +93,6 @@ def test_l2_probe_falls_back_to_zero(monkeypatch):
[((8, 9), L2, L2), ((9, 0), 50 * 1024 * 1024, 0), ((12, 0), 128 * 1024 * 1024, 0)],
)
def test_l2_probe_only_reports_measured_arches(monkeypatch, capability, l2, expected):
- """Only sm89 was measured: H100 without FBGEMM (50 MiB L2) keeps the 64 row default."""
-
class Props:
major, minor = capability
L2_cache_size = l2
@@ -116,7 +110,6 @@ def test_l2_probe_only_reports_measured_arches(monkeypatch, capability, l2, expe
[
(LARGE, True, torch.bfloat16, True),
(SMALL, False, torch.bfloat16, True),
- # Rowwise scaled_mm cannot write float32 before torch 2.14: float32 keeps dequant there.
(LARGE, False, torch.float32, True),
(LARGE, True, torch.float32, False),
],
@@ -136,6 +129,7 @@ def test_autograd_function_uses_the_same_gate(monkeypatch, shape, scaled, dtype,
monkeypatch.setattr(F8, "weight_dequant", fake_dequant)
monkeypatch.setattr(F8, "_fp8_l2_cache_bytes", lambda device: L2)
monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", out_broken)
+ monkeypatch.setattr(F8, "_scaled_mm_float32_out", lambda device: False)
N, K = shape
with FakeTensorMode(), torch.no_grad():
X = torch.empty(1, 32, K, device = "cuda", dtype = dtype)
@@ -147,8 +141,6 @@ def test_autograd_function_uses_the_same_gate(monkeypatch, shape, scaled, dtype,
def test_scaled_mm_asks_for_bf16_when_activations_are_fp16(monkeypatch):
- """torch < 2.14's rowwise CUTLASS kernel writes bf16 bits into an fp16 output when there is no
- bias: fp16 calls request bf16 and cast back; bf16 / fp32 are unchanged."""
monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)
assert F8._scaled_mm_out_dtype(torch.float16) is torch.bfloat16
assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16
@@ -156,7 +148,6 @@ def test_scaled_mm_asks_for_bf16_when_activations_are_fp16(monkeypatch):
def test_scaled_mm_keeps_fp16_output_where_torch_writes_fp16(monkeypatch):
- """torch >= 2.14 honours out.dtype; a bf16 detour would lose precision and overflow 65408..65504."""
monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", False)
assert F8._scaled_mm_out_dtype(torch.float16) is torch.float16
assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16
@@ -169,7 +160,6 @@ def test_scaled_mm_fp16_workaround_follows_the_installed_torch():
broken = torch.__version__ < "2.14.0"
assert F8._SCALED_MM_FP16_OUT_BROKEN is broken
assert F8._scaled_mm_out_dtype(torch.float16) is (torch.bfloat16 if broken else torch.float16)
- # 2.14 pre-releases keep the workaround.
assert TorchVersion("2.13.9+cu130") < "2.14.0"
assert TorchVersion("2.14.0a0+git0") < "2.14.0"
assert not TorchVersion("2.14.0+cu130") < "2.14.0"
@@ -193,7 +183,71 @@ def test_every_rowwise_scaled_mm_call_uses_the_out_dtype_helper():
for kw in node.keywords:
if kw.arg == "out_dtype":
out[node.lineno] = ast.unparse(kw.value)
- # The backend probe asks for bf16 directly; every activation-dtype call goes through the helper.
- data_calls = {line: v for line, v in out.items() if v != "torch.bfloat16"}
+ data_calls = {
+ line: v for line, v in out.items() if v not in ("torch.bfloat16", "torch.float32")
+ }
assert len(data_calls) >= 2, out
assert all(v.startswith("_scaled_mm_out_dtype(") for v in data_calls.values()), out
+
+
[email protected]("float32_out, expected", [(True, "scaled_mm"), (False, "dequant")])
+def test_float32_keeps_scaled_mm_where_rowwise_writes_float32(monkeypatch, float32_out, expected):
+ calls = []
+
+ def fake_scaled_mm(a, b, **kwargs):
+ calls.append(("scaled_mm", kwargs["out_dtype"]))
+ return torch.empty(a.shape[0], b.shape[1], device = a.device, dtype = kwargs["out_dtype"])
+
+ def fake_dequant(w, s, dtype):
+ calls.append(("dequant", dtype))
+ return torch.empty(w.shape, device = w.device, dtype = dtype)
+
+ monkeypatch.setattr(torch, "_scaled_mm", fake_scaled_mm)
+ monkeypatch.setattr(F8, "weight_dequant", fake_dequant)
+ monkeypatch.setattr(F8, "_fp8_l2_cache_bytes", lambda device: L2)
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)
+ monkeypatch.setattr(F8, "_scaled_mm_float32_out", lambda device: float32_out)
+ N, K = LARGE
+ with FakeTensorMode(), torch.no_grad():
+ X = torch.empty(1, 128, K, device = "cuda", dtype = torch.float32)
+ w = torch.empty(N, K, device = "cuda", dtype = torch.float8_e4m3fn)
+ s = torch.empty(N, 1, device = "cuda", dtype = torch.float32)
+ y = F8.FbgemmFp8Linear_matmul.apply(X, w, s, None, "scaled_mm")
+ assert y.shape == (1, 128, N) and y.dtype == torch.float32
+ assert calls == [(expected, torch.float32)]
+
+
[email protected]("cutlass", [True, False])
+def test_float32_probe_asks_scaled_mm_for_float32(monkeypatch, cutlass):
+ seen = []
+
+ def fake_scaled_mm(a, b, scale_a, scale_b, out_dtype, **kwargs):
+ seen.append((tuple(scale_a.shape), tuple(scale_b.shape), out_dtype))
+ if cutlass and out_dtype == torch.float32:
+ raise ValueError(
+ "Only bf16 and fp16 high precision output types are supported for row-wise scaling."
+ )
+ return torch.full((a.shape[0], b.shape[1]), 128.0, dtype = out_dtype)
+
+ monkeypatch.setattr(torch, "_scaled_mm", fake_scaled_mm)
+ F8._probe_scaled_mm_float32_out.cache_clear()
+ try:
+ assert F8._probe_scaled_mm_float32_out(torch.device("cpu")) is (not cutlass)
+ finally:
+ F8._probe_scaled_mm_float32_out.cache_clear()
+ assert seen == [((128, 1), (1, 128), torch.float32)]
+
+
[email protected]("rows, width", [(1, 32), (64, 32), (128, 33)])
+def test_float32_dequant_calls_do_not_probe_scaled_mm(monkeypatch, rows, width):
+ def unexpected_probe(device):
+ raise AssertionError("dequant-only calls must not run the device probe")
+
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)
+ monkeypatch.setattr(F8, "_fp8_dequant_max_rows", lambda *args: 64)
+ monkeypatch.setattr(F8, "_scaled_mm_float32_out", unexpected_probe)
+ x = torch.randn(rows, 32)
+ weight = torch.randn(width, 32).to(torch.float8_e4m3fn)
+ scales = torch.ones(width, 1)
+ result = F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")
+ torch.testing.assert_close(result, x @ weight.float().t())
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 340e9521b13..39e591a81f1 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -2008,6 +2008,26 @@ if hasattr(torch._dynamo, "assume_constant_result"):
_SCALED_MM_FP16_OUT_BROKEN = torch.__version__ < "2.14.0"
[email protected]_cache(maxsize = None)
+def _probe_scaled_mm_float32_out(device):
+ # cuBLAS and older CUTLASS differ on fp32 output (ATen/native/cuda/ScaledBlas.cpp).
+ ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)
+ return _rowwise_gemm_works(
+ lambda x: torch._scaled_mm(
+ x, x.t(), scale_a = ones((128, 1)), scale_b = ones((1, 128)), out_dtype = torch.float32
+ ),
+ device,
+ )
+
+
+def _scaled_mm_float32_out(device):
+ return _probe_scaled_mm_float32_out(device)
+
+
+if hasattr(torch._dynamo, "assume_constant_result"):
+ _scaled_mm_float32_out = torch._dynamo.assume_constant_result(_scaled_mm_float32_out)
+
+
def _scaled_mm_out_dtype(dtype):
"""Rowwise torch._scaled_mm out dtype: bf16 for fp16 on torch < 2.14 (caller casts back), else `dtype`."""
return torch.bfloat16 if dtype == torch.float16 and _SCALED_MM_FP16_OUT_BROKEN else dtype
@@ -2113,13 +2133,16 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
elif (
backend == "scaled_mm"
and rowwise
- # Before torch 2.14 rowwise outputs are bf16 / fp16 only: float32 dequantizes.
- and (x.dtype != torch.float32 or not _SCALED_MM_FP16_OUT_BROKEN)
and (weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0)
and x.shape[-1] == weight.shape[1]
# Decode-sized calls: dequant + 16-bit GEMM beats quantizing the activation unless the weight
# spills L2.
and x.numel() > _fp8_dequant_max_rows(x.device, weight.numel()) * x.shape[-1]
+ and (
+ x.dtype != torch.float32
+ or not _SCALED_MM_FP16_OUT_BROKEN
+ or _scaled_mm_float32_out(x.device)
+ )
):
output_shape = (*x.shape[:-1], -1)
x_quantized, x_scale = _quantize_fp8_per_row(