Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8: keep rowwise _scaled_mm for float32 where cuBLAS supports it (torch < 2.14) (#16393)
Daniel Han3d99bf6c5b25
diff --git a/tests/test_fp8_dequant_max_rows.py b/tests/test_fp8_dequant_max_rows.pyindex 43739592e9f..5943e3c3438 100644--- a/tests/test_fp8_dequant_max_rows.py+++ b/tests/test_fp8_dequant_max_rows.py@@ -1,8 +1,6 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.-"""scaled_mm rowwise FP8 backend: 17-64 rows go to torch._scaled_mm only when the 16-bit weight spills L2 (CPU)."""-import pytestimport torchfrom torch._subclasses.fake_tensor import FakeTensorMode@@ -16,7 +14,6 @@ LARGE = (14336, 4096) # Llama-3.1-8B MLP: 112 MiB in bf16@pytest.fixturedef route(monkeypatch):- # NVIDIA routing; on ROCm the direct path is off and every call keeps the compiled path.monkeypatch.setattr(torch.version, "hip", None, raising = False)monkeypatch.setattr(F8, "_fp8_rowwise_backend", lambda device: "scaled_mm")monkeypatch.setattr(F8, "_FP8_ROWWISE_DIRECT", True)@@ -60,7 +57,6 @@ def test_row_bounds_are_unchanged(route):def test_autocast_in_another_dtype_keeps_the_compiled_path(route, monkeypatch, autocast_dtype, expected):- # The direct path would round the dequantized weight into X's dtype before autocast's GEMM.monkeypatch.setattr(torch, "is_autocast_enabled", lambda device_type = None: autocast_dtype is not None)@@ -97,8 +93,6 @@ def test_l2_probe_falls_back_to_zero(monkeypatch):[((8, 9), L2, L2), ((9, 0), 50 * 1024 * 1024, 0), ((12, 0), 128 * 1024 * 1024, 0)],)def test_l2_probe_only_reports_measured_arches(monkeypatch, capability, l2, expected):- """Only sm89 was measured: H100 without FBGEMM (50 MiB L2) keeps the 64 row default."""-class Props:major, minor = capabilityL2_cache_size = l2@@ -116,7 +110,6 @@ def test_l2_probe_only_reports_measured_arches(monkeypatch, capability, l2, expe[(LARGE, True, torch.bfloat16, True),(SMALL, False, torch.bfloat16, True),- # Rowwise scaled_mm cannot write float32 before torch 2.14: float32 keeps dequant there.(LARGE, False, torch.float32, True),(LARGE, True, torch.float32, False),],@@ -136,6 +129,7 @@ def test_autograd_function_uses_the_same_gate(monkeypatch, shape, scaled, dtype,monkeypatch.setattr(F8, "weight_dequant", fake_dequant)monkeypatch.setattr(F8, "_fp8_l2_cache_bytes", lambda device: L2)monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", out_broken)+ monkeypatch.setattr(F8, "_scaled_mm_float32_out", lambda device: False)N, K = shapewith FakeTensorMode(), torch.no_grad():X = torch.empty(1, 32, K, device = "cuda", dtype = dtype)@@ -147,8 +141,6 @@ def test_autograd_function_uses_the_same_gate(monkeypatch, shape, scaled, dtype,def test_scaled_mm_asks_for_bf16_when_activations_are_fp16(monkeypatch):- """torch < 2.14's rowwise CUTLASS kernel writes bf16 bits into an fp16 output when there is no- bias: fp16 calls request bf16 and cast back; bf16 / fp32 are unchanged."""monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)assert F8._scaled_mm_out_dtype(torch.float16) is torch.bfloat16assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16@@ -156,7 +148,6 @@ def test_scaled_mm_asks_for_bf16_when_activations_are_fp16(monkeypatch):def test_scaled_mm_keeps_fp16_output_where_torch_writes_fp16(monkeypatch):- """torch >= 2.14 honours out.dtype; a bf16 detour would lose precision and overflow 65408..65504."""monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", False)assert F8._scaled_mm_out_dtype(torch.float16) is torch.float16assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16@@ -169,7 +160,6 @@ def test_scaled_mm_fp16_workaround_follows_the_installed_torch():broken = torch.__version__ < "2.14.0"assert F8._SCALED_MM_FP16_OUT_BROKEN is brokenassert F8._scaled_mm_out_dtype(torch.float16) is (torch.bfloat16 if broken else torch.float16)- # 2.14 pre-releases keep the workaround.assert TorchVersion("2.13.9+cu130") < "2.14.0"assert TorchVersion("2.14.0a0+git0") < "2.14.0"assert not TorchVersion("2.14.0+cu130") < "2.14.0"@@ -193,7 +183,71 @@ def test_every_rowwise_scaled_mm_call_uses_the_out_dtype_helper():for kw in node.keywords:if kw.arg == "out_dtype":out[node.lineno] = ast.unparse(kw.value)- # The backend probe asks for bf16 directly; every activation-dtype call goes through the helper.- data_calls = {line: v for line, v in out.items() if v != "torch.bfloat16"}+ data_calls = {+ line: v for line, v in out.items() if v not in ("torch.bfloat16", "torch.float32")+ }assert len(data_calls) >= 2, outassert all(v.startswith("_scaled_mm_out_dtype(") for v in data_calls.values()), out++[email protected]("float32_out, expected", [(True, "scaled_mm"), (False, "dequant")])+def test_float32_keeps_scaled_mm_where_rowwise_writes_float32(monkeypatch, float32_out, expected):+ calls = []++ def fake_scaled_mm(a, b, **kwargs):+ calls.append(("scaled_mm", kwargs["out_dtype"]))+ return torch.empty(a.shape[0], b.shape[1], device = a.device, dtype = kwargs["out_dtype"])++ def fake_dequant(w, s, dtype):+ calls.append(("dequant", dtype))+ return torch.empty(w.shape, device = w.device, dtype = dtype)++ monkeypatch.setattr(torch, "_scaled_mm", fake_scaled_mm)+ monkeypatch.setattr(F8, "weight_dequant", fake_dequant)+ monkeypatch.setattr(F8, "_fp8_l2_cache_bytes", lambda device: L2)+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)+ monkeypatch.setattr(F8, "_scaled_mm_float32_out", lambda device: float32_out)+ N, K = LARGE+ with FakeTensorMode(), torch.no_grad():+ X = torch.empty(1, 128, K, device = "cuda", dtype = torch.float32)+ w = torch.empty(N, K, device = "cuda", dtype = torch.float8_e4m3fn)+ s = torch.empty(N, 1, device = "cuda", dtype = torch.float32)+ y = F8.FbgemmFp8Linear_matmul.apply(X, w, s, None, "scaled_mm")+ assert y.shape == (1, 128, N) and y.dtype == torch.float32+ assert calls == [(expected, torch.float32)]++[email protected]("cutlass", [True, False])+def test_float32_probe_asks_scaled_mm_for_float32(monkeypatch, cutlass):+ seen = []++ def fake_scaled_mm(a, b, scale_a, scale_b, out_dtype, **kwargs):+ seen.append((tuple(scale_a.shape), tuple(scale_b.shape), out_dtype))+ if cutlass and out_dtype == torch.float32:+ raise ValueError(+ "Only bf16 and fp16 high precision output types are supported for row-wise scaling."+ )+ return torch.full((a.shape[0], b.shape[1]), 128.0, dtype = out_dtype)++ monkeypatch.setattr(torch, "_scaled_mm", fake_scaled_mm)+ F8._probe_scaled_mm_float32_out.cache_clear()+ try:+ assert F8._probe_scaled_mm_float32_out(torch.device("cpu")) is (not cutlass)+ finally:+ F8._probe_scaled_mm_float32_out.cache_clear()+ assert seen == [((128, 1), (1, 128), torch.float32)]++[email protected]("rows, width", [(1, 32), (64, 32), (128, 33)])+def test_float32_dequant_calls_do_not_probe_scaled_mm(monkeypatch, rows, width):+ def unexpected_probe(device):+ raise AssertionError("dequant-only calls must not run the device probe")++ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)+ monkeypatch.setattr(F8, "_fp8_dequant_max_rows", lambda *args: 64)+ monkeypatch.setattr(F8, "_scaled_mm_float32_out", unexpected_probe)+ x = torch.randn(rows, 32)+ weight = torch.randn(width, 32).to(torch.float8_e4m3fn)+ scales = torch.ones(width, 1)+ result = F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")+ torch.testing.assert_close(result, x @ weight.float().t())diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 340e9521b13..39e591a81f1 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -2008,6 +2008,26 @@ if hasattr(torch._dynamo, "assume_constant_result"):_SCALED_MM_FP16_OUT_BROKEN = torch.__version__ < "2.14.0"[email protected]_cache(maxsize = None)+def _probe_scaled_mm_float32_out(device):+ # cuBLAS and older CUTLASS differ on fp32 output (ATen/native/cuda/ScaledBlas.cpp).+ ones = lambda n: torch.ones(n, dtype = torch.float32, device = device)+ return _rowwise_gemm_works(+ lambda x: torch._scaled_mm(+ x, x.t(), scale_a = ones((128, 1)), scale_b = ones((1, 128)), out_dtype = torch.float32+ ),+ device,+ )+++def _scaled_mm_float32_out(device):+ return _probe_scaled_mm_float32_out(device)+++if hasattr(torch._dynamo, "assume_constant_result"):+ _scaled_mm_float32_out = torch._dynamo.assume_constant_result(_scaled_mm_float32_out)++def _scaled_mm_out_dtype(dtype):"""Rowwise torch._scaled_mm out dtype: bf16 for fp16 on torch < 2.14 (caller casts back), else `dtype`."""return torch.bfloat16 if dtype == torch.float16 and _SCALED_MM_FP16_OUT_BROKEN else dtype@@ -2113,13 +2133,16 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):elif (backend == "scaled_mm"and rowwise- # Before torch 2.14 rowwise outputs are bf16 / fp16 only: float32 dequantizes.- and (x.dtype != torch.float32 or not _SCALED_MM_FP16_OUT_BROKEN)and (weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0)and x.shape[-1] == weight.shape[1]# Decode-sized calls: dequant + 16-bit GEMM beats quantizing the activation unless the weight# spills L2.and x.numel() > _fp8_dequant_max_rows(x.device, weight.numel()) * x.shape[-1]+ and (+ x.dtype != torch.float32+ or not _SCALED_MM_FP16_OUT_BROKEN+ or _scaled_mm_float32_out(x.device)+ )):output_shape = (*x.shape[:-1], -1)x_quantized, x_scale = _quantize_fp8_per_row(