Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 scaled_mm: keep native fp16 output on torch >= 2.14 (#16063)

Daniel Han9a1ea09dc3a2
diff --git a/tests/test_fp8_dequant_max_rows.py b/tests/test_fp8_dequant_max_rows.py
index 11bffe98a06..5e16ff94a0d 100644
--- a/tests/test_fp8_dequant_max_rows.py
+++ b/tests/test_fp8_dequant_max_rows.py
@@ -121,14 +121,36 @@ def test_autograd_function_uses_the_same_gate(monkeypatch, shape, scaled):
assert calls == ["scaled_mm" if scaled else "dequant"]
-def test_scaled_mm_asks_for_bf16_when_activations_are_fp16():
- """torch 2.10's rowwise CUTLASS kernel writes bf16 bits into an fp16 output when there is no
+def test_scaled_mm_asks_for_bf16_when_activations_are_fp16(monkeypatch):
+ """torch < 2.14's rowwise CUTLASS kernel writes bf16 bits into an fp16 output when there is no
bias: fp16 calls request bf16 and cast back; bf16 / fp32 are unchanged."""
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)
assert F8._scaled_mm_out_dtype(torch.float16) is torch.bfloat16
assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16
assert F8._scaled_mm_out_dtype(torch.float32) is torch.float32
+def test_scaled_mm_keeps_fp16_output_where_torch_writes_fp16(monkeypatch):
+ """torch >= 2.14 honours out.dtype; a bf16 detour would lose precision and overflow 65408..65504."""
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", False)
+ assert F8._scaled_mm_out_dtype(torch.float16) is torch.float16
+ assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16
+ assert F8._scaled_mm_out_dtype(torch.float32) is torch.float32
+
+
+def test_scaled_mm_fp16_workaround_follows_the_installed_torch():
+ from torch.torch_version import TorchVersion
+
+ broken = torch.__version__ < "2.14.0"
+ assert F8._SCALED_MM_FP16_OUT_BROKEN is broken
+ assert F8._scaled_mm_out_dtype(torch.float16) is (torch.bfloat16 if broken else torch.float16)
+ # 2.14 pre-releases keep the workaround.
+ assert TorchVersion("2.13.9+cu130") < "2.14.0"
+ assert TorchVersion("2.14.0a0+git0") < "2.14.0"
+ assert not TorchVersion("2.14.0+cu130") < "2.14.0"
+ assert not TorchVersion("2.15.0") < "2.14.0"
+
+
def test_every_rowwise_scaled_mm_call_uses_the_out_dtype_helper():
import ast
import inspect
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 6b8d63ca72d..e6b1d17d3b1 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -1981,11 +1981,14 @@ if hasattr(torch._dynamo, "assume_constant_result"):
_fp8_rowwise_backend = torch._dynamo.assume_constant_result(_fp8_rowwise_backend)
+# torch < 2.14 rowwise CUTLASS takes its output type from the bias dtype: no bias + fp16 out = bf16 bits (garbage).
+# 2.14 dispatches on out.dtype, where a bf16 detour would lose precision and overflow 65408..65504. Pre-releases keep it.
+_SCALED_MM_FP16_OUT_BROKEN = torch.__version__ < "2.14.0"
+
+
def _scaled_mm_out_dtype(dtype):
- """Rowwise torch._scaled_mm output dtype for activations of `dtype`. torch 2.10's rowwise CUTLASS
- kernel picks its output element type from the bias dtype, so without a bias an fp16 output buffer
- receives bf16 bits (every value garbage); ask for bf16 and let the caller cast back."""
- return torch.bfloat16 if dtype == torch.float16 else dtype
+ """Rowwise torch._scaled_mm out dtype: bf16 for fp16 on torch < 2.14 (caller casts back), else `dtype`."""
+ return torch.bfloat16 if dtype == torch.float16 and _SCALED_MM_FP16_OUT_BROKEN else dtype
def _parse_fp8_dequant_max_rows(value):