Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 rowwise scaled_mm: keep fp16 output in range before torch 2.14, and drop the row guard for always-dequant dtypes when compiled (#16598)

Daniel Han1e4643cb3d08
diff --git a/tests/test_fp8_dequant_max_rows.py b/tests/test_fp8_dequant_max_rows.py
index 5943e3c3438..e1df0eade12 100644
--- a/tests/test_fp8_dequant_max_rows.py
+++ b/tests/test_fp8_dequant_max_rows.py
@@ -141,14 +141,14 @@ def test_autograd_function_uses_the_same_gate(monkeypatch, shape, scaled, dtype,
def test_scaled_mm_asks_for_bf16_when_activations_are_fp16(monkeypatch):
- monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_DEQUANT_ONLY", True)
assert F8._scaled_mm_out_dtype(torch.float16) is torch.bfloat16
assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16
assert F8._scaled_mm_out_dtype(torch.float32) is torch.float32
def test_scaled_mm_keeps_fp16_output_where_torch_writes_fp16(monkeypatch):
- monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", False)
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_DEQUANT_ONLY", False)
assert F8._scaled_mm_out_dtype(torch.float16) is torch.float16
assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16
assert F8._scaled_mm_out_dtype(torch.float32) is torch.float32
@@ -158,8 +158,11 @@ def test_scaled_mm_fp16_workaround_follows_the_installed_torch():
from torch.torch_version import TorchVersion
broken = torch.__version__ < "2.14.0"
+ detour = broken and not F8._SCALED_MM_FP16_BIAS_OUT
assert F8._SCALED_MM_FP16_OUT_BROKEN is broken
- assert F8._scaled_mm_out_dtype(torch.float16) is (torch.bfloat16 if broken else torch.float16)
+ assert F8._SCALED_MM_FP16_DEQUANT_ONLY is detour
+ assert F8._SCALED_MM_FP16_ZERO_BIAS is (broken and F8._SCALED_MM_FP16_BIAS_OUT)
+ assert F8._scaled_mm_out_dtype(torch.float16) is (torch.bfloat16 if detour else torch.float16)
assert TorchVersion("2.13.9+cu130") < "2.14.0"
assert TorchVersion("2.14.0a0+git0") < "2.14.0"
assert not TorchVersion("2.14.0+cu130") < "2.14.0"
@@ -179,15 +182,19 @@ def test_every_rowwise_scaled_mm_call_uses_the_out_dtype_helper():
and node.func.attr == "_scaled_mm"
]
out = {}
+ bias = {}
for node in calls:
for kw in node.keywords:
if kw.arg == "out_dtype":
out[node.lineno] = ast.unparse(kw.value)
+ if kw.arg == "bias":
+ bias[node.lineno] = ast.unparse(kw.value)
data_calls = {
line: v for line, v in out.items() if v not in ("torch.bfloat16", "torch.float32")
}
assert len(data_calls) >= 2, out
assert all(v.startswith("_scaled_mm_out_dtype(") for v in data_calls.values()), out
+ assert all(bias.get(line, "").startswith("_scaled_mm_fp16_bias(") for line in data_calls), bias
@pytest.mark.parametrize("float32_out, expected", [(True, "scaled_mm"), (False, "dequant")])
diff --git a/tests/test_fp8_rowwise_compile_guards.py b/tests/test_fp8_rowwise_compile_guards.py
new file mode 100644
index 00000000000..a01c0055a87
--- /dev/null
+++ b/tests/test_fp8_rowwise_compile_guards.py
@@ -0,0 +1,308 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Rowwise FP8 scaled_mm route: fp16 range before torch 2.14, and no row-count guard for calls that always
+dequantize when compiled. CPU tensors; torch._scaled_mm and the dequant GEMM run for real."""
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda
+import torch._dynamo
+from torch._subclasses.fake_tensor import FakeTensorMode
+
+from unsloth.kernels import fp8 as F8
+
+LARGE = (14336, 4096) # Llama-3.1-8B MLP: 112 MiB in bf16, spills L4's 48 MiB L2
+
+
+def _cpu_scaled_mm_rowwise_float32():
+ # CPU rowwise _scaled_mm with float32 output: torch 2.10 and older only support per-tensor scales there.
+ try:
+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)
+ torch._scaled_mm(
+ a, a.t(), scale_a = torch.ones(16, 1), scale_b = torch.ones(1, 16), out_dtype = torch.float32
+ )
+ return True
+ except Exception:
+ return False
+
+
+def _cpu_scaled_mm_fp16():
+ try:
+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)
+ ones = torch.ones(16, 1)
+ torch._scaled_mm(
+ a,
+ a.t(),
+ scale_a = ones,
+ scale_b = ones.t(),
+ bias = torch.zeros(16, dtype = torch.float16),
+ out_dtype = torch.float16,
+ )
+ return True
+ except Exception:
+ return False
+
+
+def _fp16_mode(monkeypatch, mode):
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", mode != "native")
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS", mode == "bias")
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_DEQUANT_ONLY", mode == "dequant")
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS_CACHE", {})
+
+
[email protected]("mode", ["bias", "dequant", "native"])
+def test_fp16_scaled_mm_out_dtype_and_bias_follow_the_fp16_mode(monkeypatch, mode):
+ _fp16_mode(monkeypatch, mode)
+ x = torch.empty(4, 32, dtype = torch.float16)
+ w = torch.empty(48, 32, dtype = torch.float8_e4m3fn)
+ assert F8._scaled_mm_out_dtype(torch.float16) is (
+ torch.bfloat16 if mode == "dequant" else torch.float16
+ )
+ assert F8._scaled_mm_dtype_supported(x) is (mode != "dequant")
+ bias = F8._scaled_mm_fp16_bias(x, w)
+ if mode == "bias":
+ assert bias.dtype == torch.float16 and bias.shape == (48,) and not bias.any()
+ assert F8._scaled_mm_fp16_bias(x, w) is bias
+ assert (
+ F8._scaled_mm_fp16_bias(x, torch.empty(64, 32, dtype = torch.float8_e4m3fn)) is not bias
+ )
+ assert len(F8._SCALED_MM_FP16_ZERO_BIAS_CACHE) == 2
+ else:
+ assert bias is None
+ for dtype in (torch.bfloat16, torch.float32):
+ assert F8._scaled_mm_out_dtype(dtype) is dtype
+ assert F8._scaled_mm_fp16_bias(x.to(dtype), w) is None
+
+
+def test_fp16_zero_bias_cache_is_bounded_and_skips_fake_tensors(monkeypatch):
+ _fp16_mode(monkeypatch, "bias")
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS_CACHE_MAX", 4)
+ x = torch.empty(4, 32, dtype = torch.float16)
+ for n in range(16, 16 * 11, 16):
+ F8._scaled_mm_fp16_bias(x, torch.empty(n, 32, dtype = torch.float8_e4m3fn))
+ assert len(F8._SCALED_MM_FP16_ZERO_BIAS_CACHE) <= 4
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS_CACHE", {})
+ with FakeTensorMode():
+ fx = torch.empty(4, 32, dtype = torch.float16)
+ fw = torch.empty(48, 32, dtype = torch.float8_e4m3fn)
+ assert F8._scaled_mm_fp16_bias(fx, fw) is not None
+ assert F8._SCALED_MM_FP16_ZERO_BIAS_CACHE == {}
+ with torch.inference_mode():
+ bias = F8._scaled_mm_fp16_bias(x, torch.empty(48, 32, dtype = torch.float8_e4m3fn))
+ assert not bias.is_inference()
+
+
+def test_fp16_flags_follow_the_installed_torch():
+ from torch.torch_version import TorchVersion
+
+ assert F8._SCALED_MM_FP16_BIAS_OUT is (
+ torch.__version__ >= "2.10.0" and torch.version.hip is None
+ )
+ broken = F8._SCALED_MM_FP16_OUT_BROKEN
+ assert F8._SCALED_MM_FP16_ZERO_BIAS is (broken and F8._SCALED_MM_FP16_BIAS_OUT)
+ assert F8._SCALED_MM_FP16_DEQUANT_ONLY is (broken and not F8._SCALED_MM_FP16_BIAS_OUT)
+ # 2.9 CUTLASS rowwise accepts only float32 / bf16 bias and bf16 output.
+ assert not TorchVersion("2.9.1+cu128") >= "2.10.0"
+ assert TorchVersion("2.10.0+cu128") >= "2.10.0"
+
+
[email protected]("compiled", [False, True])
[email protected]("mode", ["bias", "dequant"])
[email protected]("rows", [80, 300])
+def test_fp16_output_near_the_fp16_max_stays_finite(monkeypatch, compiled, mode, rows):
+ # Every output is exactly 65504: a bf16 detour rounds it to 65536 = fp16 inf.
+ if mode == "bias" and not _cpu_scaled_mm_fp16():
+ pytest.skip("needs CPU torch._scaled_mm with fp16 output")
+ _fp16_mode(monkeypatch, mode)
+ calls = []
+ real_scaled_mm = torch._scaled_mm
+
+ def counting_scaled_mm(*args, **kwargs):
+ calls.append(kwargs.get("out_dtype"))
+ return real_scaled_mm(*args, **kwargs)
+
+ monkeypatch.setattr(torch, "_scaled_mm", counting_scaled_mm)
+ N, K = 64, 32
+ x = torch.ones(rows, K, dtype = torch.float16)
+ weight = torch.ones(N, K).to(torch.float8_e4m3fn)
+ scales = torch.full((N, 1), 65504.0 / K)
+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")
+ if compiled:
+ torch._dynamo.reset()
+ fn = torch.compile(fn, backend = "aot_eager", fullgraph = True)
+ out = fn(x)
+ assert out.dtype == torch.float16
+ assert torch.isfinite(out).all()
+ assert (out == 65504).all()
+ if mode == "dequant":
+ assert calls == []
+ else:
+ assert calls and all(d is torch.float16 for d in calls)
+
+
[email protected]("rows", [17, 32, 64, 65, 128, 4096])
[email protected]("mode", ["bias", "dequant", "native"])
+def test_fp16_routing_decision(monkeypatch, mode, rows):
+ monkeypatch.setattr(torch.version, "hip", None, raising = False)
+ monkeypatch.setattr(F8, "_fp8_rowwise_backend", lambda device: "scaled_mm")
+ monkeypatch.setattr(F8, "_FP8_ROWWISE_DIRECT", True)
+ monkeypatch.setattr(F8, "_FP8_DEQUANT_MAX_ROWS_ENV", None)
+ monkeypatch.setattr(F8, "_FP8_DEQUANT_MAX_ROWS", 64)
+ monkeypatch.setattr(F8, "_fp8_l2_cache_bytes", lambda device: 48 * 1024 * 1024)
+ # A real pre-sm89 GPU would refuse e4m3fn here; the decision under test is the same on every card.
+ monkeypatch.setattr(F8, "_fp8_e4m3fn_unsupported_in_trace", lambda weight: False)
+ _fp16_mode(monkeypatch, mode)
+ calls = []
+
+ def fake_scaled_mm(a, b, **kwargs):
+ calls.append(("scaled_mm", kwargs["out_dtype"]))
+ return torch.empty(a.shape[0], b.shape[1], device = a.device, dtype = kwargs["out_dtype"])
+
+ def fake_dequant(w, s, dtype):
+ calls.append(("dequant", dtype))
+ return torch.empty(w.shape, device = w.device, dtype = dtype)
+
+ monkeypatch.setattr(torch, "_scaled_mm", fake_scaled_mm)
+ monkeypatch.setattr(F8, "weight_dequant", fake_dequant)
+ N, K = LARGE
+ scaled_rows = rows > 16 # the L2 gate: this weight spills 48 MiB, so 17+ rows take scaled_mm
+ for dtype in (torch.float16, torch.bfloat16):
+ expected = (
+ "scaled_mm"
+ if scaled_rows and (dtype == torch.bfloat16 or mode != "dequant")
+ else "dequant"
+ )
+ calls.clear()
+ with FakeTensorMode(), torch.no_grad():
+ X = torch.empty(1, rows, K, device = "cuda", dtype = dtype)
+ w = torch.empty(N, K, device = "cuda", dtype = torch.float8_e4m3fn)
+ s = torch.empty(N, 1, device = "cuda", dtype = torch.float32)
+ assert F8._fp8_rowwise_direct_route(X, w, s) == expected, (dtype, rows)
+ y = F8.FbgemmFp8Linear_matmul.apply(X, w, s, None, "scaled_mm")
+ assert y.shape == (1, rows, N) and y.dtype == dtype
+ assert calls == [
+ (expected, dtype if expected == "dequant" else F8._scaled_mm_out_dtype(dtype))
+ ]
+ assert F8._SCALED_MM_FP16_ZERO_BIAS_CACHE == {}
+
+
+def _compile_count(
+ fn,
+ rows_seq,
+ K,
+ dtype,
+ mark = None,
+):
+ from torch._dynamo.backends.debugging import aot_eager
+
+ graphs = []
+
+ def backend(gm, example_inputs):
+ graphs.append(gm)
+ return aot_eager(gm, example_inputs)
+
+ torch._dynamo.reset()
+ compiled = torch.compile(fn, backend = backend, dynamic = mark is None)
+ for rows in rows_seq:
+ x = torch.randn(rows, K, dtype = dtype)
+ if mark is not None:
+ torch._dynamo.mark_dynamic(x, 0, min = mark[0], max = mark[1])
+ compiled(x)
+ return len(graphs)
+
+
[email protected]("mark", [None, (2, 256)])
+def test_compiled_float32_dequant_adds_no_row_guard(monkeypatch, mark):
+ # Always-dequant dtypes must not guard on the row count (one graph, strict mark_dynamic holds).
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)
+ monkeypatch.setattr(
+ F8, "_scaled_mm_float32_out", torch._dynamo.assume_constant_result(lambda device: False)
+ )
+ N, K = 64, 32
+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)
+ scales = torch.rand(N, 1) + 0.5
+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")
+ assert _compile_count(fn, [8, 128, 64, 256], K, torch.float32, mark) == 1
+
+
+ not _cpu_scaled_mm_rowwise_float32(),
+ reason = "needs CPU rowwise torch._scaled_mm with float32 output",
+)
+def test_compiled_float32_with_float32_output_still_routes_by_rows(monkeypatch):
+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)
+ monkeypatch.setattr(
+ F8, "_scaled_mm_float32_out", torch._dynamo.assume_constant_result(lambda device: True)
+ )
+ N, K = 64, 32
+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)
+ scales = torch.rand(N, 1) + 0.5
+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")
+ assert _compile_count(fn, [8, 128, 64, 256], K, torch.float32) == 2
+
+
[email protected]("mark", [None, (2, 256)])
+def test_compiled_fp16_dequant_only_adds_no_row_guard(monkeypatch, mark):
+ _fp16_mode(monkeypatch, "dequant")
+ N, K = 64, 32
+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)
+ scales = torch.rand(N, 1) + 0.5
+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")
+ assert _compile_count(fn, [8, 128, 64, 256], K, torch.float16, mark) == 1
+
+
[email protected](not _cpu_scaled_mm_fp16(), reason = "needs CPU torch._scaled_mm with fp16 output")
+def test_compiled_fp16_zero_bias_adds_no_graphs_or_breaks(monkeypatch):
+ # Traced, the zero bias is built in the graph: no graph break, eager cache untouched.
+ from torch._dynamo.utils import counters
+
+ _fp16_mode(monkeypatch, "bias")
+ N, K = 64, 32
+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)
+ scales = torch.rand(N, 1) + 0.5
+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")
+ counters.clear()
+ fp16 = _compile_count(fn, [8, 128, 64, 256, 300], K, torch.float16)
+ breaks = sum(counters["graph_break"].values())
+ bf16 = _compile_count(fn, [8, 128, 64, 256, 300], K, torch.bfloat16)
+ assert fp16 == bf16 == 2
+ assert breaks == 0
+ assert F8._SCALED_MM_FP16_ZERO_BIAS_CACHE == {}
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+def test_fp16_zero_bias_is_not_shared_across_cuda_streams(monkeypatch):
+ # The zeros fill is asynchronous on the stream that made it: another stream must get its own bias.
+ _fp16_mode(monkeypatch, "bias")
+ x = torch.empty(4, 32, dtype = torch.float16, device = "cuda")
+ w = torch.empty(48, 32, dtype = torch.float8_e4m3fn, device = "cuda")
+ bias = F8._scaled_mm_fp16_bias(x, w)
+ side = torch.cuda.Stream()
+ with torch.cuda.stream(side):
+ other = F8._scaled_mm_fp16_bias(x, w)
+ assert F8._scaled_mm_fp16_bias(x, w) is other
+ assert other is not bias
+ assert F8._scaled_mm_fp16_bias(x, w) is bias
+
+
[email protected](not has_real_cuda(), reason = "needs a CUDA device")
+def test_fp16_zero_bias_is_graph_private_during_cuda_graph_capture(monkeypatch):
+ # Capture must get its own bias, not the evictable cached one.
+ _fp16_mode(monkeypatch, "bias")
+ x = torch.empty(4, 32, dtype = torch.float16, device = "cuda")
+ w = torch.empty(48, 32, dtype = torch.float8_e4m3fn, device = "cuda")
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ warm = F8._scaled_mm_fp16_bias(x, w)
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph, stream = side):
+ captured = F8._scaled_mm_fp16_bias(x, w)
+ out = captured + 1
+ torch.cuda.current_stream().wait_stream(side)
+ graph.replay()
+ torch.cuda.synchronize()
+ assert captured is not warm
+ assert all(v is not captured for v in F8._SCALED_MM_FP16_ZERO_BIAS_CACHE.values())
+ assert torch.equal(out, torch.ones_like(out))
diff --git a/tests/test_fp8_rowwise_direct_route.py b/tests/test_fp8_rowwise_direct_route.py
index 28c1cc1b314..7d73ede82c0 100644
--- a/tests/test_fp8_rowwise_direct_route.py
+++ b/tests/test_fp8_rowwise_direct_route.py
@@ -123,6 +123,7 @@ def test_direct_path_is_bitwise_equal_to_the_compiled_path(dtype):
if 2 * rows > F8._fp8_dequant_max_rows(X.device, N * K)
and N % 16 == 0
and K % 16 == 0
+ and not F8._scaled_mm_fp16_dequant_only(X.dtype)
else "dequant"
)
ref = F8._fbgemm_fp8_linear_compiled(X, w, s, b)
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index f3757e8fe4e..e5ce89352c6 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -2013,6 +2013,11 @@ if hasattr(torch._dynamo, "assume_constant_result"):
# torch < 2.14 rowwise CUTLASS takes its output type from the bias dtype: no bias + fp16 out = bf16 bits (garbage).
# 2.14 dispatches on out.dtype, where a bf16 detour would lose precision and overflow 65408..65504. Pre-releases keep it.
_SCALED_MM_FP16_OUT_BROKEN = torch.__version__ < "2.14.0"
+# 2.10-2.13 rowwise CUTLASS writes bf16 bits without a bias; a zero fp16 bias makes it write fp16.
+_SCALED_MM_FP16_BIAS_OUT = torch.__version__ >= "2.10.0" and torch.version.hip is None
+# torch < 2.10 and ROCm < 2.14 cannot write fp16 (bf16 detour overflows to inf), so fp16 dequantizes.
+_SCALED_MM_FP16_ZERO_BIAS = _SCALED_MM_FP16_OUT_BROKEN and _SCALED_MM_FP16_BIAS_OUT
+_SCALED_MM_FP16_DEQUANT_ONLY = _SCALED_MM_FP16_OUT_BROKEN and not _SCALED_MM_FP16_BIAS_OUT
@functools.lru_cache(maxsize = None)
@@ -2035,9 +2040,48 @@ if hasattr(torch._dynamo, "assume_constant_result"):
_scaled_mm_float32_out = torch._dynamo.assume_constant_result(_scaled_mm_float32_out)
+def _scaled_mm_fp16_dequant_only(dtype):
+ return dtype == torch.float16 and _SCALED_MM_FP16_DEQUANT_ONLY
+
+
+def _scaled_mm_dtype_supported(x):
+ dtype = x.dtype
+ if dtype != torch.float32:
+ return not _scaled_mm_fp16_dequant_only(dtype)
+ # Before torch 2.14 float32 output exists only where the device probe finds cuBLAS, not CUTLASS.
+ return not _SCALED_MM_FP16_OUT_BROKEN or _scaled_mm_float32_out(x.device)
+
+
def _scaled_mm_out_dtype(dtype):
- """Rowwise torch._scaled_mm out dtype: bf16 for fp16 on torch < 2.14 (caller casts back), else `dtype`."""
- return torch.bfloat16 if dtype == torch.float16 and _SCALED_MM_FP16_OUT_BROKEN else dtype
+ return torch.bfloat16 if _scaled_mm_fp16_dequant_only(dtype) else dtype
+
+
+# Keyed by stream (async fill); never used during CUDA graph capture (eviction under a live graph).
+_SCALED_MM_FP16_ZERO_BIAS_CACHE = {}
+_SCALED_MM_FP16_ZERO_BIAS_CACHE_MAX = 64
+
+
+def _scaled_mm_fp16_bias(x, weight):
+ if not _SCALED_MM_FP16_ZERO_BIAS or x.dtype != torch.float16:
+ return None
+ n = weight.shape[0]
+ device = x.device
+ if torch.compiler.is_compiling():
+ return torch.zeros(n, dtype = torch.float16, device = device)
+ # A fake tensor must not initialize CUDA.
+ cuda = device.type == "cuda" and torch.cuda.is_initialized()
+ if cuda and torch.cuda.is_current_stream_capturing():
+ return torch.zeros(n, dtype = torch.float16, device = device)
+ key = (n, device, torch.cuda.current_stream(device).cuda_stream if cuda else None)
+ bias = _SCALED_MM_FP16_ZERO_BIAS_CACHE.get(key)
+ if bias is None:
+ with torch.inference_mode(False), torch.no_grad():
+ bias = torch.zeros(n, dtype = torch.float16, device = device)
+ if type(bias) is torch.Tensor: # never cache a fake tensor
+ if len(_SCALED_MM_FP16_ZERO_BIAS_CACHE) >= _SCALED_MM_FP16_ZERO_BIAS_CACHE_MAX:
+ _SCALED_MM_FP16_ZERO_BIAS_CACHE.clear()
+ _SCALED_MM_FP16_ZERO_BIAS_CACHE[key] = bias
+ return bias
def _parse_fp8_dequant_max_rows(value):
@@ -2142,14 +2186,16 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
and rowwise
and (weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0)
and x.shape[-1] == weight.shape[1]
+ # Traced, an always-dequant dtype must not guard on the row count (strict mark_dynamic).
+ and (
+ x.dtype == torch.bfloat16
+ or not torch.compiler.is_compiling()
+ or _scaled_mm_dtype_supported(x)
+ )
# Decode-sized calls: dequant + 16-bit GEMM beats quantizing the activation unless the weight
# spills L2.
and x.numel() > _fp8_dequant_max_rows(x.device, weight.numel()) * x.shape[-1]
- and (
- x.dtype != torch.float32
- or not _SCALED_MM_FP16_OUT_BROKEN
- or _scaled_mm_float32_out(x.device)
- )
+ and _scaled_mm_dtype_supported(x)
):
output_shape = (*x.shape[:-1], -1)
x_quantized, x_scale = _quantize_fp8_per_row(
@@ -2162,11 +2208,12 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):
weight.t(),
scale_a = x_scale,
scale_b = weight_scale.to(torch.float32).view(1, -1),
+ bias = _scaled_mm_fp16_bias(x, weight),
out_dtype = _scaled_mm_out_dtype(x.dtype),
use_fast_accum = True,
)
output = output + bias if bias is not None else output
- # fp16 comes back as bf16 (_scaled_mm_out_dtype) and FbgemmFp8Linear keeps its bias in float32:
+ # FbgemmFp8Linear keeps its bias in float32:
# cast only when the dtype differs (a no-op .to() aliases).
if output.dtype != x.dtype:
output = output.to(x.dtype)
@@ -2341,6 +2388,7 @@ def _fp8_rowwise_direct_route(X, weight, weight_scale):
and N % 16 == 0
and K % 16 == 0
and X.numel() > _fp8_dequant_max_rows(X.device, N * K) * K
+ and not _scaled_mm_fp16_dequant_only(X.dtype)
):
scale_ub = getattr(weight, "input_scale_ub", None)
if scale_ub is not None and (
@@ -2386,6 +2434,7 @@ def _fp8_rowwise_direct(X, weight, weight_scale, bias, route):
scale_b = (
weight_scale if weight_scale.dtype == torch.float32 else weight_scale.float()
).view(1, -1),
+ bias = _scaled_mm_fp16_bias(X, weight),
out_dtype = _scaled_mm_out_dtype(X.dtype),
use_fast_accum = True,
)
@@ -2406,7 +2455,7 @@ def _fp8_rowwise_direct(X, weight, weight_scale, bias, route):
output = torch_matmul(X, W_deq.t())
if bias is not None:
output = output + bias
- # fp16 comes back as bf16 (_scaled_mm_out_dtype) and FbgemmFp8Linear keeps its bias in float32:
+ # FbgemmFp8Linear keeps its bias in float32:
# cast only when the dtype differs (a no-op .to() aliases).
if output.dtype != X.dtype:
output = output.to(X.dtype)