Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 rowwise scaled_mm: keep fp16 output in range before torch 2.14, and drop the row guard for always-dequant dtypes when compiled (#16598)
Daniel Han1e4643cb3d08
diff --git a/tests/test_fp8_dequant_max_rows.py b/tests/test_fp8_dequant_max_rows.pyindex 5943e3c3438..e1df0eade12 100644--- a/tests/test_fp8_dequant_max_rows.py+++ b/tests/test_fp8_dequant_max_rows.py@@ -141,14 +141,14 @@ def test_autograd_function_uses_the_same_gate(monkeypatch, shape, scaled, dtype,def test_scaled_mm_asks_for_bf16_when_activations_are_fp16(monkeypatch):- monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_DEQUANT_ONLY", True)assert F8._scaled_mm_out_dtype(torch.float16) is torch.bfloat16assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16assert F8._scaled_mm_out_dtype(torch.float32) is torch.float32def test_scaled_mm_keeps_fp16_output_where_torch_writes_fp16(monkeypatch):- monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", False)+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_DEQUANT_ONLY", False)assert F8._scaled_mm_out_dtype(torch.float16) is torch.float16assert F8._scaled_mm_out_dtype(torch.bfloat16) is torch.bfloat16assert F8._scaled_mm_out_dtype(torch.float32) is torch.float32@@ -158,8 +158,11 @@ def test_scaled_mm_fp16_workaround_follows_the_installed_torch():from torch.torch_version import TorchVersionbroken = torch.__version__ < "2.14.0"+ detour = broken and not F8._SCALED_MM_FP16_BIAS_OUTassert F8._SCALED_MM_FP16_OUT_BROKEN is broken- assert F8._scaled_mm_out_dtype(torch.float16) is (torch.bfloat16 if broken else torch.float16)+ assert F8._SCALED_MM_FP16_DEQUANT_ONLY is detour+ assert F8._SCALED_MM_FP16_ZERO_BIAS is (broken and F8._SCALED_MM_FP16_BIAS_OUT)+ assert F8._scaled_mm_out_dtype(torch.float16) is (torch.bfloat16 if detour else torch.float16)assert TorchVersion("2.13.9+cu130") < "2.14.0"assert TorchVersion("2.14.0a0+git0") < "2.14.0"assert not TorchVersion("2.14.0+cu130") < "2.14.0"@@ -179,15 +182,19 @@ def test_every_rowwise_scaled_mm_call_uses_the_out_dtype_helper():and node.func.attr == "_scaled_mm"]out = {}+ bias = {}for node in calls:for kw in node.keywords:if kw.arg == "out_dtype":out[node.lineno] = ast.unparse(kw.value)+ if kw.arg == "bias":+ bias[node.lineno] = ast.unparse(kw.value)data_calls = {line: v for line, v in out.items() if v not in ("torch.bfloat16", "torch.float32")}assert len(data_calls) >= 2, outassert all(v.startswith("_scaled_mm_out_dtype(") for v in data_calls.values()), out+ assert all(bias.get(line, "").startswith("_scaled_mm_fp16_bias(") for line in data_calls), bias@pytest.mark.parametrize("float32_out, expected", [(True, "scaled_mm"), (False, "dequant")])diff --git a/tests/test_fp8_rowwise_compile_guards.py b/tests/test_fp8_rowwise_compile_guards.pynew file mode 100644index 00000000000..a01c0055a87--- /dev/null+++ b/tests/test_fp8_rowwise_compile_guards.py@@ -0,0 +1,308 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Rowwise FP8 scaled_mm route: fp16 range before torch 2.14, and no row-count guard for calls that always+dequantize when compiled. CPU tensors; torch._scaled_mm and the dequant GEMM run for real."""++import pytest+import torch+from real_accelerator import has_real_cuda+import torch._dynamo+from torch._subclasses.fake_tensor import FakeTensorMode++from unsloth.kernels import fp8 as F8++LARGE = (14336, 4096) # Llama-3.1-8B MLP: 112 MiB in bf16, spills L4's 48 MiB L2+++def _cpu_scaled_mm_rowwise_float32():+ # CPU rowwise _scaled_mm with float32 output: torch 2.10 and older only support per-tensor scales there.+ try:+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)+ torch._scaled_mm(+ a, a.t(), scale_a = torch.ones(16, 1), scale_b = torch.ones(1, 16), out_dtype = torch.float32+ )+ return True+ except Exception:+ return False+++def _cpu_scaled_mm_fp16():+ try:+ a = torch.ones(16, 16).to(torch.float8_e4m3fn)+ ones = torch.ones(16, 1)+ torch._scaled_mm(+ a,+ a.t(),+ scale_a = ones,+ scale_b = ones.t(),+ bias = torch.zeros(16, dtype = torch.float16),+ out_dtype = torch.float16,+ )+ return True+ except Exception:+ return False+++def _fp16_mode(monkeypatch, mode):+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", mode != "native")+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS", mode == "bias")+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_DEQUANT_ONLY", mode == "dequant")+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS_CACHE", {})++[email protected]("mode", ["bias", "dequant", "native"])+def test_fp16_scaled_mm_out_dtype_and_bias_follow_the_fp16_mode(monkeypatch, mode):+ _fp16_mode(monkeypatch, mode)+ x = torch.empty(4, 32, dtype = torch.float16)+ w = torch.empty(48, 32, dtype = torch.float8_e4m3fn)+ assert F8._scaled_mm_out_dtype(torch.float16) is (+ torch.bfloat16 if mode == "dequant" else torch.float16+ )+ assert F8._scaled_mm_dtype_supported(x) is (mode != "dequant")+ bias = F8._scaled_mm_fp16_bias(x, w)+ if mode == "bias":+ assert bias.dtype == torch.float16 and bias.shape == (48,) and not bias.any()+ assert F8._scaled_mm_fp16_bias(x, w) is bias+ assert (+ F8._scaled_mm_fp16_bias(x, torch.empty(64, 32, dtype = torch.float8_e4m3fn)) is not bias+ )+ assert len(F8._SCALED_MM_FP16_ZERO_BIAS_CACHE) == 2+ else:+ assert bias is None+ for dtype in (torch.bfloat16, torch.float32):+ assert F8._scaled_mm_out_dtype(dtype) is dtype+ assert F8._scaled_mm_fp16_bias(x.to(dtype), w) is None+++def test_fp16_zero_bias_cache_is_bounded_and_skips_fake_tensors(monkeypatch):+ _fp16_mode(monkeypatch, "bias")+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS_CACHE_MAX", 4)+ x = torch.empty(4, 32, dtype = torch.float16)+ for n in range(16, 16 * 11, 16):+ F8._scaled_mm_fp16_bias(x, torch.empty(n, 32, dtype = torch.float8_e4m3fn))+ assert len(F8._SCALED_MM_FP16_ZERO_BIAS_CACHE) <= 4+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_ZERO_BIAS_CACHE", {})+ with FakeTensorMode():+ fx = torch.empty(4, 32, dtype = torch.float16)+ fw = torch.empty(48, 32, dtype = torch.float8_e4m3fn)+ assert F8._scaled_mm_fp16_bias(fx, fw) is not None+ assert F8._SCALED_MM_FP16_ZERO_BIAS_CACHE == {}+ with torch.inference_mode():+ bias = F8._scaled_mm_fp16_bias(x, torch.empty(48, 32, dtype = torch.float8_e4m3fn))+ assert not bias.is_inference()+++def test_fp16_flags_follow_the_installed_torch():+ from torch.torch_version import TorchVersion++ assert F8._SCALED_MM_FP16_BIAS_OUT is (+ torch.__version__ >= "2.10.0" and torch.version.hip is None+ )+ broken = F8._SCALED_MM_FP16_OUT_BROKEN+ assert F8._SCALED_MM_FP16_ZERO_BIAS is (broken and F8._SCALED_MM_FP16_BIAS_OUT)+ assert F8._SCALED_MM_FP16_DEQUANT_ONLY is (broken and not F8._SCALED_MM_FP16_BIAS_OUT)+ # 2.9 CUTLASS rowwise accepts only float32 / bf16 bias and bf16 output.+ assert not TorchVersion("2.9.1+cu128") >= "2.10.0"+ assert TorchVersion("2.10.0+cu128") >= "2.10.0"++[email protected]("compiled", [False, True])[email protected]("mode", ["bias", "dequant"])[email protected]("rows", [80, 300])+def test_fp16_output_near_the_fp16_max_stays_finite(monkeypatch, compiled, mode, rows):+ # Every output is exactly 65504: a bf16 detour rounds it to 65536 = fp16 inf.+ if mode == "bias" and not _cpu_scaled_mm_fp16():+ pytest.skip("needs CPU torch._scaled_mm with fp16 output")+ _fp16_mode(monkeypatch, mode)+ calls = []+ real_scaled_mm = torch._scaled_mm++ def counting_scaled_mm(*args, **kwargs):+ calls.append(kwargs.get("out_dtype"))+ return real_scaled_mm(*args, **kwargs)++ monkeypatch.setattr(torch, "_scaled_mm", counting_scaled_mm)+ N, K = 64, 32+ x = torch.ones(rows, K, dtype = torch.float16)+ weight = torch.ones(N, K).to(torch.float8_e4m3fn)+ scales = torch.full((N, 1), 65504.0 / K)+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")+ if compiled:+ torch._dynamo.reset()+ fn = torch.compile(fn, backend = "aot_eager", fullgraph = True)+ out = fn(x)+ assert out.dtype == torch.float16+ assert torch.isfinite(out).all()+ assert (out == 65504).all()+ if mode == "dequant":+ assert calls == []+ else:+ assert calls and all(d is torch.float16 for d in calls)++[email protected]("rows", [17, 32, 64, 65, 128, 4096])[email protected]("mode", ["bias", "dequant", "native"])+def test_fp16_routing_decision(monkeypatch, mode, rows):+ monkeypatch.setattr(torch.version, "hip", None, raising = False)+ monkeypatch.setattr(F8, "_fp8_rowwise_backend", lambda device: "scaled_mm")+ monkeypatch.setattr(F8, "_FP8_ROWWISE_DIRECT", True)+ monkeypatch.setattr(F8, "_FP8_DEQUANT_MAX_ROWS_ENV", None)+ monkeypatch.setattr(F8, "_FP8_DEQUANT_MAX_ROWS", 64)+ monkeypatch.setattr(F8, "_fp8_l2_cache_bytes", lambda device: 48 * 1024 * 1024)+ # A real pre-sm89 GPU would refuse e4m3fn here; the decision under test is the same on every card.+ monkeypatch.setattr(F8, "_fp8_e4m3fn_unsupported_in_trace", lambda weight: False)+ _fp16_mode(monkeypatch, mode)+ calls = []++ def fake_scaled_mm(a, b, **kwargs):+ calls.append(("scaled_mm", kwargs["out_dtype"]))+ return torch.empty(a.shape[0], b.shape[1], device = a.device, dtype = kwargs["out_dtype"])++ def fake_dequant(w, s, dtype):+ calls.append(("dequant", dtype))+ return torch.empty(w.shape, device = w.device, dtype = dtype)++ monkeypatch.setattr(torch, "_scaled_mm", fake_scaled_mm)+ monkeypatch.setattr(F8, "weight_dequant", fake_dequant)+ N, K = LARGE+ scaled_rows = rows > 16 # the L2 gate: this weight spills 48 MiB, so 17+ rows take scaled_mm+ for dtype in (torch.float16, torch.bfloat16):+ expected = (+ "scaled_mm"+ if scaled_rows and (dtype == torch.bfloat16 or mode != "dequant")+ else "dequant"+ )+ calls.clear()+ with FakeTensorMode(), torch.no_grad():+ X = torch.empty(1, rows, K, device = "cuda", dtype = dtype)+ w = torch.empty(N, K, device = "cuda", dtype = torch.float8_e4m3fn)+ s = torch.empty(N, 1, device = "cuda", dtype = torch.float32)+ assert F8._fp8_rowwise_direct_route(X, w, s) == expected, (dtype, rows)+ y = F8.FbgemmFp8Linear_matmul.apply(X, w, s, None, "scaled_mm")+ assert y.shape == (1, rows, N) and y.dtype == dtype+ assert calls == [+ (expected, dtype if expected == "dequant" else F8._scaled_mm_out_dtype(dtype))+ ]+ assert F8._SCALED_MM_FP16_ZERO_BIAS_CACHE == {}+++def _compile_count(+ fn,+ rows_seq,+ K,+ dtype,+ mark = None,+):+ from torch._dynamo.backends.debugging import aot_eager++ graphs = []++ def backend(gm, example_inputs):+ graphs.append(gm)+ return aot_eager(gm, example_inputs)++ torch._dynamo.reset()+ compiled = torch.compile(fn, backend = backend, dynamic = mark is None)+ for rows in rows_seq:+ x = torch.randn(rows, K, dtype = dtype)+ if mark is not None:+ torch._dynamo.mark_dynamic(x, 0, min = mark[0], max = mark[1])+ compiled(x)+ return len(graphs)++[email protected]("mark", [None, (2, 256)])+def test_compiled_float32_dequant_adds_no_row_guard(monkeypatch, mark):+ # Always-dequant dtypes must not guard on the row count (one graph, strict mark_dynamic holds).+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)+ monkeypatch.setattr(+ F8, "_scaled_mm_float32_out", torch._dynamo.assume_constant_result(lambda device: False)+ )+ N, K = 64, 32+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)+ scales = torch.rand(N, 1) + 0.5+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")+ assert _compile_count(fn, [8, 128, 64, 256], K, torch.float32, mark) == 1+++ not _cpu_scaled_mm_rowwise_float32(),+ reason = "needs CPU rowwise torch._scaled_mm with float32 output",+)+def test_compiled_float32_with_float32_output_still_routes_by_rows(monkeypatch):+ monkeypatch.setattr(F8, "_SCALED_MM_FP16_OUT_BROKEN", True)+ monkeypatch.setattr(+ F8, "_scaled_mm_float32_out", torch._dynamo.assume_constant_result(lambda device: True)+ )+ N, K = 64, 32+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)+ scales = torch.rand(N, 1) + 0.5+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")+ assert _compile_count(fn, [8, 128, 64, 256], K, torch.float32) == 2++[email protected]("mark", [None, (2, 256)])+def test_compiled_fp16_dequant_only_adds_no_row_guard(monkeypatch, mark):+ _fp16_mode(monkeypatch, "dequant")+ N, K = 64, 32+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)+ scales = torch.rand(N, 1) + 0.5+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")+ assert _compile_count(fn, [8, 128, 64, 256], K, torch.float16, mark) == 1++[email protected](not _cpu_scaled_mm_fp16(), reason = "needs CPU torch._scaled_mm with fp16 output")+def test_compiled_fp16_zero_bias_adds_no_graphs_or_breaks(monkeypatch):+ # Traced, the zero bias is built in the graph: no graph break, eager cache untouched.+ from torch._dynamo.utils import counters++ _fp16_mode(monkeypatch, "bias")+ N, K = 64, 32+ weight = (torch.randn(N, K) * 0.5).to(torch.float8_e4m3fn)+ scales = torch.rand(N, 1) + 0.5+ fn = lambda x: F8.FbgemmFp8Linear_matmul.apply(x, weight, scales, None, "scaled_mm")+ counters.clear()+ fp16 = _compile_count(fn, [8, 128, 64, 256, 300], K, torch.float16)+ breaks = sum(counters["graph_break"].values())+ bf16 = _compile_count(fn, [8, 128, 64, 256, 300], K, torch.bfloat16)+ assert fp16 == bf16 == 2+ assert breaks == 0+ assert F8._SCALED_MM_FP16_ZERO_BIAS_CACHE == {}++[email protected](not has_real_cuda(), reason = "needs a CUDA device")+def test_fp16_zero_bias_is_not_shared_across_cuda_streams(monkeypatch):+ # The zeros fill is asynchronous on the stream that made it: another stream must get its own bias.+ _fp16_mode(monkeypatch, "bias")+ x = torch.empty(4, 32, dtype = torch.float16, device = "cuda")+ w = torch.empty(48, 32, dtype = torch.float8_e4m3fn, device = "cuda")+ bias = F8._scaled_mm_fp16_bias(x, w)+ side = torch.cuda.Stream()+ with torch.cuda.stream(side):+ other = F8._scaled_mm_fp16_bias(x, w)+ assert F8._scaled_mm_fp16_bias(x, w) is other+ assert other is not bias+ assert F8._scaled_mm_fp16_bias(x, w) is bias++[email protected](not has_real_cuda(), reason = "needs a CUDA device")+def test_fp16_zero_bias_is_graph_private_during_cuda_graph_capture(monkeypatch):+ # Capture must get its own bias, not the evictable cached one.+ _fp16_mode(monkeypatch, "bias")+ x = torch.empty(4, 32, dtype = torch.float16, device = "cuda")+ w = torch.empty(48, 32, dtype = torch.float8_e4m3fn, device = "cuda")+ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ warm = F8._scaled_mm_fp16_bias(x, w)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph, stream = side):+ captured = F8._scaled_mm_fp16_bias(x, w)+ out = captured + 1+ torch.cuda.current_stream().wait_stream(side)+ graph.replay()+ torch.cuda.synchronize()+ assert captured is not warm+ assert all(v is not captured for v in F8._SCALED_MM_FP16_ZERO_BIAS_CACHE.values())+ assert torch.equal(out, torch.ones_like(out))diff --git a/tests/test_fp8_rowwise_direct_route.py b/tests/test_fp8_rowwise_direct_route.pyindex 28c1cc1b314..7d73ede82c0 100644--- a/tests/test_fp8_rowwise_direct_route.py+++ b/tests/test_fp8_rowwise_direct_route.py@@ -123,6 +123,7 @@ def test_direct_path_is_bitwise_equal_to_the_compiled_path(dtype):if 2 * rows > F8._fp8_dequant_max_rows(X.device, N * K)and N % 16 == 0and K % 16 == 0+ and not F8._scaled_mm_fp16_dequant_only(X.dtype)else "dequant")ref = F8._fbgemm_fp8_linear_compiled(X, w, s, b)diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex f3757e8fe4e..e5ce89352c6 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -2013,6 +2013,11 @@ if hasattr(torch._dynamo, "assume_constant_result"):# torch < 2.14 rowwise CUTLASS takes its output type from the bias dtype: no bias + fp16 out = bf16 bits (garbage).# 2.14 dispatches on out.dtype, where a bf16 detour would lose precision and overflow 65408..65504. Pre-releases keep it._SCALED_MM_FP16_OUT_BROKEN = torch.__version__ < "2.14.0"+# 2.10-2.13 rowwise CUTLASS writes bf16 bits without a bias; a zero fp16 bias makes it write fp16.+_SCALED_MM_FP16_BIAS_OUT = torch.__version__ >= "2.10.0" and torch.version.hip is None+# torch < 2.10 and ROCm < 2.14 cannot write fp16 (bf16 detour overflows to inf), so fp16 dequantizes.+_SCALED_MM_FP16_ZERO_BIAS = _SCALED_MM_FP16_OUT_BROKEN and _SCALED_MM_FP16_BIAS_OUT+_SCALED_MM_FP16_DEQUANT_ONLY = _SCALED_MM_FP16_OUT_BROKEN and not _SCALED_MM_FP16_BIAS_OUT@functools.lru_cache(maxsize = None)@@ -2035,9 +2040,48 @@ if hasattr(torch._dynamo, "assume_constant_result"):_scaled_mm_float32_out = torch._dynamo.assume_constant_result(_scaled_mm_float32_out)+def _scaled_mm_fp16_dequant_only(dtype):+ return dtype == torch.float16 and _SCALED_MM_FP16_DEQUANT_ONLY+++def _scaled_mm_dtype_supported(x):+ dtype = x.dtype+ if dtype != torch.float32:+ return not _scaled_mm_fp16_dequant_only(dtype)+ # Before torch 2.14 float32 output exists only where the device probe finds cuBLAS, not CUTLASS.+ return not _SCALED_MM_FP16_OUT_BROKEN or _scaled_mm_float32_out(x.device)++def _scaled_mm_out_dtype(dtype):- """Rowwise torch._scaled_mm out dtype: bf16 for fp16 on torch < 2.14 (caller casts back), else `dtype`."""- return torch.bfloat16 if dtype == torch.float16 and _SCALED_MM_FP16_OUT_BROKEN else dtype+ return torch.bfloat16 if _scaled_mm_fp16_dequant_only(dtype) else dtype+++# Keyed by stream (async fill); never used during CUDA graph capture (eviction under a live graph).+_SCALED_MM_FP16_ZERO_BIAS_CACHE = {}+_SCALED_MM_FP16_ZERO_BIAS_CACHE_MAX = 64+++def _scaled_mm_fp16_bias(x, weight):+ if not _SCALED_MM_FP16_ZERO_BIAS or x.dtype != torch.float16:+ return None+ n = weight.shape[0]+ device = x.device+ if torch.compiler.is_compiling():+ return torch.zeros(n, dtype = torch.float16, device = device)+ # A fake tensor must not initialize CUDA.+ cuda = device.type == "cuda" and torch.cuda.is_initialized()+ if cuda and torch.cuda.is_current_stream_capturing():+ return torch.zeros(n, dtype = torch.float16, device = device)+ key = (n, device, torch.cuda.current_stream(device).cuda_stream if cuda else None)+ bias = _SCALED_MM_FP16_ZERO_BIAS_CACHE.get(key)+ if bias is None:+ with torch.inference_mode(False), torch.no_grad():+ bias = torch.zeros(n, dtype = torch.float16, device = device)+ if type(bias) is torch.Tensor: # never cache a fake tensor+ if len(_SCALED_MM_FP16_ZERO_BIAS_CACHE) >= _SCALED_MM_FP16_ZERO_BIAS_CACHE_MAX:+ _SCALED_MM_FP16_ZERO_BIAS_CACHE.clear()+ _SCALED_MM_FP16_ZERO_BIAS_CACHE[key] = bias+ return biasdef _parse_fp8_dequant_max_rows(value):@@ -2142,14 +2186,16 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):and rowwiseand (weight.shape[0] % 16 == 0 and weight.shape[1] % 16 == 0)and x.shape[-1] == weight.shape[1]+ # Traced, an always-dequant dtype must not guard on the row count (strict mark_dynamic).+ and (+ x.dtype == torch.bfloat16+ or not torch.compiler.is_compiling()+ or _scaled_mm_dtype_supported(x)+ )# Decode-sized calls: dequant + 16-bit GEMM beats quantizing the activation unless the weight# spills L2.and x.numel() > _fp8_dequant_max_rows(x.device, weight.numel()) * x.shape[-1]- and (- x.dtype != torch.float32- or not _SCALED_MM_FP16_OUT_BROKEN- or _scaled_mm_float32_out(x.device)- )+ and _scaled_mm_dtype_supported(x)):output_shape = (*x.shape[:-1], -1)x_quantized, x_scale = _quantize_fp8_per_row(@@ -2162,11 +2208,12 @@ class FbgemmFp8Linear_matmul(torch.autograd.Function):weight.t(),scale_a = x_scale,scale_b = weight_scale.to(torch.float32).view(1, -1),+ bias = _scaled_mm_fp16_bias(x, weight),out_dtype = _scaled_mm_out_dtype(x.dtype),use_fast_accum = True,)output = output + bias if bias is not None else output- # fp16 comes back as bf16 (_scaled_mm_out_dtype) and FbgemmFp8Linear keeps its bias in float32:+ # FbgemmFp8Linear keeps its bias in float32:# cast only when the dtype differs (a no-op .to() aliases).if output.dtype != x.dtype:output = output.to(x.dtype)@@ -2341,6 +2388,7 @@ def _fp8_rowwise_direct_route(X, weight, weight_scale):and N % 16 == 0and K % 16 == 0and X.numel() > _fp8_dequant_max_rows(X.device, N * K) * K+ and not _scaled_mm_fp16_dequant_only(X.dtype)):scale_ub = getattr(weight, "input_scale_ub", None)if scale_ub is not None and (@@ -2386,6 +2434,7 @@ def _fp8_rowwise_direct(X, weight, weight_scale, bias, route):scale_b = (weight_scale if weight_scale.dtype == torch.float32 else weight_scale.float()).view(1, -1),+ bias = _scaled_mm_fp16_bias(X, weight),out_dtype = _scaled_mm_out_dtype(X.dtype),use_fast_accum = True,)@@ -2406,7 +2455,7 @@ def _fp8_rowwise_direct(X, weight, weight_scale, bias, route):output = torch_matmul(X, W_deq.t())if bias is not None:output = output + bias- # fp16 comes back as bf16 (_scaled_mm_out_dtype) and FbgemmFp8Linear keeps its bias in float32:+ # FbgemmFp8Linear keeps its bias in float32:# cast only when the dtype differs (a no-op .to() aliases).if output.dtype != X.dtype:output = output.to(X.dtype)