Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 block decode: keep the fused small-M kernel for fp16 when torchao is selected (#15361)

Daniel Hanbc0603f839b8
diff --git a/tests/test_fp8_small_m_torchao_route.py b/tests/test_fp8_small_m_torchao_route.py
new file mode 100644
index 00000000000..f6852aa4109
--- /dev/null
+++ b/tests/test_fp8_small_m_torchao_route.py
@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""With torchao selected, fp16 small-M decode keeps the fused kernel; bf16 goes to torchao (CPU recorders)."""
+
+import types
+
+import pytest
+import torch
+
+from unsloth.kernels import fp8 as F8
+
+
+class _CudaLike(torch.Tensor):
+ @property
+ def is_cuda(self):
+ return True
+
+
+def _operands(
+ M,
+ dtype,
+ N = 256,
+ K = 512,
+):
+ X = torch.randn(M, K).to(dtype).as_subclass(_CudaLike)
+ weight = torch.zeros(N, K, dtype = torch.float8_e4m3fn)
+ scale = torch.ones(N // 128, K // 128, dtype = torch.float32)
+ return X, weight, scale
+
+
+def calls(monkeypatch):
+ seen = []
+
+ def small_m(X, weight, weight_scale):
+ seen.append("small_m")
+ return torch.zeros(X.shape[:-1] + (weight.shape[0],), dtype = X.dtype)
+
+ def act_quant(X, block):
+ seen.append("act_quant")
+ x = torch.Tensor.as_subclass(X, torch.Tensor)
+ return x.to(torch.float8_e4m3fn), torch.ones(x.shape[0], x.shape[-1] // block)
+
+ def gemm(name):
+ def run(
+ A,
+ B,
+ As,
+ Bs,
+ *args,
+ output_dtype = torch.bfloat16,
+ **kwargs,
+ ):
+ seen.append(name)
+ return torch.zeros(A.shape[:-1] + (B.shape[0],), dtype = output_dtype)
+
+ return run
+
+ triton_gemm = gemm("triton_gemm")
+ monkeypatch.setattr(F8, "_FP8_BLOCK_SMALL_M", True)
+ monkeypatch.setattr(F8, "fp8_block_small_m_linear", small_m)
+ monkeypatch.setattr(F8, "act_quant", act_quant)
+ monkeypatch.setattr(F8, "_fp8_kernel_unsupported", lambda *a, **k: False)
+ monkeypatch.setattr(F8, "_fp8_triton_device_context", lambda *a, **k: _NullContext())
+ monkeypatch.setattr(F8, "_prefer_triton_block_matmul_at", lambda device, M: False)
+ monkeypatch.setattr(F8, "w8a8_block_fp8_matmul_triton", triton_gemm)
+ monkeypatch.setattr(
+ F8,
+ "torchao_blockwise_gemm",
+ lambda a, a_s, b, b_s, block_size: gemm("torchao")(a, b, a_s, b_s),
+ )
+ return seen
+
+
+class _NullContext:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+
+def _select(monkeypatch, backend):
+ target = F8.torchao_block_matmul if backend == "torchao" else F8.w8a8_block_fp8_matmul_triton
+ monkeypatch.setattr(F8, "fp8_block_matmul", target)
+
+
+def _forward(X, weight, scale):
+ ctx = types.SimpleNamespace()
+ return F8.FP8BlockQuantLinear.forward(ctx, X, weight, scale)
+
+
[email protected]("M", [1, 4, 16])
+def test_fp16_decode_stays_fused_with_torchao(monkeypatch, calls, M):
+ _select(monkeypatch, "torchao")
+ X, weight, scale = _operands(M, torch.float16)
+ assert F8.can_use_fp8_block_small_m(X, weight, scale, [128, 128])
+ out = _forward(X, weight, scale)
+ assert calls == ["small_m"]
+ assert out.dtype == torch.float16 and out.shape == (M, weight.shape[0])
+
+
[email protected]("M", [1, 4, 16])
+def test_bf16_decode_still_goes_to_torchao(monkeypatch, calls, M):
+ _select(monkeypatch, "torchao")
+ X, weight, scale = _operands(M, torch.bfloat16)
+ assert not F8.can_use_fp8_block_small_m(X, weight, scale, [128, 128])
+ _forward(X, weight, scale)
+ assert calls == ["act_quant", "torchao"]
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
+def test_triton_selected_uses_fused_for_both(monkeypatch, calls, dtype):
+ _select(monkeypatch, "triton")
+ X, weight, scale = _operands(2, dtype)
+ _forward(X, weight, scale)
+ assert calls == ["small_m"]
+
+
[email protected]("backend", ["torchao", "triton"])
+def test_fp16_large_m_and_kill_switch_keep_two_launches(monkeypatch, calls, backend):
+ _select(monkeypatch, backend)
+ X, weight, scale = _operands(F8._SMALL_M_MAX + 1, torch.float16)
+ _forward(X, weight, scale)
+ assert calls == ["act_quant", "triton_gemm"]
+ calls.clear()
+ monkeypatch.setattr(F8, "_FP8_BLOCK_SMALL_M", False)
+ X, weight, scale = _operands(1, torch.float16)
+ _forward(X, weight, scale)
+ assert calls == ["act_quant", "triton_gemm"]
+
+
+def test_fp16_decode_stays_fused_in_small_m_first_route(monkeypatch, calls):
+ _select(monkeypatch, "torchao")
+ monkeypatch.setattr(F8, "can_use_fp8_block_gemv", lambda *a: False)
+ monkeypatch.setattr(F8, "fp8_torch_block_quant_forward", lambda *a: calls.append("autograd"))
+ X, weight, scale = _operands(1, torch.float16)
+ with torch.no_grad():
+ F8._fp8_block_small_m_first(X, weight, scale)
+ assert calls == ["small_m"]
+ calls.clear()
+ X, weight, scale = _operands(1, torch.bfloat16)
+ with torch.no_grad():
+ F8._fp8_block_small_m_first(X, weight, scale)
+ assert calls == ["autograd"]
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
+def test_gemv_keeps_its_calls_when_torchao_is_selected(monkeypatch, calls, dtype):
+ _select(monkeypatch, "torchao")
+ monkeypatch.setattr(F8, "can_use_fp8_block_gemv", lambda *a: True)
+ monkeypatch.setattr(F8, "fp8_block_gemv", lambda *a: calls.append("gemv"))
+ X, weight, scale = _operands(4, dtype)
+ with torch.no_grad():
+ F8._fp8_block_small_m_first(X, weight, scale)
+ assert calls == ["gemv"]
+
+
+def test_triton_selected_keeps_fused_before_gemv(monkeypatch, calls):
+ _select(monkeypatch, "triton")
+ monkeypatch.setattr(F8, "can_use_fp8_block_gemv", lambda *a: True)
+ monkeypatch.setattr(F8, "fp8_block_gemv", lambda *a: calls.append("gemv"))
+ X, weight, scale = _operands(4, torch.float16)
+ with torch.no_grad():
+ F8._fp8_block_small_m_first(X, weight, scale)
+ assert calls == ["small_m"]
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index a9258f966ef..f4c4ea24fd4 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -870,9 +870,10 @@ def _small_m_config(M, N, K, device):
def can_use_fp8_block_small_m(X, weight, weight_scale, block_size):
# Eager only: under torch.compile the M bound below would guard on the symbolic M and recompile.
+ # torchao writes bf16 only (fp16 falls to act_quant + Triton GEMM), so fp16 keeps the fused kernel.
if (
not _FP8_BLOCK_SMALL_M
- or fp8_block_matmul is not w8a8_block_fp8_matmul_triton
+ or (fp8_block_matmul is not w8a8_block_fp8_matmul_triton and X.dtype != torch.float16)
or torch.compiler.is_compiling()
):
return False
@@ -1001,7 +1002,7 @@ def _prefer_triton_block_matmul_at(device, M):
def _select_fp8_block_matmul():
# torchao_block_matmul hands each call to the Triton function where that is faster; pick Triton
# outright only when every GPU prefers it at every M (sm_89), which also keeps the fused small-M
- # kernel there (can_use_fp8_block_small_m), as torchao beats it on sm_90 / sm_100 / sm_120.
+ # kernel there for bf16 (can_use_fp8_block_small_m), as torchao beats it on sm_90 / sm_100 / sm_120.
if torchao_blockwise_gemm is None:
return w8a8_block_fp8_matmul_triton
count = torch.cuda.device_count() if torch.cuda.is_available() else 0
@@ -1594,6 +1595,10 @@ def _with_fp8_block_gemv(block_forward):
def _fp8_block_small_m_first(X, weight, weight_scale):
# 100+ SM parts: the fused small-M kernel is faster, so try it first (eager only; compiled uses the GEMV).
if not (torch.is_grad_enabled() and X.requires_grad):
+ # torchao selected: GEMV first (skips act_quant, beat the fused kernel at K = 14336 on sm_120).
+ gemv_first = fp8_block_matmul is not w8a8_block_fp8_matmul_triton
+ if gemv_first and can_use_fp8_block_gemv(X, weight, weight_scale):
+ return fp8_block_gemv(X, weight, weight_scale)
if (
can_use_fp8_block_small_m(
X, weight, weight_scale, _fp8_block_size(weight, weight_scale)
@@ -1602,7 +1607,7 @@ def _fp8_block_small_m_first(X, weight, weight_scale):
and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)
):
return fp8_block_small_m_linear(X, weight, weight_scale)
- if can_use_fp8_block_gemv(X, weight, weight_scale):
+ if not gemv_first and can_use_fp8_block_gemv(X, weight, weight_scale):
return fp8_block_gemv(X, weight, weight_scale)
return fp8_torch_block_quant_forward(X, weight, weight_scale)