Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 block decode: keep the fused small-M kernel for fp16 when torchao is selected (#15361)
Daniel Hanbc0603f839b8
diff --git a/tests/test_fp8_small_m_torchao_route.py b/tests/test_fp8_small_m_torchao_route.pynew file mode 100644index 00000000000..f6852aa4109--- /dev/null+++ b/tests/test_fp8_small_m_torchao_route.py@@ -0,0 +1,167 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""With torchao selected, fp16 small-M decode keeps the fused kernel; bf16 goes to torchao (CPU recorders)."""++import types++import pytest+import torch++from unsloth.kernels import fp8 as F8+++class _CudaLike(torch.Tensor):+ @property+ def is_cuda(self):+ return True+++def _operands(+ M,+ dtype,+ N = 256,+ K = 512,+):+ X = torch.randn(M, K).to(dtype).as_subclass(_CudaLike)+ weight = torch.zeros(N, K, dtype = torch.float8_e4m3fn)+ scale = torch.ones(N // 128, K // 128, dtype = torch.float32)+ return X, weight, scale+++def calls(monkeypatch):+ seen = []++ def small_m(X, weight, weight_scale):+ seen.append("small_m")+ return torch.zeros(X.shape[:-1] + (weight.shape[0],), dtype = X.dtype)++ def act_quant(X, block):+ seen.append("act_quant")+ x = torch.Tensor.as_subclass(X, torch.Tensor)+ return x.to(torch.float8_e4m3fn), torch.ones(x.shape[0], x.shape[-1] // block)++ def gemm(name):+ def run(+ A,+ B,+ As,+ Bs,+ *args,+ output_dtype = torch.bfloat16,+ **kwargs,+ ):+ seen.append(name)+ return torch.zeros(A.shape[:-1] + (B.shape[0],), dtype = output_dtype)++ return run++ triton_gemm = gemm("triton_gemm")+ monkeypatch.setattr(F8, "_FP8_BLOCK_SMALL_M", True)+ monkeypatch.setattr(F8, "fp8_block_small_m_linear", small_m)+ monkeypatch.setattr(F8, "act_quant", act_quant)+ monkeypatch.setattr(F8, "_fp8_kernel_unsupported", lambda *a, **k: False)+ monkeypatch.setattr(F8, "_fp8_triton_device_context", lambda *a, **k: _NullContext())+ monkeypatch.setattr(F8, "_prefer_triton_block_matmul_at", lambda device, M: False)+ monkeypatch.setattr(F8, "w8a8_block_fp8_matmul_triton", triton_gemm)+ monkeypatch.setattr(+ F8,+ "torchao_blockwise_gemm",+ lambda a, a_s, b, b_s, block_size: gemm("torchao")(a, b, a_s, b_s),+ )+ return seen+++class _NullContext:+ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False+++def _select(monkeypatch, backend):+ target = F8.torchao_block_matmul if backend == "torchao" else F8.w8a8_block_fp8_matmul_triton+ monkeypatch.setattr(F8, "fp8_block_matmul", target)+++def _forward(X, weight, scale):+ ctx = types.SimpleNamespace()+ return F8.FP8BlockQuantLinear.forward(ctx, X, weight, scale)++[email protected]("M", [1, 4, 16])+def test_fp16_decode_stays_fused_with_torchao(monkeypatch, calls, M):+ _select(monkeypatch, "torchao")+ X, weight, scale = _operands(M, torch.float16)+ assert F8.can_use_fp8_block_small_m(X, weight, scale, [128, 128])+ out = _forward(X, weight, scale)+ assert calls == ["small_m"]+ assert out.dtype == torch.float16 and out.shape == (M, weight.shape[0])++[email protected]("M", [1, 4, 16])+def test_bf16_decode_still_goes_to_torchao(monkeypatch, calls, M):+ _select(monkeypatch, "torchao")+ X, weight, scale = _operands(M, torch.bfloat16)+ assert not F8.can_use_fp8_block_small_m(X, weight, scale, [128, 128])+ _forward(X, weight, scale)+ assert calls == ["act_quant", "torchao"]++[email protected]("dtype", [torch.float16, torch.bfloat16])+def test_triton_selected_uses_fused_for_both(monkeypatch, calls, dtype):+ _select(monkeypatch, "triton")+ X, weight, scale = _operands(2, dtype)+ _forward(X, weight, scale)+ assert calls == ["small_m"]++[email protected]("backend", ["torchao", "triton"])+def test_fp16_large_m_and_kill_switch_keep_two_launches(monkeypatch, calls, backend):+ _select(monkeypatch, backend)+ X, weight, scale = _operands(F8._SMALL_M_MAX + 1, torch.float16)+ _forward(X, weight, scale)+ assert calls == ["act_quant", "triton_gemm"]+ calls.clear()+ monkeypatch.setattr(F8, "_FP8_BLOCK_SMALL_M", False)+ X, weight, scale = _operands(1, torch.float16)+ _forward(X, weight, scale)+ assert calls == ["act_quant", "triton_gemm"]+++def test_fp16_decode_stays_fused_in_small_m_first_route(monkeypatch, calls):+ _select(monkeypatch, "torchao")+ monkeypatch.setattr(F8, "can_use_fp8_block_gemv", lambda *a: False)+ monkeypatch.setattr(F8, "fp8_torch_block_quant_forward", lambda *a: calls.append("autograd"))+ X, weight, scale = _operands(1, torch.float16)+ with torch.no_grad():+ F8._fp8_block_small_m_first(X, weight, scale)+ assert calls == ["small_m"]+ calls.clear()+ X, weight, scale = _operands(1, torch.bfloat16)+ with torch.no_grad():+ F8._fp8_block_small_m_first(X, weight, scale)+ assert calls == ["autograd"]++[email protected]("dtype", [torch.float16, torch.bfloat16])+def test_gemv_keeps_its_calls_when_torchao_is_selected(monkeypatch, calls, dtype):+ _select(monkeypatch, "torchao")+ monkeypatch.setattr(F8, "can_use_fp8_block_gemv", lambda *a: True)+ monkeypatch.setattr(F8, "fp8_block_gemv", lambda *a: calls.append("gemv"))+ X, weight, scale = _operands(4, dtype)+ with torch.no_grad():+ F8._fp8_block_small_m_first(X, weight, scale)+ assert calls == ["gemv"]+++def test_triton_selected_keeps_fused_before_gemv(monkeypatch, calls):+ _select(monkeypatch, "triton")+ monkeypatch.setattr(F8, "can_use_fp8_block_gemv", lambda *a: True)+ monkeypatch.setattr(F8, "fp8_block_gemv", lambda *a: calls.append("gemv"))+ X, weight, scale = _operands(4, torch.float16)+ with torch.no_grad():+ F8._fp8_block_small_m_first(X, weight, scale)+ assert calls == ["small_m"]diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex a9258f966ef..f4c4ea24fd4 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -870,9 +870,10 @@ def _small_m_config(M, N, K, device):def can_use_fp8_block_small_m(X, weight, weight_scale, block_size):# Eager only: under torch.compile the M bound below would guard on the symbolic M and recompile.+ # torchao writes bf16 only (fp16 falls to act_quant + Triton GEMM), so fp16 keeps the fused kernel.if (not _FP8_BLOCK_SMALL_M- or fp8_block_matmul is not w8a8_block_fp8_matmul_triton+ or (fp8_block_matmul is not w8a8_block_fp8_matmul_triton and X.dtype != torch.float16)or torch.compiler.is_compiling()):return False@@ -1001,7 +1002,7 @@ def _prefer_triton_block_matmul_at(device, M):def _select_fp8_block_matmul():# torchao_block_matmul hands each call to the Triton function where that is faster; pick Triton# outright only when every GPU prefers it at every M (sm_89), which also keeps the fused small-M- # kernel there (can_use_fp8_block_small_m), as torchao beats it on sm_90 / sm_100 / sm_120.+ # kernel there for bf16 (can_use_fp8_block_small_m), as torchao beats it on sm_90 / sm_100 / sm_120.if torchao_blockwise_gemm is None:return w8a8_block_fp8_matmul_tritoncount = torch.cuda.device_count() if torch.cuda.is_available() else 0@@ -1594,6 +1595,10 @@ def _with_fp8_block_gemv(block_forward):def _fp8_block_small_m_first(X, weight, weight_scale):# 100+ SM parts: the fused small-M kernel is faster, so try it first (eager only; compiled uses the GEMV).if not (torch.is_grad_enabled() and X.requires_grad):+ # torchao selected: GEMV first (skips act_quant, beat the fused kernel at K = 14336 on sm_120).+ gemv_first = fp8_block_matmul is not w8a8_block_fp8_matmul_triton+ if gemv_first and can_use_fp8_block_gemv(X, weight, weight_scale):+ return fp8_block_gemv(X, weight, weight_scale)if (can_use_fp8_block_small_m(X, weight, weight_scale, _fp8_block_size(weight, weight_scale)@@ -1602,7 +1607,7 @@ def _fp8_block_small_m_first(X, weight, weight_scale):and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)):return fp8_block_small_m_linear(X, weight, weight_scale)- if can_use_fp8_block_gemv(X, weight, weight_scale):+ if not gemv_first and can_use_fp8_block_gemv(X, weight, weight_scale):return fp8_block_gemv(X, weight, weight_scale)return fp8_torch_block_quant_forward(X, weight, weight_scale)