Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 block GEMM: find torchao 0.18 blockwise_fp8_gemm and route each call to the faster kernel (#15153)

Daniel Han8f8eaee609d6
diff --git a/tests/test_fp8_torchao_import_route.py b/tests/test_fp8_torchao_import_route.py
new file mode 100644
index 00000000000..716ba560bcf
--- /dev/null
+++ b/tests/test_fp8_torchao_import_route.py
@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""torchao blockwise_fp8_gemm: found under both module paths, and routed away from sm_100 (no GPU needed)."""
+
+import sys
+import types
+
+import pytest
+import torch
+
+from unsloth.kernels import fp8 as F8
+
+OLD = "torchao.prototype.blockwise_fp8_inference.blockwise_quantization"
+NEW = "torchao.kernel.blockwise_quantization"
+
+
+def _stub(monkeypatch, name, fn):
+ if fn is None:
+ monkeypatch.setitem(sys.modules, name, None) # the import raises ModuleNotFoundError
+ else:
+ monkeypatch.setitem(sys.modules, name, types.SimpleNamespace(blockwise_fp8_gemm = fn))
+
+
+ "old, new, expect",
+ [("old", "new", "old"), (None, "new", "new"), ("old", None, "old"), (None, None, None)],
+)
+def test_import_fallback(monkeypatch, old, new, expect):
+ fns = {"old": lambda *a, **k: "old", "new": lambda *a, **k: "new"}
+ _stub(monkeypatch, OLD, fns.get(old))
+ _stub(monkeypatch, NEW, fns.get(new))
+ got = F8._import_torchao_blockwise_gemm()
+ assert (got is None) if expect is None else (got() == expect)
+
+
+def test_import_skips_module_without_symbol(monkeypatch):
+ monkeypatch.setitem(sys.modules, OLD, types.SimpleNamespace())
+ _stub(monkeypatch, NEW, lambda: "new")
+ assert F8._import_torchao_blockwise_gemm()() == "new"
+
+
+def _devices(
+ monkeypatch,
+ tma_arch,
+ capability = (9, 0),
+):
+ # tma_arch[i]: device i is sm_100; the others report `capability`.
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: bool(tma_arch))
+ monkeypatch.setattr(torch.cuda, "device_count", lambda: len(tma_arch))
+ monkeypatch.setattr(F8, "_fp8_tma_gemm_arch_at", lambda index: tma_arch[index])
+ monkeypatch.setattr(
+ torch.cuda,
+ "get_device_capability",
+ lambda index = None: (10, 0) if tma_arch[index] else capability,
+ )
+ monkeypatch.setattr(torch.version, "hip", None)
+ monkeypatch.setattr(F8, "_FP8_TMA_GEMM", True)
+ monkeypatch.setattr(F8, "_TensorDescriptor", object)
+
+
+ "has_torchao, tma_arch, expect",
+ [
+ (False, [False], "triton"),
+ (False, [True], "triton"),
+ (True, [False], "torchao"), # H100 / sm_120
+ (True, [True], "torchao"), # B200: torchao_block_matmul hands M > 64 to the TMA kernel
+ (True, [True, True], "torchao"),
+ (True, [True, False], "torchao"),
+ (True, [], "torchao"),
+ ],
+)
+def test_select(monkeypatch, has_torchao, tma_arch, expect):
+ _devices(monkeypatch, tma_arch)
+ monkeypatch.setattr(
+ F8, "torchao_blockwise_gemm", (lambda *a, **k: None) if has_torchao else None
+ )
+ want = {"triton": F8.w8a8_block_fp8_matmul_triton, "torchao": F8.torchao_block_matmul}[expect]
+ assert F8._select_fp8_block_matmul() is want
+
+
[email protected]("switch", ["on", "kill_switch", "no_descriptor"])
+def test_tma_prefers_triton_above_64_rows(monkeypatch, switch):
+ _devices(monkeypatch, [True])
+ if switch == "kill_switch":
+ monkeypatch.setattr(F8, "_FP8_TMA_GEMM", False)
+ elif switch == "no_descriptor":
+ monkeypatch.setattr(F8, "_TensorDescriptor", None)
+ dev = torch.device("cuda", 0)
+ got = [F8._prefer_triton_block_matmul_at(dev, M) for M in (1, 16, 64, 65, 128, 8192)]
+ assert got == ([False] * 3 + [True] * 3 if switch == "on" else [False] * 6)
+
+
+def test_non_cuda_never_prefers_triton(monkeypatch):
+ _devices(monkeypatch, [True], (8, 9))
+ assert not F8._prefer_triton_block_matmul_at(torch.device("meta"), 8192)
+ assert not F8._prefer_triton_block_matmul_at(torch.device("cpu"), 8192)
+
+
+ "capability, expect",
+ [((8, 9), "triton"), ((9, 0), "torchao"), ((12, 0), "torchao"), ((8, 0), "torchao")],
+)
+def test_select_by_capability(monkeypatch, capability, expect):
+ _devices(monkeypatch, [False], capability)
+ monkeypatch.setattr(F8, "torchao_blockwise_gemm", lambda *a, **k: None)
+ want = {"triton": F8.w8a8_block_fp8_matmul_triton, "torchao": F8.torchao_block_matmul}[expect]
+ assert F8._select_fp8_block_matmul() is want
+
+
+def test_rocm_keeps_torchao(monkeypatch):
+ _devices(monkeypatch, [False], (8, 9))
+ monkeypatch.setattr(torch.version, "hip", "6.4")
+ monkeypatch.setattr(F8, "torchao_blockwise_gemm", lambda *a, **k: None)
+ assert F8._select_fp8_block_matmul() is F8.torchao_block_matmul
+
+
+def test_prefer_triton_per_device(monkeypatch):
+ _devices(monkeypatch, [False, True])
+ assert not F8._prefer_triton_block_matmul_at(torch.device("cuda", 0), 8192)
+ assert F8._prefer_triton_block_matmul_at(torch.device("cuda", 1), 8192)
+ assert not F8._prefer_triton_block_matmul_at(torch.device("cuda", 1), 64)
+ _devices(monkeypatch, [False], (8, 9))
+ assert F8._prefer_triton_block_matmul_at(torch.device("cuda", 0), 1)
+
+
+def test_torchao_block_matmul_splits_by_rows(monkeypatch):
+ # The opaque op computes M from the concrete tensor, so a compiled caller adds no guard on M.
+ calls = []
+ monkeypatch.setattr(F8, "_prefer_triton_block_matmul_at", lambda device, M: M > 64)
+ monkeypatch.setattr(F8, "w8a8_block_fp8_matmul_triton", lambda *a, **k: calls.append("triton"))
+ monkeypatch.setattr(
+ F8, "torchao_blockwise_gemm", lambda *a, **k: calls.append("torchao") or a[0]
+ )
+ meta = lambda *shape: torch.empty(*shape, device = "meta", dtype = torch.float8_e4m3fn)
+ w_s = torch.empty(2, 4, device = "meta")
+ for M in (1, 64, 65, 4096):
+ F8.torchao_block_matmul(
+ meta(M, 512), meta(256, 512), torch.empty(M, 4, device = "meta"), w_s, [128, 128]
+ )
+ assert calls == ["torchao", "torchao", "triton", "triton"]
+
+
+ "block_size, dtype, expect",
+ [
+ ([128, 128], torch.bfloat16, "torchao"),
+ ([128, 128], torch.float16, "triton"), # torchao writes bf16 only
+ ([128, 128], torch.float32, "triton"),
+ ([128, 64], torch.bfloat16, "triton"), # torchao indexes the N scale with block_size[1]
+ ],
+)
+def test_torchao_block_matmul_routes(monkeypatch, block_size, dtype, expect):
+ calls = []
+ monkeypatch.setattr(F8, "w8a8_block_fp8_matmul_triton", lambda *a, **k: calls.append("triton"))
+ monkeypatch.setattr(
+ F8,
+ "torchao_blockwise_gemm",
+ lambda *a, **k: calls.append(("torchao", k["block_size"])) or a[0],
+ )
+ meta = lambda *shape: torch.empty(*shape, device = "meta", dtype = torch.float8_e4m3fn)
+ K, N = 512, 256
+ act_s = torch.empty(64, K // block_size[1], device = "meta")
+ w_s = torch.empty(N // block_size[0], K // block_size[1], device = "meta")
+ F8.torchao_block_matmul(meta(64, K), meta(N, K), act_s, w_s, block_size, output_dtype = dtype)
+ assert calls == (["triton"] if expect == "triton" else [("torchao", block_size[1])])
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index b3cd321cfc4..8a3d01238ac 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -114,15 +114,26 @@ except:
"Unsloth: Could not find fbgemm_gpu.experimental.gemm.triton_gemm.fp8_gemm.triton_quantize_fp8_block"
)
-try:
- from torchao.prototype.blockwise_fp8_inference.blockwise_quantization import (
- blockwise_fp8_gemm as torchao_blockwise_gemm,
- )
-except:
- torchao_blockwise_gemm = None
- logger.info(
- "Unsloth: Could not find torchao.prototype.blockwise_fp8_inference.blockwise_quantization.blockwise_fp8_gemm"
- )
+
+def _import_torchao_blockwise_gemm():
+ # torchao <= 0.17 ships blockwise_fp8_gemm under prototype; 0.18 moved it to torchao.kernel.
+ try:
+ from torchao.prototype.blockwise_fp8_inference.blockwise_quantization import (
+ blockwise_fp8_gemm,
+ )
+ return blockwise_fp8_gemm
+ except Exception:
+ pass
+ try:
+ from torchao.kernel.blockwise_quantization import blockwise_fp8_gemm
+ return blockwise_fp8_gemm
+ except Exception:
+ pass
+ logger.info("Unsloth: Could not find torchao blockwise_fp8_gemm")
+ return None
+
+
+torchao_blockwise_gemm = _import_torchao_blockwise_gemm()
# ROCm Triton fuses fp32 multiply + fp16 cast into v_fma_mixlo_f16, which drops -0.0 and rounds
@@ -945,8 +956,14 @@ def torchao_block_matmul(
output_dtype: torch.dtype = torch.bfloat16,
) -> torch.Tensor:
# torchao's kernel indexes in int32 too, so route launches past 2**31 elements to the int64 one.
- if long_indexing(act_q, weight_q) or (
- act_q.numel() // act_q.shape[-1] * weight_q.shape[0] >= 2**31
+ # It also reads the weight scale with block_size[1] along N, writes bf16 only (fp16 outputs would
+ # round through bf16), and is slower on some GPUs (_prefer_triton_block_matmul_at).
+ if (
+ output_dtype != torch.bfloat16
+ or long_indexing(act_q, weight_q)
+ or (act_q.numel() // act_q.shape[-1] * weight_q.shape[0] >= 2**31)
+ or block_size[0] != block_size[1]
+ or _prefer_triton_block_matmul_at(act_q.device, act_q.numel() // act_q.shape[-1])
):
return w8a8_block_fp8_matmul_triton(
act_q, weight_q, act_scale, weight_scale, block_size, output_dtype = output_dtype
@@ -962,12 +979,39 @@ def torchao_block_matmul(
return out.to(output_dtype)
+def _prefer_triton_block_matmul_at(device, M):
+ # Outputs match bit for bit, so this is speed only (torchao 0.18, Llama-8B shapes):
+ # sm_89 (L4): torchao is 1.73x slower than the pointer kernel at K = 14336, a net loss per layer.
+ # sm_100: above 64 rows w8a8_block_fp8_matmul_triton takes BLOCK_SIZE_M = 128 and the TMA kernel
+ # (~2x torchao); at 64 rows or fewer it runs the pointer kernel, which torchao beats by 1.1-2.3x.
+ if device.type != "cuda" or torch.version.hip is not None:
+ return False
+ index = device.index if device.index is not None else torch.cuda.current_device()
+ if tuple(torch.cuda.get_device_capability(index)) == (8, 9):
+ return True
+ return (
+ M > 64 and _FP8_TMA_GEMM and _TensorDescriptor is not None and _fp8_tma_gemm_arch_at(index)
+ )
+
+
+def _select_fp8_block_matmul():
+ # torchao_block_matmul hands each call to the Triton function where that is faster; pick Triton
+ # outright only when every GPU prefers it at every M (sm_89), which also keeps the fused small-M
+ # kernel there (can_use_fp8_block_small_m), as torchao beats it on sm_90 / sm_100 / sm_120.
+ if torchao_blockwise_gemm is None:
+ return w8a8_block_fp8_matmul_triton
+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0
+ if count and all(
+ _prefer_triton_block_matmul_at(torch.device("cuda", i), 1) for i in range(count)
+ ):
+ return w8a8_block_fp8_matmul_triton
+ return torchao_block_matmul
+
+
# fbgemm <= 1.3.0 silently corrupts blockwise outputs for some shapes (fixed in 1.4.0), so never
-# use it for block FP8. Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses);
-# torchao is ~3x faster than triton but 15-30% slower than fbgemm on H100.
-fp8_block_matmul = (
- torchao_block_matmul if torchao_blockwise_gemm is not None else w8a8_block_fp8_matmul_triton
-)
+# use it for block FP8. Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses),
+# except where the Triton kernels beat torchao (_prefer_triton_block_matmul_at).
+fp8_block_matmul = _select_fp8_block_matmul()
def _rocm_fp8e4nv_kernel_supported(device):