Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 block GEMM: find torchao 0.18 blockwise_fp8_gemm and route each call to the faster kernel (#15153)
Daniel Han8f8eaee609d6
diff --git a/tests/test_fp8_torchao_import_route.py b/tests/test_fp8_torchao_import_route.pynew file mode 100644index 00000000000..716ba560bcf--- /dev/null+++ b/tests/test_fp8_torchao_import_route.py@@ -0,0 +1,167 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""torchao blockwise_fp8_gemm: found under both module paths, and routed away from sm_100 (no GPU needed)."""++import sys+import types++import pytest+import torch++from unsloth.kernels import fp8 as F8++OLD = "torchao.prototype.blockwise_fp8_inference.blockwise_quantization"+NEW = "torchao.kernel.blockwise_quantization"+++def _stub(monkeypatch, name, fn):+ if fn is None:+ monkeypatch.setitem(sys.modules, name, None) # the import raises ModuleNotFoundError+ else:+ monkeypatch.setitem(sys.modules, name, types.SimpleNamespace(blockwise_fp8_gemm = fn))+++ "old, new, expect",+ [("old", "new", "old"), (None, "new", "new"), ("old", None, "old"), (None, None, None)],+)+def test_import_fallback(monkeypatch, old, new, expect):+ fns = {"old": lambda *a, **k: "old", "new": lambda *a, **k: "new"}+ _stub(monkeypatch, OLD, fns.get(old))+ _stub(monkeypatch, NEW, fns.get(new))+ got = F8._import_torchao_blockwise_gemm()+ assert (got is None) if expect is None else (got() == expect)+++def test_import_skips_module_without_symbol(monkeypatch):+ monkeypatch.setitem(sys.modules, OLD, types.SimpleNamespace())+ _stub(monkeypatch, NEW, lambda: "new")+ assert F8._import_torchao_blockwise_gemm()() == "new"+++def _devices(+ monkeypatch,+ tma_arch,+ capability = (9, 0),+):+ # tma_arch[i]: device i is sm_100; the others report `capability`.+ monkeypatch.setattr(torch.cuda, "is_available", lambda: bool(tma_arch))+ monkeypatch.setattr(torch.cuda, "device_count", lambda: len(tma_arch))+ monkeypatch.setattr(F8, "_fp8_tma_gemm_arch_at", lambda index: tma_arch[index])+ monkeypatch.setattr(+ torch.cuda,+ "get_device_capability",+ lambda index = None: (10, 0) if tma_arch[index] else capability,+ )+ monkeypatch.setattr(torch.version, "hip", None)+ monkeypatch.setattr(F8, "_FP8_TMA_GEMM", True)+ monkeypatch.setattr(F8, "_TensorDescriptor", object)+++ "has_torchao, tma_arch, expect",+ [+ (False, [False], "triton"),+ (False, [True], "triton"),+ (True, [False], "torchao"), # H100 / sm_120+ (True, [True], "torchao"), # B200: torchao_block_matmul hands M > 64 to the TMA kernel+ (True, [True, True], "torchao"),+ (True, [True, False], "torchao"),+ (True, [], "torchao"),+ ],+)+def test_select(monkeypatch, has_torchao, tma_arch, expect):+ _devices(monkeypatch, tma_arch)+ monkeypatch.setattr(+ F8, "torchao_blockwise_gemm", (lambda *a, **k: None) if has_torchao else None+ )+ want = {"triton": F8.w8a8_block_fp8_matmul_triton, "torchao": F8.torchao_block_matmul}[expect]+ assert F8._select_fp8_block_matmul() is want++[email protected]("switch", ["on", "kill_switch", "no_descriptor"])+def test_tma_prefers_triton_above_64_rows(monkeypatch, switch):+ _devices(monkeypatch, [True])+ if switch == "kill_switch":+ monkeypatch.setattr(F8, "_FP8_TMA_GEMM", False)+ elif switch == "no_descriptor":+ monkeypatch.setattr(F8, "_TensorDescriptor", None)+ dev = torch.device("cuda", 0)+ got = [F8._prefer_triton_block_matmul_at(dev, M) for M in (1, 16, 64, 65, 128, 8192)]+ assert got == ([False] * 3 + [True] * 3 if switch == "on" else [False] * 6)+++def test_non_cuda_never_prefers_triton(monkeypatch):+ _devices(monkeypatch, [True], (8, 9))+ assert not F8._prefer_triton_block_matmul_at(torch.device("meta"), 8192)+ assert not F8._prefer_triton_block_matmul_at(torch.device("cpu"), 8192)+++ "capability, expect",+ [((8, 9), "triton"), ((9, 0), "torchao"), ((12, 0), "torchao"), ((8, 0), "torchao")],+)+def test_select_by_capability(monkeypatch, capability, expect):+ _devices(monkeypatch, [False], capability)+ monkeypatch.setattr(F8, "torchao_blockwise_gemm", lambda *a, **k: None)+ want = {"triton": F8.w8a8_block_fp8_matmul_triton, "torchao": F8.torchao_block_matmul}[expect]+ assert F8._select_fp8_block_matmul() is want+++def test_rocm_keeps_torchao(monkeypatch):+ _devices(monkeypatch, [False], (8, 9))+ monkeypatch.setattr(torch.version, "hip", "6.4")+ monkeypatch.setattr(F8, "torchao_blockwise_gemm", lambda *a, **k: None)+ assert F8._select_fp8_block_matmul() is F8.torchao_block_matmul+++def test_prefer_triton_per_device(monkeypatch):+ _devices(monkeypatch, [False, True])+ assert not F8._prefer_triton_block_matmul_at(torch.device("cuda", 0), 8192)+ assert F8._prefer_triton_block_matmul_at(torch.device("cuda", 1), 8192)+ assert not F8._prefer_triton_block_matmul_at(torch.device("cuda", 1), 64)+ _devices(monkeypatch, [False], (8, 9))+ assert F8._prefer_triton_block_matmul_at(torch.device("cuda", 0), 1)+++def test_torchao_block_matmul_splits_by_rows(monkeypatch):+ # The opaque op computes M from the concrete tensor, so a compiled caller adds no guard on M.+ calls = []+ monkeypatch.setattr(F8, "_prefer_triton_block_matmul_at", lambda device, M: M > 64)+ monkeypatch.setattr(F8, "w8a8_block_fp8_matmul_triton", lambda *a, **k: calls.append("triton"))+ monkeypatch.setattr(+ F8, "torchao_blockwise_gemm", lambda *a, **k: calls.append("torchao") or a[0]+ )+ meta = lambda *shape: torch.empty(*shape, device = "meta", dtype = torch.float8_e4m3fn)+ w_s = torch.empty(2, 4, device = "meta")+ for M in (1, 64, 65, 4096):+ F8.torchao_block_matmul(+ meta(M, 512), meta(256, 512), torch.empty(M, 4, device = "meta"), w_s, [128, 128]+ )+ assert calls == ["torchao", "torchao", "triton", "triton"]+++ "block_size, dtype, expect",+ [+ ([128, 128], torch.bfloat16, "torchao"),+ ([128, 128], torch.float16, "triton"), # torchao writes bf16 only+ ([128, 128], torch.float32, "triton"),+ ([128, 64], torch.bfloat16, "triton"), # torchao indexes the N scale with block_size[1]+ ],+)+def test_torchao_block_matmul_routes(monkeypatch, block_size, dtype, expect):+ calls = []+ monkeypatch.setattr(F8, "w8a8_block_fp8_matmul_triton", lambda *a, **k: calls.append("triton"))+ monkeypatch.setattr(+ F8,+ "torchao_blockwise_gemm",+ lambda *a, **k: calls.append(("torchao", k["block_size"])) or a[0],+ )+ meta = lambda *shape: torch.empty(*shape, device = "meta", dtype = torch.float8_e4m3fn)+ K, N = 512, 256+ act_s = torch.empty(64, K // block_size[1], device = "meta")+ w_s = torch.empty(N // block_size[0], K // block_size[1], device = "meta")+ F8.torchao_block_matmul(meta(64, K), meta(N, K), act_s, w_s, block_size, output_dtype = dtype)+ assert calls == (["triton"] if expect == "triton" else [("torchao", block_size[1])])diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex b3cd321cfc4..8a3d01238ac 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -114,15 +114,26 @@ except:"Unsloth: Could not find fbgemm_gpu.experimental.gemm.triton_gemm.fp8_gemm.triton_quantize_fp8_block")-try:- from torchao.prototype.blockwise_fp8_inference.blockwise_quantization import (- blockwise_fp8_gemm as torchao_blockwise_gemm,- )-except:- torchao_blockwise_gemm = None- logger.info(- "Unsloth: Could not find torchao.prototype.blockwise_fp8_inference.blockwise_quantization.blockwise_fp8_gemm"- )++def _import_torchao_blockwise_gemm():+ # torchao <= 0.17 ships blockwise_fp8_gemm under prototype; 0.18 moved it to torchao.kernel.+ try:+ from torchao.prototype.blockwise_fp8_inference.blockwise_quantization import (+ blockwise_fp8_gemm,+ )+ return blockwise_fp8_gemm+ except Exception:+ pass+ try:+ from torchao.kernel.blockwise_quantization import blockwise_fp8_gemm+ return blockwise_fp8_gemm+ except Exception:+ pass+ logger.info("Unsloth: Could not find torchao blockwise_fp8_gemm")+ return None+++torchao_blockwise_gemm = _import_torchao_blockwise_gemm()# ROCm Triton fuses fp32 multiply + fp16 cast into v_fma_mixlo_f16, which drops -0.0 and rounds@@ -945,8 +956,14 @@ def torchao_block_matmul(output_dtype: torch.dtype = torch.bfloat16,) -> torch.Tensor:# torchao's kernel indexes in int32 too, so route launches past 2**31 elements to the int64 one.- if long_indexing(act_q, weight_q) or (- act_q.numel() // act_q.shape[-1] * weight_q.shape[0] >= 2**31+ # It also reads the weight scale with block_size[1] along N, writes bf16 only (fp16 outputs would+ # round through bf16), and is slower on some GPUs (_prefer_triton_block_matmul_at).+ if (+ output_dtype != torch.bfloat16+ or long_indexing(act_q, weight_q)+ or (act_q.numel() // act_q.shape[-1] * weight_q.shape[0] >= 2**31)+ or block_size[0] != block_size[1]+ or _prefer_triton_block_matmul_at(act_q.device, act_q.numel() // act_q.shape[-1])):return w8a8_block_fp8_matmul_triton(act_q, weight_q, act_scale, weight_scale, block_size, output_dtype = output_dtype@@ -962,12 +979,39 @@ def torchao_block_matmul(return out.to(output_dtype)+def _prefer_triton_block_matmul_at(device, M):+ # Outputs match bit for bit, so this is speed only (torchao 0.18, Llama-8B shapes):+ # sm_89 (L4): torchao is 1.73x slower than the pointer kernel at K = 14336, a net loss per layer.+ # sm_100: above 64 rows w8a8_block_fp8_matmul_triton takes BLOCK_SIZE_M = 128 and the TMA kernel+ # (~2x torchao); at 64 rows or fewer it runs the pointer kernel, which torchao beats by 1.1-2.3x.+ if device.type != "cuda" or torch.version.hip is not None:+ return False+ index = device.index if device.index is not None else torch.cuda.current_device()+ if tuple(torch.cuda.get_device_capability(index)) == (8, 9):+ return True+ return (+ M > 64 and _FP8_TMA_GEMM and _TensorDescriptor is not None and _fp8_tma_gemm_arch_at(index)+ )+++def _select_fp8_block_matmul():+ # torchao_block_matmul hands each call to the Triton function where that is faster; pick Triton+ # outright only when every GPU prefers it at every M (sm_89), which also keeps the fused small-M+ # kernel there (can_use_fp8_block_small_m), as torchao beats it on sm_90 / sm_100 / sm_120.+ if torchao_blockwise_gemm is None:+ return w8a8_block_fp8_matmul_triton+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0+ if count and all(+ _prefer_triton_block_matmul_at(torch.device("cuda", i), 1) for i in range(count)+ ):+ return w8a8_block_fp8_matmul_triton+ return torchao_block_matmul++# fbgemm <= 1.3.0 silently corrupts blockwise outputs for some shapes (fixed in 1.4.0), so never-# use it for block FP8. Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses);-# torchao is ~3x faster than triton but 15-30% slower than fbgemm on H100.-fp8_block_matmul = (- torchao_block_matmul if torchao_blockwise_gemm is not None else w8a8_block_fp8_matmul_triton-)+# use it for block FP8. Preference: fbgemm (>=1.4.0) > torchao > triton (similar outputs/losses),+# except where the Triton kernels beat torchao (_prefer_triton_block_matmul_at).+fp8_block_matmul = _select_fp8_block_matmul()def _rocm_fp8e4nv_kernel_supported(device):