Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Fix fp8 block dequant fallback on pre-sm89 GPUs (#10790)
小云a6766d2c0802
diff --git a/tests/test_fp8_pre_sm89_dequant_fallback.py b/tests/test_fp8_pre_sm89_dequant_fallback.pynew file mode 100644index 00000000000..3109527817c--- /dev/null+++ b/tests/test_fp8_pre_sm89_dequant_fallback.py@@ -0,0 +1,270 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""The block dequant fallback must divert on exactly the GPUs triton cannot compile fp8e4nv for.++triton adds fp8e4nv (torch.float8_e4m3fn) to its supported dtypes at compute capability >= 89+(third_party/nvidia/backend/compiler.py), so sm80 has to take the torch path or it raises+instead of falling back. sm89 -- 4090, L40S, L4 -- is on the supported side of that line, and a+major-only "< 9" test silently drops all of Ada onto a path that costs several times the memory.++The capability is monkeypatched and the triton entry point is stubbed, so the routing cases run+on any CUDA GPU including the pre-sm89 ones under discussion, which cannot compile the kernel+they are asserted to select. Only the value comparison needs real fp8 hardware, and it skips.+"""++import pytest+import torch++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")++BLOCK = [128, 128]++[email protected](autouse = True)+def _clear_capability_cache():+ # The guard caches per device, so a simulated capability would otherwise be ignored.+ from unsloth.kernels import fp8++ fp8._fp8_device_lacks_kernel.cache_clear()+ yield+ fp8._fp8_device_lacks_kernel.cache_clear()+++def _make(+ dtype = torch.float8_e4m3fn,+ m = 256,+ n = 256,+):+ torch.manual_seed(0)+ weight = (torch.randn(m, n, device = "cuda") * 0.4).to(dtype)+ scale = torch.rand(m // BLOCK[0], n // BLOCK[1], device = "cuda", dtype = torch.float32) + 0.5+ return weight, scale+++def _route(+ monkeypatch,+ capability,+ weight,+ scale,+ hip = None,+):+ """Return "triton" or "torch" for a simulated device.++ The kernel is stubbed rather than wrapped: these cases assert which branch is chosen, and+ letting the real one run would need a GPU that can compile fp8e4nv, so every "routes to+ triton" case would fail on exactly the pre-sm89 cards this file is about. Values are+ covered separately by test_fallback_matches_the_triton_kernel_bit_for_bit, which skips+ when the hardware cannot run both sides.+ """+ from unsloth.kernels import fp8++ calls = []++ def _stub(+ x,+ s,+ block_size = 128,+ dtype = torch.bfloat16,+ ):+ calls.append(1)+ return torch.empty(x.shape, dtype = dtype, device = x.device)++ monkeypatch.setattr(fp8, "weight_dequant_block", _stub)+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: capability)+ monkeypatch.setattr(torch.version, "hip", hip)+ fp8._blockwise_weight_dequant_any_shape(weight, scale, BLOCK, torch.bfloat16)+ return "triton" if calls else "torch"+++ "capability, expected",+ [+ ((7, 0), "torch"), # V100+ ((8, 0), "torch"), # A100, the reported failure+ ((8, 6), "torch"), # A10 / 3090+ ((8, 7), "torch"), # Jetson Orin+ ((8, 9), "triton"), # 4090 / L40S / L4: fp8e4nv IS supported here+ ((9, 0), "triton"), # H100+ ((10, 0), "triton"), # B200+ ((12, 0), "triton"), # RTX 5090+ ],+)+def test_divert_boundary_is_sm89_not_sm90(monkeypatch, capability, expected):+ weight, scale = _make()+ assert _route(monkeypatch, capability, weight, scale) == expected++[email protected]("capability", [(7, 0), (8, 0), (8, 9), (9, 0)])+def test_float8_e5m2_is_never_diverted(monkeypatch, capability):+ # fp8e5 compiles on every arch triton supports, so the guard must not touch it.+ weight, scale = _make(dtype = torch.float8_e5m2)+ assert _route(monkeypatch, capability, weight, scale) == "triton"++[email protected]("capability", [(8, 0), (9, 0), (11, 5)])+def test_rocm_is_never_diverted(monkeypatch, capability):+ # get_device_capability is gfx-derived on ROCm, not an SM number, and AMD's triton backend+ # lists fp8e4nv unconditionally, so the NVIDIA-only guard must not fire there.+ weight, scale = _make()+ assert _route(monkeypatch, capability, weight, scale, hip = "6.2.0") == "triton"+++ "shape, block",+ [+ ((256, 256), [128, 128]),+ ((1024, 2048), [128, 128]),+ ((256, 256), [64, 64]),+ ((300, 256), [128, 128]),+ ((256, 300), [128, 128]),+ ((1, 128), [128, 128]),+ ],+)+def test_fallback_matches_the_triton_kernel_bit_for_bit(monkeypatch, shape, block):+ # Diverting must not change a single value, in either direction.+ if torch.cuda.get_device_capability() < (8, 9):+ pytest.skip("needs a GPU that can run the triton fp8e4nv path to compare against")+ m, n = shape+ torch.manual_seed(0)+ weight = (torch.randn(m, n, device = "cuda") * 0.4).to(torch.float8_e4m3fn)+ scale = (+ torch.rand(-(-m // block[0]), -(-n // block[1]), device = "cuda", dtype = torch.float32) + 0.5+ )++ from unsloth.kernels import fp8++ # Both capabilities are simulated in one test, so clear between them too: the cache is+ # keyed on the device, and a stale entry would quietly compare triton against itself.+ with monkeypatch.context() as mp:+ mp.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (9, 0))+ fp8._fp8_device_lacks_kernel.cache_clear()+ assert not fp8._fp8_kernel_unsupported(weight)+ via_triton = fp8._blockwise_weight_dequant_any_shape(weight, scale, block, torch.bfloat16)+ with monkeypatch.context() as mp:+ mp.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (8, 0))+ fp8._fp8_device_lacks_kernel.cache_clear()+ assert fp8._fp8_kernel_unsupported(weight)+ via_torch = fp8._blockwise_weight_dequant_any_shape(weight, scale, block, torch.bfloat16)++ assert torch.equal(via_triton, via_torch)+++def test_fallback_does_not_materialize_a_full_size_scale(monkeypatch):+ # The whole-weight expansion needed two m*n float32 temporaries; the chunked one must stay+ # well under that or a 40GB A100 trades a CompilationError for an OOM.+ from unsloth.kernels import fp8++ m, n = 4096, 8192+ weight = (torch.randn(m, n, device = "cuda") * 0.3).to(torch.float8_e4m3fn)+ scale = torch.rand(m // 128, n // 128, device = "cuda", dtype = torch.float32) + 0.5++ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (8, 0))+ monkeypatch.setattr(fp8, "_DEQUANT_CHUNK_ELEMS", 1 << 20) # keep the test tensor small+ torch.cuda.synchronize()+ torch.cuda.empty_cache()+ torch.cuda.reset_peak_memory_stats()+ before = torch.cuda.memory_allocated()+ out = fp8._blockwise_weight_dequant_any_shape(weight, scale, BLOCK, torch.bfloat16)+ torch.cuda.synchronize()+ peak = torch.cuda.max_memory_allocated() - before++ assert out.shape == (m, n)+ # Two m*n float32 tensors plus the output is 10+ bytes/element; chunking holds it near the+ # 2-byte output itself.+ assert peak < 5 * m * n, f"fallback peaked at {peak / m / n:.1f} bytes/element"++[email protected]("weight_dtype", [torch.float8_e4m3fn, torch.float8_e5m2])+def test_pre_sm89_forward_never_reaches_the_fp8_only_kernels(monkeypatch, weight_dtype):+ # Fixing the weight dequant alone is not enough to run a model: the forward also calls+ # act_quant and the w8a8 gemm, and both take fp8e4nv pointers, so on a pre-sm89 card the+ # public path kept raising with the dequant helper already fixed. Assert the forward+ # reaches neither, rather than only that it returns. e5m2 weights are covered because+ # act_quant emits e4m3fn regardless, so keying the decision on the weight alone lets them+ # through to a kernel this device cannot compile.+ from unsloth.kernels import fp8++ called = []+ monkeypatch.setattr(+ fp8,+ "act_quant",+ lambda *a, **k: called.append("act_quant")+ or (_ for _ in ()).throw(AssertionError("act_quant reached on a pre-sm89 device")),+ )+ monkeypatch.setattr(+ fp8,+ "fp8_block_matmul",+ lambda *a, **k: called.append("fp8_block_matmul")+ or (_ for _ in ()).throw(AssertionError("fp8_block_matmul reached on a pre-sm89 device")),+ )+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (8, 0))++ torch.manual_seed(0)+ bs, m, k = 128, 512, 1024+ weight = (torch.randn(m, k, device = "cuda") * 0.3).to(weight_dtype)+ scale = torch.rand(m // bs, k // bs, device = "cuda", dtype = torch.float32) + 0.5+ scale.block_size = [bs, bs]+ X = (torch.randn(32, k, device = "cuda", dtype = torch.bfloat16) * 0.5).requires_grad_(True)++ out = fp8.FP8BlockQuantLinear.apply(X, weight, scale)+ out.float().pow(2).mean().backward()+ torch.cuda.synchronize()++ assert called == [], f"pre-sm89 forward reached fp8-only kernels: {called}"+ assert out.shape == (32, m) and torch.isfinite(out.float()).all()+ assert X.grad is not None and torch.isfinite(X.grad).all()+++def test_grouped_fp8_eval_is_guarded_in_source():+ # The grouped layer only exists in newer transformers, so the runtime test below skips on+ # most installs. Assert statically that eval does not bypass the guard, or admitting these+ # checkpoints at load time just moves the failure to the first grouped forward.+ import ast+ import inspect++ from unsloth.kernels import fp8++ tree = ast.parse(inspect.getsource(fp8))+ fn = next(+ n+ for n in ast.walk(tree)+ if isinstance(n, ast.FunctionDef) and n.name == "_fp8_grouped_forward"+ )+ guard = ast.dump(fn.body[0].test)+ assert "_fp8_kernel_unsupported" in guard, "grouped eval no longer consults the guard"+ assert "training" in guard, "grouped forward no longer distinguishes training from eval"+++def test_grouped_fp8_eval_uses_the_fallback_on_pre_sm89(monkeypatch):+ from unsloth.kernels import fp8++ if getattr(fp8, "FP8GroupedLinear", None) is None:+ pytest.skip("this transformers has no FP8GroupedLinear")++ calls = []+ monkeypatch.setattr(+ fp8,+ "_blockwise_weight_dequant_any_shape",+ lambda w, s, b, d, _o = fp8._blockwise_weight_dequant_any_shape: (+ calls.append(1) or _o(w, s, b, d)+ ),+ )+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda *a, **k: (8, 0))++ n_groups, out_per, hidden, bs = 2, 256, 256, 128+ layer = type("Stub", (), {})()+ layer.weight = (torch.randn(n_groups * out_per, hidden, device = "cuda") * 0.3).to(+ torch.float8_e4m3fn+ )+ layer.weight_scale_inv = (+ torch.rand(n_groups * out_per // bs, hidden // bs, device = "cuda", dtype = torch.float32) + 0.5+ )+ layer.n_groups, layer.block_size, layer.has_bias = n_groups, [bs, bs], False+ layer.bias, layer.training = None, False++ out = fp8.FP8GroupedLinear.forward(+ layer, torch.randn(4, n_groups, hidden, device = "cuda", dtype = torch.bfloat16)+ )++ assert calls, "grouped eval bypassed the guarded dequant on a pre-sm89 device"+ assert torch.isfinite(out.float()).all()diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 0a3dca1bf14..80912efc56d 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -9,6 +9,7 @@# See the License for the specific language governing permissions and# limitations under the License.import os+import functoolsfrom contextlib import nullcontextimport torchimport torch.nn as nn@@ -131,8 +132,9 @@ def weight_dequant(raise ValueError(f"Incompatible shapes {x.shape = }, {s.shape = }")return yelse:- # Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n)).- return weight_dequant_block(x, s, dtype = dtype)+ # Block quantized weight: scale shape is (ceil(m/block_m), ceil(n/block_n)). Go through the+ # any-shape helper so fast_dequantize's callers get the pre-sm89 fallback too.+ return _blockwise_weight_dequant_any_shape(x, s, [128, 128], dtype)# Copied from huggingface.co/deepseek-ai/DeepSeek-V3 inference/kernel.py@@ -340,9 +342,58 @@ fp8_block_matmul = ()[email protected]_cache(maxsize = None)+def _fp8_device_lacks_kernel(device):+ # A device's capability cannot change, so look it up once. Tests that simulate another+ # card must call _fp8_device_lacks_kernel.cache_clear().+ return torch.version.hip is None and torch.cuda.get_device_capability(device) < (8, 9)+++def _fp8_kernel_unsupported(tensor, kernel_dtype = None):+ """True when triton here cannot compile this fp8 dtype, so kernels taking it must be avoided.++ triton adds fp8e4nv only from sm89 on, so compare the full (major, minor): sm89 (4090,+ L40S, L4) does support it. ROCm lists fp8e4nv always and its capability is gfx-derived.++ kernel_dtype is the fp8 dtype the kernel really handles. It defaults to the tensor's own,+ which is right for a dequant, but a forward must pass e4m3fn: act_quant emits e4m3fn+ whatever the weight dtype is, so e5m2 weights hit the same unsupported kernel.+ """+ return (+ tensor.is_cuda+ and (tensor.dtype if kernel_dtype is None else kernel_dtype) == torch.float8_e4m3fn+ and _fp8_device_lacks_kernel(tensor.device)+ )+++# Expanding the scale over the whole weight needs two m*n float32 temporaries, ~6x the triton+# kernel's peak; chunk by block-rows so the scratch stays bounded. Values are identical.+_DEQUANT_CHUNK_ELEMS = 8 * 1024 * 1024+++def _torch_blockwise_dequant(weight, weight_scale, block_size, out_dtype):+ # no_grad to match the triton kernel this stands in for, which builds no graph. Not+ # torch.compiled: the callers are already compiled regions, and compiling the loop here+ # unrolls one graph per chunk, which measured slower than eager.+ with torch.no_grad():+ m, n = weight.shape+ out = torch.empty(m, n, dtype = out_dtype, device = weight.device)+ rows = max(+ block_size[0], (_DEQUANT_CHUNK_ELEMS // max(n, 1)) // block_size[0] * block_size[0]+ )+ for i in range(0, m, rows):+ j = min(i + rows, m)+ s = weight_scale[i // block_size[0] : -(-j // block_size[0])]+ s = s.repeat_interleave(block_size[0], 0)[: j - i]+ s = s.repeat_interleave(block_size[1], 1)[:, :n]+ out[i:j] = weight[i:j].to(torch.float32) * s+ return out++def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dtype):"""Blockwise fp8 weight dequant for any shape: triton when the weight tiles- evenly into block_size, else a torch-native per-block scale expansion."""+ evenly into block_size and this GPU can compile its fp8 dtype, else a+ torch-native per-block scale expansion."""m, n = weight.shapeif weight_scale.dtype not in (torch.float32, torch.float16, torch.bfloat16):weight_scale = weight_scale.to(torch.float32) # e.g. float8_e8m0fnu scales break triton@@ -351,12 +402,17 @@ def _blockwise_weight_dequant_any_shape(weight, weight_scale, block_size, out_dt# Per-tensor scale: the normal forward stashes the un-expanded scalar, which repeat_interleave# cannot grow to (m, n).return (weight.to(torch.float32) * weight_scale.float()).to(out_dtype)- if m % block_size[0] != 0 or n % block_size[1] != 0 or block_size[0] != block_size[1]:- # Uneven tiling, or rectangular blocks: the triton kernel uses a single BLOCK_SIZE for both axes- # and derives the column scale stride from it, so it mis-indexes when block_size[0] != [1].- s_full = weight_scale.repeat_interleave(block_size[0], 0)[:m]- s_full = s_full.repeat_interleave(block_size[1], 1)[:, :n]- return (weight.to(torch.float32) * s_full).to(out_dtype)+ # An fp8 dtype this device cannot compile: fall back instead of raising from the fallback.+ kernel_fp8_unsupported = _fp8_kernel_unsupported(weight)+ if (+ m % block_size[0] != 0+ or n % block_size[1] != 0+ or block_size[0] != block_size[1]+ or kernel_fp8_unsupported+ ):+ # Uneven tiling, rectangular blocks, or an uncompilable fp8 dtype: the triton kernel uses one+ # BLOCK_SIZE for both axes, so it mis-indexes when block_size[0] != [1].+ return _torch_blockwise_dequant(weight, weight_scale, block_size, out_dtype)# Even tiling with square blocks: pass the real block size, since weight_dequant would silently# default to 128 and dequantize wrongly.return weight_dequant_block(weight, weight_scale, block_size = block_size[0], dtype = out_dtype)@@ -402,9 +458,11 @@ class FP8BlockQuantLinear(torch.autograd.Function):if not weight.is_contiguous():weight = weight.contiguous()- if X.shape[-1] % block_size[1] != 0:- # Hidden dim not divisible by the activation block: dequant plus plain matmul, using the un-expanded- # scale so a scalar per-tensor scale keeps the fast path in forward and backward.+ if X.shape[-1] % block_size[1] != 0 or _fp8_kernel_unsupported(weight, torch.float8_e4m3fn):+ # Hidden dim not divisible by the activation block, or a dtype this device cannot compile+ # (act_quant and the gemm below both take fp8e4nv pointers, so pre-sm89 must skip both):+ # dequant plus plain matmul, using the un-expanded scale so a scalar per-tensor scale+ # keeps the fast path in forward and backward.W_deq = _blockwise_weight_dequant_any_shape(weight, original_weight_scale, block_size, X.dtype)@@ -558,6 +616,8 @@ class FP8_fbgemm_block_linear(torch.autograd.Function):# fbgemm 1.4.0) in_features % 16 == 0, out_features % 8 == 0; anything else raises.kernel_supported = (not per_tensor+ # triton_quantize_fp8_block below emits fp8e4nv, so pre-sm89 needs the dequant path too.+ and not _fp8_kernel_unsupported(weight, torch.float8_e4m3fn)and weight_scale.dtype == torch.float32and (bs_m, bs_n, bs_k) == (128, 128, 128)and X.shape[-1] % 16 == 0@@ -760,7 +820,11 @@ if FP8GroupedLinear is not None:return grad_x, None, None, None, None, grad_biasdef _fp8_grouped_forward(self, x):- if self.weight.element_size() > 1 or not self.training:+ # The upstream eval path runs its own fp8e4nv kernels, so pre-sm89 has to use the+ # dequant path here too, not only while training.+ if self.weight.element_size() > 1 or (+ not self.training and not _fp8_kernel_unsupported(self.weight, torch.float8_e4m3fn)+ ):return _fp8_grouped_forward_orig(self, x)bias = self.bias if self.has_bias else Nonereturn _FP8GroupedMM.apply(