Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 dequant under compile: keep the torch path for bf16 on cards without native bf16 (#15071)
Daniel Hanea73d3758109
diff --git a/tests/test_fp8_pre_sm89_byte_dequant.py b/tests/test_fp8_pre_sm89_byte_dequant.pyindex d1f71ddd932..c73cc17489f 100644--- a/tests/test_fp8_pre_sm89_byte_dequant.py+++ b/tests/test_fp8_pre_sm89_byte_dequant.py@@ -351,14 +351,23 @@ def test_fp8_block_linear_forward_backward_unchanged(pre_sm89):assert torch.equal(X.grad, Xr.grad)+def _compile_dtype():+ # Inductor skips (SkipFrame) a graph with a bf16 output on a card without native bf16 (T4), and Dynamo then+ # traces the op body as its own frame, so fullgraph fails there. fp16 is what Unsloth runs on such cards.+ try:+ native = torch.cuda.is_bf16_supported(including_emulation = False)+ except TypeError: # torch without the argument+ native = torch.cuda.get_device_capability()[0] >= 8+ return torch.bfloat16 if native else torch.float16++@needs_cudadef test_under_torch_compile(pre_sm89):+ dtype = _compile_dtype()w = _weight(256, 384, "cuda")s = _scale(256, 384, [128, 128], torch.float32, "cuda")- f = torch.compile(- lambda w, s: fp8._e4m3fn_bytes_dequant(w, s, 128, 128, torch.bfloat16), fullgraph = True- )- _same_bits(f(w, s), fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16))+ f = torch.compile(lambda w, s: fp8._e4m3fn_bytes_dequant(w, s, 128, 128, dtype), fullgraph = True)+ _same_bits(f(w, s), fp8._torch_blockwise_dequant(w, s, [128, 128], dtype))def test_cpu_keeps_torch_path(monkeypatch):diff --git a/tests/test_fp8_scale_dequant_one_pass.py b/tests/test_fp8_scale_dequant_one_pass.pyindex dc67c4bac6f..0cc033e914a 100644--- a/tests/test_fp8_scale_dequant_one_pass.py+++ b/tests/test_fp8_scale_dequant_one_pass.py@@ -171,7 +171,9 @@ def test_compiled_dequant_below_sm89_is_one_opaque_op(F, monkeypatch, dtype, layelse:x, xs, ref = q, s[5:6], _reference(q, s[5:6].view(1, 1), dtype)out, ops = _compiled_graph_ops(lambda a, b: F.weight_dequant(a, b, dtype), x, xs)- assert any("fp8_scale_dequant" in op for op in ops), ops+ # bf16 on a card without native bf16 (T4): Inductor skips the graph, so the torch expression stays.+ opaque = not F._bf16_skipped_by_inductor(x, dtype)+ assert any("fp8_scale_dequant" in op for op in ops) == opaque, opsassert out.stride() == ref.stride()_assert_same_bits(out, ref)@@ -199,3 +201,17 @@ def test_compiled_dequant_reads_no_device_capability(F, monkeypatch):out, ops = _compiled_graph_ops(lambda x, y: F.weight_dequant(x, y, torch.float16), q, s)assert any("fp8_scale_dequant" in op for op in ops), ops_assert_same_bits(out, _reference(q, s, torch.float16))++[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("layout", ["row", "tensor"])+def test_inductor_fullgraph_dequant_matches_eager(F, dtype, layout):+ # Real Inductor, no forced gate. On T4, bf16 graphs are skipped by Inductor, so routing them to the+ # opaque op made Dynamo trace its body and fail fullgraph (device_count()); fp16 must stay one graph.+ q, s = _weight(96, 160)+ if layout == "tensor":+ s = s[5:6]+ ref = _reference(q, s.view(1, 1) if layout == "tensor" else s, dtype)+ torch._dynamo.reset()+ out = torch.compile(lambda a, b: F.weight_dequant(a, b, dtype), fullgraph = True)(q, s)+ _assert_same_bits(out, ref)diff --git a/tests/test_moe_dw_accumulate.py b/tests/test_moe_dw_accumulate.pyindex c83d36ea95c..6d2a2e2e448 100644--- a/tests/test_moe_dw_accumulate.py+++ b/tests/test_moe_dw_accumulate.py@@ -30,6 +30,19 @@ except Exception as exc: # pragma: no cover - depends on the installed stackpytest.skip(f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)DEV = "cuda"+++def _compile_dtype():+ # Inductor skips (SkipFrame) any graph with bf16 inputs or outputs on a card without native bf16 (sm_75), so+ # Dynamo then traces each nested Python frame, op bodies included, and fullgraph fails on the first one.+ # fp16, the dtype Unsloth trains in there, compiles as one graph.+ try:+ native = torch.cuda.is_bf16_supported(including_emulation = False)+ except TypeError: # torch without the argument+ native = torch.cuda.get_device_capability()[0] >= 8+ return torch.bfloat16 if native else torch.float16++TMA_STORE = [False, True] if gg.supports_tma() else [False]@@ -384,8 +397,9 @@ def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):from torch._dynamo.utils import countersE, N, K, T, topk = 4, 128, 128, 64, 2+ dtype = _compile_dtype()cfgs = _cfgs()- c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)+ c = _Case(E, N, K, T, topk, "permute_x", dtype)def fn(X, W):return _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)@@ -395,7 +409,7 @@ def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):compiled = torch.compile(fn, fullgraph = True)grads = []for f in (fn, compiled):- W = _param(E, N, K, torch.bfloat16)+ W = _param(E, N, K, dtype)g0 = torch.randn_like(W)W.grad = g0.clone()X = c.X.clone().requires_grad_(True)@@ -414,7 +428,7 @@ def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):return bufcounters.clear()- buf_e = torch.randn(E, N, K, device = DEV).bfloat16()+ buf_e = torch.randn(E, N, K, device = DEV).to(dtype)buf_c = buf_e.clone()acc(c.X, c.G, buf_e)torch.compile(acc, fullgraph = True)(c.X, c.G, buf_c)diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex dca43f5d4e5..9775287ff7b 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -258,7 +258,8 @@ _SCALE_DEQUANT_MIN_NUMEL = 1 << 22def _scale_dequant_one_pass_ok(x, s, dtype):# NVIDIA e4m3fn only; under compile only below sm89, where Inductor cannot emit fp8e4nv.if torch.compiler.is_compiling():- if not _fp8_e4m3fn_unsupported_in_trace(x):+ # bf16 below sm80: Inductor never compiles the graph, so the torch expression runs eagerly as before.+ if not _fp8_e4m3fn_unsupported_in_trace(x) or _bf16_skipped_by_inductor(x, dtype):return Falseelif x.numel() <= _SCALE_DEQUANT_MIN_NUMEL:return False@@ -862,6 +863,26 @@ def _fp8_e4m3fn_unsupported_in_trace(tensor):return index < len(_FP8_DEVICE_LACKS_KERNEL) and _FP8_DEVICE_LACKS_KERNEL[index]+def _native_bf16_per_device():+ try:+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0+ return tuple(torch.cuda.get_device_capability(i)[0] >= 8 for i in range(count))+ except Exception:+ return ()+++# Inductor skips (SkipFrame) any graph with a bf16 input or output on a card without native bf16 (sm_75):+# Dynamo then traces each nested frame alone, and an opaque op's body breaks fullgraph there.+_CUDA_NATIVE_BF16 = _native_bf16_per_device()+++def _bf16_skipped_by_inductor(tensor, dtype):+ if dtype != torch.bfloat16 or not tensor.is_cuda or torch.version.hip is not None:+ return False+ index = tensor.device.index if tensor.device.index is not None else torch.cuda.current_device()+ return index < len(_CUDA_NATIVE_BF16) and not _CUDA_NATIVE_BF16[index]++@triton.jitdef _fp8_rowwise_gemv_kernel(x_ptr,