Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 dequant under compile: keep the torch path for bf16 on cards without native bf16 (#15071)

Daniel Hanea73d3758109
diff --git a/tests/test_fp8_pre_sm89_byte_dequant.py b/tests/test_fp8_pre_sm89_byte_dequant.py
index d1f71ddd932..c73cc17489f 100644
--- a/tests/test_fp8_pre_sm89_byte_dequant.py
+++ b/tests/test_fp8_pre_sm89_byte_dequant.py
@@ -351,14 +351,23 @@ def test_fp8_block_linear_forward_backward_unchanged(pre_sm89):
assert torch.equal(X.grad, Xr.grad)
+def _compile_dtype():
+ # Inductor skips (SkipFrame) a graph with a bf16 output on a card without native bf16 (T4), and Dynamo then
+ # traces the op body as its own frame, so fullgraph fails there. fp16 is what Unsloth runs on such cards.
+ try:
+ native = torch.cuda.is_bf16_supported(including_emulation = False)
+ except TypeError: # torch without the argument
+ native = torch.cuda.get_device_capability()[0] >= 8
+ return torch.bfloat16 if native else torch.float16
+
+
@needs_cuda
def test_under_torch_compile(pre_sm89):
+ dtype = _compile_dtype()
w = _weight(256, 384, "cuda")
s = _scale(256, 384, [128, 128], torch.float32, "cuda")
- f = torch.compile(
- lambda w, s: fp8._e4m3fn_bytes_dequant(w, s, 128, 128, torch.bfloat16), fullgraph = True
- )
- _same_bits(f(w, s), fp8._torch_blockwise_dequant(w, s, [128, 128], torch.bfloat16))
+ f = torch.compile(lambda w, s: fp8._e4m3fn_bytes_dequant(w, s, 128, 128, dtype), fullgraph = True)
+ _same_bits(f(w, s), fp8._torch_blockwise_dequant(w, s, [128, 128], dtype))
def test_cpu_keeps_torch_path(monkeypatch):
diff --git a/tests/test_fp8_scale_dequant_one_pass.py b/tests/test_fp8_scale_dequant_one_pass.py
index dc67c4bac6f..0cc033e914a 100644
--- a/tests/test_fp8_scale_dequant_one_pass.py
+++ b/tests/test_fp8_scale_dequant_one_pass.py
@@ -171,7 +171,9 @@ def test_compiled_dequant_below_sm89_is_one_opaque_op(F, monkeypatch, dtype, lay
else:
x, xs, ref = q, s[5:6], _reference(q, s[5:6].view(1, 1), dtype)
out, ops = _compiled_graph_ops(lambda a, b: F.weight_dequant(a, b, dtype), x, xs)
- assert any("fp8_scale_dequant" in op for op in ops), ops
+ # bf16 on a card without native bf16 (T4): Inductor skips the graph, so the torch expression stays.
+ opaque = not F._bf16_skipped_by_inductor(x, dtype)
+ assert any("fp8_scale_dequant" in op for op in ops) == opaque, ops
assert out.stride() == ref.stride()
_assert_same_bits(out, ref)
@@ -199,3 +201,17 @@ def test_compiled_dequant_reads_no_device_capability(F, monkeypatch):
out, ops = _compiled_graph_ops(lambda x, y: F.weight_dequant(x, y, torch.float16), q, s)
assert any("fp8_scale_dequant" in op for op in ops), ops
_assert_same_bits(out, _reference(q, s, torch.float16))
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("layout", ["row", "tensor"])
+def test_inductor_fullgraph_dequant_matches_eager(F, dtype, layout):
+ # Real Inductor, no forced gate. On T4, bf16 graphs are skipped by Inductor, so routing them to the
+ # opaque op made Dynamo trace its body and fail fullgraph (device_count()); fp16 must stay one graph.
+ q, s = _weight(96, 160)
+ if layout == "tensor":
+ s = s[5:6]
+ ref = _reference(q, s.view(1, 1) if layout == "tensor" else s, dtype)
+ torch._dynamo.reset()
+ out = torch.compile(lambda a, b: F.weight_dequant(a, b, dtype), fullgraph = True)(q, s)
+ _assert_same_bits(out, ref)
diff --git a/tests/test_moe_dw_accumulate.py b/tests/test_moe_dw_accumulate.py
index c83d36ea95c..6d2a2e2e448 100644
--- a/tests/test_moe_dw_accumulate.py
+++ b/tests/test_moe_dw_accumulate.py
@@ -30,6 +30,19 @@ except Exception as exc: # pragma: no cover - depends on the installed stack
pytest.skip(f"grouped_gemm is unimportable here: {exc}", allow_module_level = True)
DEV = "cuda"
+
+
+def _compile_dtype():
+ # Inductor skips (SkipFrame) any graph with bf16 inputs or outputs on a card without native bf16 (sm_75), so
+ # Dynamo then traces each nested Python frame, op bodies included, and fullgraph fails on the first one.
+ # fp16, the dtype Unsloth trains in there, compiles as one graph.
+ try:
+ native = torch.cuda.is_bf16_supported(including_emulation = False)
+ except TypeError: # torch without the argument
+ native = torch.cuda.get_device_capability()[0] >= 8
+ return torch.bfloat16 if native else torch.float16
+
+
TMA_STORE = [False, True] if gg.supports_tma() else [False]
@@ -384,8 +397,9 @@ def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):
from torch._dynamo.utils import counters
E, N, K, T, topk = 4, 128, 128, 64, 2
+ dtype = _compile_dtype()
cfgs = _cfgs()
- c = _Case(E, N, K, T, topk, "permute_x", torch.bfloat16)
+ c = _Case(E, N, K, T, topk, "permute_x", dtype)
def fn(X, W):
return _gemm(X, W, c.m_sizes, c.gather, topk, c.mode, cfgs)
@@ -395,7 +409,7 @@ def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):
compiled = torch.compile(fn, fullgraph = True)
grads = []
for f in (fn, compiled):
- W = _param(E, N, K, torch.bfloat16)
+ W = _param(E, N, K, dtype)
g0 = torch.randn_like(W)
W.grad = g0.clone()
X = c.X.clone().requires_grad_(True)
@@ -414,7 +428,7 @@ def test_compiled_fullgraph_zero_breaks_and_matches_eager(monkeypatch):
return buf
counters.clear()
- buf_e = torch.randn(E, N, K, device = DEV).bfloat16()
+ buf_e = torch.randn(E, N, K, device = DEV).to(dtype)
buf_c = buf_e.clone()
acc(c.X, c.G, buf_e)
torch.compile(acc, fullgraph = True)(c.X, c.G, buf_c)
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index dca43f5d4e5..9775287ff7b 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -258,7 +258,8 @@ _SCALE_DEQUANT_MIN_NUMEL = 1 << 22
def _scale_dequant_one_pass_ok(x, s, dtype):
# NVIDIA e4m3fn only; under compile only below sm89, where Inductor cannot emit fp8e4nv.
if torch.compiler.is_compiling():
- if not _fp8_e4m3fn_unsupported_in_trace(x):
+ # bf16 below sm80: Inductor never compiles the graph, so the torch expression runs eagerly as before.
+ if not _fp8_e4m3fn_unsupported_in_trace(x) or _bf16_skipped_by_inductor(x, dtype):
return False
elif x.numel() <= _SCALE_DEQUANT_MIN_NUMEL:
return False
@@ -862,6 +863,26 @@ def _fp8_e4m3fn_unsupported_in_trace(tensor):
return index < len(_FP8_DEVICE_LACKS_KERNEL) and _FP8_DEVICE_LACKS_KERNEL[index]
+def _native_bf16_per_device():
+ try:
+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0
+ return tuple(torch.cuda.get_device_capability(i)[0] >= 8 for i in range(count))
+ except Exception:
+ return ()
+
+
+# Inductor skips (SkipFrame) any graph with a bf16 input or output on a card without native bf16 (sm_75):
+# Dynamo then traces each nested frame alone, and an opaque op's body breaks fullgraph there.
+_CUDA_NATIVE_BF16 = _native_bf16_per_device()
+
+
+def _bf16_skipped_by_inductor(tensor, dtype):
+ if dtype != torch.bfloat16 or not tensor.is_cuda or torch.version.hip is not None:
+ return False
+ index = tensor.device.index if tensor.device.index is not None else torch.cuda.current_device()
+ return index < len(_CUDA_NATIVE_BF16) and not _CUDA_NATIVE_BF16[index]
+
+
@triton.jit
def _fp8_rowwise_gemv_kernel(
x_ptr,