Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 per-tensor / rowwise dequant under torch.compile below sm89: one opaque byte-decode op (#14913)

Daniel Handa109fd877c0
diff --git a/tests/test_fp8_scale_dequant_one_pass.py b/tests/test_fp8_scale_dequant_one_pass.py
index 4b479b987f5..a0225d4257b 100644
--- a/tests/test_fp8_scale_dequant_one_pass.py
+++ b/tests/test_fp8_scale_dequant_one_pass.py
@@ -133,13 +133,47 @@ def test_strided_transposed_view_keeps_the_torch_path(F, monkeypatch):
assert not calls
-def test_compiled_dequant_keeps_the_fused_expression(F, monkeypatch):
- calls = _count_launches(F, monkeypatch)
+def _compiled_graph_ops(fn, *args):
+ ops = []
+
+ def backend(gm, example_inputs):
+ ops.extend(str(node.target) for node in gm.graph.nodes if node.op == "call_function")
+ return gm.forward
+
+ torch._dynamo.reset()
+ out = torch.compile(fn, backend = backend, fullgraph = True)(*args)
+ return out, ops
+
+
+def test_compiled_dequant_keeps_the_fused_expression(F):
+ # sm89+: Inductor fuses the torch expression, so the graph holds no opaque dequant op.
+ if F._fp8_e4m3fn_unsupported_in_trace(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn)):
+ pytest.skip("covered by the pre-sm89 test")
q, s = _weight(64, 128)
+ out, ops = _compiled_graph_ops(lambda x, y: F.weight_dequant(x, y, torch.float16), q, s)
+ _assert_same_bits(out, _reference(q, s, torch.float16))
+ assert not any("fp8_scale_dequant" in op for op in ops)
torch._dynamo.reset()
- fn = torch.compile(lambda x, y: F.weight_dequant(x, y, torch.float16), fullgraph = True)
- _assert_same_bits(fn(q, s), _reference(q, s, torch.float16))
- assert not calls
+ inductor = torch.compile(lambda x, y: F.weight_dequant(x, y, torch.float16), fullgraph = True)
+ _assert_same_bits(inductor(q, s), _reference(q, s, torch.float16))
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("layout", ["row", "row_t", "tensor"])
+def test_compiled_dequant_below_sm89_is_one_opaque_op(F, monkeypatch, dtype, layout):
+ # Inductor cannot emit fp8e4nv below sm89, so the fused expression fails to compile there.
+ monkeypatch.setattr(F, "_FP8_DEVICE_LACKS_KERNEL", (True,) * torch.cuda.device_count())
+ q, s = _weight(96, 160)
+ if layout == "row":
+ x, xs, ref = q, s, _reference(q, s, dtype)
+ elif layout == "row_t":
+ x, xs, ref = q.t(), s, _reference(q, s, dtype).t()
+ else:
+ x, xs, ref = q, s[5:6], _reference(q, s[5:6].view(1, 1), dtype)
+ out, ops = _compiled_graph_ops(lambda a, b: F.weight_dequant(a, b, dtype), x, xs)
+ assert any("fp8_scale_dequant" in op for op in ops), ops
+ assert out.stride() == ref.stride()
+ _assert_same_bits(out, ref)
def test_small_weights_keep_the_torch_path(fp8, monkeypatch):
@@ -150,3 +184,16 @@ def test_small_weights_keep_the_torch_path(fp8, monkeypatch):
q, s = _weight(2049, 2048)
_assert_same_bits(fp8.weight_dequant(q, s, torch.bfloat16), _reference(q, s, torch.bfloat16))
assert len(calls) == 1
+
+
+def test_compiled_dequant_reads_no_device_capability(F, monkeypatch):
+ # The compiled gate reads capabilities cached at import, never the device.
+ def no_query(*args, **kwargs):
+ raise AssertionError("device capability queried while tracing")
+
+ monkeypatch.setattr(F, "_fp8_device_lacks_kernel", no_query)
+ monkeypatch.setattr(F, "_FP8_DEVICE_LACKS_KERNEL", (True,) * torch.cuda.device_count(), raising = False)
+ q, s = _weight(64, 128)
+ out, ops = _compiled_graph_ops(lambda x, y: F.weight_dequant(x, y, torch.float16), q, s)
+ assert any("fp8_scale_dequant" in op for op in ops), ops
+ _assert_same_bits(out, _reference(q, s, torch.float16))
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 7a6b40838df..693c17e5337 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -205,10 +205,14 @@ _SCALE_DEQUANT_MIN_NUMEL = 1 << 22
def _scale_dequant_one_pass_ok(x, s, dtype):
- # Eager NVIDIA e4m3fn only; compiled regions already fuse the torch expression.
+ # NVIDIA e4m3fn only; under compile only below sm89, where Inductor cannot emit fp8e4nv.
+ if torch.compiler.is_compiling():
+ if not _fp8_e4m3fn_unsupported_in_trace(x):
+ return False
+ elif x.numel() <= _SCALE_DEQUANT_MIN_NUMEL:
+ return False
return (
- x.numel() > _SCALE_DEQUANT_MIN_NUMEL
- and x.is_cuda
+ x.is_cuda
and torch.version.hip is None
and x.dtype == torch.float8_e4m3fn
and dtype in (torch.float16, torch.bfloat16, torch.float32)
@@ -218,11 +222,14 @@ def _scale_dequant_one_pass_ok(x, s, dtype):
and not (torch.is_grad_enabled() and (x.requires_grad or s.requires_grad))
# The kernel indexes dense rows, so a transposed view must be a dense transpose.
and (x.is_contiguous() or (_is_transposed_view(x) and x.t().is_contiguous()))
- and not torch.compiler.is_compiling()
)
-def _scale_dequant_one_pass(x, s, dtype):
+@_opaque_under_compile(
+ "fp8_scale_dequant",
+ lambda x, s, dtype: torch.empty_like(x, dtype = dtype),
+)
+def _scale_dequant_one_pass(x: torch.Tensor, s: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:
transposed = _is_transposed_view(x)
if transposed:
x = x.t()
@@ -764,6 +771,25 @@ def _fp8_kernel_unsupported(tensor, kernel_dtype = None):
)
+def _fp8_lacks_kernel_per_device():
+ try:
+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0
+ return tuple(_fp8_device_lacks_kernel(torch.device("cuda", i)) for i in range(count))
+ except Exception:
+ return ()
+
+
+# Read once: a capability query while tracing can graph-break a compiled region.
+_FP8_DEVICE_LACKS_KERNEL = _fp8_lacks_kernel_per_device()
+
+
+def _fp8_e4m3fn_unsupported_in_trace(tensor):
+ if not tensor.is_cuda or tensor.dtype != torch.float8_e4m3fn:
+ return False
+ index = tensor.device.index if tensor.device.index is not None else torch.cuda.current_device()
+ return index < len(_FP8_DEVICE_LACKS_KERNEL) and _FP8_DEVICE_LACKS_KERNEL[index]
+
+
@triton.jit
def _fp8_rowwise_gemv_kernel(
x_ptr,