Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 per-tensor / rowwise dequant under torch.compile below sm89: one opaque byte-decode op (#14913)
Daniel Handa109fd877c0
diff --git a/tests/test_fp8_scale_dequant_one_pass.py b/tests/test_fp8_scale_dequant_one_pass.pyindex 4b479b987f5..a0225d4257b 100644--- a/tests/test_fp8_scale_dequant_one_pass.py+++ b/tests/test_fp8_scale_dequant_one_pass.py@@ -133,13 +133,47 @@ def test_strided_transposed_view_keeps_the_torch_path(F, monkeypatch):assert not calls-def test_compiled_dequant_keeps_the_fused_expression(F, monkeypatch):- calls = _count_launches(F, monkeypatch)+def _compiled_graph_ops(fn, *args):+ ops = []++ def backend(gm, example_inputs):+ ops.extend(str(node.target) for node in gm.graph.nodes if node.op == "call_function")+ return gm.forward++ torch._dynamo.reset()+ out = torch.compile(fn, backend = backend, fullgraph = True)(*args)+ return out, ops+++def test_compiled_dequant_keeps_the_fused_expression(F):+ # sm89+: Inductor fuses the torch expression, so the graph holds no opaque dequant op.+ if F._fp8_e4m3fn_unsupported_in_trace(torch.empty(1, device = "cuda", dtype = torch.float8_e4m3fn)):+ pytest.skip("covered by the pre-sm89 test")q, s = _weight(64, 128)+ out, ops = _compiled_graph_ops(lambda x, y: F.weight_dequant(x, y, torch.float16), q, s)+ _assert_same_bits(out, _reference(q, s, torch.float16))+ assert not any("fp8_scale_dequant" in op for op in ops)torch._dynamo.reset()- fn = torch.compile(lambda x, y: F.weight_dequant(x, y, torch.float16), fullgraph = True)- _assert_same_bits(fn(q, s), _reference(q, s, torch.float16))- assert not calls+ inductor = torch.compile(lambda x, y: F.weight_dequant(x, y, torch.float16), fullgraph = True)+ _assert_same_bits(inductor(q, s), _reference(q, s, torch.float16))++[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("layout", ["row", "row_t", "tensor"])+def test_compiled_dequant_below_sm89_is_one_opaque_op(F, monkeypatch, dtype, layout):+ # Inductor cannot emit fp8e4nv below sm89, so the fused expression fails to compile there.+ monkeypatch.setattr(F, "_FP8_DEVICE_LACKS_KERNEL", (True,) * torch.cuda.device_count())+ q, s = _weight(96, 160)+ if layout == "row":+ x, xs, ref = q, s, _reference(q, s, dtype)+ elif layout == "row_t":+ x, xs, ref = q.t(), s, _reference(q, s, dtype).t()+ else:+ x, xs, ref = q, s[5:6], _reference(q, s[5:6].view(1, 1), dtype)+ out, ops = _compiled_graph_ops(lambda a, b: F.weight_dequant(a, b, dtype), x, xs)+ assert any("fp8_scale_dequant" in op for op in ops), ops+ assert out.stride() == ref.stride()+ _assert_same_bits(out, ref)def test_small_weights_keep_the_torch_path(fp8, monkeypatch):@@ -150,3 +184,16 @@ def test_small_weights_keep_the_torch_path(fp8, monkeypatch):q, s = _weight(2049, 2048)_assert_same_bits(fp8.weight_dequant(q, s, torch.bfloat16), _reference(q, s, torch.bfloat16))assert len(calls) == 1+++def test_compiled_dequant_reads_no_device_capability(F, monkeypatch):+ # The compiled gate reads capabilities cached at import, never the device.+ def no_query(*args, **kwargs):+ raise AssertionError("device capability queried while tracing")++ monkeypatch.setattr(F, "_fp8_device_lacks_kernel", no_query)+ monkeypatch.setattr(F, "_FP8_DEVICE_LACKS_KERNEL", (True,) * torch.cuda.device_count(), raising = False)+ q, s = _weight(64, 128)+ out, ops = _compiled_graph_ops(lambda x, y: F.weight_dequant(x, y, torch.float16), q, s)+ assert any("fp8_scale_dequant" in op for op in ops), ops+ _assert_same_bits(out, _reference(q, s, torch.float16))diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 7a6b40838df..693c17e5337 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -205,10 +205,14 @@ _SCALE_DEQUANT_MIN_NUMEL = 1 << 22def _scale_dequant_one_pass_ok(x, s, dtype):- # Eager NVIDIA e4m3fn only; compiled regions already fuse the torch expression.+ # NVIDIA e4m3fn only; under compile only below sm89, where Inductor cannot emit fp8e4nv.+ if torch.compiler.is_compiling():+ if not _fp8_e4m3fn_unsupported_in_trace(x):+ return False+ elif x.numel() <= _SCALE_DEQUANT_MIN_NUMEL:+ return Falsereturn (- x.numel() > _SCALE_DEQUANT_MIN_NUMEL- and x.is_cuda+ x.is_cudaand torch.version.hip is Noneand x.dtype == torch.float8_e4m3fnand dtype in (torch.float16, torch.bfloat16, torch.float32)@@ -218,11 +222,14 @@ def _scale_dequant_one_pass_ok(x, s, dtype):and not (torch.is_grad_enabled() and (x.requires_grad or s.requires_grad))# The kernel indexes dense rows, so a transposed view must be a dense transpose.and (x.is_contiguous() or (_is_transposed_view(x) and x.t().is_contiguous()))- and not torch.compiler.is_compiling())-def _scale_dequant_one_pass(x, s, dtype):+@_opaque_under_compile(+ "fp8_scale_dequant",+ lambda x, s, dtype: torch.empty_like(x, dtype = dtype),+)+def _scale_dequant_one_pass(x: torch.Tensor, s: torch.Tensor, dtype: torch.dtype) -> torch.Tensor:transposed = _is_transposed_view(x)if transposed:x = x.t()@@ -764,6 +771,25 @@ def _fp8_kernel_unsupported(tensor, kernel_dtype = None):)+def _fp8_lacks_kernel_per_device():+ try:+ count = torch.cuda.device_count() if torch.cuda.is_available() else 0+ return tuple(_fp8_device_lacks_kernel(torch.device("cuda", i)) for i in range(count))+ except Exception:+ return ()+++# Read once: a capability query while tracing can graph-break a compiled region.+_FP8_DEVICE_LACKS_KERNEL = _fp8_lacks_kernel_per_device()+++def _fp8_e4m3fn_unsupported_in_trace(tensor):+ if not tensor.is_cuda or tensor.dtype != torch.float8_e4m3fn:+ return False+ index = tensor.device.index if tensor.device.index is not None else torch.cuda.current_device()+ return index < len(_FP8_DEVICE_LACKS_KERNEL) and _FP8_DEVICE_LACKS_KERNEL[index]++@triton.jitdef _fp8_rowwise_gemv_kernel(x_ptr,