Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

FP8 fp16 dequant: keep autograd for a trainable weight or scale (#15354)

Daniel Hand9b50dacbccd
diff --git a/tests/test_fp8_fp16_scale_rounding.py b/tests/test_fp8_fp16_scale_rounding.py
index 9636fc2e02a..6e91a4c3bd0 100644
--- a/tests/test_fp8_fp16_scale_rounding.py
+++ b/tests/test_fp8_fp16_scale_rounding.py
@@ -90,3 +90,26 @@ def test_other_dtypes_keep_native_expression(F, dtype, layout):
out = F.weight_dequant(q.t().contiguous(), s, dtype)
assert out.dtype == dtype
assert torch.equal(out, expected)
+
+
[email protected]("layout", ["tensor", "row"])
[email protected]("trainable", ["scale", "weight"])
+def test_fp16_dequant_keeps_autograd_for_trainable_inputs(F, layout, trainable):
+ """The fp16 out= rounding has no autograd: a trainable scale or weight must still backprop."""
+ q = _q(448.0, (4, 8))
+ s = torch.tensor(1.234567e-4) if layout == "tensor" else torch.full((4, 1), 1.234567e-4)
+ if trainable == "scale":
+ s.requires_grad_()
+ else:
+ q = q.to(torch.float32).requires_grad_()
+ y = F.weight_dequant(q, s, torch.float16)
+ assert y.dtype == torch.float16
+ assert torch.equal(
+ y.detach(), _oracle(q.detach().to(torch.float8_e4m3fn), s.detach(), torch.float16)
+ )
+ y.float().sum().backward()
+ grad = s.grad if trainable == "scale" else q.grad
+ assert grad is not None and torch.isfinite(grad).all()
+ if trainable == "scale":
+ # d(sum q * s) / ds = sum of q over what each scale covers.
+ assert torch.allclose(grad, torch.full_like(s, 448.0 * (32 if layout == "tensor" else 8)))
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 4a5d7871f18..a9258f966ef 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -256,7 +256,10 @@ def _scale_mul(x, s, dtype):
return _scale_dequant_one_pass(x, s, dtype)
if dtype != torch.float16:
return x.to(dtype) * s.to(dtype)
- if torch.compiler.is_compiling():
+ # out= has no autograd: a trainable weight or scale takes the fp32 expression (same bits).
+ if torch.compiler.is_compiling() or (
+ torch.is_grad_enabled() and (x.requires_grad or s.requires_grad)
+ ):
return (x.to(torch.float32) * s.to(torch.float32)).to(dtype)
x16 = x.to(dtype)
return torch.mul(x16, s.to(torch.float32), out = torch.empty_like(x16))