Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8 fp16 dequant: keep autograd for a trainable weight or scale (#15354)
Daniel Hand9b50dacbccd
diff --git a/tests/test_fp8_fp16_scale_rounding.py b/tests/test_fp8_fp16_scale_rounding.pyindex 9636fc2e02a..6e91a4c3bd0 100644--- a/tests/test_fp8_fp16_scale_rounding.py+++ b/tests/test_fp8_fp16_scale_rounding.py@@ -90,3 +90,26 @@ def test_other_dtypes_keep_native_expression(F, dtype, layout):out = F.weight_dequant(q.t().contiguous(), s, dtype)assert out.dtype == dtypeassert torch.equal(out, expected)++[email protected]("layout", ["tensor", "row"])[email protected]("trainable", ["scale", "weight"])+def test_fp16_dequant_keeps_autograd_for_trainable_inputs(F, layout, trainable):+ """The fp16 out= rounding has no autograd: a trainable scale or weight must still backprop."""+ q = _q(448.0, (4, 8))+ s = torch.tensor(1.234567e-4) if layout == "tensor" else torch.full((4, 1), 1.234567e-4)+ if trainable == "scale":+ s.requires_grad_()+ else:+ q = q.to(torch.float32).requires_grad_()+ y = F.weight_dequant(q, s, torch.float16)+ assert y.dtype == torch.float16+ assert torch.equal(+ y.detach(), _oracle(q.detach().to(torch.float8_e4m3fn), s.detach(), torch.float16)+ )+ y.float().sum().backward()+ grad = s.grad if trainable == "scale" else q.grad+ assert grad is not None and torch.isfinite(grad).all()+ if trainable == "scale":+ # d(sum q * s) / ds = sum of q over what each scale covers.+ assert torch.allclose(grad, torch.full_like(s, 448.0 * (32 if layout == "tensor" else 8)))diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 4a5d7871f18..a9258f966ef 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -256,7 +256,10 @@ def _scale_mul(x, s, dtype):return _scale_dequant_one_pass(x, s, dtype)if dtype != torch.float16:return x.to(dtype) * s.to(dtype)- if torch.compiler.is_compiling():+ # out= has no autograd: a trainable weight or scale takes the fp32 expression (same bits).+ if torch.compiler.is_compiling() or (+ torch.is_grad_enabled() and (x.requires_grad or s.requires_grad)+ ):return (x.to(torch.float32) * s.to(torch.float32)).to(dtype)x16 = x.to(dtype)return torch.mul(x16, s.to(torch.float32), out = torch.empty_like(x16))