Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
mxfp4_packed_matmul: exact gradients for grad-requiring inputs (#15060)
Daniel Hanb60f1a1b4c32
diff --git a/tests/test_mxfp4_packed_low_memory.py b/tests/test_mxfp4_packed_low_memory.pyindex 1ece6165003..0a8edc3432f 100644--- a/tests/test_mxfp4_packed_low_memory.py+++ b/tests/test_mxfp4_packed_low_memory.py@@ -246,3 +246,89 @@ def test_bias_never_writes_out_whatever_the_memory(sm75, monkeypatch, compiling)got = mcl.mxfp4_packed_matmul(x, packed, scale, out = out, bias = b)assert got.data_ptr() != out.data_ptr() and bool((out == 7.0).all())torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())+++def _grad_oracle(x, packed, scale, bias, transpose):+ # fp64 oracle for d(sum(y)) / dx (and / dbias) of y = x @ W^T + b (or x @ W).+ w = mcl.dequantize_mxfp4_packed(packed, scale, torch.float32).double()+ xd = x.detach().double().requires_grad_()+ bd = None if bias is None else bias.detach().double().requires_grad_()+ y = xd @ w if transpose else torch.nn.functional.linear(xd, w, bd)+ y.sum().backward()+ return xd.grad, None if bd is None else bd.grad+++def _assert_grad_close(got, want):+ err = (got.double() - want).norm() / want.norm()+ assert err.item() < 1e-2, err.item()++[email protected]("path", ["chunks", "tile", "single_shot"])[email protected]("bias", [False, True])+def test_direct_helper_backward_is_exact_on_every_path(sm75, path, bias):+ # Each chunk's F.linear saved a view of the one reused decode buffer: backward read only the last chunk.+ device = sm75+ N, K = 512, 256+ packed, scale = _weight(N, K, device, seed = 8)+ x = torch.randn(3, K, dtype = torch.float16, device = device).requires_grad_()+ b = torch.randn(N, dtype = torch.float16, device = device).requires_grad_() if bias else None+ if path == "chunks":+ mcl._DENSE_BUDGET[device.index] = [128 * K * x.element_size(), 1024] # four 128-row chunks+ elif path == "tile":+ mcl._DENSE_BUDGET[device.index] = [64 * K * x.element_size(), 1024] # below the chunk floor+ y = mcl.mxfp4_packed_matmul(x, packed, scale, bias = b)+ assert y.requires_grad+ y.float().sum().backward()+ want_x, want_b = _grad_oracle(x, packed, scale, b, False)+ _assert_grad_close(x.grad, want_x)+ if bias:+ _assert_grad_close(b.grad, want_b)+++def test_direct_helper_transpose_backward(sm75):+ device = sm75+ N, K = 512, 256+ packed, scale = _weight(N, K, device, seed = 9)+ g = torch.randn(3, N, dtype = torch.float16, device = device).requires_grad_()+ mcl._DENSE_BUDGET[device.index] = [+ 128 * K * g.element_size(),+ 1024,+ ] # its backward is a chunked forward+ mcl.mxfp4_packed_matmul(g, packed, scale, transpose = True).float().sum().backward()+ want, _ = _grad_oracle(g, packed, scale, None, True)+ _assert_grad_close(g.grad, want)+++def test_compiled_direct_helper_backward(sm75):+ # The bounded custom op has no autograd formula of its own: a grad-requiring call goes through the Function.+ device = sm75+ N, K = 512, 256+ packed, scale = _weight(N, K, device, seed = 10)+ x = torch.randn(3, K, dtype = torch.float16, device = device).requires_grad_()+ torch._dynamo.reset()+ f = torch.compile(lambda a: mcl.mxfp4_packed_matmul(a, packed, scale), fullgraph = True)+ f(x).float().sum().backward()+ want, _ = _grad_oracle(x, packed, scale, None, False)+ _assert_grad_close(x.grad, want)++[email protected]("bias", [False, True])+def test_direct_helper_with_grad_keeps_the_out_contract(sm75, bias):+ # As without grad: a bias returns a new tensor and leaves ``out`` alone; no bias fills ``out``.+ device = sm75+ N, K = 512, 256+ packed, scale = _weight(N, K, device, seed = 11)+ x = torch.randn(3, K, dtype = torch.float16, device = device).requires_grad_()+ b = torch.randn(N, dtype = torch.float16, device = device) if bias else None+ mcl._DENSE_BUDGET[device.index] = [128 * K * x.element_size(), 1024]+ out = torch.full((3, N), 7.0, dtype = torch.float16, device = device)+ got = mcl.mxfp4_packed_matmul(x, packed, scale, out = out, bias = b)+ want = _single_shot(x.detach(), packed, scale, b)+ torch.testing.assert_close(got.detach(), want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())+ if bias:+ assert got.data_ptr() != out.data_ptr() and bool((out == 7.0).all())+ else:+ assert got.data_ptr() == out.data_ptr()+ got.float().sum().backward()+ want_x, _ = _grad_oracle(x, packed, scale, b, False)+ _assert_grad_close(x.grad, want_x)diff --git a/unsloth/models/mxfp4_compressed_linear.py b/unsloth/models/mxfp4_compressed_linear.pyindex 78947b0d14c..f1c6e6e5f09 100644--- a/unsloth/models/mxfp4_compressed_linear.py+++ b/unsloth/models/mxfp4_compressed_linear.py@@ -122,7 +122,20 @@ def mxfp4_packed_matmul():"""``x @ W^T (+ bias)`` (``transpose=False``) or ``x @ W`` for the packed ``[out, in / 2]`` weight, in x's dtype.Skinny forward inputs (decode, small batches) never build the 16-bit weight on sm_80+; below sm_80 they decode it- for cuBLAS only as far as memory allows (row chunks, else the tile kernel); the rest decode it once for cuBLAS."""+ for cuBLAS only as far as memory allows (row chunks, else the tile kernel); the rest decode it once for cuBLAS.+ Differentiable in ``x`` / ``bias``: the kernels and the reused chunk buffer have no autograd of their own."""+ if torch.is_grad_enabled() and (+ x.requires_grad or (bias is not None and not transpose and bias.requires_grad)+ ):+ y = _Mxfp4PackedMatmulFunction.apply(x, packed, scale, transpose, bias)+ # As _linear: a used bias never writes out.+ if out is None or (bias is not None and not transpose):+ return y+ return out.copy_(y)+ return _mxfp4_packed_matmul(x, packed, scale, transpose, out, bias)+++def _mxfp4_packed_matmul(x, packed, scale, transpose, out, bias):rows = x.numel() // x.shape[-1] if x.shape[-1] else 0# dX (transpose) stays on decode + cuBLAS: the fused kernel only wins it for a few wide shapes.if not transpose and _zoo_mxfp4_matmul is not None and rows <= _FUSED_MAX_ROWS and x.is_cuda:@@ -138,6 +151,26 @@ def mxfp4_packed_matmul(return _linear(x, weight, out, bias)+class _Mxfp4PackedMatmulFunction(torch.autograd.Function):+ @staticmethod+ def forward(ctx, x, packed, scale, transpose, bias):+ ctx.save_for_backward(packed, scale)+ ctx.transpose, ctx.x_dtype = transpose, x.dtype+ return _mxfp4_packed_matmul(x, packed, scale, transpose, None, None if transpose else bias)++ @staticmethod+ def backward(ctx, grad_output):+ packed, scale = ctx.saved_tensors+ grad_x = grad_bias = None+ if ctx.needs_input_grad[0]:+ grad_x = _mxfp4_packed_matmul(+ grad_output, packed, scale, not ctx.transpose, None, None+ ).to(ctx.x_dtype)+ if ctx.needs_input_grad[4] and not ctx.transpose:+ grad_bias = grad_output.reshape(-1, grad_output.shape[-1]).sum(0)+ return grad_x, None, None, None, grad_bias++def _linear(x, weight, out, bias):if bias is not None or out is None:return torch.nn.functional.linear(x, weight, bias)