Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MXFP4 packed matmul: bound the dX decode like the forward below sm_80 (#15376)

Daniel Hanfeef4b78dfef
diff --git a/tests/test_mxfp4_packed_low_memory.py b/tests/test_mxfp4_packed_low_memory.py
index 0a8edc3432f..b3ce0dfb7d6 100644
--- a/tests/test_mxfp4_packed_low_memory.py
+++ b/tests/test_mxfp4_packed_low_memory.py
@@ -332,3 +332,147 @@ def test_direct_helper_with_grad_keeps_the_out_contract(sm75, bias):
got.float().sum().backward()
want_x, _ = _grad_oracle(x, packed, scale, b, False)
_assert_grad_close(x.grad, want_x)
+
+
+def test_t4_backward_fits_in_64mib_headroom(sm75):
+ # dX = dY @ W decoded the whole 112 MiB weight even where the forward ran in chunks.
+ device = sm75
+ N, K = 14336, 4096
+ packed, scale = _weight(N, K, device, seed = 12)
+ x = torch.randn(1, K, dtype = torch.float16, device = device).requires_grad_()
+ mcl.mxfp4_packed_matmul(x.detach(), packed, scale) # warm kernels and their self-checks
+ with _headroom(64, device):
+ start = torch.cuda.memory_allocated(device)
+ torch.cuda.reset_peak_memory_stats(device)
+ mcl.mxfp4_packed_matmul(x, packed, scale).float().sum().backward()
+ torch.cuda.synchronize(device)
+ peak = torch.cuda.max_memory_allocated(device) - start
+ assert peak < 64 * 2**20
+ want, _ = _grad_oracle(x, packed, scale, None, False)
+ _assert_grad_close(x.grad, want)
+
+
+_COLD_COMPILED_BACKWARD = """
+import torch
+import unsloth # noqa: F401
+import unsloth.models.mxfp4_compressed_linear as mcl
+import unsloth_zoo.mxfp4_dequant as md
+device = torch.device("cuda", torch.cuda.current_device())
+g = torch.Generator(device = device).manual_seed(10)
+packed = torch.randint(0, 256, (512, 128), dtype = torch.uint8, device = device, generator = g)
+scale = torch.randint(118, 128, (512, 8), dtype = torch.uint8, device = device, generator = g)
+x = torch.randn(3, 256, dtype = torch.float16, device = device).requires_grad_()
+md._KERNEL_VERIFIED.clear() # nothing decoded eagerly yet
+f = torch.compile(lambda a: mcl.mxfp4_packed_matmul(a, packed, scale), fullgraph = True)
+f(x).float().sum().backward()
+w = mcl.dequantize_mxfp4_packed(packed, scale, torch.float32).double()
+want = torch.ones(3, 512, dtype = torch.float64, device = device) @ w
+err = (x.grad.double() - want).norm() / want.norm()
+assert err.item() < 1e-2, err.item()
+print("OK")
+"""
+
+
+def test_cold_compiled_direct_helper_backward():
+ # The first decode of a process can be the one traced in a compiled backward: no warm self-check to lean on.
+ import os
+ import subprocess
+ import sys
+
+ try:
+ import unsloth_zoo.mxfp4_dequant # noqa: F401
+ except Exception:
+ pytest.skip("needs unsloth_zoo.mxfp4_dequant")
+ result = subprocess.run(
+ [sys.executable, "-c", _COLD_COMPILED_BACKWARD],
+ env = dict(os.environ),
+ capture_output = True,
+ text = True,
+ timeout = 900,
+ )
+ if result.returncode != 0 or "OK" not in result.stdout:
+ pytest.fail(result.stdout[-2000:] + "\n" + result.stderr[-4000:], pytrace = False)
+
+
+def test_chunked_dx_does_not_overflow_in_a_16bit_chunk_product():
+ # 256 rows all decoding to 6.0, two 128-row chunks with dY = +100 then -100: each fp16 chunk
+ # product is 76800 (> 65504) but the full reduction is exactly 0.
+ device = torch.device("cuda", torch.cuda.current_device())
+ N, K = 256, 64
+ packed = torch.full((N, K // 2), 0x77, dtype = torch.uint8, device = device)
+ scale = torch.full((N, K // 32), 127, dtype = torch.uint8, device = device)
+ x = torch.cat([torch.full((1, 128), 100.0), torch.full((1, 128), -100.0)], dim = 1)
+ x = x.to(device = device, dtype = torch.float16)
+ saved = mcl._DENSE_BUDGET.get(device.index)
+ mcl._DENSE_BUDGET[device.index] = [
+ 128 * K * (x.element_size() + 4),
+ 1 << 20,
+ ] # decode + fp32 copy per row
+
+ def _no_tile(*a, **k):
+ raise AssertionError("tile kernel taken: the two 128-row chunks must run")
+
+ real_tile, mcl._zoo_mxfp4_matmul = mcl._zoo_mxfp4_matmul, _no_tile
+ try:
+ y = mcl._bounded_dense_matmul_t_eager(x, packed, scale)
+ finally:
+ mcl._zoo_mxfp4_matmul = real_tile
+ if saved is None:
+ mcl._DENSE_BUDGET.pop(device.index, None)
+ else:
+ mcl._DENSE_BUDGET[device.index] = saved
+ assert torch.isfinite(y).all()
+ assert torch.equal(y, torch.zeros_like(y))
+
+
[email protected](not torch.cuda.is_available(), reason = "needs a CUDA device")
+def test_chunked_dx_stays_inside_its_own_budget(monkeypatch):
+ """Each dX chunk row holds its 16-bit decode and the fp32 copy the fp32 product reads."""
+ import unsloth # noqa: F401
+ import unsloth.models.mxfp4_compressed_linear as mcl
+ import unsloth_zoo.mxfp4_gemm as mg
+
+ if not hasattr(mcl, "_bounded_dense_matmul_t_eager"):
+ pytest.fail("no bounded dX on this tree")
+ dev = torch.device("cuda", torch.cuda.current_device())
+ for dt in (torch.float16, torch.bfloat16):
+ if not mg.mxfp4_grouped_gemv_available(dev, dt):
+ pytest.skip("MXFP4 kernel does not self-check here")
+ monkeypatch.setitem(mg._AVAILABLE_DTYPE, (str(dev), dt, "tiles"), False)
+ monkeypatch.setattr(mcl, "_DENSE_BUDGET", {})
+ budgets = []
+ real_free = mcl._free_bytes
+ monkeypatch.setattr(
+ mcl, "_free_bytes", lambda d: budgets.append(real_free(d) // 2) or real_free(d)
+ )
+ monkeypatch.setattr(
+ mcl, "_zoo_mxfp4_matmul", lambda *a, **k: pytest.fail("tile kernel taken: chunks must fit")
+ )
+ N, K = 14336, 4096
+ g = torch.Generator(device = dev).manual_seed(12)
+ packed = torch.randint(0, 256, (N, K // 2), dtype = torch.uint8, device = dev, generator = g)
+ scale = torch.randint(118, 128, (N, K // 32), dtype = torch.uint8, device = dev, generator = g)
+ x = torch.randn(1, N, dtype = torch.float16, device = dev)
+ gc.collect()
+ torch.cuda.synchronize(dev)
+ torch.cuda.empty_cache()
+ start = torch.cuda.memory_allocated(dev)
+ total = torch.cuda.get_device_properties(dev).total_memory
+ torch.cuda.set_per_process_memory_fraction((start + 64 * 2**20) / total, dev)
+ try:
+ torch.cuda.reset_peak_memory_stats(dev)
+ y = mcl._bounded_dense_matmul_t_eager(x, packed, scale)
+ torch.cuda.synchronize(dev)
+ peak = torch.cuda.max_memory_allocated(dev) - start
+ finally:
+ torch.cuda.set_per_process_memory_fraction(1.0, dev)
+ torch.cuda.empty_cache()
+ assert len(budgets) == 1, budgets
+ state = mcl._DENSE_BUDGET[dev.index]
+ assert (
+ state[0] == budgets[0]
+ ), f"a chunk attempt overran the budget and was retried: {budgets[0]} -> {state[0]}"
+ assert peak <= budgets[0] + 2 * 2**20, (peak, budgets[0])
+ w = mcl.dequantize_mxfp4_packed(packed, scale, torch.float32).double()
+ want = x.double() @ w
+ assert ((y.double() - want).norm() / want.norm()).item() < 1e-3
diff --git a/unsloth/models/mxfp4_compressed_linear.py b/unsloth/models/mxfp4_compressed_linear.py
index eaa7a63a783..8b4fe99b733 100644
--- a/unsloth/models/mxfp4_compressed_linear.py
+++ b/unsloth/models/mxfp4_compressed_linear.py
@@ -145,12 +145,20 @@ def mxfp4_packed_matmul(
def _mxfp4_packed_matmul(x, packed, scale, transpose, out, bias):
rows = x.numel() // x.shape[-1] if x.shape[-1] else 0
# dX (transpose) stays on decode + cuBLAS: the fused kernel only wins it for a few wide shapes.
- if not transpose and _zoo_mxfp4_matmul is not None and rows <= _FUSED_MAX_ROWS and x.is_cuda:
- if mxfp4_gemm_available(x.device, x.dtype):
+ if _zoo_mxfp4_matmul is not None and rows <= _FUSED_MAX_ROWS and x.is_cuda:
+ fused = mxfp4_gemm_available(x.device, x.dtype)
+ if fused and not transpose:
return _zoo_mxfp4_matmul(x, packed, scale, trans = transpose, out = out, bias = bias)
- if _zoo_mxfp4_kernel_verified is not None and _zoo_mxfp4_kernel_verified(x.device, x.dtype):
+ if (
+ not fused
+ and _zoo_mxfp4_kernel_verified is not None
+ and _zoo_mxfp4_kernel_verified(x.device, x.dtype)
+ ):
# Below sm_80 decode + cuBLAS beats the tile kernel, but only the tile kernel never builds the
- # 16-bit weight: decode in chunks of output rows when the weight does not fit.
+ # 16-bit weight: decode in chunks of weight rows when the weight does not fit. dX (the backward of
+ # such a forward) is bounded the same way.
+ if transpose:
+ return _bounded_dense_matmul_t(x, packed, scale, out)
return _bounded_dense_matmul(x, packed, scale, out, bias)
weight = dequantize_mxfp4_packed(packed, scale, x.dtype)
if transpose:
@@ -271,7 +279,60 @@ def _bounded_dense_matmul_eager(
state[0] = step * row_bytes // 2
+def _bounded_dense_matmul_t_eager(
+ x,
+ packed,
+ scale,
+ out = None,
+):
+ """``x @ W`` (the dX of ``x @ W^T``) under the same budget as ``_bounded_dense_matmul_eager``: the whole W
+ when it allocates, else row chunks of W whose products are summed in fp32, else the tile kernel."""
+ N, half = packed.shape
+ K = half * 2
+ dtype, device = x.dtype, x.device
+ row_bytes = K * x.element_size()
+ key = device.index
+ state = _DENSE_BUDGET.get(key)
+ if state is not None:
+ state[1] -= 1
+ if state[1] <= 0:
+ _DENSE_BUDGET.pop(key, None)
+ state = None
+ if state is None or N * row_bytes <= state[0]:
+ try:
+ weight = torch.empty((N, K), dtype = dtype, device = device)
+ weight = dequantize_mxfp4_packed(packed, scale, dtype, out = weight)
+ return torch.matmul(x, weight, out = out)
+ except torch.cuda.OutOfMemoryError:
+ weight = None
+ state = _DENSE_BUDGET[key] = [_free_bytes(device) // 2, _DENSE_RETRY_CALLS]
+ lead = x.shape[:-1]
+ x2 = x.reshape(-1, N)
+ # Each chunk holds its 16-bit decode and the fp32 copy the fp32 product reads.
+ chunk_row_bytes = row_bytes + K * 4
+ while True:
+ step = min(N, state[0] // chunk_row_bytes)
+ if step < _DENSE_MIN_CHUNK_ROWS:
+ return _zoo_mxfp4_matmul(x, packed, scale, trans = True, out = out)
+ try:
+ buffer = torch.empty((step, K), dtype = dtype, device = device)
+ acc = torch.zeros((x2.shape[0], K), dtype = torch.float32, device = device)
+ for r0 in range(0, N, step):
+ r1 = min(N, r0 + step)
+ weight = dequantize_mxfp4_packed(
+ packed[r0:r1], scale[r0:r1], dtype, out = buffer[: r1 - r0]
+ )
+ # fp32 products: a 16-bit chunk product can overflow even when the sum fits.
+ acc.addmm_(x2[:, r0:r1].float(), weight.float())
+ y = acc.to(dtype).view(*lead, K)
+ return y if out is None else out.copy_(y)
+ except torch.cuda.OutOfMemoryError:
+ buffer = weight = acc = None
+ state[0] = step * chunk_row_bytes // 2
+
+
_bounded_dense_matmul_op = None
+_bounded_dense_matmul_t_op = None
try:
@torch.library.custom_op("unsloth::mxfp4_bounded_dense_matmul", mutates_args = ())
@@ -286,6 +347,29 @@ try:
except Exception:
_bounded_dense_matmul_op = None
+try:
+
+ @torch.library.custom_op("unsloth::mxfp4_bounded_dense_matmul_t", mutates_args = ())
+ def _bounded_dense_matmul_t_op(
+ x: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor
+ ) -> torch.Tensor:
+ return _bounded_dense_matmul_t_eager(x, packed, scale, None)
+
+ @_bounded_dense_matmul_t_op.register_fake
+ def _(x, packed, scale):
+ return x.new_empty((*x.shape[:-1], packed.shape[1] * 2))
+
+except Exception:
+ _bounded_dense_matmul_t_op = None
+
+
+def _bounded_dense_matmul_t(x, packed, scale, out):
+ if torch.compiler.is_compiling():
+ if _bounded_dense_matmul_t_op is None:
+ return torch.matmul(x, dequantize_mxfp4_packed(packed, scale, x.dtype), out = out)
+ y = _bounded_dense_matmul_t_op(x, packed, scale)
+ return y if out is None or out.shape != y.shape else out.copy_(y)
+ return _bounded_dense_matmul_t_eager(x, packed, scale, out)
def _bounded_dense_matmul(x, packed, scale, out, bias):