Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MXFP4 packed matmul: bound the dX decode like the forward below sm_80 (#15376)
Daniel Hanfeef4b78dfef
diff --git a/tests/test_mxfp4_packed_low_memory.py b/tests/test_mxfp4_packed_low_memory.pyindex 0a8edc3432f..b3ce0dfb7d6 100644--- a/tests/test_mxfp4_packed_low_memory.py+++ b/tests/test_mxfp4_packed_low_memory.py@@ -332,3 +332,147 @@ def test_direct_helper_with_grad_keeps_the_out_contract(sm75, bias):got.float().sum().backward()want_x, _ = _grad_oracle(x, packed, scale, b, False)_assert_grad_close(x.grad, want_x)+++def test_t4_backward_fits_in_64mib_headroom(sm75):+ # dX = dY @ W decoded the whole 112 MiB weight even where the forward ran in chunks.+ device = sm75+ N, K = 14336, 4096+ packed, scale = _weight(N, K, device, seed = 12)+ x = torch.randn(1, K, dtype = torch.float16, device = device).requires_grad_()+ mcl.mxfp4_packed_matmul(x.detach(), packed, scale) # warm kernels and their self-checks+ with _headroom(64, device):+ start = torch.cuda.memory_allocated(device)+ torch.cuda.reset_peak_memory_stats(device)+ mcl.mxfp4_packed_matmul(x, packed, scale).float().sum().backward()+ torch.cuda.synchronize(device)+ peak = torch.cuda.max_memory_allocated(device) - start+ assert peak < 64 * 2**20+ want, _ = _grad_oracle(x, packed, scale, None, False)+ _assert_grad_close(x.grad, want)+++_COLD_COMPILED_BACKWARD = """+import torch+import unsloth # noqa: F401+import unsloth.models.mxfp4_compressed_linear as mcl+import unsloth_zoo.mxfp4_dequant as md+device = torch.device("cuda", torch.cuda.current_device())+g = torch.Generator(device = device).manual_seed(10)+packed = torch.randint(0, 256, (512, 128), dtype = torch.uint8, device = device, generator = g)+scale = torch.randint(118, 128, (512, 8), dtype = torch.uint8, device = device, generator = g)+x = torch.randn(3, 256, dtype = torch.float16, device = device).requires_grad_()+md._KERNEL_VERIFIED.clear() # nothing decoded eagerly yet+f = torch.compile(lambda a: mcl.mxfp4_packed_matmul(a, packed, scale), fullgraph = True)+f(x).float().sum().backward()+w = mcl.dequantize_mxfp4_packed(packed, scale, torch.float32).double()+want = torch.ones(3, 512, dtype = torch.float64, device = device) @ w+err = (x.grad.double() - want).norm() / want.norm()+assert err.item() < 1e-2, err.item()+print("OK")+"""+++def test_cold_compiled_direct_helper_backward():+ # The first decode of a process can be the one traced in a compiled backward: no warm self-check to lean on.+ import os+ import subprocess+ import sys++ try:+ import unsloth_zoo.mxfp4_dequant # noqa: F401+ except Exception:+ pytest.skip("needs unsloth_zoo.mxfp4_dequant")+ result = subprocess.run(+ [sys.executable, "-c", _COLD_COMPILED_BACKWARD],+ env = dict(os.environ),+ capture_output = True,+ text = True,+ timeout = 900,+ )+ if result.returncode != 0 or "OK" not in result.stdout:+ pytest.fail(result.stdout[-2000:] + "\n" + result.stderr[-4000:], pytrace = False)+++def test_chunked_dx_does_not_overflow_in_a_16bit_chunk_product():+ # 256 rows all decoding to 6.0, two 128-row chunks with dY = +100 then -100: each fp16 chunk+ # product is 76800 (> 65504) but the full reduction is exactly 0.+ device = torch.device("cuda", torch.cuda.current_device())+ N, K = 256, 64+ packed = torch.full((N, K // 2), 0x77, dtype = torch.uint8, device = device)+ scale = torch.full((N, K // 32), 127, dtype = torch.uint8, device = device)+ x = torch.cat([torch.full((1, 128), 100.0), torch.full((1, 128), -100.0)], dim = 1)+ x = x.to(device = device, dtype = torch.float16)+ saved = mcl._DENSE_BUDGET.get(device.index)+ mcl._DENSE_BUDGET[device.index] = [+ 128 * K * (x.element_size() + 4),+ 1 << 20,+ ] # decode + fp32 copy per row++ def _no_tile(*a, **k):+ raise AssertionError("tile kernel taken: the two 128-row chunks must run")++ real_tile, mcl._zoo_mxfp4_matmul = mcl._zoo_mxfp4_matmul, _no_tile+ try:+ y = mcl._bounded_dense_matmul_t_eager(x, packed, scale)+ finally:+ mcl._zoo_mxfp4_matmul = real_tile+ if saved is None:+ mcl._DENSE_BUDGET.pop(device.index, None)+ else:+ mcl._DENSE_BUDGET[device.index] = saved+ assert torch.isfinite(y).all()+ assert torch.equal(y, torch.zeros_like(y))++[email protected](not torch.cuda.is_available(), reason = "needs a CUDA device")+def test_chunked_dx_stays_inside_its_own_budget(monkeypatch):+ """Each dX chunk row holds its 16-bit decode and the fp32 copy the fp32 product reads."""+ import unsloth # noqa: F401+ import unsloth.models.mxfp4_compressed_linear as mcl+ import unsloth_zoo.mxfp4_gemm as mg++ if not hasattr(mcl, "_bounded_dense_matmul_t_eager"):+ pytest.fail("no bounded dX on this tree")+ dev = torch.device("cuda", torch.cuda.current_device())+ for dt in (torch.float16, torch.bfloat16):+ if not mg.mxfp4_grouped_gemv_available(dev, dt):+ pytest.skip("MXFP4 kernel does not self-check here")+ monkeypatch.setitem(mg._AVAILABLE_DTYPE, (str(dev), dt, "tiles"), False)+ monkeypatch.setattr(mcl, "_DENSE_BUDGET", {})+ budgets = []+ real_free = mcl._free_bytes+ monkeypatch.setattr(+ mcl, "_free_bytes", lambda d: budgets.append(real_free(d) // 2) or real_free(d)+ )+ monkeypatch.setattr(+ mcl, "_zoo_mxfp4_matmul", lambda *a, **k: pytest.fail("tile kernel taken: chunks must fit")+ )+ N, K = 14336, 4096+ g = torch.Generator(device = dev).manual_seed(12)+ packed = torch.randint(0, 256, (N, K // 2), dtype = torch.uint8, device = dev, generator = g)+ scale = torch.randint(118, 128, (N, K // 32), dtype = torch.uint8, device = dev, generator = g)+ x = torch.randn(1, N, dtype = torch.float16, device = dev)+ gc.collect()+ torch.cuda.synchronize(dev)+ torch.cuda.empty_cache()+ start = torch.cuda.memory_allocated(dev)+ total = torch.cuda.get_device_properties(dev).total_memory+ torch.cuda.set_per_process_memory_fraction((start + 64 * 2**20) / total, dev)+ try:+ torch.cuda.reset_peak_memory_stats(dev)+ y = mcl._bounded_dense_matmul_t_eager(x, packed, scale)+ torch.cuda.synchronize(dev)+ peak = torch.cuda.max_memory_allocated(dev) - start+ finally:+ torch.cuda.set_per_process_memory_fraction(1.0, dev)+ torch.cuda.empty_cache()+ assert len(budgets) == 1, budgets+ state = mcl._DENSE_BUDGET[dev.index]+ assert (+ state[0] == budgets[0]+ ), f"a chunk attempt overran the budget and was retried: {budgets[0]} -> {state[0]}"+ assert peak <= budgets[0] + 2 * 2**20, (peak, budgets[0])+ w = mcl.dequantize_mxfp4_packed(packed, scale, torch.float32).double()+ want = x.double() @ w+ assert ((y.double() - want).norm() / want.norm()).item() < 1e-3diff --git a/unsloth/models/mxfp4_compressed_linear.py b/unsloth/models/mxfp4_compressed_linear.pyindex eaa7a63a783..8b4fe99b733 100644--- a/unsloth/models/mxfp4_compressed_linear.py+++ b/unsloth/models/mxfp4_compressed_linear.py@@ -145,12 +145,20 @@ def mxfp4_packed_matmul(def _mxfp4_packed_matmul(x, packed, scale, transpose, out, bias):rows = x.numel() // x.shape[-1] if x.shape[-1] else 0# dX (transpose) stays on decode + cuBLAS: the fused kernel only wins it for a few wide shapes.- if not transpose and _zoo_mxfp4_matmul is not None and rows <= _FUSED_MAX_ROWS and x.is_cuda:- if mxfp4_gemm_available(x.device, x.dtype):+ if _zoo_mxfp4_matmul is not None and rows <= _FUSED_MAX_ROWS and x.is_cuda:+ fused = mxfp4_gemm_available(x.device, x.dtype)+ if fused and not transpose:return _zoo_mxfp4_matmul(x, packed, scale, trans = transpose, out = out, bias = bias)- if _zoo_mxfp4_kernel_verified is not None and _zoo_mxfp4_kernel_verified(x.device, x.dtype):+ if (+ not fused+ and _zoo_mxfp4_kernel_verified is not None+ and _zoo_mxfp4_kernel_verified(x.device, x.dtype)+ ):# Below sm_80 decode + cuBLAS beats the tile kernel, but only the tile kernel never builds the- # 16-bit weight: decode in chunks of output rows when the weight does not fit.+ # 16-bit weight: decode in chunks of weight rows when the weight does not fit. dX (the backward of+ # such a forward) is bounded the same way.+ if transpose:+ return _bounded_dense_matmul_t(x, packed, scale, out)return _bounded_dense_matmul(x, packed, scale, out, bias)weight = dequantize_mxfp4_packed(packed, scale, x.dtype)if transpose:@@ -271,7 +279,60 @@ def _bounded_dense_matmul_eager(state[0] = step * row_bytes // 2+def _bounded_dense_matmul_t_eager(+ x,+ packed,+ scale,+ out = None,+):+ """``x @ W`` (the dX of ``x @ W^T``) under the same budget as ``_bounded_dense_matmul_eager``: the whole W+ when it allocates, else row chunks of W whose products are summed in fp32, else the tile kernel."""+ N, half = packed.shape+ K = half * 2+ dtype, device = x.dtype, x.device+ row_bytes = K * x.element_size()+ key = device.index+ state = _DENSE_BUDGET.get(key)+ if state is not None:+ state[1] -= 1+ if state[1] <= 0:+ _DENSE_BUDGET.pop(key, None)+ state = None+ if state is None or N * row_bytes <= state[0]:+ try:+ weight = torch.empty((N, K), dtype = dtype, device = device)+ weight = dequantize_mxfp4_packed(packed, scale, dtype, out = weight)+ return torch.matmul(x, weight, out = out)+ except torch.cuda.OutOfMemoryError:+ weight = None+ state = _DENSE_BUDGET[key] = [_free_bytes(device) // 2, _DENSE_RETRY_CALLS]+ lead = x.shape[:-1]+ x2 = x.reshape(-1, N)+ # Each chunk holds its 16-bit decode and the fp32 copy the fp32 product reads.+ chunk_row_bytes = row_bytes + K * 4+ while True:+ step = min(N, state[0] // chunk_row_bytes)+ if step < _DENSE_MIN_CHUNK_ROWS:+ return _zoo_mxfp4_matmul(x, packed, scale, trans = True, out = out)+ try:+ buffer = torch.empty((step, K), dtype = dtype, device = device)+ acc = torch.zeros((x2.shape[0], K), dtype = torch.float32, device = device)+ for r0 in range(0, N, step):+ r1 = min(N, r0 + step)+ weight = dequantize_mxfp4_packed(+ packed[r0:r1], scale[r0:r1], dtype, out = buffer[: r1 - r0]+ )+ # fp32 products: a 16-bit chunk product can overflow even when the sum fits.+ acc.addmm_(x2[:, r0:r1].float(), weight.float())+ y = acc.to(dtype).view(*lead, K)+ return y if out is None else out.copy_(y)+ except torch.cuda.OutOfMemoryError:+ buffer = weight = acc = None+ state[0] = step * chunk_row_bytes // 2++_bounded_dense_matmul_op = None+_bounded_dense_matmul_t_op = Nonetry:@torch.library.custom_op("unsloth::mxfp4_bounded_dense_matmul", mutates_args = ())@@ -286,6 +347,29 @@ try:except Exception:_bounded_dense_matmul_op = None+try:++ @torch.library.custom_op("unsloth::mxfp4_bounded_dense_matmul_t", mutates_args = ())+ def _bounded_dense_matmul_t_op(+ x: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor+ ) -> torch.Tensor:+ return _bounded_dense_matmul_t_eager(x, packed, scale, None)++ @_bounded_dense_matmul_t_op.register_fake+ def _(x, packed, scale):+ return x.new_empty((*x.shape[:-1], packed.shape[1] * 2))++except Exception:+ _bounded_dense_matmul_t_op = None+++def _bounded_dense_matmul_t(x, packed, scale, out):+ if torch.compiler.is_compiling():+ if _bounded_dense_matmul_t_op is None:+ return torch.matmul(x, dequantize_mxfp4_packed(packed, scale, x.dtype), out = out)+ y = _bounded_dense_matmul_t_op(x, packed, scale)+ return y if out is None or out.shape != y.shape else out.copy_(y)+ return _bounded_dense_matmul_t_eager(x, packed, scale, out)def _bounded_dense_matmul(x, packed, scale, out, bias):