Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
MXFP4 packed Linear: bound the below-sm_80 weight decode by free memory (#14784)
Daniel Han4077567c5e1f
diff --git a/tests/test_mxfp4_packed_low_memory.py b/tests/test_mxfp4_packed_low_memory.pynew file mode 100644index 00000000000..635a24d3480--- /dev/null+++ b/tests/test_mxfp4_packed_low_memory.py@@ -0,0 +1,231 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Below sm_80 a packed MXFP4 forward decodes the 16-bit weight for cuBLAS; it must not need more memory than the+tile kernel it replaced when the whole weight does not fit (T4 with little headroom)."""++import contextlib+import gc++import pytest+from llama_patch_isolation import restore_llama_patches # noqa: F401+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py+import torch++import unsloth # noqa: F401+import unsloth.models.mxfp4_compressed_linear as mcl++pytestmark = [+ pytest.mark.usefixtures("restore_llama_patches"),+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device"),+]+++def _zoo():+ try:+ import unsloth_zoo.mxfp4_gemm as mg+ except Exception:+ pytest.skip("needs unsloth_zoo.mxfp4_gemm")+ if not hasattr(mg, "mxfp4_grouped_gemv_available"):+ pytest.skip("unsloth_zoo before the sm_80 tile GEMM floor: the tile kernel still runs here")+ return mg+++def sm75(monkeypatch):+ """The sm_75 policy on whatever GPU runs this: the verified tile GEMM is kept off, real kernels run."""+ mg = _zoo()+ device = torch.device("cuda", torch.cuda.current_device())+ monkeypatch.delenv("UNSLOTH_MXFP4_FUSED_GEMM", raising = False)+ for dtype in (torch.float16, torch.bfloat16):+ if not mg.mxfp4_grouped_gemv_available(device, dtype):+ pytest.skip("the fused MXFP4 kernel does not self-check on this GPU")+ monkeypatch.setitem(mg._AVAILABLE_DTYPE, (str(device), dtype, "tiles"), False)+ assert not mg.mxfp4_gemm_available(device, dtype)+ monkeypatch.setattr(mcl, "_DENSE_BUDGET", {}, raising = False)+ return device+++def _headroom(mib, device):+ """Let the caching allocator hand out at most ``mib`` MiB more than is allocated now."""+ gc.collect()+ torch.cuda.synchronize(device)+ torch.cuda.empty_cache()+ total = torch.cuda.get_device_properties(device).total_memory+ cap = torch.cuda.memory_allocated(device) + mib * 2**20+ torch.cuda.set_per_process_memory_fraction(cap / total, device)+ try:+ yield+ finally:+ torch.cuda.set_per_process_memory_fraction(1.0, device)+ torch.cuda.empty_cache()+++def _weight(N, K, device, seed = 0):+ g = torch.Generator(device = device).manual_seed(seed)+ packed = torch.randint(0, 256, (N, K // 2), dtype = torch.uint8, device = device, generator = g)+ scale = torch.randint(118, 128, (N, K // 32), dtype = torch.uint8, device = device, generator = g)+ return packed, scale+++def _single_shot(x, packed, scale, bias = None):+ # What #1899 runs below sm_80: the whole 16-bit weight, one cuBLAS call.+ weight = mcl.dequantize_mxfp4_packed(packed, scale, x.dtype)+ return torch.nn.functional.linear(x, weight, bias)+++def test_t4_decode_fits_in_64mib_headroom(sm75):+ # 14336 x 4096 decodes to 112 MiB of fp16: the tile kernel needed 28 KiB, dequantize + cuBLAS must not OOM.+ device = sm75+ N, K = 14336, 4096+ packed, scale = _weight(N, K, device)+ x = torch.randn(1, K, dtype = torch.float16, device = device)+ want = _single_shot(x, packed, scale)+ mcl.mxfp4_packed_matmul(x, packed, scale) # warm kernels and their self-checks+ with _headroom(64, device):+ start = torch.cuda.memory_allocated(device)+ torch.cuda.reset_peak_memory_stats(device)+ got = mcl.mxfp4_packed_matmul(x, packed, scale)+ torch.cuda.synchronize(device)+ peak = torch.cuda.max_memory_allocated(device) - start+ assert peak < 64 * 2**20+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())+ # The next calls go straight to chunks: no second allocation failure.+ ooms = torch.cuda.memory_stats(device).get("num_ooms", 0)+ with _headroom(64, device):+ mcl.mxfp4_packed_matmul(x, packed, scale)+ assert torch.cuda.memory_stats(device).get("num_ooms", 0) == ooms+++def test_weight_that_fits_keeps_the_single_shot_decode(sm75):+ device = sm75+ packed, scale = _weight(1024, 512, device, seed = 1)+ x = torch.randn(3, 512, dtype = torch.bfloat16, device = device)+ assert torch.equal(mcl.mxfp4_packed_matmul(x, packed, scale), _single_shot(x, packed, scale))+ assert not mcl._DENSE_BUDGET++[email protected]("dtype", [torch.float16, torch.bfloat16])[email protected]("rows", [1, 7, 64])[email protected]("bias", [False, True])[email protected]("lead", [(), (2,)])+def test_row_chunks_match_the_single_shot_decode(sm75, dtype, rows, bias, lead):+ device = sm75+ N, K = 1000, 384 # 1000 rows: the last chunk is ragged+ packed, scale = _weight(N, K, device, seed = rows)+ x = torch.randn(*lead, rows, K, dtype = dtype, device = device)+ b = torch.randn(N, dtype = dtype, device = device) if bias else None+ want = _single_shot(x, packed, scale, b)+ row_bytes = K * x.element_size()+ mcl._DENSE_BUDGET[device.index] = [300 * row_bytes, 1024] # 300-row chunks+ got = mcl.mxfp4_packed_matmul(x, packed, scale, bias = b)+ assert got.shape == want.shape and got.dtype == dtype+ # Same decoded rows, same math; cuBLAS may pick another reduction order for the narrower chunk (1 ulp).+ tol = 1.6e-2 if dtype == torch.bfloat16 else 2e-3+ torch.testing.assert_close(got, want, rtol = tol, atol = tol * want.abs().max().item())+ out = torch.empty_like(want)+ mcl._DENSE_BUDGET[device.index] = [300 * row_bytes, 1024]+ if not bias:+ assert mcl.mxfp4_packed_matmul(x, packed, scale, out = out).data_ptr() == out.data_ptr()+ torch.testing.assert_close(out, want, rtol = tol, atol = tol * want.abs().max().item())+++def test_no_room_for_a_chunk_takes_the_tile_kernel(sm75, monkeypatch):+ device = sm75+ N, K = 4096, 4096+ packed, scale = _weight(N, K, device, seed = 3)+ x = torch.randn(2, K, dtype = torch.float16, device = device)+ want = _single_shot(x, packed, scale)+ calls = []+ tile = mcl._zoo_mxfp4_matmul+ monkeypatch.setattr(mcl, "_zoo_mxfp4_matmul", lambda *a, **k: calls.append(1) or tile(*a, **k))+ mcl._DENSE_BUDGET[device.index] = [64 * K * 2, 1024] # 64 rows < the 128-row chunk floor+ got = mcl.mxfp4_packed_matmul(x, packed, scale)+ assert calls+ torch.testing.assert_close(got, want, rtol = 2e-2, atol = 2e-2 * want.abs().max().item())+++def test_budget_is_retried_after_a_while(sm75):+ device = sm75+ packed, scale = _weight(512, 256, device, seed = 4)+ x = torch.randn(1, 256, dtype = torch.float16, device = device)+ mcl._DENSE_BUDGET[device.index] = [128 * 256 * 2, 2]+ mcl.mxfp4_packed_matmul(x, packed, scale)+ assert device.index in mcl._DENSE_BUDGET+ mcl.mxfp4_packed_matmul(x, packed, scale)+ assert device.index not in mcl._DENSE_BUDGET # whole weight fitted again+++def test_compiled_forward_is_bounded_without_graph_breaks(sm75):+ from torch._dynamo.utils import counters++ device = sm75+ N, K = 14336, 4096+ module = mcl.make_mxfp4_packed_linear(K, N, device = device)+ module.weight_packed.data, module.weight_scale.data = _weight(N, K, device, seed = 5)+ module.compute_dtype = torch.float16+ x = torch.randn(1, K, dtype = torch.float16, device = device)+ with torch.no_grad():+ want = module(x)+ torch._dynamo.reset()+ counters.clear()+ compiled = torch.compile(module, fullgraph = True)+ torch.testing.assert_close(compiled(x), want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())+ assert sum(counters["graph_break"].values()) == 0+ with _headroom(64, device):+ got = compiled(x)+ torch.cuda.synchronize(device)+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())+ torch._dynamo.reset()+++def _fake_oom_dequant(monkeypatch, fail_rows):+ """``dequantize_mxfp4_packed`` raising an allocator OOM once for each entry of ``fail_rows`` (row counts, in+ order), as when the decoded weight allocated but its scratch, the output or the cuBLAS workspace did not."""+ real = mcl.dequantize_mxfp4_packed+ pending, seen = list(fail_rows), []++ def fake(packed, scale, dtype = torch.bfloat16, out = None):+ seen.append(packed.shape[0])+ if pending and packed.shape[0] == pending[0]:+ pending.pop(0)+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (fake)")+ return real(packed, scale, dtype, out = out)++ monkeypatch.setattr(mcl, "dequantize_mxfp4_packed", fake)+ return seen, pending+++def test_oom_after_the_weight_allocated_falls_back_to_chunks(sm75, monkeypatch):+ device = sm75+ N, K = 1024, 512+ packed, scale = _weight(N, K, device, seed = 6)+ x = torch.randn(2, K, dtype = torch.float16, device = device)+ want = _single_shot(x, packed, scale)+ # Whole weight attempt fails late, then the first (full-height) chunk fails too: half-height chunks finish.+ seen, pending = _fake_oom_dequant(monkeypatch, [N, N])+ got = mcl.mxfp4_packed_matmul(x, packed, scale)+ assert not pending and seen[:2] == [N, N] and seen[2] == N // 2+ assert device.index in mcl._DENSE_BUDGET+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())++[email protected]("compiling", [False, True])+def test_bias_never_writes_out_whatever_the_memory(sm75, monkeypatch, compiling):+ # The full-weight path with a bias returns a new tensor and leaves ``out`` alone: so must the fallbacks.+ device = sm75+ N, K = 1000, 384+ packed, scale = _weight(N, K, device, seed = 7)+ x = torch.randn(3, K, dtype = torch.float16, device = device)+ b = torch.randn(N, dtype = torch.float16, device = device)+ want = _single_shot(x, packed, scale, b)+ out = torch.full_like(want, 7.0)+ full = mcl.mxfp4_packed_matmul(x, packed, scale, out = out, bias = b)+ assert full.data_ptr() != out.data_ptr() and bool((out == 7.0).all())+ if compiling:+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)+ mcl._DENSE_BUDGET[device.index] = [300 * K * x.element_size(), 1024] # 300-row chunks+ got = mcl.mxfp4_packed_matmul(x, packed, scale, out = out, bias = b)+ assert got.data_ptr() != out.data_ptr() and bool((out == 7.0).all())+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())diff --git a/unsloth/models/mxfp4_compressed_linear.py b/unsloth/models/mxfp4_compressed_linear.pyindex 14eb65910c9..7c6b76d8b6e 100644--- a/unsloth/models/mxfp4_compressed_linear.py+++ b/unsloth/models/mxfp4_compressed_linear.py@@ -51,6 +51,13 @@ try:from unsloth_zoo.mxfp4_gemm import mxfp4_gemm_available, mxfp4_matmul as _zoo_mxfp4_matmulexcept Exception:_zoo_mxfp4_matmul = None+try:+ # Self-check only, no sm_80 floor (zoo #1899+): below sm_80 the tile kernel is the low-memory last resort.+ from unsloth_zoo.mxfp4_gemm import (+ mxfp4_grouped_gemv_available as _zoo_mxfp4_kernel_verified,+ )+except Exception:+ _zoo_mxfp4_kernel_verified = None_FP4_VALUES = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,@@ -80,13 +87,18 @@ def dequantize_mxfp4_packed(packed,scale,dtype = torch.bfloat16,+ out = None,):- """Packed ``[out, in / 2]`` + E8M0 scales ``[out, in / 32]`` -> ``[out, in]``; zoo kernel if self-verified, else torch."""+ """Packed ``[out, in / 2]`` + E8M0 scales ``[out, in / 32]`` -> ``[out, in]``; zoo kernel if self-verified, else torch.+ ``out``: a contiguous ``[out, in]`` buffer to decode into."""out_features, half = packed.shapeblocks = packed.view(out_features, half // 16, 16)if _zoo_mxfp4_dequantize is not None and dtype in (torch.bfloat16, torch.float16):- return _zoo_mxfp4_dequantize(blocks, scale, dtype = dtype)- return _dequantize_torch(blocks, scale, dtype)+ if out is None:+ return _zoo_mxfp4_dequantize(blocks, scale, dtype = dtype)+ return _zoo_mxfp4_dequantize(blocks, scale, dtype = dtype, out = out)+ weight = _dequantize_torch(blocks, scale, dtype)+ return weight if out is None else out.copy_(weight)def _fused_max_rows():@@ -109,7 +121,8 @@ def mxfp4_packed_matmul(bias = None,):"""``x @ W^T (+ bias)`` (``transpose=False``) or ``x @ W`` for the packed ``[out, in / 2]`` weight, in x's dtype.- Skinny forward inputs (decode, small batches) never build the 16-bit weight; the rest decode it once for cuBLAS."""+ Skinny forward inputs (decode, small batches) never build the 16-bit weight on sm_80+; below sm_80 they decode it+ for cuBLAS only as far as memory allows (row chunks, else the tile kernel); the rest decode it once for cuBLAS."""rows = x.numel() // x.shape[-1] if x.shape[-1] else 0# dX (transpose) stays on decode + cuBLAS: the fused kernel only wins it for a few wide shapes.if (@@ -117,17 +130,140 @@ def mxfp4_packed_matmul(and _zoo_mxfp4_matmul is not Noneand rows <= _FUSED_MAX_ROWSand x.is_cuda- and mxfp4_gemm_available(x.device, x.dtype)):- return _zoo_mxfp4_matmul(x, packed, scale, trans = transpose, out = out, bias = bias)+ if mxfp4_gemm_available(x.device, x.dtype):+ return _zoo_mxfp4_matmul(x, packed, scale, trans = transpose, out = out, bias = bias)+ if _zoo_mxfp4_kernel_verified is not None and _zoo_mxfp4_kernel_verified(+ x.device, x.dtype+ ):+ # Below sm_80 decode + cuBLAS beats the tile kernel, but only the tile kernel never builds the+ # 16-bit weight: decode in chunks of output rows when the weight does not fit.+ return _bounded_dense_matmul(x, packed, scale, out, bias)weight = dequantize_mxfp4_packed(packed, scale, x.dtype)if transpose:return torch.matmul(x, weight, out = out)+ return _linear(x, weight, out, bias)+++def _linear(x, weight, out, bias):if bias is not None or out is None:return torch.nn.functional.linear(x, weight, bias)return torch.matmul(x, weight.t(), out = out)+# Below sm_80 (`_bounded_dense_matmul`): per GPU, the bytes a decoded chunk may take once the whole 16-bit+# weight failed to allocate, and the calls left before the whole weight is tried again.+_DENSE_BUDGET = {}+_DENSE_RETRY_CALLS = 1024+# Fewer decoded rows than this per chunk: the tile kernel (output only) instead.+_DENSE_MIN_CHUNK_ROWS = 128+++def _free_bytes(device):+ """What the caching allocator can still hand out: driver free (capped by a per-process memory fraction)+ plus its own cached free blocks. Only queried after an allocation failed: the queries cost ~100 us."""+ free, total = torch.cuda.mem_get_info(device)+ reserved = torch.cuda.memory_reserved(device)+ try:+ cap = int(torch.cuda.get_per_process_memory_fraction(device) * total)+ free = min(free, cap - reserved)+ except Exception:+ pass+ return max(0, free) + max(0, reserved - torch.cuda.memory_allocated(device))+++def _bounded_dense_matmul_eager(x, packed, scale, out = None, bias = None):+ """``x @ W^T (+ bias)`` without ever holding more of the 16-bit W than fits: the whole W when it allocates+ (one decode + one cuBLAS call), else row chunks of W written to ``y[..., r0:r1]`` (no cross-chunk sums),+ else the tile kernel."""+ N, half = packed.shape+ K = half * 2+ dtype, device = x.dtype, x.device+ row_bytes = K * x.element_size()+ key = device.index+ state = _DENSE_BUDGET.get(key)+ if state is not None:+ state[1] -= 1+ if state[1] <= 0:+ _DENSE_BUDGET.pop(key, None)+ state = None+ if state is None or N * row_bytes <= state[0]:+ # The weight, the decode's scratch, the output and the cuBLAS workspace: any of them may be the one+ # that does not fit, so the whole attempt is guarded and its tensors are dropped before the retry.+ try:+ weight = torch.empty((N, K), dtype = dtype, device = device)+ return _linear(x, dequantize_mxfp4_packed(packed, scale, dtype, out = weight), out, bias)+ except torch.cuda.OutOfMemoryError:+ weight = None+ state = _DENSE_BUDGET[key] = [_free_bytes(device) // 2, _DENSE_RETRY_CALLS]+ lead = x.shape[:-1]+ x2 = x.reshape(-1, K)+ M = x2.shape[0]+ # Like _linear: ``out`` is only written when there is no bias, whichever path runs.+ if (+ bias is None+ and out is not None+ and out.dtype == dtype+ and out.is_contiguous()+ and out.numel() == M * N+ ):+ y = out.view(M, N)+ else:+ y = torch.empty((M, N), dtype = dtype, device = device)+ while True:+ step = min(N, state[0] // row_bytes)+ if step < _DENSE_MIN_CHUNK_ROWS:+ # Not even a small chunk fits: the tile kernel only needs the output.+ return _zoo_mxfp4_matmul(x, packed, scale, trans = False, out = y, bias = bias)+ try:+ buffer = torch.empty((step, K), dtype = dtype, device = device)+ for r0 in range(0, N, step):+ r1 = min(N, r0 + step)+ weight = dequantize_mxfp4_packed(+ packed[r0:r1], scale[r0:r1], dtype, out = buffer[: r1 - r0]+ )+ y[:, r0:r1] = torch.nn.functional.linear(+ x2, weight, None if bias is None else bias[r0:r1]+ )+ return y.view(*lead, N)+ except torch.cuda.OutOfMemoryError:+ # Chunks are rewritten whole, so a retry with half the rows restarts cleanly.+ buffer = weight = None+ state[0] = step * row_bytes // 2+++_bounded_dense_matmul_op = None+try:++ @torch.library.custom_op("unsloth::mxfp4_bounded_dense_matmul", mutates_args = ())+ def _bounded_dense_matmul_op(+ x: torch.Tensor,+ packed: torch.Tensor,+ scale: torch.Tensor,+ bias: Optional[torch.Tensor],+ ) -> torch.Tensor:+ return _bounded_dense_matmul_eager(x, packed, scale, None, bias)++ @_bounded_dense_matmul_op.register_fake+ def _(x, packed, scale, bias):+ return x.new_empty((*x.shape[:-1], packed.shape[0]))++except Exception:+ _bounded_dense_matmul_op = None+++def _bounded_dense_matmul(x, packed, scale, out, bias):+ if torch.compiler.is_compiling():+ # Opaque to the trace: the allocation fallbacks must run at call time, not be traced once.+ if _bounded_dense_matmul_op is None:+ return _linear(x, dequantize_mxfp4_packed(packed, scale, x.dtype), out, bias)+ y = _bounded_dense_matmul_op(x, packed, scale, bias)+ if out is None or bias is not None or out.shape != y.shape:+ return y # with a bias _linear never writes ``out`` either+ return out.copy_(y)+ return _bounded_dense_matmul_eager(x, packed, scale, out, bias)++def _compute_dtype(x):device_type = x.device.typetry: