Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MXFP4 packed Linear: bound the below-sm_80 weight decode by free memory (#14784)

Daniel Han4077567c5e1f
diff --git a/tests/test_mxfp4_packed_low_memory.py b/tests/test_mxfp4_packed_low_memory.py
new file mode 100644
index 00000000000..635a24d3480
--- /dev/null
+++ b/tests/test_mxfp4_packed_low_memory.py
@@ -0,0 +1,231 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Below sm_80 a packed MXFP4 forward decodes the 16-bit weight for cuBLAS; it must not need more memory than the
+tile kernel it replaced when the whole weight does not fit (T4 with little headroom)."""
+
+import contextlib
+import gc
+
+import pytest
+from llama_patch_isolation import restore_llama_patches # noqa: F401
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+import torch
+
+import unsloth # noqa: F401
+import unsloth.models.mxfp4_compressed_linear as mcl
+
+pytestmark = [
+ pytest.mark.usefixtures("restore_llama_patches"),
+ pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device"),
+]
+
+
+def _zoo():
+ try:
+ import unsloth_zoo.mxfp4_gemm as mg
+ except Exception:
+ pytest.skip("needs unsloth_zoo.mxfp4_gemm")
+ if not hasattr(mg, "mxfp4_grouped_gemv_available"):
+ pytest.skip("unsloth_zoo before the sm_80 tile GEMM floor: the tile kernel still runs here")
+ return mg
+
+
+def sm75(monkeypatch):
+ """The sm_75 policy on whatever GPU runs this: the verified tile GEMM is kept off, real kernels run."""
+ mg = _zoo()
+ device = torch.device("cuda", torch.cuda.current_device())
+ monkeypatch.delenv("UNSLOTH_MXFP4_FUSED_GEMM", raising = False)
+ for dtype in (torch.float16, torch.bfloat16):
+ if not mg.mxfp4_grouped_gemv_available(device, dtype):
+ pytest.skip("the fused MXFP4 kernel does not self-check on this GPU")
+ monkeypatch.setitem(mg._AVAILABLE_DTYPE, (str(device), dtype, "tiles"), False)
+ assert not mg.mxfp4_gemm_available(device, dtype)
+ monkeypatch.setattr(mcl, "_DENSE_BUDGET", {}, raising = False)
+ return device
+
+
+def _headroom(mib, device):
+ """Let the caching allocator hand out at most ``mib`` MiB more than is allocated now."""
+ gc.collect()
+ torch.cuda.synchronize(device)
+ torch.cuda.empty_cache()
+ total = torch.cuda.get_device_properties(device).total_memory
+ cap = torch.cuda.memory_allocated(device) + mib * 2**20
+ torch.cuda.set_per_process_memory_fraction(cap / total, device)
+ try:
+ yield
+ finally:
+ torch.cuda.set_per_process_memory_fraction(1.0, device)
+ torch.cuda.empty_cache()
+
+
+def _weight(N, K, device, seed = 0):
+ g = torch.Generator(device = device).manual_seed(seed)
+ packed = torch.randint(0, 256, (N, K // 2), dtype = torch.uint8, device = device, generator = g)
+ scale = torch.randint(118, 128, (N, K // 32), dtype = torch.uint8, device = device, generator = g)
+ return packed, scale
+
+
+def _single_shot(x, packed, scale, bias = None):
+ # What #1899 runs below sm_80: the whole 16-bit weight, one cuBLAS call.
+ weight = mcl.dequantize_mxfp4_packed(packed, scale, x.dtype)
+ return torch.nn.functional.linear(x, weight, bias)
+
+
+def test_t4_decode_fits_in_64mib_headroom(sm75):
+ # 14336 x 4096 decodes to 112 MiB of fp16: the tile kernel needed 28 KiB, dequantize + cuBLAS must not OOM.
+ device = sm75
+ N, K = 14336, 4096
+ packed, scale = _weight(N, K, device)
+ x = torch.randn(1, K, dtype = torch.float16, device = device)
+ want = _single_shot(x, packed, scale)
+ mcl.mxfp4_packed_matmul(x, packed, scale) # warm kernels and their self-checks
+ with _headroom(64, device):
+ start = torch.cuda.memory_allocated(device)
+ torch.cuda.reset_peak_memory_stats(device)
+ got = mcl.mxfp4_packed_matmul(x, packed, scale)
+ torch.cuda.synchronize(device)
+ peak = torch.cuda.max_memory_allocated(device) - start
+ assert peak < 64 * 2**20
+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())
+ # The next calls go straight to chunks: no second allocation failure.
+ ooms = torch.cuda.memory_stats(device).get("num_ooms", 0)
+ with _headroom(64, device):
+ mcl.mxfp4_packed_matmul(x, packed, scale)
+ assert torch.cuda.memory_stats(device).get("num_ooms", 0) == ooms
+
+
+def test_weight_that_fits_keeps_the_single_shot_decode(sm75):
+ device = sm75
+ packed, scale = _weight(1024, 512, device, seed = 1)
+ x = torch.randn(3, 512, dtype = torch.bfloat16, device = device)
+ assert torch.equal(mcl.mxfp4_packed_matmul(x, packed, scale), _single_shot(x, packed, scale))
+ assert not mcl._DENSE_BUDGET
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16])
[email protected]("rows", [1, 7, 64])
[email protected]("bias", [False, True])
[email protected]("lead", [(), (2,)])
+def test_row_chunks_match_the_single_shot_decode(sm75, dtype, rows, bias, lead):
+ device = sm75
+ N, K = 1000, 384 # 1000 rows: the last chunk is ragged
+ packed, scale = _weight(N, K, device, seed = rows)
+ x = torch.randn(*lead, rows, K, dtype = dtype, device = device)
+ b = torch.randn(N, dtype = dtype, device = device) if bias else None
+ want = _single_shot(x, packed, scale, b)
+ row_bytes = K * x.element_size()
+ mcl._DENSE_BUDGET[device.index] = [300 * row_bytes, 1024] # 300-row chunks
+ got = mcl.mxfp4_packed_matmul(x, packed, scale, bias = b)
+ assert got.shape == want.shape and got.dtype == dtype
+ # Same decoded rows, same math; cuBLAS may pick another reduction order for the narrower chunk (1 ulp).
+ tol = 1.6e-2 if dtype == torch.bfloat16 else 2e-3
+ torch.testing.assert_close(got, want, rtol = tol, atol = tol * want.abs().max().item())
+ out = torch.empty_like(want)
+ mcl._DENSE_BUDGET[device.index] = [300 * row_bytes, 1024]
+ if not bias:
+ assert mcl.mxfp4_packed_matmul(x, packed, scale, out = out).data_ptr() == out.data_ptr()
+ torch.testing.assert_close(out, want, rtol = tol, atol = tol * want.abs().max().item())
+
+
+def test_no_room_for_a_chunk_takes_the_tile_kernel(sm75, monkeypatch):
+ device = sm75
+ N, K = 4096, 4096
+ packed, scale = _weight(N, K, device, seed = 3)
+ x = torch.randn(2, K, dtype = torch.float16, device = device)
+ want = _single_shot(x, packed, scale)
+ calls = []
+ tile = mcl._zoo_mxfp4_matmul
+ monkeypatch.setattr(mcl, "_zoo_mxfp4_matmul", lambda *a, **k: calls.append(1) or tile(*a, **k))
+ mcl._DENSE_BUDGET[device.index] = [64 * K * 2, 1024] # 64 rows < the 128-row chunk floor
+ got = mcl.mxfp4_packed_matmul(x, packed, scale)
+ assert calls
+ torch.testing.assert_close(got, want, rtol = 2e-2, atol = 2e-2 * want.abs().max().item())
+
+
+def test_budget_is_retried_after_a_while(sm75):
+ device = sm75
+ packed, scale = _weight(512, 256, device, seed = 4)
+ x = torch.randn(1, 256, dtype = torch.float16, device = device)
+ mcl._DENSE_BUDGET[device.index] = [128 * 256 * 2, 2]
+ mcl.mxfp4_packed_matmul(x, packed, scale)
+ assert device.index in mcl._DENSE_BUDGET
+ mcl.mxfp4_packed_matmul(x, packed, scale)
+ assert device.index not in mcl._DENSE_BUDGET # whole weight fitted again
+
+
+def test_compiled_forward_is_bounded_without_graph_breaks(sm75):
+ from torch._dynamo.utils import counters
+
+ device = sm75
+ N, K = 14336, 4096
+ module = mcl.make_mxfp4_packed_linear(K, N, device = device)
+ module.weight_packed.data, module.weight_scale.data = _weight(N, K, device, seed = 5)
+ module.compute_dtype = torch.float16
+ x = torch.randn(1, K, dtype = torch.float16, device = device)
+ with torch.no_grad():
+ want = module(x)
+ torch._dynamo.reset()
+ counters.clear()
+ compiled = torch.compile(module, fullgraph = True)
+ torch.testing.assert_close(compiled(x), want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())
+ assert sum(counters["graph_break"].values()) == 0
+ with _headroom(64, device):
+ got = compiled(x)
+ torch.cuda.synchronize(device)
+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())
+ torch._dynamo.reset()
+
+
+def _fake_oom_dequant(monkeypatch, fail_rows):
+ """``dequantize_mxfp4_packed`` raising an allocator OOM once for each entry of ``fail_rows`` (row counts, in
+ order), as when the decoded weight allocated but its scratch, the output or the cuBLAS workspace did not."""
+ real = mcl.dequantize_mxfp4_packed
+ pending, seen = list(fail_rows), []
+
+ def fake(packed, scale, dtype = torch.bfloat16, out = None):
+ seen.append(packed.shape[0])
+ if pending and packed.shape[0] == pending[0]:
+ pending.pop(0)
+ raise torch.cuda.OutOfMemoryError("CUDA out of memory (fake)")
+ return real(packed, scale, dtype, out = out)
+
+ monkeypatch.setattr(mcl, "dequantize_mxfp4_packed", fake)
+ return seen, pending
+
+
+def test_oom_after_the_weight_allocated_falls_back_to_chunks(sm75, monkeypatch):
+ device = sm75
+ N, K = 1024, 512
+ packed, scale = _weight(N, K, device, seed = 6)
+ x = torch.randn(2, K, dtype = torch.float16, device = device)
+ want = _single_shot(x, packed, scale)
+ # Whole weight attempt fails late, then the first (full-height) chunk fails too: half-height chunks finish.
+ seen, pending = _fake_oom_dequant(monkeypatch, [N, N])
+ got = mcl.mxfp4_packed_matmul(x, packed, scale)
+ assert not pending and seen[:2] == [N, N] and seen[2] == N // 2
+ assert device.index in mcl._DENSE_BUDGET
+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())
+
+
[email protected]("compiling", [False, True])
+def test_bias_never_writes_out_whatever_the_memory(sm75, monkeypatch, compiling):
+ # The full-weight path with a bias returns a new tensor and leaves ``out`` alone: so must the fallbacks.
+ device = sm75
+ N, K = 1000, 384
+ packed, scale = _weight(N, K, device, seed = 7)
+ x = torch.randn(3, K, dtype = torch.float16, device = device)
+ b = torch.randn(N, dtype = torch.float16, device = device)
+ want = _single_shot(x, packed, scale, b)
+ out = torch.full_like(want, 7.0)
+ full = mcl.mxfp4_packed_matmul(x, packed, scale, out = out, bias = b)
+ assert full.data_ptr() != out.data_ptr() and bool((out == 7.0).all())
+ if compiling:
+ monkeypatch.setattr(torch.compiler, "is_compiling", lambda: True)
+ mcl._DENSE_BUDGET[device.index] = [300 * K * x.element_size(), 1024] # 300-row chunks
+ got = mcl.mxfp4_packed_matmul(x, packed, scale, out = out, bias = b)
+ assert got.data_ptr() != out.data_ptr() and bool((out == 7.0).all())
+ torch.testing.assert_close(got, want, rtol = 2e-3, atol = 2e-3 * want.abs().max().item())
diff --git a/unsloth/models/mxfp4_compressed_linear.py b/unsloth/models/mxfp4_compressed_linear.py
index 14eb65910c9..7c6b76d8b6e 100644
--- a/unsloth/models/mxfp4_compressed_linear.py
+++ b/unsloth/models/mxfp4_compressed_linear.py
@@ -51,6 +51,13 @@ try:
from unsloth_zoo.mxfp4_gemm import mxfp4_gemm_available, mxfp4_matmul as _zoo_mxfp4_matmul
except Exception:
_zoo_mxfp4_matmul = None
+try:
+ # Self-check only, no sm_80 floor (zoo #1899+): below sm_80 the tile kernel is the low-memory last resort.
+ from unsloth_zoo.mxfp4_gemm import (
+ mxfp4_grouped_gemv_available as _zoo_mxfp4_kernel_verified,
+ )
+except Exception:
+ _zoo_mxfp4_kernel_verified = None
_FP4_VALUES = (
0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
@@ -80,13 +87,18 @@ def dequantize_mxfp4_packed(
packed,
scale,
dtype = torch.bfloat16,
+ out = None,
):
- """Packed ``[out, in / 2]`` + E8M0 scales ``[out, in / 32]`` -> ``[out, in]``; zoo kernel if self-verified, else torch."""
+ """Packed ``[out, in / 2]`` + E8M0 scales ``[out, in / 32]`` -> ``[out, in]``; zoo kernel if self-verified, else torch.
+ ``out``: a contiguous ``[out, in]`` buffer to decode into."""
out_features, half = packed.shape
blocks = packed.view(out_features, half // 16, 16)
if _zoo_mxfp4_dequantize is not None and dtype in (torch.bfloat16, torch.float16):
- return _zoo_mxfp4_dequantize(blocks, scale, dtype = dtype)
- return _dequantize_torch(blocks, scale, dtype)
+ if out is None:
+ return _zoo_mxfp4_dequantize(blocks, scale, dtype = dtype)
+ return _zoo_mxfp4_dequantize(blocks, scale, dtype = dtype, out = out)
+ weight = _dequantize_torch(blocks, scale, dtype)
+ return weight if out is None else out.copy_(weight)
def _fused_max_rows():
@@ -109,7 +121,8 @@ def mxfp4_packed_matmul(
bias = None,
):
"""``x @ W^T (+ bias)`` (``transpose=False``) or ``x @ W`` for the packed ``[out, in / 2]`` weight, in x's dtype.
- Skinny forward inputs (decode, small batches) never build the 16-bit weight; the rest decode it once for cuBLAS."""
+ Skinny forward inputs (decode, small batches) never build the 16-bit weight on sm_80+; below sm_80 they decode it
+ for cuBLAS only as far as memory allows (row chunks, else the tile kernel); the rest decode it once for cuBLAS."""
rows = x.numel() // x.shape[-1] if x.shape[-1] else 0
# dX (transpose) stays on decode + cuBLAS: the fused kernel only wins it for a few wide shapes.
if (
@@ -117,17 +130,140 @@ def mxfp4_packed_matmul(
and _zoo_mxfp4_matmul is not None
and rows <= _FUSED_MAX_ROWS
and x.is_cuda
- and mxfp4_gemm_available(x.device, x.dtype)
):
- return _zoo_mxfp4_matmul(x, packed, scale, trans = transpose, out = out, bias = bias)
+ if mxfp4_gemm_available(x.device, x.dtype):
+ return _zoo_mxfp4_matmul(x, packed, scale, trans = transpose, out = out, bias = bias)
+ if _zoo_mxfp4_kernel_verified is not None and _zoo_mxfp4_kernel_verified(
+ x.device, x.dtype
+ ):
+ # Below sm_80 decode + cuBLAS beats the tile kernel, but only the tile kernel never builds the
+ # 16-bit weight: decode in chunks of output rows when the weight does not fit.
+ return _bounded_dense_matmul(x, packed, scale, out, bias)
weight = dequantize_mxfp4_packed(packed, scale, x.dtype)
if transpose:
return torch.matmul(x, weight, out = out)
+ return _linear(x, weight, out, bias)
+
+
+def _linear(x, weight, out, bias):
if bias is not None or out is None:
return torch.nn.functional.linear(x, weight, bias)
return torch.matmul(x, weight.t(), out = out)
+# Below sm_80 (`_bounded_dense_matmul`): per GPU, the bytes a decoded chunk may take once the whole 16-bit
+# weight failed to allocate, and the calls left before the whole weight is tried again.
+_DENSE_BUDGET = {}
+_DENSE_RETRY_CALLS = 1024
+# Fewer decoded rows than this per chunk: the tile kernel (output only) instead.
+_DENSE_MIN_CHUNK_ROWS = 128
+
+
+def _free_bytes(device):
+ """What the caching allocator can still hand out: driver free (capped by a per-process memory fraction)
+ plus its own cached free blocks. Only queried after an allocation failed: the queries cost ~100 us."""
+ free, total = torch.cuda.mem_get_info(device)
+ reserved = torch.cuda.memory_reserved(device)
+ try:
+ cap = int(torch.cuda.get_per_process_memory_fraction(device) * total)
+ free = min(free, cap - reserved)
+ except Exception:
+ pass
+ return max(0, free) + max(0, reserved - torch.cuda.memory_allocated(device))
+
+
+def _bounded_dense_matmul_eager(x, packed, scale, out = None, bias = None):
+ """``x @ W^T (+ bias)`` without ever holding more of the 16-bit W than fits: the whole W when it allocates
+ (one decode + one cuBLAS call), else row chunks of W written to ``y[..., r0:r1]`` (no cross-chunk sums),
+ else the tile kernel."""
+ N, half = packed.shape
+ K = half * 2
+ dtype, device = x.dtype, x.device
+ row_bytes = K * x.element_size()
+ key = device.index
+ state = _DENSE_BUDGET.get(key)
+ if state is not None:
+ state[1] -= 1
+ if state[1] <= 0:
+ _DENSE_BUDGET.pop(key, None)
+ state = None
+ if state is None or N * row_bytes <= state[0]:
+ # The weight, the decode's scratch, the output and the cuBLAS workspace: any of them may be the one
+ # that does not fit, so the whole attempt is guarded and its tensors are dropped before the retry.
+ try:
+ weight = torch.empty((N, K), dtype = dtype, device = device)
+ return _linear(x, dequantize_mxfp4_packed(packed, scale, dtype, out = weight), out, bias)
+ except torch.cuda.OutOfMemoryError:
+ weight = None
+ state = _DENSE_BUDGET[key] = [_free_bytes(device) // 2, _DENSE_RETRY_CALLS]
+ lead = x.shape[:-1]
+ x2 = x.reshape(-1, K)
+ M = x2.shape[0]
+ # Like _linear: ``out`` is only written when there is no bias, whichever path runs.
+ if (
+ bias is None
+ and out is not None
+ and out.dtype == dtype
+ and out.is_contiguous()
+ and out.numel() == M * N
+ ):
+ y = out.view(M, N)
+ else:
+ y = torch.empty((M, N), dtype = dtype, device = device)
+ while True:
+ step = min(N, state[0] // row_bytes)
+ if step < _DENSE_MIN_CHUNK_ROWS:
+ # Not even a small chunk fits: the tile kernel only needs the output.
+ return _zoo_mxfp4_matmul(x, packed, scale, trans = False, out = y, bias = bias)
+ try:
+ buffer = torch.empty((step, K), dtype = dtype, device = device)
+ for r0 in range(0, N, step):
+ r1 = min(N, r0 + step)
+ weight = dequantize_mxfp4_packed(
+ packed[r0:r1], scale[r0:r1], dtype, out = buffer[: r1 - r0]
+ )
+ y[:, r0:r1] = torch.nn.functional.linear(
+ x2, weight, None if bias is None else bias[r0:r1]
+ )
+ return y.view(*lead, N)
+ except torch.cuda.OutOfMemoryError:
+ # Chunks are rewritten whole, so a retry with half the rows restarts cleanly.
+ buffer = weight = None
+ state[0] = step * row_bytes // 2
+
+
+_bounded_dense_matmul_op = None
+try:
+
+ @torch.library.custom_op("unsloth::mxfp4_bounded_dense_matmul", mutates_args = ())
+ def _bounded_dense_matmul_op(
+ x: torch.Tensor,
+ packed: torch.Tensor,
+ scale: torch.Tensor,
+ bias: Optional[torch.Tensor],
+ ) -> torch.Tensor:
+ return _bounded_dense_matmul_eager(x, packed, scale, None, bias)
+
+ @_bounded_dense_matmul_op.register_fake
+ def _(x, packed, scale, bias):
+ return x.new_empty((*x.shape[:-1], packed.shape[0]))
+
+except Exception:
+ _bounded_dense_matmul_op = None
+
+
+def _bounded_dense_matmul(x, packed, scale, out, bias):
+ if torch.compiler.is_compiling():
+ # Opaque to the trace: the allocation fallbacks must run at call time, not be traced once.
+ if _bounded_dense_matmul_op is None:
+ return _linear(x, dequantize_mxfp4_packed(packed, scale, x.dtype), out, bias)
+ y = _bounded_dense_matmul_op(x, packed, scale, bias)
+ if out is None or bias is not None or out.shape != y.shape:
+ return y # with a bias _linear never writes ``out`` either
+ return out.copy_(y)
+ return _bounded_dense_matmul_eager(x, packed, scale, out, bias)
+
+
def _compute_dtype(x):
device_type = x.device.type
try: