Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

MXFP4 torch decode fallback: run ldexp on the operands' GPU (#16062)

Daniel Haneb6f10485b6d
diff --git a/tests/test_mxfp4_compressed_linear.py b/tests/test_mxfp4_compressed_linear.py
index 5f8bf94c50a..40c52c7a8c0 100644
--- a/tests/test_mxfp4_compressed_linear.py
+++ b/tests/test_mxfp4_compressed_linear.py
@@ -456,6 +456,61 @@ def test_torch_fallback_without_the_zoo_kernel(monkeypatch):
assert torch.equal(dequantize_mxfp4_packed(packed, scale, torch.bfloat16), with_zoo)
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_torch_fallback_runs_ldexp_on_the_operands_gpu(dtype, monkeypatch):
+ # float32 always takes this path (the zoo decoder serves only fp16 / bf16).
+ import contextlib
+ from torch._subclasses.fake_tensor import FakeTensorMode
+ import unsloth.models.mxfp4_compressed_linear as mcl
+
+ current = ["cuda:0"]
+
+ @contextlib.contextmanager
+ def fake_cuda_device(device):
+ previous, current[0] = current[0], str(torch.device(device))
+ try:
+ yield
+ finally:
+ current[0] = previous
+
+ seen = []
+ real_ldexp = torch.ldexp
+
+ def spy_ldexp(x, other, *args, **kwargs):
+ seen.append((str(x.device), current[0]))
+ return real_ldexp(x, other, *args, **kwargs)
+
+ monkeypatch.setattr(torch.cuda, "device", fake_cuda_device)
+ monkeypatch.setattr(torch, "ldexp", spy_ldexp)
+ monkeypatch.setattr(mcl, "_LUT_CACHE", {})
+ monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
+ with FakeTensorMode(allow_non_fake_inputs = True):
+ packed = torch.empty(8, 32, dtype = torch.uint8, device = "cuda:1")
+ scale = torch.empty(8, 2, dtype = torch.uint8, device = "cuda:1")
+ out = dequantize_mxfp4_packed(packed, scale, dtype)
+ assert out.shape == (8, 64) and out.dtype == dtype
+ assert seen == [("cuda:1", "cuda:1")]
+ assert current == ["cuda:0"]
+
+
+ not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two CUDA devices"
+)
[email protected]("dtype", [torch.float32, torch.float16, torch.bfloat16])
+def test_torch_fallback_on_a_non_current_gpu_matches_cpu(dtype, monkeypatch):
+ import unsloth.models.mxfp4_compressed_linear as mcl
+
+ monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
+ packed, scale = _random_packed(64, 128, low = 100, high = 140)
+ want = dequantize_mxfp4_packed(packed, scale, dtype)
+ with torch.cuda.device(0):
+ got = dequantize_mxfp4_packed(packed.to("cuda:1"), scale.to("cuda:1"), dtype)
+ torch.cuda.synchronize(1)
+ assert torch.cuda.current_device() == 0
+ assert got.device == torch.device("cuda:1")
+ assert torch.equal(got.cpu(), want)
+
+
def test_dtype_casts_keep_the_packed_bytes_and_set_the_decode_dtype():
module = _filled(32, 64, dtype = torch.float16)
module.weight_scale.data.clamp_(110, 130) # stay inside fp16's range
diff --git a/unsloth/models/mxfp4_compressed_linear.py b/unsloth/models/mxfp4_compressed_linear.py
index d1b2822154b..4d41febea76 100644
--- a/unsloth/models/mxfp4_compressed_linear.py
+++ b/unsloth/models/mxfp4_compressed_linear.py
@@ -13,6 +13,7 @@
"""Trains mxfp4-pack-quantized Linears in place; frozen weight dequantized per use, no 16-bit copy."""
+import contextlib
import functools
import os
import types
@@ -72,7 +73,22 @@ def mxfp4_keep_packed_enabled() -> bool:
return os.environ.get(MXFP4_KEEP_PACKED_ENV, "1").strip() != "0"
+def _operand_device_guard(tensor):
+ """Make ``tensor``'s GPU current: torch.ldexp (>= 2.11) launches on the current device, so split-model operands
+ decode garbage or fault otherwise. Compiled code gets Inductor's own guard."""
+ device = tensor.device
+ if device.index is None or device.type not in ("cuda", "xpu") or torch.compiler.is_compiling():
+ return contextlib.nullcontext()
+ guard = getattr(getattr(torch, device.type, None), "device", None)
+ return contextlib.nullcontext() if guard is None else guard(device)
+
+
def _dequantize_torch(blocks, scales, dtype):
+ with _operand_device_guard(blocks):
+ return _dequantize_torch_impl(blocks, scales, dtype)
+
+
+def _dequantize_torch_impl(blocks, scales, dtype):
work = torch.float32 if dtype == torch.float16 and _LDEXP_IN_INPUT_DTYPE else dtype
key = (blocks.device, work)
lut = _LUT_CACHE.get(key)