Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
"""compressed-tensors ``mxfp4-pack-quantized`` Linears stay packed and train through."""
import copy
import json
import os
import shutil
import pytest
from llama_patch_isolation import restore_llama_patches # noqa: F401
from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
import torch
from torch import nn
import unsloth # noqa: F401
from unsloth.models.mxfp4_compressed_linear import (
Mxfp4PackedLinear,
_dequantize_torch,
adopt_compressed_mxfp4_modules,
dequantize_mxfp4_packed,
make_mxfp4_packed_linear,
)
try:
from compressed_tensors.config import CompressionFormat
# Releases before MXFP4 (0.12.x, which vllm 0.11.2 pins) reject its config outright.
HAS_CT = any(f.value == "mxfp4-pack-quantized" for f in CompressionFormat)
if HAS_CT:
# compressed-tensors < 0.16 (the newest on torch 2.6) has the format but decompress raises.
from compressed_tensors.compressors import BaseCompressor
from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
BaseCompressor.get_value_from_registry("mxfp4-pack-quantized").decompress(
{
"weight_packed": torch.zeros(1, 16, dtype = torch.uint8),
"weight_scale": torch.full((1, 1), 127, dtype = torch.uint8),
},
QuantizationScheme(
targets = ["Linear"],
weights = QuantizationArgs(
num_bits = 4,
type = "float",
strategy = "group",
group_size = 32,
symmetric = True,
scale_dtype = torch.uint8,
),
format = "mxfp4-pack-quantized",
),
)
except Exception:
HAS_CT = False
from unsloth.models.compressed_tensors_bnb import _transformers_supports_weight_converters
pytestmark = pytest.mark.usefixtures("restore_llama_patches")
HAS_CONVERTERS = _transformers_supports_weight_converters()
try:
from unsloth_zoo.temporary_patches import mxfp4 as _zoo_mxfp4
HAS_ZOO_PACKED_SAVE = hasattr(_zoo_mxfp4, "_densified_module_names")
except Exception:
HAS_ZOO_PACKED_SAVE = False
# Without the zoo packed save support the packed route declines by design.
needs_zoo_packed_save = pytest.mark.skipif(
not HAS_ZOO_PACKED_SAVE, reason = "needs unsloth-zoo's packed-module save support"
)
DEVICES = ["cpu"] + (["cuda"] if has_real_cuda() else [])
def _cuda_amp_dtype():
# CUDA autocast refuses bf16 below sm_80 (T4), where Unsloth trains in fp16.
return torch.bfloat16 if torch.cuda.get_device_capability() >= (8, 0) else torch.float16
_E2M1 = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
def _random_packed(
out_features,
in_features,
seed = 0,
low = 100,
high = 150,
):
g = torch.Generator().manual_seed(seed)
packed = torch.randint(0, 256, (out_features, in_features // 2), generator = g, dtype = torch.uint8)
scale = torch.randint(
low, high, (out_features, in_features // 32), generator = g, dtype = torch.uint8
)
return packed, scale
def _reference_decode(packed, scale):
"""Element by element, in float64: low nibble first, sign in bit 3, times 2**(e - 127)."""
out_features, half = packed.shape
w = torch.empty(out_features, half * 2, dtype = torch.float64)
for i in range(out_features):
for j in range(half):
byte = int(packed[i, j])
for k, nib in enumerate((byte & 0x0F, byte >> 4)):
col = 2 * j + k
v = _E2M1[nib & 0x07] * (-1.0 if nib & 0x08 else 1.0)
w[i, col] = v * 2.0 ** (int(scale[i, col // 32]) - 127)
return w
@pytest.mark.parametrize("device", DEVICES)
def test_dequant_matches_an_independent_float64_decode(device):
packed, scale = _random_packed(8, 64, low = 1, high = 254)
packed[0, :16] = torch.arange(0, 256, 16, dtype = torch.uint8)
packed[1, :16] = torch.arange(0, 16, dtype = torch.uint8)
want = _reference_decode(packed, scale)
for dtype in (torch.bfloat16, torch.float32):
got = dequantize_mxfp4_packed(packed.to(device), scale.to(device), dtype).cpu()
assert got.dtype == dtype and got.shape == (8, 64)
assert torch.equal(got.double(), want.to(dtype).double()), dtype
@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
@pytest.mark.parametrize("device", DEVICES)
def test_dequant_matches_compressed_tensors_decompress(device):
from compressed_tensors.compressors import BaseCompressor
from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
scheme = QuantizationScheme(
targets = ["Linear"],
weights = QuantizationArgs(
num_bits = 4,
type = "float",
strategy = "group",
group_size = 32,
symmetric = True,
scale_dtype = torch.uint8,
),
format = "mxfp4-pack-quantized",
)
comp = BaseCompressor.get_value_from_registry("mxfp4-pack-quantized")
for seed, (o, i) in enumerate([(128, 128), (96, 256), (3, 64)]):
packed, scale = _random_packed(o, i, seed = seed, low = 1, high = 254)
state = {
"weight_packed": packed,
"weight_scale": scale,
"weight_shape": torch.tensor([o, i]),
}
want = comp.decompress(state, scheme)["weight"].to(torch.bfloat16)
got = dequantize_mxfp4_packed(packed.to(device), scale.to(device), torch.bfloat16).cpu()
assert torch.equal(got.view(torch.int16), want.view(torch.int16)), (o, i)
@pytest.mark.parametrize("device", DEVICES)
def test_torch_fp16_decode_rounds_once_for_every_finite_scale(device):
# E8M0 1..254 (0xFF is NaN): past fp16's range a value is inf, a zero nibble stays +-0, small ones go subnormal.
# torch < 2.12's ldexp built 2^e in fp16, so 0 * 2^16 was NaN, 2^-10 * 2^20 inf and 6 * 2^-25 zero.
packed, scale = _random_packed(8, 64, low = 1, high = 255)
packed[0, :16] = torch.arange(0, 256, 16, dtype = torch.uint8)
packed[1, :16] = torch.arange(0, 16, dtype = torch.uint8)
scale[0, 0], scale[0, 1], scale[1, 0], scale[1, 1] = 143, 147, 102, 254
want = _reference_decode(packed, scale).to(torch.float16)
got = _dequantize_torch(packed.to(device).view(8, 2, 16), scale.to(device), torch.float16).cpu()
assert got.dtype == torch.float16 and got.shape == (8, 64)
assert torch.equal(got.view(torch.int16), want.view(torch.int16))
@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def test_zoo_kernel_and_torch_path_agree():
packed, scale = _random_packed(256, 512, low = 1, high = 254)
packed, scale = packed.cuda(), scale.cuda()
blocks = packed.view(256, 16, 16)
for dtype in (torch.bfloat16, torch.float16):
a = dequantize_mxfp4_packed(packed, scale, dtype)
b = _dequantize_torch(blocks, scale, dtype)
assert torch.equal(a.view(torch.int16), b.view(torch.int16)), dtype
@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("bias", [False, True])
def test_forward_and_input_gradient_are_exact_and_save_only_packed_bytes(device, bias, monkeypatch):
import unsloth.models.mxfp4_compressed_linear as mcl
# The decode + cuBLAS route is bit-exact; skinny CUDA inputs take the fused GEMM.
monkeypatch.setattr(mcl, "_FUSED_MAX_ROWS", 0)
torch.manual_seed(0)
out_f, in_f = 96, 64
packed, scale = _random_packed(out_f, in_f)
module = make_mxfp4_packed_linear(in_f, out_f, device = device)
module.weight_packed.data.copy_(packed)
module.weight_scale.data.copy_(scale)
ref = nn.Linear(in_f, out_f, bias = bias, device = device, dtype = torch.bfloat16)
ref.weight.data.copy_(module.dequantize_weight(torch.bfloat16))
ref.weight.requires_grad_(False)
if bias:
module.bias = nn.Parameter(ref.bias.detach().clone())
x = torch.randn(5, 7, in_f, device = device, dtype = torch.bfloat16)
xa, xb = x.clone().requires_grad_(), x.clone().requires_grad_()
saved = []
with torch.autograd.graph.saved_tensors_hooks(lambda t: saved.append(t) or t, lambda t: t):
ya = module(xa)
yb = ref(xb)
assert torch.equal(ya, yb)
assert saved and all(t.dtype == torch.uint8 for t in saved), [t.dtype for t in saved]
g = torch.randn_like(ya)
ya.backward(g)
yb.backward(g)
assert torch.equal(xa.grad, xb.grad)
if bias:
assert torch.allclose(module.bias.grad.float(), ref.bias.grad.float(), rtol = 1e-2, atol = 1e-2)
assert module.weight_packed.grad is None and not module.weight_packed.requires_grad
@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def test_autocast_runs_the_matmul_in_the_autocast_dtype():
amp = _cuda_amp_dtype()
packed, scale = _random_packed(64, 64)
module = make_mxfp4_packed_linear(64, 64, device = "cuda")
module.weight_packed.data.copy_(packed)
module.weight_scale.data.copy_(scale)
if amp == torch.float16:
module.weight_scale.data.clamp_(118, 126) # fp16's range: the decode stays finite and exact
ref = nn.Linear(64, 64, bias = False, device = "cuda", dtype = torch.float32)
ref.weight.data.copy_(module.dequantize_weight(torch.float32))
x = torch.randn(4, 64, device = "cuda", requires_grad = True)
x2 = x.detach().clone().requires_grad_()
with torch.autocast("cuda", dtype = amp):
ya, yb = module(x), ref(x2)
assert ya.dtype == yb.dtype == amp
assert torch.equal(ya, yb)
ya.float().sum().backward()
yb.float().sum().backward()
assert x.grad.dtype == torch.float32
assert torch.equal(x.grad, x2.grad)
def test_weight_property_is_an_exact_read_only_decode():
packed, scale = _random_packed(32, 64)
module = make_mxfp4_packed_linear(64, 32)
module.weight_packed.data.copy_(packed)
module.weight_scale.data.copy_(scale)
w = module.weight
assert w.dtype == torch.bfloat16 and w.shape == (32, 64)
assert torch.equal(w.double(), _reference_decode(packed, scale).to(torch.bfloat16).double())
assert "weight" not in dict(module.named_parameters())
assert {n for n, _ in module.named_parameters()} == {"weight_packed", "weight_scale"}
with pytest.raises(ValueError):
make_mxfp4_packed_linear(48, 32)
def _ct_compressed_model(targets_ignore = (), groups = None):
from compressed_tensors.compressors import ModelCompressor
from compressed_tensors.quantization import QuantizationConfig, apply_quantization_config
torch.manual_seed(0)
class Tiny(nn.Module):
def __init__(self):
super().__init__()
self.a = nn.Linear(64, 32, bias = False)
self.b = nn.Linear(32, 64, bias = False)
self.latent = nn.Linear(64, 64, bias = False)
def forward(self, x):
return self.latent(self.b(self.a(x)))
model = Tiny().to(torch.bfloat16)
quant = {
"quant_method": "compressed-tensors",
"format": "mxfp4-pack-quantized",
"quantization_status": "compressed",
"ignore": list(targets_ignore),
"config_groups": {
"group_0": {
"targets": ["Linear"],
"weights": {
"num_bits": 4,
"type": "float",
"strategy": "group",
"group_size": 32,
"symmetric": True,
"scale_dtype": "torch.uint8",
},
}
},
}
if groups is not None:
quant["config_groups"] = groups
qc = QuantizationConfig.model_validate(quant)
apply_quantization_config(model, qc, run_compressed = False)
for m in (model.a, model.b, model.latent):
m.weight_scale.data.fill_(125 if m.weight_scale.dtype == torch.uint8 else 0.01)
ModelCompressor(quantization_config = qc).compress_model(model)
return model
_MXFP4_WEIGHTS = {
"num_bits": 4,
"type": "float",
"strategy": "group",
"group_size": 32,
"symmetric": True,
"scale_dtype": "torch.uint8",
}
@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adopting_compressed_tensors_modules_keeps_them_packed_and_matches_its_decompress():
model = _ct_compressed_model()
assert hasattr(model, "ct_decompress_hook")
assert "weight_packed" in model.a._parameters and "weight" not in model.a._parameters
reference = copy.deepcopy(model)
x = torch.randn(3, 64, dtype = torch.bfloat16)
with torch.no_grad():
want = reference(x)
assert "weight" in reference.a._parameters
n = adopt_compressed_mxfp4_modules(model, packed_names = {"a", "b"}, dtype = torch.bfloat16)
assert n == 2
assert type(model.a) is Mxfp4PackedLinear and type(model.b) is Mxfp4PackedLinear
assert type(model.latent) is nn.Linear and "weight" in model.latent._parameters
assert "weight_packed" not in model.latent._parameters
assert not hasattr(model.latent, "quantization_scheme")
assert all("forward" not in m.__dict__ for m in (model.a, model.b, model.latent))
assert not hasattr(model, "ct_decompress_hook")
model.latent.weight.data.copy_(reference.latent.weight.data)
with torch.no_grad():
got = model(x)
assert torch.equal(got, want)
assert "weight_packed" in model.a._parameters
@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adoption_is_all_or_nothing(monkeypatch):
model = _ct_compressed_model()
model.b.__class__ = type("Other", (nn.Linear,), {})
assert adopt_compressed_mxfp4_modules(model) == 0
assert type(model.a) is nn.Linear and hasattr(model, "ct_decompress_hook")
model = _ct_compressed_model()
monkeypatch.setenv("UNSLOTH_MXFP4_KEEP_PACKED", "0")
assert adopt_compressed_mxfp4_modules(model) == 0
assert type(model.a) is nn.Linear
def _nested_model():
class Block(nn.Module):
def __init__(self):
super().__init__()
self.w1 = nn.Linear(64, 32, bias = False)
self.proj = nn.Linear(64, 64, bias = False)
class Inner(nn.Module):
def __init__(self):
super().__init__()
self.layers = nn.ModuleList([Block(), Block()])
class Outer(nn.Module):
def __init__(self):
super().__init__()
self.model = Inner()
return Outer()
@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_loader_matches_packed_checkpoint_names_and_swaps_in_empty_packed_modules():
from unsloth.models.compressed_tensors_bnb import (
_build_quantization_config,
_keep_packed_linears,
_match_packed_linears,
plan_mxfp4_keep_packed,
)
with torch.device("meta"):
model = _nested_model()
keys = [
"language_model.model.layers.0.w1.weight_packed",
"language_model.model.layers.0.w1.weight_scale",
"language_model.model.layers.1.w1.weight_packed",
"language_model.model.layers.1.w1.weight_scale",
"language_model.model.layers.0.proj.weight",
]
assert set(_match_packed_linears(model, keys)) == {"model.layers.0.w1", "model.layers.1.w1"}
# A packed module with no per-Linear home (a merged expert stack) keeps the old route.
assert _match_packed_linears(model, keys + ["model.experts.gate_up_proj.weight_packed"]) is None
assert _match_packed_linears(model, ["model.layers.0.proj.weight"]) is None
plan = plan_mxfp4_keep_packed(model, keys)
assert plan.blocks == [] and plan.linears == ["model.layers.0.w1", "model.layers.1.w1"]
assert (
plan_mxfp4_keep_packed(model, keys + ["model.experts.gate_up_proj.weight_packed"]) is None
)
assert plan_mxfp4_keep_packed(model, ["model.layers.0.proj.weight"]) is None
ct_plan = {
"quant_method": "compressed-tensors",
"format": "mxfp4-pack-quantized",
"config_groups": {
"group_0": {
"targets": ["Linear"],
"weights": {
"num_bits": 4,
"type": "float",
"strategy": "group",
"group_size": 32,
"symmetric": True,
"scale_dtype": "torch.uint8",
},
}
},
}
config = _build_quantization_config(ct_plan)
assert _keep_packed_linears(model, plan.linears, config, torch.float16) == 2
w1 = model.model.layers[1].w1
assert type(w1) is Mxfp4PackedLinear and w1.compute_dtype == torch.float16
assert w1.weight_packed.shape == (32, 32) and w1.weight_packed.dtype == torch.uint8
assert w1.weight_scale.shape == (32, 2) and w1.weight_packed.is_meta
assert type(model.model.layers[0].proj) is nn.Linear
assert str(getattr(w1.quantization_status, "value", w1.quantization_status)) == "compressed"
def _filled(
out_features,
in_features,
seed = 0,
bias = False,
device = "cpu",
dtype = None,
):
packed, scale = _random_packed(out_features, in_features, seed = seed)
module = make_mxfp4_packed_linear(in_features, out_features, device = device, dtype = dtype)
module.weight_packed.data.copy_(packed)
module.weight_scale.data.copy_(scale)
if bias:
module.bias = nn.Parameter(torch.randn(out_features, device = device, dtype = torch.bfloat16))
return module
def test_torch_fallback_without_the_zoo_kernel(monkeypatch):
import unsloth.models.mxfp4_compressed_linear as mcl
packed, scale = _random_packed(16, 64, low = 1, high = 254)
with_zoo = dequantize_mxfp4_packed(packed, scale, torch.bfloat16)
monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
assert torch.equal(dequantize_mxfp4_packed(packed, scale, torch.bfloat16), with_zoo)
def _fake_current_cuda_device(monkeypatch):
"""Replace torch.cuda.device with a guard that only records the current device."""
import contextlib
current = ["cuda:0"]
@contextlib.contextmanager
def fake_cuda_device(device):
previous, current[0] = current[0], str(torch.device(device))
try:
yield
finally:
current[0] = previous
monkeypatch.setattr(torch.cuda, "device", fake_cuda_device)
return current
def test_the_torch_decode_runs_inside_its_operands_device_guard(monkeypatch):
# Any torch build, CPU-only included: the fake-tensor test below needs a CUDA-linked one.
import types
import unsloth.models.mxfp4_compressed_linear as mcl
current = _fake_current_cuda_device(monkeypatch)
seen = []
def spy_impl(blocks, scales, dtype):
seen.append(current[0])
return "decoded"
monkeypatch.setattr(mcl, "_dequantize_torch_impl", spy_impl)
on_gpu1 = types.SimpleNamespace(device = torch.device("cuda:1"))
assert mcl._dequantize_torch(on_gpu1, None, torch.float32) == "decoded"
assert seen == ["cuda:1"] and current == ["cuda:0"]
# A CPU operand, or a CUDA one with no index, keeps the current device.
for device in ("cpu", "cuda"):
seen.clear()
mcl._dequantize_torch(types.SimpleNamespace(device = torch.device(device)), None, None)
assert seen == ["cuda:0"], device
@pytest.mark.skipif(
not torch.backends.cuda.is_built(),
reason = "FakeTensorMode places a tensor on cuda only in a CUDA-linked torch build",
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_torch_fallback_runs_ldexp_on_the_operands_gpu(dtype, monkeypatch):
# float32 always takes this path (the zoo decoder serves only fp16 / bf16).
from torch._subclasses.fake_tensor import FakeTensorMode
import unsloth.models.mxfp4_compressed_linear as mcl
current = _fake_current_cuda_device(monkeypatch)
seen = []
real_ldexp = torch.ldexp
def spy_ldexp(x, other, *args, **kwargs):
seen.append((str(x.device), current[0]))
return real_ldexp(x, other, *args, **kwargs)
monkeypatch.setattr(torch, "ldexp", spy_ldexp)
monkeypatch.setattr(mcl, "_LUT_CACHE", {})
monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
with FakeTensorMode(allow_non_fake_inputs = True):
packed = torch.empty(8, 32, dtype = torch.uint8, device = "cuda:1")
scale = torch.empty(8, 2, dtype = torch.uint8, device = "cuda:1")
out = dequantize_mxfp4_packed(packed, scale, dtype)
assert out.shape == (8, 64) and out.dtype == dtype
assert seen == [("cuda:1", "cuda:1")]
assert current == ["cuda:0"]
@pytest.mark.skipif(
not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two CUDA devices"
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_torch_fallback_on_a_non_current_gpu_matches_cpu(dtype, monkeypatch):
import unsloth.models.mxfp4_compressed_linear as mcl
monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
packed, scale = _random_packed(64, 128, low = 100, high = 140)
want = dequantize_mxfp4_packed(packed, scale, dtype)
with torch.cuda.device(0):
got = dequantize_mxfp4_packed(packed.to("cuda:1"), scale.to("cuda:1"), dtype)
torch.cuda.synchronize(1)
assert torch.cuda.current_device() == 0
assert got.device == torch.device("cuda:1")
assert torch.equal(got.cpu(), want)
def test_dtype_casts_keep_the_packed_bytes_and_set_the_decode_dtype():
module = _filled(32, 64, dtype = torch.float16)
module.weight_scale.data.clamp_(110, 130) # stay inside fp16's range
before = module.weight_packed.clone(), module.weight_scale.clone()
module.to(torch.bfloat16).half()
assert module.weight_packed.dtype == torch.uint8 and torch.equal(
module.weight_packed, before[0]
)
assert torch.equal(module.weight_scale, before[1])
# `weight` (read by PEFT and the fast paths) decodes in the model's dtype, not always bf16.
assert module.weight.dtype == torch.float16
assert torch.equal(module.weight.float(), module.dequantize_weight(torch.bfloat16).float())
@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def test_device_move_carries_bytes_and_scales():
module = _filled(32, 64).cuda()
assert module.weight_packed.is_cuda and module.weight_scale.is_cuda
assert module.weight.is_cuda
def test_fast_lora_paths_get_the_packed_bytes_and_a_decoder_not_a_16bit_weight():
peft = pytest.importorskip("peft")
from unsloth.kernels.utils import (
fast_dequantize,
get_lora_parameters,
get_lora_parameters_bias,
has_mxfp4_base,
)
class Block(nn.Module):
def __init__(self):
super().__init__()
self.proj = _filled(32, 64)
self.dense = nn.Linear(64, 64, bias = False)
model = peft.get_peft_model(Block(), peft.LoraConfig(r = 4, target_modules = ["proj", "dense"]))
proj = model.base_model.model.proj
W, quant_state, A, B, scale = get_lora_parameters(proj)
# W saved for backward must be the packed bytes, never a 16-bit decode.
assert W is proj.base_layer.weight_packed and A is not None
assert getattr(quant_state, "_unsloth_packed_weight_state", False)
dense = proj.base_layer.dequantize_weight()
assert torch.equal(fast_dequantize(W, quant_state), dense)
assert torch.equal(fast_dequantize(W.t(), quant_state), dense.t())
assert get_lora_parameters_bias(proj)[1] is quant_state
assert not has_mxfp4_base(proj) and not has_mxfp4_base(model.base_model.model.dense)
def _mlp_lora_pair(
hidden,
inter,
dtype = torch.bfloat16,
):
peft = pytest.importorskip("peft")
class MLP(nn.Module):
def __init__(self, gate, up, down):
super().__init__()
self.gate_proj, self.up_proj, self.down_proj = gate, up, down
packed = [
_filled(inter, hidden, seed = 1, dtype = dtype),
_filled(inter, hidden, seed = 2, dtype = dtype),
_filled(hidden, inter, seed = 3, dtype = dtype),
]
for module in packed:
module.weight_scale.data.clamp_(118, 126)
dense = []
for module in packed:
linear = nn.Linear(module.in_features, module.out_features, bias = False, dtype = dtype)
linear.weight.data.copy_(module.dequantize_weight())
dense.append(linear)
models = []
for layers in (packed, dense):
model = peft.get_peft_model(
MLP(*layers),
peft.LoraConfig(
r = 8, lora_alpha = 16, target_modules = ["gate_proj", "up_proj", "down_proj"]
),
).cuda()
g = torch.Generator().manual_seed(5)
with torch.no_grad():
for name, param in sorted(model.named_parameters()):
if "lora_" in name:
param.copy_((torch.randn(param.shape, generator = g) * 0.05).to(param.dtype))
models.append(model.base_model.model)
return models
@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
@pytest.mark.parametrize("rows", [8, 256])
def test_fused_lora_mlp_on_a_packed_base_matches_the_dense_base(rows):
"""8 rows run the fused MXFP4 GEMM (bf16-rounding-level), 256 decode for cuBLAS (bit-exact)."""
from unsloth.kernels.fast_lora import apply_lora_mlp_swiglu, apply_lora_o
amp = _cuda_amp_dtype()
packed, dense = _mlp_lora_pair(64, 128, amp)
x = torch.randn(
1, rows, 64, device = "cuda", generator = torch.Generator("cuda").manual_seed(0)
).to(amp)
results = []
for mlp in (packed, dense):
xa = x.clone().requires_grad_()
with torch.autocast("cuda", dtype = amp):
y = apply_lora_mlp_swiglu(mlp, xa, inplace = False)
y = y + apply_lora_o(
type("O", (), {"o_proj": mlp.down_proj})(), y.new_ones(1, rows, 128) * 0.01
)
# fp16 grads of this loss pass 65504; a power-of-two seed scale is exact and keeps them finite.
(y.float().square().sum() * (2.0**-14 if amp == torch.float16 else 1.0)).backward()
lora_grads = [p.grad.float() for n, p in sorted(mlp.named_parameters()) if "lora_" in n]
results.append((y.float(), xa.grad.float(), lora_grads))
(ya, ga, la), (yb, gb, lb) = results
assert len(la) == len(lb) == 6
assert all(torch.isfinite(g).all() and g.abs().max() > 0 for g in (ga, *la))
if rows > 64:
assert torch.equal(ya, yb) and torch.equal(ga, gb)
assert all(torch.equal(a, b) for a, b in zip(la, lb))
else:
rel = lambda a, b: ((a - b).abs().max() / b.abs().max().clamp_min(1e-6)).item() # noqa: E731
assert rel(ya, yb) < 2e-2 and rel(ga, gb) < 2e-2
assert all(rel(a, b) < 3e-2 for a, b in zip(la, lb))
@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
@pytest.mark.parametrize("rows", [1, 3, 64, 65, 300])
@pytest.mark.parametrize("bias", [False, True])
def test_packed_linear_forward_backward_close_to_the_dense_decode_on_cuda(rows, bias):
module = _filled(96, 160, bias = bias).cuda()
module.weight_scale.data.clamp_(118, 126)
dense = module.dequantize_weight()
x = torch.randn(rows, 160, device = "cuda", generator = torch.Generator("cuda").manual_seed(1)).to(
torch.bfloat16
)
xa, xb = x.clone().requires_grad_(), x.clone().requires_grad_()
ya = module(xa)
yb = torch.nn.functional.linear(
xb, dense, None if module.bias is None else module.bias.to(torch.bfloat16)
)
go = torch.randn_like(ya)
ya.backward(go)
yb.backward(go)
tol = dict(atol = 1e-1, rtol = 2e-2) if rows <= 64 else dict(atol = 0, rtol = 0)
torch.testing.assert_close(ya.float(), yb.float(), **tol)
torch.testing.assert_close(xa.grad.float(), xb.grad.float(), **tol)
with torch.no_grad():
torch.testing.assert_close(module(x).float(), ya.float(), atol = 0, rtol = 0)
def _lora_pair(bias = False, scale_range = None):
peft = pytest.importorskip("peft")
packed = _filled(32, 64, bias = bias)
if scale_range is not None:
packed.weight_scale.data.clamp_(*scale_range)
dense = nn.Linear(64, 32, bias = bias, dtype = torch.bfloat16)
dense.weight.data.copy_(packed.dequantize_weight())
if bias:
dense.bias.data.copy_(packed.bias.data)
models = []
for base in (packed, dense):
holder = nn.Sequential(base)
model = peft.get_peft_model(
holder, peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["0"])
)
g = torch.Generator().manual_seed(3)
with torch.no_grad():
for name, param in sorted(model.named_parameters()):
if "lora_" in name:
param.copy_((torch.randn(param.shape, generator = g) * 0.1).to(param.dtype))
models.append(model)
return models
@pytest.mark.parametrize("bias", [False, True])
def test_peft_lora_on_a_packed_linear_matches_the_dense_decode(bias):
packed_model, dense_model = _lora_pair(bias)
x = torch.randn(3, 64, dtype = torch.bfloat16)
xa, xb = x.clone().requires_grad_(), x.clone().requires_grad_()
ya, yb = packed_model(xa), dense_model(xb)
assert torch.equal(ya, yb)
ya.float().sum().backward()
yb.float().sum().backward()
assert torch.equal(xa.grad, xb.grad)
grads = lambda m: {n: p.grad for n, p in m.named_parameters() if "lora_" in n} # noqa: E731
ga, gb = grads(packed_model), grads(dense_model)
assert ga.keys() == gb.keys() and all(torch.equal(ga[n], gb[n]) for n in ga)
def test_peft_merge_densifies_exactly_and_unmerge_restores_the_packed_bytes():
packed_model, dense_model = _lora_pair(scale_range = (118, 134))
x = torch.randn(3, 64, generator = torch.Generator().manual_seed(0)).to(torch.bfloat16)
base = packed_model.base_model.model[0].base_layer
packed_bytes, scale = base.weight_packed, base.weight_scale
with torch.no_grad():
with_lora = packed_model(x)
packed_model.merge_adapter()
dense_model.merge_adapter()
merged_base = packed_model.base_model.model[0].base_layer
# The delta must land in a real weight: a write into a fresh decode would be lost.
assert type(merged_base) is nn.Linear and "weight_packed" not in merged_base._parameters
assert torch.equal(merged_base.weight, dense_model.base_model.model[0].base_layer.weight)
merged = packed_model(x)
packed_model.unmerge_adapter()
restored = packed_model.base_model.model[0].base_layer
assert type(restored) is Mxfp4PackedLinear
assert restored.weight_packed is packed_bytes and restored.weight_scale is scale
assert "weight" not in restored._parameters
assert torch.equal(packed_model(x), with_lora)
assert not torch.equal(merged, packed_model.base_model.model[0].base_layer(x))
torch.testing.assert_close(merged.float(), with_lora.float(), atol = 2e-2, rtol = 2e-2)
unloaded = packed_model.merge_and_unload()
assert type(unloaded[0]) is nn.Linear
assert torch.equal(unloaded[0].weight, dense_model.base_model.model[0].base_layer.weight)
def _write_tiny_mxfp4_llama(root):
from safetensors.torch import save_file
from transformers import LlamaConfig, LlamaForCausalLM
from compressed_tensors.compressors import BaseCompressor
from compressed_tensors.quantization import QuantizationConfig
torch.manual_seed(0)
config = LlamaConfig(
hidden_size = 64,
num_hidden_layers = 2,
num_attention_heads = 4,
num_key_value_heads = 2,
intermediate_size = 128,
vocab_size = 32000,
max_position_embeddings = 128,
tie_word_embeddings = False,
)
model = LlamaForCausalLM(config).to(torch.bfloat16)
quant = {
"quant_method": "compressed-tensors",
"format": "mxfp4-pack-quantized",
"quantization_status": "compressed",
"ignore": ["lm_head"],
"config_groups": {
"group_0": {
"targets": ["Linear"],
"weights": {
"num_bits": 4,
"type": "float",
"strategy": "group",
"group_size": 32,
"symmetric": True,
"scale_dtype": "torch.uint8",
"dynamic": False,
"observer": "minmax",
"observer_kwargs": {},
"actorder": None,
"block_structure": None,
},
"input_activations": None,
"output_activations": None,
}
},
}
scheme = list(QuantizationConfig.model_validate(quant).config_groups.values())[0]
comp = BaseCompressor.get_value_from_registry("mxfp4-pack-quantized")
linears = {n for n, m in model.named_modules() if isinstance(m, nn.Linear)}
packed, plain = {}, {}
for k, w in model.state_dict().items():
mod = k[: -len(".weight")] if k.endswith(".weight") else None
if mod is None or mod not in linears or mod == "lm_head":
packed[k] = plain[k] = w.contiguous()
continue
out_f, in_f = w.shape
amax = w.float().reshape(out_f, in_f // 32, 32).abs().amax(-1).clamp_min(1e-8)
exp = (torch.ceil(torch.log2(amax / 6.0)) + 127).clamp(1, 254).to(torch.uint8)
state = {"weight": w.float(), "weight_scale": 2.0 ** (exp.float() - 127)}
out = comp.compress(state, scheme)
out["weight_scale"] = exp
for pk, pv in out.items():
if pk == "weight_shape":
continue
packed[mod + "." + pk] = pv.contiguous()
plain[k] = (
comp.decompress(
{
"weight_packed": out["weight_packed"],
"weight_scale": exp,
"weight_shape": torch.tensor([out_f, in_f]),
},
scheme,
)["weight"]
.to(torch.bfloat16)
.contiguous()
)
packed_dir, bf16_dir = os.path.join(root, "packed"), os.path.join(root, "bf16")
os.makedirs(packed_dir)
os.makedirs(bf16_dir)
save_file(packed, os.path.join(packed_dir, "model.safetensors"), metadata = {"format": "pt"})
save_file(plain, os.path.join(bf16_dir, "model.safetensors"), metadata = {"format": "pt"})
cfg = config.to_dict()
json.dump(cfg, open(os.path.join(bf16_dir, "config.json"), "w"))
cfg["quantization_config"] = quant
json.dump(cfg, open(os.path.join(packed_dir, "config.json"), "w"))
return packed_dir, bf16_dir
def _add_tokenizer(path, root):
from transformers import AutoTokenizer
tok_dir = os.path.join(root, "tok")
if not os.path.isdir(tok_dir):
AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer").save_pretrained(
tok_dir
)
for f in os.listdir(tok_dir):
if not os.path.exists(os.path.join(path, f)):
shutil.copy(os.path.join(tok_dir, f), os.path.join(path, f))
def _lora_losses(model, steps = 3):
from unsloth import FastLanguageModel
model = FastLanguageModel.get_peft_model(
model,
r = 4,
lora_alpha = 8,
random_state = 3407,
target_modules = [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"gate_proj",
"up_proj",
"down_proj",
],
)
model.train()
opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr = 1e-2)
torch.manual_seed(1)
ids = torch.randint(0, 256, (2, 24), device = "cuda:0")
losses = []
for _ in range(steps):
loss = model(input_ids = ids, labels = ids).loss
loss.backward()
opt.step()
opt.zero_grad(set_to_none = True)
losses.append(loss.item())
return losses
needs_gpu_loader_any = [
pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device"),
pytest.mark.skipif(
not HAS_CT
or not hasattr(__import__("transformers"), "__version__")
or int(__import__("transformers").__version__.split(".")[0]) < 5,
reason = "needs compressed-tensors and transformers 5",
),
]
needs_gpu_loader = [
pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device"),
pytest.mark.skipif(
not (HAS_CT and HAS_CONVERTERS),
reason = "needs compressed-tensors and the transformers 5 loader",
),
]
def _apply(marks):
def wrap(fn):
for m in marks:
fn = m(fn)
return fn
return wrap
@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
@pytest.mark.parametrize("load_in_4bit", [True, False])
def test_mxfp4_checkpoint_stays_packed_and_matches_its_bf16_decode(
load_in_4bit, tmp_path, monkeypatch
):
import unsloth.models.llama as llama_module
from unsloth import FastLanguageModel
packed_dir, bf16_dir = _write_tiny_mxfp4_llama(str(tmp_path))
for d in (packed_dir, bf16_dir):
_add_tokenizer(d, str(tmp_path))
kw = dict(max_seq_length = 64, dtype = torch.bfloat16)
model_a, _ = FastLanguageModel.from_pretrained(packed_dir, load_in_4bit = load_in_4bit, **kw)
kept = [n for n, m in model_a.named_modules() if isinstance(m, Mxfp4PackedLinear)]
assert len(kept) == 2 * 7, kept
assert all(
p.dtype == torch.uint8 for n, p in model_a.named_parameters() if "weight_packed" in n
)
model_b, _ = FastLanguageModel.from_pretrained(bf16_dir, load_in_4bit = False, **kw)
ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
import unsloth.models.mxfp4_compressed_linear as mcl
with torch.no_grad():
# 16 rows run the fused MXFP4 GEMM (bf16-rounding level); decode + cuBLAS is bit-exact.
fused = model_a(input_ids = ids).logits
want = model_b(input_ids = ids).logits
torch.testing.assert_close(fused.float(), want.float(), atol = 5e-2, rtol = 2e-2)
monkeypatch.setattr(mcl, "_FUSED_MAX_ROWS", 0)
assert torch.equal(model_a(input_ids = ids).logits, want)
losses_a = _lora_losses(model_a)
from unsloth.kernels import apply_lora_o, apply_lora_qkv
for layer in model_a.model.layers:
assert "_unsloth_forward" in layer.mlp.__dict__ or "forward" in layer.mlp.__dict__
assert getattr(layer.self_attn, "apply_qkv", None) is apply_lora_qkv
assert getattr(layer.self_attn, "apply_o", None) is apply_lora_o
assert losses_a == _lora_losses(model_b)
@_apply(needs_gpu_loader)
def test_keep_packed_off_restores_the_bitsandbytes_route(tmp_path, monkeypatch):
import bitsandbytes as bnb
from unsloth import FastLanguageModel
monkeypatch.setenv("UNSLOTH_MXFP4_KEEP_PACKED", "0")
packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
_add_tokenizer(packed_dir, str(tmp_path))
model, _ = FastLanguageModel.from_pretrained(
packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True
)
assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())
assert sum(isinstance(m, bnb.nn.Linear4bit) for m in model.modules()) == 2 * 7
def _plain_logits(path, ids):
from transformers import AutoModelForCausalLM
model = AutoModelForCausalLM.from_pretrained(path, dtype = torch.bfloat16, device_map = {"": 0})
with torch.no_grad():
return model(input_ids = ids).logits
@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
def test_full_and_merged_saves_reload_in_plain_transformers(tmp_path, monkeypatch):
from safetensors import safe_open
from unsloth import FastLanguageModel
packed_dir, bf16_dir = _write_tiny_mxfp4_llama(str(tmp_path))
_add_tokenizer(packed_dir, str(tmp_path))
model, tokenizer = FastLanguageModel.from_pretrained(
packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = False
)
ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
with torch.no_grad():
base_logits = model(input_ids = ids).logits
full = str(tmp_path / "full")
model.save_pretrained(full)
assert sum(isinstance(m, Mxfp4PackedLinear) for m in model.modules()) == 2 * 7
with safe_open(os.path.join(full, "model.safetensors"), "pt") as f:
keys = list(f.keys())
assert not any("weight_packed" in k or "weight_scale" in k for k in keys)
assert "quantization_config" not in json.load(open(os.path.join(full, "config.json")))
assert torch.equal(_plain_logits(full, ids), _plain_logits(bf16_dir, ids))
model = FastLanguageModel.get_peft_model(
model,
r = 4,
lora_alpha = 8,
random_state = 3407,
target_modules = ["q_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
)
with torch.no_grad():
for name, param in model.named_parameters():
if "lora_B" in name:
param.normal_(0, 0.05)
merged_dir = str(tmp_path / "merged")
model.save_pretrained_merged(merged_dir, tokenizer, save_method = "merged_16bit")
unloaded = model.merge_and_unload()
assert [n for n, m in unloaded.named_modules() if isinstance(m, Mxfp4PackedLinear)] == [
f"model.layers.{i}.self_attn.k_proj" for i in range(2)
]
with torch.no_grad():
unloaded_logits = unloaded(input_ids = ids).logits
assert not torch.equal(unloaded_logits, base_logits)
with safe_open(os.path.join(merged_dir, "model.safetensors"), "pt") as f:
assert not any("weight_packed" in k or "weight_scale" in k for k in f.keys())
unloaded_dir = str(tmp_path / "unloaded")
unloaded.save_pretrained(unloaded_dir)
merged_logits, unloaded_reload = (
_plain_logits(merged_dir, ids),
_plain_logits(unloaded_dir, ids),
)
torch.testing.assert_close(merged_logits, unloaded_reload, atol = 2e-2, rtol = 2e-2)
torch.testing.assert_close(unloaded_reload, unloaded_logits, atol = 2e-2, rtol = 2e-2)
assert not torch.equal(merged_logits, _plain_logits(full, ids))
@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
def test_full_finetuning_keeps_the_stock_compressed_tensors_route(tmp_path, monkeypatch):
from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed
from transformers import AutoModelForCausalLM
packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
assert install_compressed_tensors_keep_packed()
monkeypatch.setenv("UNSLOTH_ENABLE_FULL_FINETUNING", "1")
model = AutoModelForCausalLM.from_pretrained(packed_dir, dtype = torch.bfloat16)
assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())
monkeypatch.setenv("UNSLOTH_ENABLE_FULL_FINETUNING", "0")
model = AutoModelForCausalLM.from_pretrained(packed_dir, dtype = torch.bfloat16)
assert sum(isinstance(m, Mxfp4PackedLinear) for m in model.modules()) == 2 * 7
def _decompress_requests():
"""The ways this transformers asks compressed-tensors to decompress at load."""
try:
import inspect
from transformers import CompressedTensorsConfig
except Exception:
return []
params = inspect.signature(CompressedTensorsConfig).parameters
return [{k: v} for k, v in (("run_compressed", False), ("dequantize", True)) if k in params]
@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
@pytest.mark.parametrize("request_kwargs", _decompress_requests(), ids = lambda kw: next(iter(kw)))
def test_an_explicit_decompress_request_keeps_the_stock_route(request_kwargs, tmp_path):
from transformers import AutoModelForCausalLM, CompressedTensorsConfig
from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed
packed_dir, bf16_dir = _write_tiny_mxfp4_llama(str(tmp_path))
assert install_compressed_tensors_keep_packed()
# GPU when present: an earlier FastLanguageModel load patches Llama's forward.
device = "cuda:0" if has_real_cuda() else "cpu"
model = AutoModelForCausalLM.from_pretrained(
packed_dir,
dtype = torch.bfloat16,
device_map = {"": device},
quantization_config = CompressedTensorsConfig(**request_kwargs),
)
assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())
reference = AutoModelForCausalLM.from_pretrained(
bf16_dir, dtype = torch.bfloat16, device_map = {"": device}
)
ids = torch.randint(0, 256, (1, 12), device = device)
with torch.no_grad():
assert torch.equal(model(input_ids = ids).logits, reference(input_ids = ids).logits)
@pytest.mark.parametrize("cast", [torch.float16, torch.float32])
def test_a_dtype_cast_reaches_the_merge_and_unmerge(cast):
packed_model, dense_model = _lora_pair(scale_range = (118, 134)) # inside fp16's range
packed_model.to(cast)
dense_model.to(cast)
base = packed_model.base_model.model[0].base_layer
assert base.compute_dtype == cast and base.weight.dtype == cast
x = torch.randn(3, 64, dtype = cast)
with torch.no_grad():
packed_model.merge_adapter()
dense_model.merge_adapter()
assert packed_model.base_model.model[0].base_layer.weight.dtype == cast
assert torch.equal(packed_model(x), dense_model(x))
# Cast while merged, then unmerge: the packed module decodes in the model's new dtype.
packed_model.to(torch.bfloat16)
packed_model.unmerge_adapter()
restored = packed_model.base_model.model[0].base_layer
assert type(restored) is Mxfp4PackedLinear and restored.compute_dtype == torch.bfloat16
packed_model.merge_adapter()
assert packed_model(x.to(torch.bfloat16)).dtype == torch.bfloat16
@needs_zoo_packed_save
@pytest.mark.skipif(
not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
)
def test_the_sixteen_bit_route_decodes_in_the_load_dtype(tmp_path):
peft = pytest.importorskip("peft")
from transformers import AutoModelForCausalLM
from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed
packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
assert install_compressed_tensors_keep_packed()
device = "cuda:0" if has_real_cuda() else "cpu"
model = AutoModelForCausalLM.from_pretrained(
packed_dir, dtype = torch.float16, device_map = {"": device}
)
packed = [m for m in model.modules() if isinstance(m, Mxfp4PackedLinear)]
assert len(packed) == 2 * 7
assert {m.compute_dtype for m in packed} == {torch.float16}
assert {m.weight.dtype for m in packed} == {torch.float16}
assert model.model.layers[0].input_layernorm.weight.dtype == torch.float16
model = peft.get_peft_model(model, peft.LoraConfig(r = 2, target_modules = ["q_proj"]))
merged = model.merge_and_unload()
with torch.no_grad():
assert merged(input_ids = torch.tensor([[1, 2, 3, 4]], device = device)).logits.isfinite().all()
@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adoption_declines_a_scheme_that_also_quantizes_activations():
from unsloth.models.mxfp4_compressed_linear import is_mxfp4_scheme
activations = {
"num_bits": 8,
"type": "int",
"strategy": "token",
"dynamic": True,
"symmetric": True,
}
groups = {
"group_0": {
"targets": ["Linear"],
"weights": _MXFP4_WEIGHTS,
"input_activations": activations,
}
}
model = _ct_compressed_model(groups = groups)
assert not is_mxfp4_scheme(model.a.quantization_scheme, "mxfp4-pack-quantized")
reference = copy.deepcopy(model)
assert adopt_compressed_mxfp4_modules(model, dtype = torch.bfloat16) == 0
assert type(model.a) is nn.Linear and hasattr(model, "ct_decompress_hook")
x = torch.randn(3, 64, dtype = torch.bfloat16)
with torch.no_grad():
assert torch.equal(model(x), reference(x))
@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adoption_declines_when_another_compressed_format_needs_the_hook():
fp8 = {
"num_bits": 8,
"type": "float",
"strategy": "channel",
"symmetric": True,
"dynamic": False,
}
groups = {
"group_0": {
"targets": ["re:^a$", "re:^b$"],
"weights": _MXFP4_WEIGHTS,
"format": "mxfp4-pack-quantized",
},
"group_1": {"targets": ["re:^latent$"], "weights": fp8, "format": "float-quantized"},
}
model = _ct_compressed_model(groups = groups)
assert model.latent.weight.dtype == torch.float8_e4m3fn
reference = copy.deepcopy(model)
assert adopt_compressed_mxfp4_modules(model, dtype = torch.bfloat16) == 0
assert type(model.a) is nn.Linear and hasattr(model, "ct_decompress_hook")
x = torch.randn(3, 64, dtype = torch.bfloat16)
with torch.no_grad():
assert torch.equal(model(x), reference(x))
@pytest.mark.parametrize("init", ["pissa", "olora"])
def test_initialisers_that_rewrite_the_base_weight_refuse_a_packed_base(init):
peft = pytest.importorskip("peft")
holder = nn.Sequential(_filled(32, 64))
with pytest.raises(NotImplementedError, match = "packed in MXFP4"):
peft.get_peft_model(
holder, peft.LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = init)
)
peft.get_peft_model(nn.Sequential(_filled(32, 64)), peft.LoraConfig(r = 4, target_modules = ["0"]))
dense = nn.Sequential(nn.Linear(64, 32, bias = False))
peft.get_peft_model(dense, peft.LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = init))
@_apply(needs_gpu_loader_any)
def test_the_sixteen_bit_route_declines_without_the_zoo_full_save_support(tmp_path, monkeypatch):
from transformers import AutoModelForCausalLM
from unsloth_zoo.temporary_patches import mxfp4 as zoo_mxfp4
from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed
packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
assert install_compressed_tensors_keep_packed()
monkeypatch.delattr(zoo_mxfp4, "_densified_module_names", raising = False)
model = AutoModelForCausalLM.from_pretrained(
packed_dir, dtype = torch.bfloat16, device_map = {"": 0}
)
assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())
@needs_zoo_packed_save
@_apply(needs_gpu_loader)
def test_a_partly_merged_bitsandbytes_route_model_saves_and_reloads(tmp_path):
from transformers import AutoModelForCausalLM
from unsloth import FastLanguageModel
packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
_add_tokenizer(packed_dir, str(tmp_path))
model, _ = FastLanguageModel.from_pretrained(
packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True
)
model = FastLanguageModel.get_peft_model(model, r = 4, lora_alpha = 8, target_modules = ["q_proj"])
with torch.no_grad():
for name, param in model.named_parameters():
if "lora_B" in name:
param.normal_(0, 0.05)
unloaded = model.merge_and_unload()
ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
with torch.no_grad():
want = unloaded(input_ids = ids).logits
out = str(tmp_path / "unloaded")
unloaded.save_pretrained(out)
# want.dtype: Unsloth loads fp16 where bf16 is unsupported (T4) even when asked for bf16
reloaded = AutoModelForCausalLM.from_pretrained(out, dtype = want.dtype, device_map = {"": 0})
attn = reloaded.model.layers[0].self_attn
assert type(attn.q_proj) is nn.Linear and type(attn.k_proj) is nn.Linear
with torch.no_grad():
got = reloaded(input_ids = ids).logits
torch.testing.assert_close(got, want, atol = 2e-2, rtol = 2e-2)
def test_merge_and_unmerge_under_an_accelerate_hook():
pytest.importorskip("accelerate")
from accelerate.hooks import AlignDevicesHook, add_hook_to_module
packed_model, dense_model = _lora_pair(scale_range = (118, 134))
add_hook_to_module(
packed_model.base_model.model[0].base_layer, AlignDevicesHook(io_same_device = True)
)
x = torch.randn(3, 64, generator = torch.Generator().manual_seed(0)).to(torch.bfloat16)
with torch.no_grad():
with_lora = packed_model(x)
packed_model.merge_adapter()
dense_model.merge_adapter()
assert torch.equal(packed_model(x), dense_model(x))
packed_model.unmerge_adapter()
assert torch.equal(packed_model(x), with_lora)
@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
def test_the_sixteen_bit_route_plans_only_mxfp4_checkpoints(tmp_path, monkeypatch):
import importlib.util
import unsloth.models.compressed_tensors_bnb as ctb
from transformers import AutoModelForCausalLM
from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed
path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "test_compressed_tensors_bnb.py"
)
spec = importlib.util.spec_from_file_location("_k3s_ct_bnb_tests", path)
ct_bnb_tests = importlib.util.module_from_spec(spec)
spec.loader.exec_module(ct_bnb_tests)
calls = []
real = ctb._checkpoint_keys
monkeypatch.setattr(ctb, "_checkpoint_keys", lambda files: calls.append(files) or real(files))
assert install_compressed_tensors_keep_packed()
int4_dir, _ = ct_bnb_tests._write_tiny_packed_llama(str(tmp_path / "int4"))
AutoModelForCausalLM.from_pretrained(int4_dir, dtype = torch.bfloat16, device_map = {"": 0})
assert calls == []
mxfp4_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path / "mxfp4"))
model = AutoModelForCausalLM.from_pretrained(
mxfp4_dir, dtype = torch.bfloat16, device_map = {"": 0}
)
assert (
len(calls) == 1 and sum(isinstance(m, Mxfp4PackedLinear) for m in model.modules()) == 2 * 7
)
@pytest.mark.parametrize("spelling", ["compressed-tensors", "compressed_tensors", "sparseml"])
def test_every_compressed_tensors_spelling_installs_the_keep_packed_hook(spelling, monkeypatch):
from types import SimpleNamespace
from unsloth.models import loader_utils, mxfp4_compressed_linear
calls = []
monkeypatch.setattr(
mxfp4_compressed_linear, "install_compressed_tensors_keep_packed", lambda: calls.append(1)
)
config = SimpleNamespace(
quantization_config = {"quant_method": spelling, "format": "mxfp4-pack-quantized"}
)
loader_utils.check_and_disable_bitsandbytes_loading(
config,
load_in_4bit = False,
verbose = False,
requantize_packed = False,
)
assert calls == [1]