Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

54.2 KB
Raw Blame
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.

"""compressed-tensors ``mxfp4-pack-quantized`` Linears stay packed and train through."""

import copy
import json
import os
import shutil

import pytest
from llama_patch_isolation import restore_llama_patches  # noqa: F401
from real_accelerator import has_real_cuda  # tests/_shared, on sys.path via tests/conftest.py
import torch
from torch import nn

import unsloth  # noqa: F401
from unsloth.models.mxfp4_compressed_linear import (
    Mxfp4PackedLinear,
    _dequantize_torch,
    adopt_compressed_mxfp4_modules,
    dequantize_mxfp4_packed,
    make_mxfp4_packed_linear,
)

try:
    from compressed_tensors.config import CompressionFormat

    # Releases before MXFP4 (0.12.x, which vllm 0.11.2 pins) reject its config outright.
    HAS_CT = any(f.value == "mxfp4-pack-quantized" for f in CompressionFormat)
    if HAS_CT:
        # compressed-tensors < 0.16 (the newest on torch 2.6) has the format but decompress raises.
        from compressed_tensors.compressors import BaseCompressor
        from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
        BaseCompressor.get_value_from_registry("mxfp4-pack-quantized").decompress(
            {
                "weight_packed": torch.zeros(1, 16, dtype = torch.uint8),
                "weight_scale": torch.full((1, 1), 127, dtype = torch.uint8),
            },
            QuantizationScheme(
                targets = ["Linear"],
                weights = QuantizationArgs(
                    num_bits = 4,
                    type = "float",
                    strategy = "group",
                    group_size = 32,
                    symmetric = True,
                    scale_dtype = torch.uint8,
                ),
                format = "mxfp4-pack-quantized",
            ),
        )
except Exception:
    HAS_CT = False

from unsloth.models.compressed_tensors_bnb import _transformers_supports_weight_converters

pytestmark = pytest.mark.usefixtures("restore_llama_patches")

HAS_CONVERTERS = _transformers_supports_weight_converters()

try:
    from unsloth_zoo.temporary_patches import mxfp4 as _zoo_mxfp4
    HAS_ZOO_PACKED_SAVE = hasattr(_zoo_mxfp4, "_densified_module_names")
except Exception:
    HAS_ZOO_PACKED_SAVE = False
# Without the zoo packed save support the packed route declines by design.
needs_zoo_packed_save = pytest.mark.skipif(
    not HAS_ZOO_PACKED_SAVE, reason = "needs unsloth-zoo's packed-module save support"
)

DEVICES = ["cpu"] + (["cuda"] if has_real_cuda() else [])


def _cuda_amp_dtype():
    # CUDA autocast refuses bf16 below sm_80 (T4), where Unsloth trains in fp16.
    return torch.bfloat16 if torch.cuda.get_device_capability() >= (8, 0) else torch.float16


_E2M1 = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]


def _random_packed(
    out_features,
    in_features,
    seed = 0,
    low = 100,
    high = 150,
):
    g = torch.Generator().manual_seed(seed)
    packed = torch.randint(0, 256, (out_features, in_features // 2), generator = g, dtype = torch.uint8)
    scale = torch.randint(
        low, high, (out_features, in_features // 32), generator = g, dtype = torch.uint8
    )
    return packed, scale


def _reference_decode(packed, scale):
    """Element by element, in float64: low nibble first, sign in bit 3, times 2**(e - 127)."""
    out_features, half = packed.shape
    w = torch.empty(out_features, half * 2, dtype = torch.float64)
    for i in range(out_features):
        for j in range(half):
            byte = int(packed[i, j])
            for k, nib in enumerate((byte & 0x0F, byte >> 4)):
                col = 2 * j + k
                v = _E2M1[nib & 0x07] * (-1.0 if nib & 0x08 else 1.0)
                w[i, col] = v * 2.0 ** (int(scale[i, col // 32]) - 127)
    return w


@pytest.mark.parametrize("device", DEVICES)
def test_dequant_matches_an_independent_float64_decode(device):
    packed, scale = _random_packed(8, 64, low = 1, high = 254)
    packed[0, :16] = torch.arange(0, 256, 16, dtype = torch.uint8)
    packed[1, :16] = torch.arange(0, 16, dtype = torch.uint8)
    want = _reference_decode(packed, scale)
    for dtype in (torch.bfloat16, torch.float32):
        got = dequantize_mxfp4_packed(packed.to(device), scale.to(device), dtype).cpu()
        assert got.dtype == dtype and got.shape == (8, 64)
        assert torch.equal(got.double(), want.to(dtype).double()), dtype


@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
@pytest.mark.parametrize("device", DEVICES)
def test_dequant_matches_compressed_tensors_decompress(device):
    from compressed_tensors.compressors import BaseCompressor
    from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme

    scheme = QuantizationScheme(
        targets = ["Linear"],
        weights = QuantizationArgs(
            num_bits = 4,
            type = "float",
            strategy = "group",
            group_size = 32,
            symmetric = True,
            scale_dtype = torch.uint8,
        ),
        format = "mxfp4-pack-quantized",
    )
    comp = BaseCompressor.get_value_from_registry("mxfp4-pack-quantized")
    for seed, (o, i) in enumerate([(128, 128), (96, 256), (3, 64)]):
        packed, scale = _random_packed(o, i, seed = seed, low = 1, high = 254)
        state = {
            "weight_packed": packed,
            "weight_scale": scale,
            "weight_shape": torch.tensor([o, i]),
        }
        want = comp.decompress(state, scheme)["weight"].to(torch.bfloat16)
        got = dequantize_mxfp4_packed(packed.to(device), scale.to(device), torch.bfloat16).cpu()
        assert torch.equal(got.view(torch.int16), want.view(torch.int16)), (o, i)


@pytest.mark.parametrize("device", DEVICES)
def test_torch_fp16_decode_rounds_once_for_every_finite_scale(device):
    # E8M0 1..254 (0xFF is NaN): past fp16's range a value is inf, a zero nibble stays +-0, small ones go subnormal.
    # torch < 2.12's ldexp built 2^e in fp16, so 0 * 2^16 was NaN, 2^-10 * 2^20 inf and 6 * 2^-25 zero.
    packed, scale = _random_packed(8, 64, low = 1, high = 255)
    packed[0, :16] = torch.arange(0, 256, 16, dtype = torch.uint8)
    packed[1, :16] = torch.arange(0, 16, dtype = torch.uint8)
    scale[0, 0], scale[0, 1], scale[1, 0], scale[1, 1] = 143, 147, 102, 254
    want = _reference_decode(packed, scale).to(torch.float16)
    got = _dequantize_torch(packed.to(device).view(8, 2, 16), scale.to(device), torch.float16).cpu()
    assert got.dtype == torch.float16 and got.shape == (8, 64)
    assert torch.equal(got.view(torch.int16), want.view(torch.int16))


@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def test_zoo_kernel_and_torch_path_agree():
    packed, scale = _random_packed(256, 512, low = 1, high = 254)
    packed, scale = packed.cuda(), scale.cuda()
    blocks = packed.view(256, 16, 16)
    for dtype in (torch.bfloat16, torch.float16):
        a = dequantize_mxfp4_packed(packed, scale, dtype)
        b = _dequantize_torch(blocks, scale, dtype)
        assert torch.equal(a.view(torch.int16), b.view(torch.int16)), dtype


@pytest.mark.parametrize("device", DEVICES)
@pytest.mark.parametrize("bias", [False, True])
def test_forward_and_input_gradient_are_exact_and_save_only_packed_bytes(device, bias, monkeypatch):
    import unsloth.models.mxfp4_compressed_linear as mcl

    # The decode + cuBLAS route is bit-exact; skinny CUDA inputs take the fused GEMM.
    monkeypatch.setattr(mcl, "_FUSED_MAX_ROWS", 0)
    torch.manual_seed(0)
    out_f, in_f = 96, 64
    packed, scale = _random_packed(out_f, in_f)
    module = make_mxfp4_packed_linear(in_f, out_f, device = device)
    module.weight_packed.data.copy_(packed)
    module.weight_scale.data.copy_(scale)
    ref = nn.Linear(in_f, out_f, bias = bias, device = device, dtype = torch.bfloat16)
    ref.weight.data.copy_(module.dequantize_weight(torch.bfloat16))
    ref.weight.requires_grad_(False)
    if bias:
        module.bias = nn.Parameter(ref.bias.detach().clone())
    x = torch.randn(5, 7, in_f, device = device, dtype = torch.bfloat16)
    xa, xb = x.clone().requires_grad_(), x.clone().requires_grad_()

    saved = []
    with torch.autograd.graph.saved_tensors_hooks(lambda t: saved.append(t) or t, lambda t: t):
        ya = module(xa)
    yb = ref(xb)
    assert torch.equal(ya, yb)
    assert saved and all(t.dtype == torch.uint8 for t in saved), [t.dtype for t in saved]
    g = torch.randn_like(ya)
    ya.backward(g)
    yb.backward(g)
    assert torch.equal(xa.grad, xb.grad)
    if bias:
        assert torch.allclose(module.bias.grad.float(), ref.bias.grad.float(), rtol = 1e-2, atol = 1e-2)
    assert module.weight_packed.grad is None and not module.weight_packed.requires_grad


@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def test_autocast_runs_the_matmul_in_the_autocast_dtype():
    amp = _cuda_amp_dtype()
    packed, scale = _random_packed(64, 64)
    module = make_mxfp4_packed_linear(64, 64, device = "cuda")
    module.weight_packed.data.copy_(packed)
    module.weight_scale.data.copy_(scale)
    if amp == torch.float16:
        module.weight_scale.data.clamp_(118, 126)  # fp16's range: the decode stays finite and exact
    ref = nn.Linear(64, 64, bias = False, device = "cuda", dtype = torch.float32)
    ref.weight.data.copy_(module.dequantize_weight(torch.float32))
    x = torch.randn(4, 64, device = "cuda", requires_grad = True)
    x2 = x.detach().clone().requires_grad_()
    with torch.autocast("cuda", dtype = amp):
        ya, yb = module(x), ref(x2)
    assert ya.dtype == yb.dtype == amp
    assert torch.equal(ya, yb)
    ya.float().sum().backward()
    yb.float().sum().backward()
    assert x.grad.dtype == torch.float32
    assert torch.equal(x.grad, x2.grad)


def test_weight_property_is_an_exact_read_only_decode():
    packed, scale = _random_packed(32, 64)
    module = make_mxfp4_packed_linear(64, 32)
    module.weight_packed.data.copy_(packed)
    module.weight_scale.data.copy_(scale)
    w = module.weight
    assert w.dtype == torch.bfloat16 and w.shape == (32, 64)
    assert torch.equal(w.double(), _reference_decode(packed, scale).to(torch.bfloat16).double())
    assert "weight" not in dict(module.named_parameters())
    assert {n for n, _ in module.named_parameters()} == {"weight_packed", "weight_scale"}
    with pytest.raises(ValueError):
        make_mxfp4_packed_linear(48, 32)


def _ct_compressed_model(targets_ignore = (), groups = None):
    from compressed_tensors.compressors import ModelCompressor
    from compressed_tensors.quantization import QuantizationConfig, apply_quantization_config

    torch.manual_seed(0)

    class Tiny(nn.Module):
        def __init__(self):
            super().__init__()
            self.a = nn.Linear(64, 32, bias = False)
            self.b = nn.Linear(32, 64, bias = False)
            self.latent = nn.Linear(64, 64, bias = False)

        def forward(self, x):
            return self.latent(self.b(self.a(x)))

    model = Tiny().to(torch.bfloat16)
    quant = {
        "quant_method": "compressed-tensors",
        "format": "mxfp4-pack-quantized",
        "quantization_status": "compressed",
        "ignore": list(targets_ignore),
        "config_groups": {
            "group_0": {
                "targets": ["Linear"],
                "weights": {
                    "num_bits": 4,
                    "type": "float",
                    "strategy": "group",
                    "group_size": 32,
                    "symmetric": True,
                    "scale_dtype": "torch.uint8",
                },
            }
        },
    }
    if groups is not None:
        quant["config_groups"] = groups
    qc = QuantizationConfig.model_validate(quant)
    apply_quantization_config(model, qc, run_compressed = False)
    for m in (model.a, model.b, model.latent):
        m.weight_scale.data.fill_(125 if m.weight_scale.dtype == torch.uint8 else 0.01)
    ModelCompressor(quantization_config = qc).compress_model(model)
    return model


_MXFP4_WEIGHTS = {
    "num_bits": 4,
    "type": "float",
    "strategy": "group",
    "group_size": 32,
    "symmetric": True,
    "scale_dtype": "torch.uint8",
}


@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adopting_compressed_tensors_modules_keeps_them_packed_and_matches_its_decompress():
    model = _ct_compressed_model()
    assert hasattr(model, "ct_decompress_hook")
    assert "weight_packed" in model.a._parameters and "weight" not in model.a._parameters
    reference = copy.deepcopy(model)
    x = torch.randn(3, 64, dtype = torch.bfloat16)
    with torch.no_grad():
        want = reference(x)
    assert "weight" in reference.a._parameters

    n = adopt_compressed_mxfp4_modules(model, packed_names = {"a", "b"}, dtype = torch.bfloat16)
    assert n == 2
    assert type(model.a) is Mxfp4PackedLinear and type(model.b) is Mxfp4PackedLinear
    assert type(model.latent) is nn.Linear and "weight" in model.latent._parameters
    assert "weight_packed" not in model.latent._parameters
    assert not hasattr(model.latent, "quantization_scheme")
    assert all("forward" not in m.__dict__ for m in (model.a, model.b, model.latent))
    assert not hasattr(model, "ct_decompress_hook")
    model.latent.weight.data.copy_(reference.latent.weight.data)
    with torch.no_grad():
        got = model(x)
    assert torch.equal(got, want)
    assert "weight_packed" in model.a._parameters


@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adoption_is_all_or_nothing(monkeypatch):
    model = _ct_compressed_model()
    model.b.__class__ = type("Other", (nn.Linear,), {})
    assert adopt_compressed_mxfp4_modules(model) == 0
    assert type(model.a) is nn.Linear and hasattr(model, "ct_decompress_hook")
    model = _ct_compressed_model()
    monkeypatch.setenv("UNSLOTH_MXFP4_KEEP_PACKED", "0")
    assert adopt_compressed_mxfp4_modules(model) == 0
    assert type(model.a) is nn.Linear


def _nested_model():
    class Block(nn.Module):
        def __init__(self):
            super().__init__()
            self.w1 = nn.Linear(64, 32, bias = False)
            self.proj = nn.Linear(64, 64, bias = False)

    class Inner(nn.Module):
        def __init__(self):
            super().__init__()
            self.layers = nn.ModuleList([Block(), Block()])

    class Outer(nn.Module):
        def __init__(self):
            super().__init__()
            self.model = Inner()

    return Outer()


@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_loader_matches_packed_checkpoint_names_and_swaps_in_empty_packed_modules():
    from unsloth.models.compressed_tensors_bnb import (
        _build_quantization_config,
        _keep_packed_linears,
        _match_packed_linears,
        plan_mxfp4_keep_packed,
    )

    with torch.device("meta"):
        model = _nested_model()
    keys = [
        "language_model.model.layers.0.w1.weight_packed",
        "language_model.model.layers.0.w1.weight_scale",
        "language_model.model.layers.1.w1.weight_packed",
        "language_model.model.layers.1.w1.weight_scale",
        "language_model.model.layers.0.proj.weight",
    ]
    assert set(_match_packed_linears(model, keys)) == {"model.layers.0.w1", "model.layers.1.w1"}
    # A packed module with no per-Linear home (a merged expert stack) keeps the old route.
    assert _match_packed_linears(model, keys + ["model.experts.gate_up_proj.weight_packed"]) is None
    assert _match_packed_linears(model, ["model.layers.0.proj.weight"]) is None
    plan = plan_mxfp4_keep_packed(model, keys)
    assert plan.blocks == [] and plan.linears == ["model.layers.0.w1", "model.layers.1.w1"]
    assert (
        plan_mxfp4_keep_packed(model, keys + ["model.experts.gate_up_proj.weight_packed"]) is None
    )
    assert plan_mxfp4_keep_packed(model, ["model.layers.0.proj.weight"]) is None

    ct_plan = {
        "quant_method": "compressed-tensors",
        "format": "mxfp4-pack-quantized",
        "config_groups": {
            "group_0": {
                "targets": ["Linear"],
                "weights": {
                    "num_bits": 4,
                    "type": "float",
                    "strategy": "group",
                    "group_size": 32,
                    "symmetric": True,
                    "scale_dtype": "torch.uint8",
                },
            }
        },
    }
    config = _build_quantization_config(ct_plan)
    assert _keep_packed_linears(model, plan.linears, config, torch.float16) == 2
    w1 = model.model.layers[1].w1
    assert type(w1) is Mxfp4PackedLinear and w1.compute_dtype == torch.float16
    assert w1.weight_packed.shape == (32, 32) and w1.weight_packed.dtype == torch.uint8
    assert w1.weight_scale.shape == (32, 2) and w1.weight_packed.is_meta
    assert type(model.model.layers[0].proj) is nn.Linear
    assert str(getattr(w1.quantization_status, "value", w1.quantization_status)) == "compressed"


def _filled(
    out_features,
    in_features,
    seed = 0,
    bias = False,
    device = "cpu",
    dtype = None,
):
    packed, scale = _random_packed(out_features, in_features, seed = seed)
    module = make_mxfp4_packed_linear(in_features, out_features, device = device, dtype = dtype)
    module.weight_packed.data.copy_(packed)
    module.weight_scale.data.copy_(scale)
    if bias:
        module.bias = nn.Parameter(torch.randn(out_features, device = device, dtype = torch.bfloat16))
    return module


def test_torch_fallback_without_the_zoo_kernel(monkeypatch):
    import unsloth.models.mxfp4_compressed_linear as mcl

    packed, scale = _random_packed(16, 64, low = 1, high = 254)
    with_zoo = dequantize_mxfp4_packed(packed, scale, torch.bfloat16)
    monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
    assert torch.equal(dequantize_mxfp4_packed(packed, scale, torch.bfloat16), with_zoo)


def _fake_current_cuda_device(monkeypatch):
    """Replace torch.cuda.device with a guard that only records the current device."""
    import contextlib

    current = ["cuda:0"]

    @contextlib.contextmanager
    def fake_cuda_device(device):
        previous, current[0] = current[0], str(torch.device(device))
        try:
            yield
        finally:
            current[0] = previous

    monkeypatch.setattr(torch.cuda, "device", fake_cuda_device)
    return current


def test_the_torch_decode_runs_inside_its_operands_device_guard(monkeypatch):
    # Any torch build, CPU-only included: the fake-tensor test below needs a CUDA-linked one.
    import types
    import unsloth.models.mxfp4_compressed_linear as mcl

    current = _fake_current_cuda_device(monkeypatch)
    seen = []

    def spy_impl(blocks, scales, dtype):
        seen.append(current[0])
        return "decoded"

    monkeypatch.setattr(mcl, "_dequantize_torch_impl", spy_impl)
    on_gpu1 = types.SimpleNamespace(device = torch.device("cuda:1"))
    assert mcl._dequantize_torch(on_gpu1, None, torch.float32) == "decoded"
    assert seen == ["cuda:1"] and current == ["cuda:0"]
    # A CPU operand, or a CUDA one with no index, keeps the current device.
    for device in ("cpu", "cuda"):
        seen.clear()
        mcl._dequantize_torch(types.SimpleNamespace(device = torch.device(device)), None, None)
        assert seen == ["cuda:0"], device


@pytest.mark.skipif(
    not torch.backends.cuda.is_built(),
    reason = "FakeTensorMode places a tensor on cuda only in a CUDA-linked torch build",
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_torch_fallback_runs_ldexp_on_the_operands_gpu(dtype, monkeypatch):
    # float32 always takes this path (the zoo decoder serves only fp16 / bf16).
    from torch._subclasses.fake_tensor import FakeTensorMode
    import unsloth.models.mxfp4_compressed_linear as mcl

    current = _fake_current_cuda_device(monkeypatch)
    seen = []
    real_ldexp = torch.ldexp

    def spy_ldexp(x, other, *args, **kwargs):
        seen.append((str(x.device), current[0]))
        return real_ldexp(x, other, *args, **kwargs)

    monkeypatch.setattr(torch, "ldexp", spy_ldexp)
    monkeypatch.setattr(mcl, "_LUT_CACHE", {})
    monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
    with FakeTensorMode(allow_non_fake_inputs = True):
        packed = torch.empty(8, 32, dtype = torch.uint8, device = "cuda:1")
        scale = torch.empty(8, 2, dtype = torch.uint8, device = "cuda:1")
        out = dequantize_mxfp4_packed(packed, scale, dtype)
    assert out.shape == (8, 64) and out.dtype == dtype
    assert seen == [("cuda:1", "cuda:1")]
    assert current == ["cuda:0"]


@pytest.mark.skipif(
    not has_real_cuda() or torch.cuda.device_count() < 2, reason = "needs two CUDA devices"
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_torch_fallback_on_a_non_current_gpu_matches_cpu(dtype, monkeypatch):
    import unsloth.models.mxfp4_compressed_linear as mcl

    monkeypatch.setattr(mcl, "_zoo_mxfp4_dequantize", None)
    packed, scale = _random_packed(64, 128, low = 100, high = 140)
    want = dequantize_mxfp4_packed(packed, scale, dtype)
    with torch.cuda.device(0):
        got = dequantize_mxfp4_packed(packed.to("cuda:1"), scale.to("cuda:1"), dtype)
        torch.cuda.synchronize(1)
        assert torch.cuda.current_device() == 0
    assert got.device == torch.device("cuda:1")
    assert torch.equal(got.cpu(), want)


def test_dtype_casts_keep_the_packed_bytes_and_set_the_decode_dtype():
    module = _filled(32, 64, dtype = torch.float16)
    module.weight_scale.data.clamp_(110, 130)  # stay inside fp16's range
    before = module.weight_packed.clone(), module.weight_scale.clone()
    module.to(torch.bfloat16).half()
    assert module.weight_packed.dtype == torch.uint8 and torch.equal(
        module.weight_packed, before[0]
    )
    assert torch.equal(module.weight_scale, before[1])
    # `weight` (read by PEFT and the fast paths) decodes in the model's dtype, not always bf16.
    assert module.weight.dtype == torch.float16
    assert torch.equal(module.weight.float(), module.dequantize_weight(torch.bfloat16).float())


@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
def test_device_move_carries_bytes_and_scales():
    module = _filled(32, 64).cuda()
    assert module.weight_packed.is_cuda and module.weight_scale.is_cuda
    assert module.weight.is_cuda


def test_fast_lora_paths_get_the_packed_bytes_and_a_decoder_not_a_16bit_weight():
    peft = pytest.importorskip("peft")
    from unsloth.kernels.utils import (
        fast_dequantize,
        get_lora_parameters,
        get_lora_parameters_bias,
        has_mxfp4_base,
    )

    class Block(nn.Module):
        def __init__(self):
            super().__init__()
            self.proj = _filled(32, 64)
            self.dense = nn.Linear(64, 64, bias = False)

    model = peft.get_peft_model(Block(), peft.LoraConfig(r = 4, target_modules = ["proj", "dense"]))
    proj = model.base_model.model.proj
    W, quant_state, A, B, scale = get_lora_parameters(proj)
    # W saved for backward must be the packed bytes, never a 16-bit decode.
    assert W is proj.base_layer.weight_packed and A is not None
    assert getattr(quant_state, "_unsloth_packed_weight_state", False)
    dense = proj.base_layer.dequantize_weight()
    assert torch.equal(fast_dequantize(W, quant_state), dense)
    assert torch.equal(fast_dequantize(W.t(), quant_state), dense.t())
    assert get_lora_parameters_bias(proj)[1] is quant_state
    assert not has_mxfp4_base(proj) and not has_mxfp4_base(model.base_model.model.dense)


def _mlp_lora_pair(
    hidden,
    inter,
    dtype = torch.bfloat16,
):
    peft = pytest.importorskip("peft")

    class MLP(nn.Module):
        def __init__(self, gate, up, down):
            super().__init__()
            self.gate_proj, self.up_proj, self.down_proj = gate, up, down

    packed = [
        _filled(inter, hidden, seed = 1, dtype = dtype),
        _filled(inter, hidden, seed = 2, dtype = dtype),
        _filled(hidden, inter, seed = 3, dtype = dtype),
    ]
    for module in packed:
        module.weight_scale.data.clamp_(118, 126)
    dense = []
    for module in packed:
        linear = nn.Linear(module.in_features, module.out_features, bias = False, dtype = dtype)
        linear.weight.data.copy_(module.dequantize_weight())
        dense.append(linear)
    models = []
    for layers in (packed, dense):
        model = peft.get_peft_model(
            MLP(*layers),
            peft.LoraConfig(
                r = 8, lora_alpha = 16, target_modules = ["gate_proj", "up_proj", "down_proj"]
            ),
        ).cuda()
        g = torch.Generator().manual_seed(5)
        with torch.no_grad():
            for name, param in sorted(model.named_parameters()):
                if "lora_" in name:
                    param.copy_((torch.randn(param.shape, generator = g) * 0.05).to(param.dtype))
        models.append(model.base_model.model)
    return models


@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
@pytest.mark.parametrize("rows", [8, 256])
def test_fused_lora_mlp_on_a_packed_base_matches_the_dense_base(rows):
    """8 rows run the fused MXFP4 GEMM (bf16-rounding-level), 256 decode for cuBLAS (bit-exact)."""
    from unsloth.kernels.fast_lora import apply_lora_mlp_swiglu, apply_lora_o

    amp = _cuda_amp_dtype()
    packed, dense = _mlp_lora_pair(64, 128, amp)
    x = torch.randn(
        1, rows, 64, device = "cuda", generator = torch.Generator("cuda").manual_seed(0)
    ).to(amp)
    results = []
    for mlp in (packed, dense):
        xa = x.clone().requires_grad_()
        with torch.autocast("cuda", dtype = amp):
            y = apply_lora_mlp_swiglu(mlp, xa, inplace = False)
            y = y + apply_lora_o(
                type("O", (), {"o_proj": mlp.down_proj})(), y.new_ones(1, rows, 128) * 0.01
            )
        # fp16 grads of this loss pass 65504; a power-of-two seed scale is exact and keeps them finite.
        (y.float().square().sum() * (2.0**-14 if amp == torch.float16 else 1.0)).backward()
        lora_grads = [p.grad.float() for n, p in sorted(mlp.named_parameters()) if "lora_" in n]
        results.append((y.float(), xa.grad.float(), lora_grads))
    (ya, ga, la), (yb, gb, lb) = results
    assert len(la) == len(lb) == 6
    assert all(torch.isfinite(g).all() and g.abs().max() > 0 for g in (ga, *la))
    if rows > 64:
        assert torch.equal(ya, yb) and torch.equal(ga, gb)
        assert all(torch.equal(a, b) for a, b in zip(la, lb))
    else:
        rel = lambda a, b: ((a - b).abs().max() / b.abs().max().clamp_min(1e-6)).item()  # noqa: E731
        assert rel(ya, yb) < 2e-2 and rel(ga, gb) < 2e-2
        assert all(rel(a, b) < 3e-2 for a, b in zip(la, lb))


@pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device")
@pytest.mark.parametrize("rows", [1, 3, 64, 65, 300])
@pytest.mark.parametrize("bias", [False, True])
def test_packed_linear_forward_backward_close_to_the_dense_decode_on_cuda(rows, bias):
    module = _filled(96, 160, bias = bias).cuda()
    module.weight_scale.data.clamp_(118, 126)
    dense = module.dequantize_weight()
    x = torch.randn(rows, 160, device = "cuda", generator = torch.Generator("cuda").manual_seed(1)).to(
        torch.bfloat16
    )
    xa, xb = x.clone().requires_grad_(), x.clone().requires_grad_()
    ya = module(xa)
    yb = torch.nn.functional.linear(
        xb, dense, None if module.bias is None else module.bias.to(torch.bfloat16)
    )
    go = torch.randn_like(ya)
    ya.backward(go)
    yb.backward(go)
    tol = dict(atol = 1e-1, rtol = 2e-2) if rows <= 64 else dict(atol = 0, rtol = 0)
    torch.testing.assert_close(ya.float(), yb.float(), **tol)
    torch.testing.assert_close(xa.grad.float(), xb.grad.float(), **tol)
    with torch.no_grad():
        torch.testing.assert_close(module(x).float(), ya.float(), atol = 0, rtol = 0)


def _lora_pair(bias = False, scale_range = None):
    peft = pytest.importorskip("peft")
    packed = _filled(32, 64, bias = bias)
    if scale_range is not None:
        packed.weight_scale.data.clamp_(*scale_range)
    dense = nn.Linear(64, 32, bias = bias, dtype = torch.bfloat16)
    dense.weight.data.copy_(packed.dequantize_weight())
    if bias:
        dense.bias.data.copy_(packed.bias.data)
    models = []
    for base in (packed, dense):
        holder = nn.Sequential(base)
        model = peft.get_peft_model(
            holder, peft.LoraConfig(r = 4, lora_alpha = 8, target_modules = ["0"])
        )
        g = torch.Generator().manual_seed(3)
        with torch.no_grad():
            for name, param in sorted(model.named_parameters()):
                if "lora_" in name:
                    param.copy_((torch.randn(param.shape, generator = g) * 0.1).to(param.dtype))
        models.append(model)
    return models


@pytest.mark.parametrize("bias", [False, True])
def test_peft_lora_on_a_packed_linear_matches_the_dense_decode(bias):
    packed_model, dense_model = _lora_pair(bias)
    x = torch.randn(3, 64, dtype = torch.bfloat16)
    xa, xb = x.clone().requires_grad_(), x.clone().requires_grad_()
    ya, yb = packed_model(xa), dense_model(xb)
    assert torch.equal(ya, yb)
    ya.float().sum().backward()
    yb.float().sum().backward()
    assert torch.equal(xa.grad, xb.grad)
    grads = lambda m: {n: p.grad for n, p in m.named_parameters() if "lora_" in n}  # noqa: E731
    ga, gb = grads(packed_model), grads(dense_model)
    assert ga.keys() == gb.keys() and all(torch.equal(ga[n], gb[n]) for n in ga)


def test_peft_merge_densifies_exactly_and_unmerge_restores_the_packed_bytes():
    packed_model, dense_model = _lora_pair(scale_range = (118, 134))
    x = torch.randn(3, 64, generator = torch.Generator().manual_seed(0)).to(torch.bfloat16)
    base = packed_model.base_model.model[0].base_layer
    packed_bytes, scale = base.weight_packed, base.weight_scale
    with torch.no_grad():
        with_lora = packed_model(x)
        packed_model.merge_adapter()
        dense_model.merge_adapter()
        merged_base = packed_model.base_model.model[0].base_layer
        # The delta must land in a real weight: a write into a fresh decode would be lost.
        assert type(merged_base) is nn.Linear and "weight_packed" not in merged_base._parameters
        assert torch.equal(merged_base.weight, dense_model.base_model.model[0].base_layer.weight)
        merged = packed_model(x)
        packed_model.unmerge_adapter()
        restored = packed_model.base_model.model[0].base_layer
        assert type(restored) is Mxfp4PackedLinear
        assert restored.weight_packed is packed_bytes and restored.weight_scale is scale
        assert "weight" not in restored._parameters
        assert torch.equal(packed_model(x), with_lora)
    assert not torch.equal(merged, packed_model.base_model.model[0].base_layer(x))
    torch.testing.assert_close(merged.float(), with_lora.float(), atol = 2e-2, rtol = 2e-2)
    unloaded = packed_model.merge_and_unload()
    assert type(unloaded[0]) is nn.Linear
    assert torch.equal(unloaded[0].weight, dense_model.base_model.model[0].base_layer.weight)


def _write_tiny_mxfp4_llama(root):
    from safetensors.torch import save_file
    from transformers import LlamaConfig, LlamaForCausalLM
    from compressed_tensors.compressors import BaseCompressor
    from compressed_tensors.quantization import QuantizationConfig

    torch.manual_seed(0)
    config = LlamaConfig(
        hidden_size = 64,
        num_hidden_layers = 2,
        num_attention_heads = 4,
        num_key_value_heads = 2,
        intermediate_size = 128,
        vocab_size = 32000,
        max_position_embeddings = 128,
        tie_word_embeddings = False,
    )
    model = LlamaForCausalLM(config).to(torch.bfloat16)
    quant = {
        "quant_method": "compressed-tensors",
        "format": "mxfp4-pack-quantized",
        "quantization_status": "compressed",
        "ignore": ["lm_head"],
        "config_groups": {
            "group_0": {
                "targets": ["Linear"],
                "weights": {
                    "num_bits": 4,
                    "type": "float",
                    "strategy": "group",
                    "group_size": 32,
                    "symmetric": True,
                    "scale_dtype": "torch.uint8",
                    "dynamic": False,
                    "observer": "minmax",
                    "observer_kwargs": {},
                    "actorder": None,
                    "block_structure": None,
                },
                "input_activations": None,
                "output_activations": None,
            }
        },
    }
    scheme = list(QuantizationConfig.model_validate(quant).config_groups.values())[0]
    comp = BaseCompressor.get_value_from_registry("mxfp4-pack-quantized")
    linears = {n for n, m in model.named_modules() if isinstance(m, nn.Linear)}
    packed, plain = {}, {}
    for k, w in model.state_dict().items():
        mod = k[: -len(".weight")] if k.endswith(".weight") else None
        if mod is None or mod not in linears or mod == "lm_head":
            packed[k] = plain[k] = w.contiguous()
            continue
        out_f, in_f = w.shape
        amax = w.float().reshape(out_f, in_f // 32, 32).abs().amax(-1).clamp_min(1e-8)
        exp = (torch.ceil(torch.log2(amax / 6.0)) + 127).clamp(1, 254).to(torch.uint8)
        state = {"weight": w.float(), "weight_scale": 2.0 ** (exp.float() - 127)}
        out = comp.compress(state, scheme)
        out["weight_scale"] = exp
        for pk, pv in out.items():
            if pk == "weight_shape":
                continue
            packed[mod + "." + pk] = pv.contiguous()
        plain[k] = (
            comp.decompress(
                {
                    "weight_packed": out["weight_packed"],
                    "weight_scale": exp,
                    "weight_shape": torch.tensor([out_f, in_f]),
                },
                scheme,
            )["weight"]
            .to(torch.bfloat16)
            .contiguous()
        )
    packed_dir, bf16_dir = os.path.join(root, "packed"), os.path.join(root, "bf16")
    os.makedirs(packed_dir)
    os.makedirs(bf16_dir)
    save_file(packed, os.path.join(packed_dir, "model.safetensors"), metadata = {"format": "pt"})
    save_file(plain, os.path.join(bf16_dir, "model.safetensors"), metadata = {"format": "pt"})
    cfg = config.to_dict()
    json.dump(cfg, open(os.path.join(bf16_dir, "config.json"), "w"))
    cfg["quantization_config"] = quant
    json.dump(cfg, open(os.path.join(packed_dir, "config.json"), "w"))
    return packed_dir, bf16_dir


def _add_tokenizer(path, root):
    from transformers import AutoTokenizer

    tok_dir = os.path.join(root, "tok")
    if not os.path.isdir(tok_dir):
        AutoTokenizer.from_pretrained("hf-internal-testing/llama-tokenizer").save_pretrained(
            tok_dir
        )
    for f in os.listdir(tok_dir):
        if not os.path.exists(os.path.join(path, f)):
            shutil.copy(os.path.join(tok_dir, f), os.path.join(path, f))


def _lora_losses(model, steps = 3):
    from unsloth import FastLanguageModel

    model = FastLanguageModel.get_peft_model(
        model,
        r = 4,
        lora_alpha = 8,
        random_state = 3407,
        target_modules = [
            "q_proj",
            "k_proj",
            "v_proj",
            "o_proj",
            "gate_proj",
            "up_proj",
            "down_proj",
        ],
    )
    model.train()
    opt = torch.optim.AdamW([p for p in model.parameters() if p.requires_grad], lr = 1e-2)
    torch.manual_seed(1)
    ids = torch.randint(0, 256, (2, 24), device = "cuda:0")
    losses = []
    for _ in range(steps):
        loss = model(input_ids = ids, labels = ids).loss
        loss.backward()
        opt.step()
        opt.zero_grad(set_to_none = True)
        losses.append(loss.item())
    return losses


needs_gpu_loader_any = [
    pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device"),
    pytest.mark.skipif(
        not HAS_CT
        or not hasattr(__import__("transformers"), "__version__")
        or int(__import__("transformers").__version__.split(".")[0]) < 5,
        reason = "needs compressed-tensors and transformers 5",
    ),
]
needs_gpu_loader = [
    pytest.mark.skipif(not has_real_cuda(), reason = "needs a CUDA device"),
    pytest.mark.skipif(
        not (HAS_CT and HAS_CONVERTERS),
        reason = "needs compressed-tensors and the transformers 5 loader",
    ),
]


def _apply(marks):
    def wrap(fn):
        for m in marks:
            fn = m(fn)
        return fn

    return wrap


@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
@pytest.mark.parametrize("load_in_4bit", [True, False])
def test_mxfp4_checkpoint_stays_packed_and_matches_its_bf16_decode(
    load_in_4bit, tmp_path, monkeypatch
):
    import unsloth.models.llama as llama_module
    from unsloth import FastLanguageModel

    packed_dir, bf16_dir = _write_tiny_mxfp4_llama(str(tmp_path))
    for d in (packed_dir, bf16_dir):
        _add_tokenizer(d, str(tmp_path))
    kw = dict(max_seq_length = 64, dtype = torch.bfloat16)
    model_a, _ = FastLanguageModel.from_pretrained(packed_dir, load_in_4bit = load_in_4bit, **kw)
    kept = [n for n, m in model_a.named_modules() if isinstance(m, Mxfp4PackedLinear)]
    assert len(kept) == 2 * 7, kept
    assert all(
        p.dtype == torch.uint8 for n, p in model_a.named_parameters() if "weight_packed" in n
    )
    model_b, _ = FastLanguageModel.from_pretrained(bf16_dir, load_in_4bit = False, **kw)
    ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
    import unsloth.models.mxfp4_compressed_linear as mcl

    with torch.no_grad():
        # 16 rows run the fused MXFP4 GEMM (bf16-rounding level); decode + cuBLAS is bit-exact.
        fused = model_a(input_ids = ids).logits
        want = model_b(input_ids = ids).logits
        torch.testing.assert_close(fused.float(), want.float(), atol = 5e-2, rtol = 2e-2)
        monkeypatch.setattr(mcl, "_FUSED_MAX_ROWS", 0)
        assert torch.equal(model_a(input_ids = ids).logits, want)
    losses_a = _lora_losses(model_a)
    from unsloth.kernels import apply_lora_o, apply_lora_qkv

    for layer in model_a.model.layers:
        assert "_unsloth_forward" in layer.mlp.__dict__ or "forward" in layer.mlp.__dict__
        assert getattr(layer.self_attn, "apply_qkv", None) is apply_lora_qkv
        assert getattr(layer.self_attn, "apply_o", None) is apply_lora_o
    assert losses_a == _lora_losses(model_b)


@_apply(needs_gpu_loader)
def test_keep_packed_off_restores_the_bitsandbytes_route(tmp_path, monkeypatch):
    import bitsandbytes as bnb
    from unsloth import FastLanguageModel

    monkeypatch.setenv("UNSLOTH_MXFP4_KEEP_PACKED", "0")
    packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
    _add_tokenizer(packed_dir, str(tmp_path))
    model, _ = FastLanguageModel.from_pretrained(
        packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True
    )
    assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())
    assert sum(isinstance(m, bnb.nn.Linear4bit) for m in model.modules()) == 2 * 7


def _plain_logits(path, ids):
    from transformers import AutoModelForCausalLM
    model = AutoModelForCausalLM.from_pretrained(path, dtype = torch.bfloat16, device_map = {"": 0})
    with torch.no_grad():
        return model(input_ids = ids).logits


@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
def test_full_and_merged_saves_reload_in_plain_transformers(tmp_path, monkeypatch):
    from safetensors import safe_open
    from unsloth import FastLanguageModel

    packed_dir, bf16_dir = _write_tiny_mxfp4_llama(str(tmp_path))
    _add_tokenizer(packed_dir, str(tmp_path))
    model, tokenizer = FastLanguageModel.from_pretrained(
        packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = False
    )
    ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
    with torch.no_grad():
        base_logits = model(input_ids = ids).logits
    full = str(tmp_path / "full")
    model.save_pretrained(full)
    assert sum(isinstance(m, Mxfp4PackedLinear) for m in model.modules()) == 2 * 7
    with safe_open(os.path.join(full, "model.safetensors"), "pt") as f:
        keys = list(f.keys())
    assert not any("weight_packed" in k or "weight_scale" in k for k in keys)
    assert "quantization_config" not in json.load(open(os.path.join(full, "config.json")))
    assert torch.equal(_plain_logits(full, ids), _plain_logits(bf16_dir, ids))

    model = FastLanguageModel.get_peft_model(
        model,
        r = 4,
        lora_alpha = 8,
        random_state = 3407,
        target_modules = ["q_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
    )
    with torch.no_grad():
        for name, param in model.named_parameters():
            if "lora_B" in name:
                param.normal_(0, 0.05)
    merged_dir = str(tmp_path / "merged")
    model.save_pretrained_merged(merged_dir, tokenizer, save_method = "merged_16bit")
    unloaded = model.merge_and_unload()
    assert [n for n, m in unloaded.named_modules() if isinstance(m, Mxfp4PackedLinear)] == [
        f"model.layers.{i}.self_attn.k_proj" for i in range(2)
    ]
    with torch.no_grad():
        unloaded_logits = unloaded(input_ids = ids).logits
    assert not torch.equal(unloaded_logits, base_logits)
    with safe_open(os.path.join(merged_dir, "model.safetensors"), "pt") as f:
        assert not any("weight_packed" in k or "weight_scale" in k for k in f.keys())
    unloaded_dir = str(tmp_path / "unloaded")
    unloaded.save_pretrained(unloaded_dir)
    merged_logits, unloaded_reload = (
        _plain_logits(merged_dir, ids),
        _plain_logits(unloaded_dir, ids),
    )
    torch.testing.assert_close(merged_logits, unloaded_reload, atol = 2e-2, rtol = 2e-2)
    torch.testing.assert_close(unloaded_reload, unloaded_logits, atol = 2e-2, rtol = 2e-2)
    assert not torch.equal(merged_logits, _plain_logits(full, ids))


@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
def test_full_finetuning_keeps_the_stock_compressed_tensors_route(tmp_path, monkeypatch):
    from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed
    from transformers import AutoModelForCausalLM

    packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
    assert install_compressed_tensors_keep_packed()
    monkeypatch.setenv("UNSLOTH_ENABLE_FULL_FINETUNING", "1")
    model = AutoModelForCausalLM.from_pretrained(packed_dir, dtype = torch.bfloat16)
    assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())
    monkeypatch.setenv("UNSLOTH_ENABLE_FULL_FINETUNING", "0")
    model = AutoModelForCausalLM.from_pretrained(packed_dir, dtype = torch.bfloat16)
    assert sum(isinstance(m, Mxfp4PackedLinear) for m in model.modules()) == 2 * 7


def _decompress_requests():
    """The ways this transformers asks compressed-tensors to decompress at load."""
    try:
        import inspect
        from transformers import CompressedTensorsConfig
    except Exception:
        return []
    params = inspect.signature(CompressedTensorsConfig).parameters
    return [{k: v} for k, v in (("run_compressed", False), ("dequantize", True)) if k in params]


@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
@pytest.mark.parametrize("request_kwargs", _decompress_requests(), ids = lambda kw: next(iter(kw)))
def test_an_explicit_decompress_request_keeps_the_stock_route(request_kwargs, tmp_path):
    from transformers import AutoModelForCausalLM, CompressedTensorsConfig
    from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed

    packed_dir, bf16_dir = _write_tiny_mxfp4_llama(str(tmp_path))
    assert install_compressed_tensors_keep_packed()
    # GPU when present: an earlier FastLanguageModel load patches Llama's forward.
    device = "cuda:0" if has_real_cuda() else "cpu"
    model = AutoModelForCausalLM.from_pretrained(
        packed_dir,
        dtype = torch.bfloat16,
        device_map = {"": device},
        quantization_config = CompressedTensorsConfig(**request_kwargs),
    )
    assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())
    reference = AutoModelForCausalLM.from_pretrained(
        bf16_dir, dtype = torch.bfloat16, device_map = {"": device}
    )
    ids = torch.randint(0, 256, (1, 12), device = device)
    with torch.no_grad():
        assert torch.equal(model(input_ids = ids).logits, reference(input_ids = ids).logits)


@pytest.mark.parametrize("cast", [torch.float16, torch.float32])
def test_a_dtype_cast_reaches_the_merge_and_unmerge(cast):
    packed_model, dense_model = _lora_pair(scale_range = (118, 134))  # inside fp16's range
    packed_model.to(cast)
    dense_model.to(cast)
    base = packed_model.base_model.model[0].base_layer
    assert base.compute_dtype == cast and base.weight.dtype == cast
    x = torch.randn(3, 64, dtype = cast)
    with torch.no_grad():
        packed_model.merge_adapter()
        dense_model.merge_adapter()
        assert packed_model.base_model.model[0].base_layer.weight.dtype == cast
        assert torch.equal(packed_model(x), dense_model(x))
        # Cast while merged, then unmerge: the packed module decodes in the model's new dtype.
        packed_model.to(torch.bfloat16)
        packed_model.unmerge_adapter()
        restored = packed_model.base_model.model[0].base_layer
        assert type(restored) is Mxfp4PackedLinear and restored.compute_dtype == torch.bfloat16
        packed_model.merge_adapter()
        assert packed_model(x.to(torch.bfloat16)).dtype == torch.bfloat16


@needs_zoo_packed_save
@pytest.mark.skipif(
    not (HAS_CT and HAS_CONVERTERS), reason = "needs compressed-tensors and the transformers 5 loader"
)
def test_the_sixteen_bit_route_decodes_in_the_load_dtype(tmp_path):
    peft = pytest.importorskip("peft")
    from transformers import AutoModelForCausalLM
    from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed

    packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
    assert install_compressed_tensors_keep_packed()
    device = "cuda:0" if has_real_cuda() else "cpu"
    model = AutoModelForCausalLM.from_pretrained(
        packed_dir, dtype = torch.float16, device_map = {"": device}
    )
    packed = [m for m in model.modules() if isinstance(m, Mxfp4PackedLinear)]
    assert len(packed) == 2 * 7
    assert {m.compute_dtype for m in packed} == {torch.float16}
    assert {m.weight.dtype for m in packed} == {torch.float16}
    assert model.model.layers[0].input_layernorm.weight.dtype == torch.float16
    model = peft.get_peft_model(model, peft.LoraConfig(r = 2, target_modules = ["q_proj"]))
    merged = model.merge_and_unload()
    with torch.no_grad():
        assert merged(input_ids = torch.tensor([[1, 2, 3, 4]], device = device)).logits.isfinite().all()


@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adoption_declines_a_scheme_that_also_quantizes_activations():
    from unsloth.models.mxfp4_compressed_linear import is_mxfp4_scheme

    activations = {
        "num_bits": 8,
        "type": "int",
        "strategy": "token",
        "dynamic": True,
        "symmetric": True,
    }
    groups = {
        "group_0": {
            "targets": ["Linear"],
            "weights": _MXFP4_WEIGHTS,
            "input_activations": activations,
        }
    }
    model = _ct_compressed_model(groups = groups)
    assert not is_mxfp4_scheme(model.a.quantization_scheme, "mxfp4-pack-quantized")
    reference = copy.deepcopy(model)
    assert adopt_compressed_mxfp4_modules(model, dtype = torch.bfloat16) == 0
    assert type(model.a) is nn.Linear and hasattr(model, "ct_decompress_hook")
    x = torch.randn(3, 64, dtype = torch.bfloat16)
    with torch.no_grad():
        assert torch.equal(model(x), reference(x))


@pytest.mark.skipif(not HAS_CT, reason = "needs compressed-tensors")
def test_adoption_declines_when_another_compressed_format_needs_the_hook():
    fp8 = {
        "num_bits": 8,
        "type": "float",
        "strategy": "channel",
        "symmetric": True,
        "dynamic": False,
    }
    groups = {
        "group_0": {
            "targets": ["re:^a$", "re:^b$"],
            "weights": _MXFP4_WEIGHTS,
            "format": "mxfp4-pack-quantized",
        },
        "group_1": {"targets": ["re:^latent$"], "weights": fp8, "format": "float-quantized"},
    }
    model = _ct_compressed_model(groups = groups)
    assert model.latent.weight.dtype == torch.float8_e4m3fn
    reference = copy.deepcopy(model)
    assert adopt_compressed_mxfp4_modules(model, dtype = torch.bfloat16) == 0
    assert type(model.a) is nn.Linear and hasattr(model, "ct_decompress_hook")
    x = torch.randn(3, 64, dtype = torch.bfloat16)
    with torch.no_grad():
        assert torch.equal(model(x), reference(x))


@pytest.mark.parametrize("init", ["pissa", "olora"])
def test_initialisers_that_rewrite_the_base_weight_refuse_a_packed_base(init):
    peft = pytest.importorskip("peft")
    holder = nn.Sequential(_filled(32, 64))
    with pytest.raises(NotImplementedError, match = "packed in MXFP4"):
        peft.get_peft_model(
            holder, peft.LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = init)
        )
    peft.get_peft_model(nn.Sequential(_filled(32, 64)), peft.LoraConfig(r = 4, target_modules = ["0"]))
    dense = nn.Sequential(nn.Linear(64, 32, bias = False))
    peft.get_peft_model(dense, peft.LoraConfig(r = 4, target_modules = ["0"], init_lora_weights = init))


@_apply(needs_gpu_loader_any)
def test_the_sixteen_bit_route_declines_without_the_zoo_full_save_support(tmp_path, monkeypatch):
    from transformers import AutoModelForCausalLM
    from unsloth_zoo.temporary_patches import mxfp4 as zoo_mxfp4
    from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed

    packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
    assert install_compressed_tensors_keep_packed()
    monkeypatch.delattr(zoo_mxfp4, "_densified_module_names", raising = False)
    model = AutoModelForCausalLM.from_pretrained(
        packed_dir, dtype = torch.bfloat16, device_map = {"": 0}
    )
    assert not any(isinstance(m, Mxfp4PackedLinear) for m in model.modules())


@needs_zoo_packed_save
@_apply(needs_gpu_loader)
def test_a_partly_merged_bitsandbytes_route_model_saves_and_reloads(tmp_path):
    from transformers import AutoModelForCausalLM
    from unsloth import FastLanguageModel

    packed_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path))
    _add_tokenizer(packed_dir, str(tmp_path))
    model, _ = FastLanguageModel.from_pretrained(
        packed_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True
    )
    model = FastLanguageModel.get_peft_model(model, r = 4, lora_alpha = 8, target_modules = ["q_proj"])
    with torch.no_grad():
        for name, param in model.named_parameters():
            if "lora_B" in name:
                param.normal_(0, 0.05)
    unloaded = model.merge_and_unload()
    ids = torch.randint(0, 256, (1, 16), device = "cuda:0")
    with torch.no_grad():
        want = unloaded(input_ids = ids).logits
    out = str(tmp_path / "unloaded")
    unloaded.save_pretrained(out)
    # want.dtype: Unsloth loads fp16 where bf16 is unsupported (T4) even when asked for bf16
    reloaded = AutoModelForCausalLM.from_pretrained(out, dtype = want.dtype, device_map = {"": 0})
    attn = reloaded.model.layers[0].self_attn
    assert type(attn.q_proj) is nn.Linear and type(attn.k_proj) is nn.Linear
    with torch.no_grad():
        got = reloaded(input_ids = ids).logits
    torch.testing.assert_close(got, want, atol = 2e-2, rtol = 2e-2)


def test_merge_and_unmerge_under_an_accelerate_hook():
    pytest.importorskip("accelerate")
    from accelerate.hooks import AlignDevicesHook, add_hook_to_module

    packed_model, dense_model = _lora_pair(scale_range = (118, 134))
    add_hook_to_module(
        packed_model.base_model.model[0].base_layer, AlignDevicesHook(io_same_device = True)
    )
    x = torch.randn(3, 64, generator = torch.Generator().manual_seed(0)).to(torch.bfloat16)
    with torch.no_grad():
        with_lora = packed_model(x)
        packed_model.merge_adapter()
        dense_model.merge_adapter()
        assert torch.equal(packed_model(x), dense_model(x))
        packed_model.unmerge_adapter()
        assert torch.equal(packed_model(x), with_lora)


@needs_zoo_packed_save
@_apply(needs_gpu_loader_any)
def test_the_sixteen_bit_route_plans_only_mxfp4_checkpoints(tmp_path, monkeypatch):
    import importlib.util
    import unsloth.models.compressed_tensors_bnb as ctb
    from transformers import AutoModelForCausalLM
    from unsloth.models.mxfp4_compressed_linear import install_compressed_tensors_keep_packed

    path = os.path.join(
        os.path.dirname(os.path.abspath(__file__)), "test_compressed_tensors_bnb.py"
    )
    spec = importlib.util.spec_from_file_location("_k3s_ct_bnb_tests", path)
    ct_bnb_tests = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(ct_bnb_tests)

    calls = []
    real = ctb._checkpoint_keys
    monkeypatch.setattr(ctb, "_checkpoint_keys", lambda files: calls.append(files) or real(files))
    assert install_compressed_tensors_keep_packed()
    int4_dir, _ = ct_bnb_tests._write_tiny_packed_llama(str(tmp_path / "int4"))
    AutoModelForCausalLM.from_pretrained(int4_dir, dtype = torch.bfloat16, device_map = {"": 0})
    assert calls == []
    mxfp4_dir, _ = _write_tiny_mxfp4_llama(str(tmp_path / "mxfp4"))
    model = AutoModelForCausalLM.from_pretrained(
        mxfp4_dir, dtype = torch.bfloat16, device_map = {"": 0}
    )
    assert (
        len(calls) == 1 and sum(isinstance(m, Mxfp4PackedLinear) for m in model.modules()) == 2 * 7
    )


@pytest.mark.parametrize("spelling", ["compressed-tensors", "compressed_tensors", "sparseml"])
def test_every_compressed_tensors_spelling_installs_the_keep_packed_hook(spelling, monkeypatch):
    from types import SimpleNamespace
    from unsloth.models import loader_utils, mxfp4_compressed_linear

    calls = []
    monkeypatch.setattr(
        mxfp4_compressed_linear, "install_compressed_tensors_keep_packed", lambda: calls.append(1)
    )
    config = SimpleNamespace(
        quantization_config = {"quant_method": spelling, "format": "mxfp4-pack-quantized"}
    )
    loader_utils.check_and_disable_bitsandbytes_loading(
        config,
        load_in_4bit = False,
        verbose = False,
        requantize_packed = False,
    )
    assert calls == [1]