Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Train LoRA on NVFP4 compressed-tensors checkpoints with the weights kept packed (W4A16) (#12185)
Daniel Han3cffce234b76
diff --git a/tests/_nvfp4_fixtures.py b/tests/_nvfp4_fixtures.pynew file mode 100644index 00000000000..779e2c5a1ad--- /dev/null+++ b/tests/_nvfp4_fixtures.py@@ -0,0 +1,374 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""Tiny compressed-tensors NVFP4 + FP8 mixed-precision checkpoints, laid out like unsloth/Qwen3.8-27B-NVFP4.++MLPs are NVFP4 (weight_packed U8, weight_scale F8_E4M3 per 16 columns, weight_global_scale and input_global_scale F32[1]),+attention and the last layer's MLP are FP8 per channel (weight F8_E4M3, weight_scale BF16 [out, 1]). Built without+llmcompressor: scales are computed here and the tensors come from compressed-tensors' own compressors.++ python tests/_nvfp4_fixtures.py OUT_DIR [--arch qwen3|qwen3_5]+"""++import argparse+import copy+import json+import os+import re+import sys++import torch++QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"+QWEN3_5 = "trl-internal-testing/tiny-Qwen3_5ForConditionalGeneration-NoThink"++_FP8_ACT = {+ "actorder": None,+ "block_structure": None,+ "dynamic": True,+ "group_size": None,+ "num_bits": 8,+ "observer": None,+ "observer_kwargs": {},+ "scale_dtype": None,+ "strategy": "token",+ "symmetric": True,+ "type": "float",+ "zp_dtype": None,+}+_FP8_W = {+ "actorder": None,+ "block_structure": None,+ "dynamic": False,+ "group_size": None,+ "num_bits": 8,+ "observer": "memoryless_minmax",+ "observer_kwargs": {},+ "scale_dtype": None,+ "strategy": "channel",+ "symmetric": True,+ "type": "float",+ "zp_dtype": None,+}+_FP4 = {+ "actorder": None,+ "block_structure": None,+ "dynamic": False,+ "group_size": 16,+ "num_bits": 4,+ "observer_kwargs": {},+ "scale_dtype": "torch.float8_e4m3fn",+ "strategy": "tensor_group",+ "symmetric": True,+ "type": "float",+ "zp_dtype": None,+}+++def _quant_config(+ num_layers,+ lm_head,+ attn_targets,+ ignore,+ lm_head_kind = "fp8",+):+ last = str(num_layers - 1)+ fp8_targets = list(attn_targets) + [r"re:.*layers\.(" + last + r")\.mlp\.(gate|up|down)_proj$"]+ fp4_targets = [r"re:.*mlp\.(gate|up|down)_proj$"]+ if lm_head:+ (fp8_targets if lm_head_kind == "fp8" else fp4_targets).append("re:.*lm_head")+ fp4_act = dict(_FP4, dynamic = "local", observer = "static_minmax")+ return {+ "config_groups": {+ "group_0": {+ "format": "float-quantized",+ "input_activations": _FP8_ACT,+ "output_activations": None,+ "targets": fp8_targets,+ "weights": _FP8_W,+ },+ "group_1": {+ "format": "nvfp4-pack-quantized",+ "input_activations": fp4_act,+ "output_activations": None,+ "targets": fp4_targets,+ "weights": dict(_FP4, actorder = "static"),+ },+ },+ "format": "mixed-precision",+ "global_compression_ratio": None,+ "ignore": list(ignore),+ "kv_cache_scheme": None,+ "quant_method": "compressed-tensors",+ "quantization_status": "compressed",+ "sparsity_config": {},+ "transform_config": {},+ "version": "0.17.0",+ }+++def _matches(name, targets):+ return any(re.match(t[3:], name) if t.startswith("re:") else name == t for t in targets)+++def nvfp4_tensors(w):+ """weight_packed, weight_scale (fp8), weight_global_scale for w (out, in), via compressed-tensors' compressor."""+ from compressed_tensors.compressors import NVFP4PackedCompressor+ from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme++ w = w.float()+ out_f, in_f = w.shape+ assert in_f % 16 == 0, in_f+ global_scale = (448.0 * 6.0 / w.abs().amax().clamp(min = 1e-12)).reshape(1).float()+ group_amax = w.reshape(out_f, in_f // 16, 16).abs().amax(-1)+ # Rounded to fp8 values but kept float: the compressor divides by the global scale, then stores fp8.+ scale = (global_scale * (group_amax / 6.0)).clamp(max = 448.0).to(torch.float8_e4m3fn).float()+ scheme = QuantizationScheme(+ targets = ["Linear"],+ weights = QuantizationArgs(+ **{k: v for k, v in _FP4.items() if v is not None and k not in ("observer_kwargs",)}+ ),+ )+ out = NVFP4PackedCompressor.compress(+ {"weight": w, "weight_scale": scale, "weight_global_scale": global_scale}, scheme+ )+ return out["weight_packed"], out["weight_scale"], global_scale+++def fp8_channel_tensors(w, scale_dtype = torch.bfloat16):+ w = w.float()+ scale = (w.abs().amax(1, keepdim = True) / 448.0).clamp(min = 1e-12)+ q = (w / scale).clamp(-448, 448).to(torch.float8_e4m3fn)+ return q, scale.to(scale_dtype)+++def dequant_nvfp4_reference(+ packed,+ scale,+ global_scale,+ dtype = torch.bfloat16,+):+ from compressed_tensors.compressors import NVFP4PackedCompressor+ from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme++ scheme = QuantizationScheme(+ targets = ["Linear"],+ weights = QuantizationArgs(+ **{k: v for k, v in _FP4.items() if v is not None and k not in ("observer_kwargs",)}+ ),+ )+ out = NVFP4PackedCompressor.decompress(+ {"weight_packed": packed, "weight_scale": scale, "weight_global_scale": global_scale},+ scheme,+ )+ return out["weight"].to(dtype)+++def build(+ out_dir,+ arch = "qwen3",+ seed = 3407,+ lm_head_kind = "fp8",+):+ """Write a tiny mixed NVFP4/FP8 checkpoint to out_dir; returns {name: kind} for every quantized Linear."""+ from safetensors.torch import save_file+ from transformers import (+ AutoConfig,+ AutoModelForCausalLM,+ AutoModelForImageTextToText,+ AutoTokenizer,+ )++ torch.manual_seed(seed)+ if arch == "qwen3":+ repo = QWEN3+ config = AutoConfig.from_pretrained(repo)+ # hidden 8 cannot hold 16-column NVFP4 groups.+ config.update(+ dict(+ hidden_size = 64,+ intermediate_size = 128,+ head_dim = 16,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ num_hidden_layers = 2,+ )+ )+ config.torch_dtype = torch.bfloat16+ model = AutoModelForCausalLM.from_config(config, dtype = torch.bfloat16)+ attn = [r"re:.*self_attn\.(q|k|v|o)_proj$"]+ ignore = ["model.layers.0.self_attn.o_proj"]+ lm_head = True+ elif arch == "qwen3_5":+ repo = QWEN3_5+ config = AutoConfig.from_pretrained(repo)+ config.torch_dtype = torch.bfloat16+ model = AutoModelForImageTextToText.from_config(config, dtype = torch.bfloat16)+ attn = [+ r"re:.*self_attn\.(q|k|v|o)_proj$",+ r"re:.*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$",+ ]+ ignore = [+ n+ for n, m in model.named_modules()+ if isinstance(m, torch.nn.Linear)+ and (".visual." in f".{n}" or n.endswith(("in_proj_a", "in_proj_b")))+ ]+ lm_head = False # tied to the embeddings+ else:+ raise ValueError(arch)++ num_layers = config.get_text_config().num_hidden_layers+ qcfg = _quant_config(num_layers, lm_head, attn, ignore, lm_head_kind)+ fp8_targets = qcfg["config_groups"]["group_0"]["targets"]+ fp4_targets = qcfg["config_groups"]["group_1"]["targets"]++ state = {k: v.detach().clone().contiguous() for k, v in model.state_dict().items()}+ kinds = {}+ for name, module in model.named_modules():+ if not isinstance(module, torch.nn.Linear) or name in ignore:+ continue+ w = state.pop(name + ".weight", None)+ if w is None:+ continue+ # Same precedence as the real checkpoint: the FP8 override wins over the broad MLP NVFP4 target.+ if _matches(name, fp8_targets):+ q, s = fp8_channel_tensors(w)+ state[name + ".weight"] = q+ state[name + ".weight_scale"] = s+ kinds[name] = "fp8"+ elif _matches(name, fp4_targets):+ packed, scale, gscale = nvfp4_tensors(w)+ state[name + ".weight_packed"] = packed+ state[name + ".weight_scale"] = scale+ state[name + ".weight_global_scale"] = gscale+ state[name + ".input_global_scale"] = torch.tensor(+ [448.0 * 6.0 / 4.0], dtype = torch.float32+ )+ kinds[name] = "nvfp4"+ else:+ state[name + ".weight"] = w+ if getattr(config, "tie_word_embeddings", False) or getattr(+ config.get_text_config(), "tie_word_embeddings", False+ ):+ state = {k: v for k, v in state.items() if not k.endswith("lm_head.weight")}++ os.makedirs(out_dir, exist_ok = True)+ save_file(state, os.path.join(out_dir, "model.safetensors"), metadata = {"format": "pt"})+ cfg = copy.deepcopy(config.to_dict())+ cfg["quantization_config"] = qcfg+ cfg["torch_dtype"] = "bfloat16"+ with open(os.path.join(out_dir, "config.json"), "w", encoding = "utf-8") as f:+ json.dump(cfg, f, indent = 2)+ AutoTokenizer.from_pretrained(repo).save_pretrained(out_dir)+ try:+ from transformers import AutoProcessor+ AutoProcessor.from_pretrained(repo).save_pretrained(out_dir)+ except Exception:+ pass+ with open(os.path.join(out_dir, "nvfp4_fixture_kinds.json"), "w", encoding = "utf-8") as f:+ json.dump(kinds, f, indent = 1)+ return kinds+++def run_lora_case(path, arch, api, out_dir):+ """Load with Fast{LanguageModel,Model}, LoRA, 5 steps, save, reload; returns a JSON-able report. Run in a fresh process."""+ dtype = getattr(torch, os.environ.get("NVFP4_CASE_DTYPE", "bfloat16"))+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ import unsloth+ import unsloth.kernels.nvfp4 as nv+ from peft import PeftModel++ with open(os.path.join(path, "nvfp4_fixture_kinds.json"), encoding = "utf-8") as f:+ kinds = json.load(f)+ name = next(n for n, k in kinds.items() if k == "nvfp4")+ seen = []+ real = nv.nvfp4_linear++ def recording(X, *args, **kwargs):+ seen.append(str(X.dtype))+ return real(X, *args, **kwargs)++ nv.nvfp4_linear = recording+ Fast = getattr(unsloth, api)+ model, _ = Fast.from_pretrained(path, max_seq_length = 64, load_in_4bit = False, dtype = dtype)+ model = Fast.get_peft_model(+ model,+ r = 8,+ lora_alpha = 16,+ lora_dropout = 0,+ bias = "none",+ random_state = 3407,+ target_modules = [+ "q_proj",+ "k_proj",+ "v_proj",+ "o_proj",+ "gate_proj",+ "up_proj",+ "down_proj",+ ],+ )+ base = next(+ m+ for n, m in model.named_modules()+ if n.endswith((name, name + ".base_layer")) and hasattr(m, "weight_packed")+ )+ packed = base.weight_packed.clone()+ lora_b = lambda: next(p for n, p in model.named_parameters() if "lora_B" in n).detach().clone()+ b0 = lora_b()+ params = [p for p in model.parameters() if p.requires_grad]+ opt = torch.optim.AdamW(params, lr = 1e-3)+ torch.manual_seed(0)+ ids = torch.randint(0, 1000, (2, 32), device = "cuda")+ losses = []+ for _ in range(5):+ loss = model(input_ids = ids, labels = ids).loss+ loss.backward()+ opt.step()+ opt.zero_grad(set_to_none = True)+ losses.append(loss.item())+ model.eval()++ # The first eval calls recompile after training and can run eager meanwhile; compiled and eager bf16 LoRA+ # rounding differ by ~0.3%, so compare settled forwards (logits are bit-identical with TORCHDYNAMO_DISABLE=1).+ def settled(m):+ with torch.no_grad():+ for _ in range(3):+ m(input_ids = ids)+ return m(input_ids = ids).logits.float()++ want = settled(model)+ model.save_pretrained(os.path.join(out_dir, "lora"))+ fresh, _ = Fast.from_pretrained(path, max_seq_length = 64, load_in_4bit = False, dtype = dtype)+ fresh = PeftModel.from_pretrained(fresh, os.path.join(out_dir, "lora"), torch_device = "cuda")+ fresh.eval()+ got = settled(fresh)+ adapters = lambda m: {+ k.replace(".default", ""): v for k, v in m.state_dict().items() if "lora_" in k+ }+ trained, reloaded = adapters(model), adapters(fresh)+ return {+ "losses": losses,+ "input_dtypes": sorted(set(seen)),+ "packed_unchanged": torch.equal(base.weight_packed, packed),+ "packed_dtype": str(base.weight_packed.dtype),+ "lora_changed": not torch.equal(lora_b(), b0),+ "saved": sorted(os.listdir(os.path.join(out_dir, "lora"))),+ "reload_max_abs": (got - want).abs().max().item(),+ "adapters_equal": trained.keys() == reloaded.keys()+ and all(torch.equal(trained[k], reloaded[k]) for k in trained),+ "peak_gb": torch.cuda.max_memory_allocated() / 2**30,+ }+++if __name__ == "__main__":+ if len(sys.argv) > 1 and sys.argv[1] == "--lora-case":+ report = run_lora_case(*sys.argv[2:6])+ print("LORA_CASE " + json.dumps(report))+ sys.exit(0)+ ap = argparse.ArgumentParser()+ ap.add_argument("out_dir")+ ap.add_argument("--arch", default = "qwen3", choices = ["qwen3", "qwen3_5"])+ a = ap.parse_args()+ k = build(a.out_dir, a.arch)+ print({v: sum(1 for x in k.values() if x == v) for v in set(k.values())})diff --git a/tests/test_nvfp4_kernel.py b/tests/test_nvfp4_kernel.pynew file mode 100644index 00000000000..f483bb5a4b2--- /dev/null+++ b/tests/test_nvfp4_kernel.py@@ -0,0 +1,203 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Packed NVFP4 (compressed-tensors nvfp4-pack-quantized) must dequantize exactly as compressed-tensors does and train as W4A16.+import os++import pytest++torch = pytest.importorskip("torch")+ct_helpers = pytest.importorskip("compressed_tensors.compressors.nvfp4.helpers")+ct_forward = pytest.importorskip("compressed_tensors.quantization.lifecycle.forward")++CUDA = torch.cuda.is_available()+needs_cuda = pytest.mark.skipif(not CUDA, reason = "needs CUDA")++[email protected](scope = "module")+def N():+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ from unsloth.kernels import nvfp4+ return nvfp4+++def _random_nvfp4(+ rows,+ cols,+ device,+ global_scale = 7.3,+ seed = 0,+):+ g = torch.Generator(device = "cpu").manual_seed(seed)+ packed = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, generator = g)+ scale = (torch.rand(rows, cols // 16, generator = g) * 3 + 0.01).to(torch.float8_e4m3fn)+ return packed.to(device), scale.to(device), torch.tensor([global_scale], device = device)+++def _reference(packed, scale, global_scale, dtype):+ from compressed_tensors.quantization import QuantizationArgs++ rows, half = packed.shape+ x_q = ct_helpers.unpack_fp4_from_uint8(packed, rows, half * 2, dtype = dtype)+ # The weight args NVFP4PackedCompressor.decompress passes; compressed-tensors >= 0.19 no longer infers them.+ args = QuantizationArgs(num_bits = 4, type = "float", strategy = "tensor_group", group_size = 16)+ return ct_forward.dequantize(+ x_q = x_q, scale = scale.to(dtype), global_scale = global_scale, dtype = dtype, args = args+ )+++def _devices():+ return ["cpu", "cuda"] if CUDA else ["cpu"]++[email protected]("device", _devices())+def test_every_code_and_sign(N, device):+ # Byte b holds code b & 15 in the even column and b >> 4 in the odd one.+ packed = torch.arange(256, dtype = torch.uint8).reshape(16, 16).to(device)+ scale = torch.ones(16, 2, device = device).to(torch.float8_e4m3fn)+ out = N.nvfp4_dequantize(packed, scale, torch.ones(1, device = device), torch.float32)+ lut = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]+ codes = [(b & 15, b >> 4) for b in range(256)]+ expect = torch.tensor(+ [[(-1 if c & 8 else 1) * lut[c & 7] for c in pair] for pair in codes]+ ).reshape(16, 32)+ assert torch.equal(out.cpu(), expect)+ assert torch.signbit(out.cpu()[0, 16]) # code 8 is -0.0, as in compressed-tensors+++@needs_cuda+def test_every_e4m3_group_scale(N):+ # Before sm_89 Triton has no fp8e4nv, so the kernel decodes the scale bytes itself: all 256 codes, incl.+ # subnormals, -0.0 and NaN, must equal torch's own fp8 conversion.+ codes = torch.arange(256, dtype = torch.uint8, device = "cuda").reshape(16, 16)+ scale = codes.view(torch.float8_e4m3fn)+ packed = torch.full((16, 128), 0x22, dtype = torch.uint8, device = "cuda") # every value 1.0+ out = N.nvfp4_dequantize(packed, scale, torch.ones(1, device = "cuda"), torch.float32)+ expect = scale.float().repeat_interleave(16, dim = 1)+ assert torch.equal(out.isnan(), expect.isnan())+ assert torch.equal(out.nan_to_num(), expect.nan_to_num())+ finite = ~expect.isnan()+ assert torch.equal(+ out.signbit()[finite], expect.signbit()[finite]+ ) # -0.0 (0x80) keeps its sign++[email protected]("device", _devices())[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("shape", [(64, 256), (48, 400), (257, 96)])+def test_matches_compressed_tensors_bit_exact(N, device, dtype, shape):+ packed, scale, gs = _random_nvfp4(*shape, device, global_scale = 0.37)+ out = N.nvfp4_dequantize(packed, scale, gs, dtype)+ assert out.dtype == dtype and out.shape == shape+ assert torch.equal(out, _reference(packed, scale, gs, dtype))+++@needs_cuda+def test_triton_equals_torch_reference(N):+ packed, scale, gs = _random_nvfp4(320, 1024, "cuda")+ tri = N.nvfp4_dequantize(packed, scale, gs)+ ref = N._nvfp4_dequantize_torch(packed, scale, gs, torch.bfloat16)+ assert torch.equal(tri, ref)++[email protected]("device", _devices())+def test_transposed_view_is_the_transposed_weight(N, device):+ packed, scale, gs = _random_nvfp4(96, 256, device)+ dense = N.nvfp4_dequantize(packed, scale, gs)+ assert torch.equal(N.nvfp4_dequantize(packed.t(), scale, gs), dense.t())+++def test_too_few_scales_is_an_error(N):+ packed, scale, gs = _random_nvfp4(16, 64, "cpu")+ with pytest.raises(ValueError):+ N.nvfp4_dequantize(packed, scale[:, :2], gs)+++def _dense_reference_grads(W, X, dY, bias):+ X = X.detach().clone().requires_grad_(True)+ b = bias.detach().clone().requires_grad_(True) if bias is not None else None+ out = X @ W.t()+ if b is not None:+ out = out + b+ out.backward(dY)+ return out.detach(), X.grad, None if b is None else b.grad++[email protected]("device", _devices())[email protected]("with_bias", [False, True])+def test_autograd_matches_the_dense_weight(N, device, with_bias):+ dtype = torch.bfloat16 if device == "cuda" else torch.float32+ packed, scale, gs = _random_nvfp4(192, 128, device)+ W = N.nvfp4_dequantize(packed, scale, gs, dtype)+ X = torch.randn(2, 5, 128, device = device, dtype = dtype, requires_grad = True)+ dY = torch.randn(2, 5, 192, device = device, dtype = dtype)+ bias = torch.randn(192, device = device, dtype = dtype, requires_grad = True) if with_bias else None+ out = N.nvfp4_linear(X, packed, scale, gs, bias)+ out.backward(dY)+ ref_out, ref_dX, ref_db = _dense_reference_grads(W, X, dY, bias)+ assert out.dtype == dtype+ torch.testing.assert_close(out, ref_out)+ torch.testing.assert_close(X.grad, ref_dX)+ assert X.grad.abs().sum() > 0+ if with_bias:+ torch.testing.assert_close(bias.grad, ref_db)+++def test_float32_bias_keeps_the_activation_dtype(N):+ packed, scale, gs = _random_nvfp4(32, 64, "cpu")+ X = torch.randn(3, 64, dtype = torch.bfloat16)+ out = N.nvfp4_linear(X, packed, scale, gs, torch.randn(32, dtype = torch.float32))+ assert out.dtype == torch.bfloat16+++def test_backward_saves_only_the_packed_weight(N):+ device = "cuda" if CUDA else "cpu"+ packed, scale, gs = _random_nvfp4(256, 512, device)+ X = torch.randn(4, 512, device = device, requires_grad = True)+ saved = []++ def pack_hook(t):+ saved.append((tuple(t.shape), t.dtype))+ return t++ with torch.autograd.graph.saved_tensors_hooks(pack_hook, lambda t: t):+ N.nvfp4_linear(X, packed, scale, gs)+ dense = [s for s in saved if s[0] == (256, 512) or s[0] == (512, 256)]+ assert dense == [], saved+++@needs_cuda+def test_fullgraph_compile_has_no_breaks_and_matches_eager(N):+ import torch._dynamo++ packed, scale, gs = _random_nvfp4(384, 256, "cuda")+ step = lambda X: N.nvfp4_linear(X, packed, scale, gs).float().square().mean()+ # T4 (sm_75) trains in fp16: Inductor has no native bf16 there.+ dtype = torch.bfloat16 if torch.cuda.get_device_capability() >= (8, 0) else torch.float16+ X = torch.randn(64, 256, device = "cuda", dtype = dtype, requires_grad = True)+ step(X).backward()+ eager = X.grad.clone()+ X.grad = None+ torch._dynamo.reset()+ assert torch._dynamo.explain(step)(X).graph_break_count == 0+ torch._dynamo.reset()+ compiled = torch.compile(step, fullgraph = True)+ for rows in (16, 48, 64): # several shapes: dynamic-shape recompiles must stay correct+ Xr = torch.randn(rows, 256, device = "cuda", dtype = dtype, requires_grad = True)+ compiled(Xr).backward()+ compiled(X).backward()+ assert X.grad.abs().sum() > 0+ torch.testing.assert_close(X.grad, eager)+++@needs_cuda+def test_outputs_past_2_31_elements_use_64_bit_offsets(N):+ # 131080 x 16384 outputs: int32 offsets of the last rows wrap negative.+ rows, cols = 2**17 + 8, 16384+ if torch.cuda.mem_get_info()[0] < 8 * 2**30:+ pytest.skip("needs 8 GB free")+ packed = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, device = "cuda")+ scale = (torch.rand(rows, cols // 16, device = "cuda") + 0.5).to(torch.float8_e4m3fn)+ gs = torch.tensor([2.0], device = "cuda")+ W = N.nvfp4_dequantize(packed, scale, gs, torch.bfloat16)+ tail = slice(rows - 16, rows)+ want = N._nvfp4_dequantize_torch(packed[tail], scale[tail], gs, torch.bfloat16)+ assert torch.equal(W[tail], want)diff --git a/tests/test_nvfp4_loader.py b/tests/test_nvfp4_loader.pynew file mode 100644index 00000000000..744678d6cd1--- /dev/null+++ b/tests/test_nvfp4_loader.py@@ -0,0 +1,506 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+# NVFP4 compressed-tensors checkpoints (unsloth/Qwen3.8-27B-NVFP4 layout) stay packed and train W4A16 with LoRA.+import os+import sys++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("compressed_tensors")++pytestmark = pytest.mark.skipif(+ not torch.cuda.is_available(), reason = "NVFP4 routing needs CUDA weights"+)++sys.path.insert(0, os.path.dirname(__file__))+import _nvfp4_fixtures as fx # noqa: E402+++class _Checkpoints(dict):+ # Built on first use, so a transformers without qwen3_5 (4.57.x) still runs the qwen3 cases.+ def __init__(self, tmp_path_factory):+ super().__init__()+ self.tmp_path_factory = tmp_path_factory++ def __missing__(self, arch):+ from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES++ if arch not in CONFIG_MAPPING_NAMES:+ pytest.skip(f"transformers has no {arch}")+ d = self.tmp_path_factory.mktemp(arch)+ self[arch] = (str(d), fx.build(str(d), arch))+ return self[arch]++[email protected](scope = "module")+def ckpt(tmp_path_factory):+ return _Checkpoints(tmp_path_factory)+++def _load_raw(path, arch):+ from transformers import AutoModelForCausalLM, AutoModelForImageTextToText+ cls = AutoModelForCausalLM if arch == "qwen3" else AutoModelForImageTextToText+ return cls.from_pretrained(path, device_map = "cuda", dtype = torch.bfloat16)+++def _module(model, name):+ return dict(model.named_modules())[name]++[email protected]("arch", ["qwen3", "qwen3_5"])+def test_fixture_decompresses_to_the_packed_weights(ckpt, arch):+ path, kinds = ckpt[arch]+ from safetensors import safe_open++ f = safe_open(os.path.join(path, "model.safetensors"), "pt")+ name = next(n for n, k in kinds.items() if k == "nvfp4")+ packed, scale, gs = (+ f.get_tensor(f"{name}.{s}")+ for s in ("weight_packed", "weight_scale", "weight_global_scale")+ )+ from unsloth.kernels.nvfp4 import nvfp4_dequantize++ ref = fx.dequant_nvfp4_reference(packed, scale, gs)+ assert torch.equal(nvfp4_dequantize(packed.cuda(), scale.cuda(), gs.cuda()).cpu(), ref)+ model = _load_raw(path, arch)+ module = _module(model, name)+ assert (+ str(module.quantization_status.value) == "compressed"+ and module.weight_packed.dtype == torch.uint8+ )+ # The broad MLP target overlaps the last layer's FP8 override; the override wins, as in the real checkpoint.+ last = next(n for n, k in kinds.items() if k == "fp8" and ".mlp." in n)+ assert _module(model, last).quantization_scheme.weights.num_bits == 8++[email protected]("arch", ["qwen3", "qwen3_5"])+def test_nvfp4_layers_stay_packed_and_run_w4a16(ckpt, arch, monkeypatch):+ from safetensors import safe_open+ from unsloth.kernels.nvfp4 import nvfp4_dequantize+ from unsloth.models import loader_utils++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ path, kinds = ckpt[arch]+ model = _load_raw(path, arch)+ loader_utils._prepare_compressed_tensors_model(model)+ nvfp4 = [n for n, k in kinds.items() if k == "nvfp4"]+ assert model._unsloth_compressed_tensors_nvfp4 == len(nvfp4)+ assert getattr(model, "ct_decompress_hook", None) is None and not model._forward_pre_hooks+ keys = set(safe_open(os.path.join(path, "model.safetensors"), "pt").keys())+ # Same keys as the checkpoint, except that decompressed layers are dense and drop their scales+ # (a tied lm_head may add its alias).+ dense = [n for n, k in kinds.items() if k == "fp8"] + ["lm_head"]+ keys -= {n + ".weight_scale" for n in dense}+ assert set(model.state_dict()) - {"lm_head.weight"} == keys - {"lm_head.weight"}+ assert not any(+ n.endswith(".weight") and n[: -len(".weight")] in nvfp4 for n, _ in model.named_parameters()+ )+ for name, kind in kinds.items():+ module = _module(model, name)+ if kind == "fp8":+ assert module.weight.dtype == torch.bfloat16 # decompressed per module+ with torch.no_grad():+ model(input_ids = torch.randint(0, 1000, (2, 8), device = "cuda"))+ for name in nvfp4:+ module = _module(model, name)+ assert module.weight_packed.dtype == torch.uint8 and module.weight.dtype == torch.uint8+ assert module.weight.quant_state.shape == (module.out_features, module.in_features)+ x = torch.randn(3, module.in_features, device = "cuda", dtype = torch.bfloat16)+ W = nvfp4_dequantize(+ module.weight_packed, module.weight_scale, module.weight_global_scale, torch.bfloat16+ )+ with torch.no_grad():+ # A16: exactly the dense matmul, no activation fake-quantization.+ assert torch.equal(module(x), torch.nn.functional.linear(x, W))+++def test_dtype_casts_keep_the_nvfp4_storage_dtypes(ckpt, monkeypatch):+ # model.to(fp16) must not cast the scales: an fp16 global scale overflows past 65504 and zeroes the layer.+ from unsloth.models import loader_utils++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ module = _module(model, next(n for n, k in kinds.items() if k == "nvfp4"))+ dtypes = {n: p.dtype for n, p in module.named_parameters()}+ x = torch.randn(3, module.in_features, device = "cuda", dtype = torch.float16)+ with torch.no_grad():+ module.weight_global_scale.fill_(1e5)+ want = module(x)+ model.to(torch.float16)+ assert {n: p.dtype for n, p in module.named_parameters()} == dtypes+ assert torch.equal(module(x), want) and want.abs().max() > 0+ # Fused LoRA paths dequantize through the quant state, which must follow the cast.+ assert module.weight.quant_state.dtype == torch.float16+++def _fp8_route_available():+ from unsloth.models import loader_utils+ return loader_utils._zoo_peft_forward_keeps_fp8_inputs()++[email protected]("ct_helper", [True, False])[email protected]("kind", ["nvfp4", "fp8"])+def test_peft_merge_dequantizes_the_routed_base(ckpt, kind, ct_helper, monkeypatch):+ # merge_and_unload / merge_adapter / merged_4bit add the dense delta into base_layer.weight.+ from peft import LoraConfig, get_peft_model+ from unsloth.models import loader_utils++ if kind == "fp8" and not _fp8_route_available():+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1" if kind == "fp8" else "0")+ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ name = next(+ n+ for n, k in kinds.items()+ if k == kind and ("layers.0." if kind == "nvfp4" else "layers.1.") in n+ )+ model = get_peft_model(model, LoraConfig(r = 4, target_modules = [name.split(".")[-1]]))+ lora = _module(model, "base_model.model." + name)+ torch.nn.init.normal_(lora.lora_B["default"].weight, std = 0.02)+ want = loader_utils._routed_dense_weight(lora.get_base_layer())+ want += lora.get_delta_weight("default")+ if not ct_helper:+ # compressed-tensors < 0.13 has no module-level decompress_module.+ import compressed_tensors.compressors as ct_compressors+ monkeypatch.delattr(ct_compressors, "decompress_module")+ merged = model.merge_and_unload()+ layer = _module(merged, name)+ assert type(layer) is torch.nn.Linear and layer.weight.dtype == torch.bfloat16+ assert torch.equal(layer.weight, want)++[email protected]("kind", ["nvfp4", "fp8"])+def test_dora_reads_the_dense_routed_weight(ckpt, kind, monkeypatch):+ from peft import LoraConfig, get_peft_model+ from unsloth.models import loader_utils++ if kind == "fp8" and not _fp8_route_available():+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1" if kind == "fp8" else "0")+ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ name = next(+ n+ for n, k in kinds.items()+ if k == kind and ("layers.0." if kind == "nvfp4" else "layers.1.") in n+ )+ model = get_peft_model(+ model, LoraConfig(r = 4, target_modules = [name.split(".")[-1]], use_dora = True)+ )+ lora = _module(model, "base_model.model." + name)+ W = loader_utils._routed_dense_weight(lora.get_base_layer()).float()+ magnitude = lora.lora_magnitude_vector["default"].weight+ delta = lora.get_delta_weight("default").float()+ assert torch.allclose(magnitude.float(), (W + delta).norm(dim = 1), rtol = 1e-2)+ out = model(input_ids = torch.randint(0, 1000, (1, 8), device = "cuda")).logits+ assert torch.isfinite(out).all()++[email protected]("merge", [False, True])+def test_saved_checkpoint_reloads_with_plain_transformers(ckpt, merge, tmp_path, monkeypatch):+ # save_pretrained (after merge_and_unload or not): per-module decompressed layers such as lm_head and the+ # merged ones are dense, so they must join the ignore list, or the reload reads them as FP8 / packed.+ from peft import LoraConfig, get_peft_model+ from transformers import AutoModelForCausalLM+ from unsloth.models import loader_utils++ if not _fp8_route_available():+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1")+ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ if merge:+ model = get_peft_model(model, LoraConfig(r = 4, target_modules = ["gate_proj", "q_proj"]))+ for n, p in model.named_parameters():+ if "lora_B" in n:+ torch.nn.init.normal_(p, std = 0.02)+ model = model.merge_and_unload()+ ids = torch.randint(0, 1000, (1, 16), device = "cuda")+ with torch.no_grad():+ want = model(input_ids = ids).logits.float()+ model.save_pretrained(str(tmp_path / "out"))+ reloaded = AutoModelForCausalLM.from_pretrained(+ str(tmp_path / "out"), device_map = "cuda", dtype = torch.bfloat16+ )+ with torch.no_grad():+ got = reloaded(input_ids = ids).logits.float()+ # Plain compressed-tensors fake-quantizes activations (W4A4 / W8A8), so only close, not equal;+ # a layer read in the wrong format gives logits near zero.+ assert (got - want).abs().max() < 0.1 * want.abs().max()++[email protected]("init", ["pissa_niter_2", "olora", "loftq", "orthogonal"])+def test_svd_style_lora_inits_see_the_dense_base(ckpt, init, monkeypatch):+ # PiSSA / OLoRA rewrite base_layer.weight from its SVD / QR and orthogonal init builds adapters in its dtype,+ # so the packed base goes dense first.+ from peft import LoraConfig, get_peft_model+ from unsloth.models import loader_utils++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ name = next(n for n, k in kinds.items() if k == "nvfp4")+ extra = {}+ if init == "loftq":+ pytest.importorskip("scipy")+ from peft import LoftQConfig+ extra["loftq_config"] = LoftQConfig(loftq_bits = 4, loftq_iter = 1)+ model = get_peft_model(+ model,+ LoraConfig(r = 4, target_modules = [name.split(".")[-1]], init_lora_weights = init, **extra),+ )+ base = _module(model, "base_model.model." + name).get_base_layer()+ assert type(base) is torch.nn.Linear and base.weight.dtype == torch.bfloat16+ out = model(input_ids = torch.randint(0, 1000, (1, 8), device = "cuda")).logits+ assert torch.isfinite(out).all()+++def test_lora_ga_preprocess_sees_the_dense_base(ckpt, monkeypatch):+ # preprocess_loraga sets requires_grad on each target weight before get_peft_model runs.+ peft = pytest.importorskip("peft")+ if not hasattr(peft, "preprocess_loraga"):+ pytest.skip("PEFT without LoRA-GA")+ from peft import LoraConfig, LoraGAConfig, get_peft_model, preprocess_loraga+ from unsloth.models import loader_utils++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ name = next(n for n, k in kinds.items() if k == "nvfp4")+ config = LoraConfig(+ r = 4,+ target_modules = [name.split(".")[-1]],+ init_lora_weights = "lora_ga",+ lora_ga_config = LoraGAConfig(),+ )+ ids = torch.randint(0, 1000, (1, 8), device = "cuda")++ def train_step():+ model(input_ids = ids, labels = ids).loss.backward()++ preprocess_loraga(model, config, train_step)+ assert _module(model, name).weight.dtype == torch.bfloat16+ model = get_peft_model(model, config)+ out = model(input_ids = ids).logits+ assert torch.isfinite(out).all()+++def test_corda_preprocess_sees_the_dense_base(ckpt, monkeypatch):+ # preprocess_corda multiplies each target weight by its input covariance before get_peft_model runs.+ corda = pytest.importorskip("peft.tuners.lora.corda")+ from peft import LoraConfig, get_peft_model+ from peft.tuners.lora.config import CordaConfig+ from unsloth.models import loader_utils++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ name = next(n for n, k in kinds.items() if k == "nvfp4")+ config = LoraConfig(+ r = 4,+ target_modules = [name.split(".")[-1]],+ init_lora_weights = "corda",+ corda_config = CordaConfig(),+ )+ ids = torch.randint(0, 1000, (1, 64), device = "cuda")++ def run_model():+ # CorDA's covariance hook squeezes a batch of one.+ for _ in range(4):+ model(input_ids = torch.randint(0, 1000, (1, 64), device = "cuda"))++ corda.preprocess_corda(model, config, run_model = run_model)+ assert _module(model, name).weight.dtype == torch.bfloat16+ model = get_peft_model(model, config)+ out = model(input_ids = ids).logits+ assert torch.isfinite(out).all()+++def test_decompressed_layers_join_a_missing_ignore_list(ckpt, monkeypatch):+ from unsloth.models import loader_utils++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ path, _ = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ model.config.quantization_config.quantization_config.ignore = None+ loader_utils._prepare_compressed_tensors_model(model)+ assert "lm_head" in model.config.quantization_config.quantization_config.ignore+++def test_nvfp4_lm_head_is_decompressed(tmp_path, monkeypatch):+ # Decode and the fused CE loss read lm_head.weight directly, so it cannot stay packed.+ from unsloth.models import loader_utils++ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")+ kinds = fx.build(str(tmp_path), "qwen3", lm_head_kind = "nvfp4")+ assert kinds["lm_head"] == "nvfp4"+ model = _load_raw(str(tmp_path), "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ head = model.get_output_embeddings()+ assert head.weight.dtype == torch.bfloat16+ assert tuple(head.weight.shape) == (model.config.vocab_size, model.config.hidden_size)+ assert model._unsloth_compressed_tensors_nvfp4 == len(kinds) - 1 - sum(+ k == "fp8" for k in kinds.values()+ )+++def test_opt_out_keeps_the_full_decompress(ckpt, monkeypatch):+ from unsloth.models import loader_utils++ path, kinds = ckpt["qwen3"]+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_NVFP4_KERNELS", "0")+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ name = next(n for n, k in kinds.items() if k == "nvfp4")+ assert _module(model, name).weight.dtype == torch.bfloat16+ assert not getattr(model, "_unsloth_compressed_tensors_nvfp4", 0)+++def test_full_finetuning_decompresses_everything(ckpt):+ from unsloth.models import loader_utils++ path, kinds = ckpt["qwen3"]+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model, full_finetuning = True)+ assert all(_module(model, n).weight.dtype == torch.bfloat16 for n in kinds)+++def test_fp8_group_routes_with_the_fp8_kernels_by_default(ckpt, monkeypatch):+ from unsloth.models import loader_utils++ path, kinds = ckpt["qwen3"]+ monkeypatch.delenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", raising = False)+ monkeypatch.setattr(loader_utils, "_zoo_peft_forward_keeps_fp8_inputs", lambda: True)+ model = _load_raw(path, "qwen3")+ loader_utils._prepare_compressed_tensors_model(model)+ lm_head = model.get_output_embeddings()+ for name, kind in kinds.items():+ module = _module(model, name)+ if module is lm_head:+ # The fused CE loss reads lm_head.weight directly, so it is decompressed, not routed.+ assert module.weight.dtype == torch.bfloat16+ assert not getattr(module, "_unsloth_compressed_tensors_fp8", False)+ elif kind == "fp8":+ assert (+ module.weight.dtype == torch.float8_e4m3fn+ and module._unsloth_compressed_tensors_fp8+ )+ else:+ assert module._unsloth_compressed_tensors_nvfp4+++ "arch, api, fp8_kernels",+ [+ ("qwen3", "FastLanguageModel", False),+ ("qwen3", "FastModel", False),+ ("qwen3_5", "FastModel", False),+ ("qwen3_5", "FastLanguageModel", False),+ # The qwen3 fixture has an untied FP8 lm_head, which the FP8 route once handed to the fused CE loss.+ ("qwen3", "FastModel", True),+ ],+)+def test_lora_trains_on_the_packed_base_and_reloads(ckpt, arch, api, fp8_kernels, tmp_path):+ _check_lora_case(ckpt, arch, api, fp8_kernels, tmp_path)++[email protected]("api", ["FastLanguageModel", "FastModel"])+def test_lora_trains_in_float16(ckpt, api, tmp_path):+ # T4 / V100 load in fp16: the NVFP4 quant state must dequantize to fp16, and the bf16-only FP8 kernels stay off.+ _check_lora_case(ckpt, "qwen3", api, None, tmp_path, dtype = "float16")+++def _check_lora_case(+ ckpt,+ arch,+ api,+ fp8_kernels,+ tmp_path,+ dtype = "bfloat16",+):+ import json+ import subprocess++ if fp8_kernels:+ from unsloth.models import loader_utils+ if not loader_utils._zoo_peft_forward_keeps_fp8_inputs():+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")+ path, _ = ckpt[arch]+ root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ # A fresh process per case: patches and the compiled module cache from one loader leak into the next.+ env = dict(+ os.environ,+ PYTHONPATH = root + os.pathsep + os.environ.get("PYTHONPATH", ""),+ UNSLOTH_COMPILE_LOCATION = str(tmp_path / "compiled"),+ UNSLOTH_IS_PRESENT = "1",+ NVFP4_CASE_DTYPE = dtype,+ )+ if fp8_kernels is not None:+ env["UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS"] = "1" if fp8_kernels else "0"+ else:+ env.pop("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", None)+ run = subprocess.run(+ [+ sys.executable,+ os.path.join(os.path.dirname(__file__), "_nvfp4_fixtures.py"),+ "--lora-case",+ path,+ arch,+ api,+ str(tmp_path),+ ],+ env = env,+ capture_output = True,+ text = True,+ timeout = 1200,+ )+ lines = [l for l in run.stdout.splitlines() if l.startswith("LORA_CASE ")]+ assert run.returncode == 0 and lines, run.stderr[-3000:]+ r = json.loads(lines[-1][len("LORA_CASE ") :])+ assert all(torch.isfinite(torch.tensor(r["losses"]))) and r["losses"][-1] < r["losses"][0]+ # The packed base is never cast into, and activations reach it in a float dtype (W4A16, not uint8).+ assert r["input_dtypes"] and set(r["input_dtypes"]) <= {+ "torch.bfloat16",+ "torch.float16",+ "torch.float32",+ }+ assert r["packed_unchanged"] and r["packed_dtype"] == "torch.uint8" and r["lora_changed"]+ assert "adapter_model.safetensors" in r["saved"]+ assert r["adapters_equal"]+ # FastLanguageModel trains through fused LoRA kernels; a plain PEFT reload differs by ~0.04 on bf16 main too.+ if api == "FastModel":+ assert r["reload_max_abs"] <= 1e-2+++def test_float8_lora_adapters_are_upcast_on_every_peft():+ # PEFT < 0.19 gives an FP8 base float8 LoRA weights, on get_peft_model and on PeftModel.from_pretrained;+ # the FP8 route beside NVFP4 relies on Unsloth's upcast.+ import unsloth # noqa: F401+ from peft import LoraConfig, get_peft_model++ base = torch.nn.Sequential(torch.nn.Linear(32, 32, bias = False)).cuda()+ base[0].weight.data = base[0].weight.data.to(torch.float8_e4m3fn)+ model = get_peft_model(base, LoraConfig(r = 4, target_modules = ["0"]))+ dtypes = {n: p.dtype for n, p in model.named_parameters()}+ assert all(d == torch.float32 for n, d in dtypes.items() if "lora_" in n), dtypes+ assert model.base_model.model[0].base_layer.weight.dtype == torch.float8_e4m3fn++ import tempfile+ from peft import PeftModel++ with tempfile.TemporaryDirectory() as d:+ model.save_pretrained(d)+ fresh = torch.nn.Sequential(torch.nn.Linear(32, 32, bias = False)).cuda()+ fresh[0].weight.data = fresh[0].weight.data.to(torch.float8_e4m3fn)+ fresh = PeftModel.from_pretrained(fresh, d, torch_device = "cuda")+ assert all(p.dtype == torch.float32 for n, p in fresh.named_parameters() if "lora_" in n)diff --git a/tests/test_nvfp4_lora.py b/tests/test_nvfp4_lora.pynew file mode 100644index 00000000000..a5260646ae1--- /dev/null+++ b/tests/test_nvfp4_lora.py@@ -0,0 +1,175 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+# Packed NVFP4 bases (plain `weight` attribute = weight_packed with an NVFP4QuantState) through Unsloth's fused LoRA kernels.+import os++import pytest++torch = pytest.importorskip("torch")++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")++[email protected](scope = "module")+def mods():+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")+ from unsloth.kernels import fast_lora, nvfp4, utils+ return fast_lora, nvfp4, utils+++def _packed(out_features, in_features, seed):+ g = torch.Generator(device = "cuda").manual_seed(seed)+ packed = torch.randint(+ 0, 256, (out_features, in_features // 2), generator = g, device = "cuda", dtype = torch.uint8+ )+ # Asymmetric group scales so a wrong group axis or orientation shows up.+ scale = torch.rand(out_features, in_features // 16, generator = g, device = "cuda") * 3 + 0.25+ scale = scale * torch.linspace(0.5, 2.0, in_features // 16, device = "cuda")+ return packed, scale.to(torch.float8_e4m3fn), torch.tensor([37.5], device = "cuda")+++def _lora_model(+ mods,+ names_shapes,+ r = 8,+ seed = 0,+):+ """PEFT LoRA over nn.Linears whose base weights are swapped for packed NVFP4; returns (nvfp4 model, dense twin)."""+ from peft import LoraConfig, get_peft_model++ _, nvfp4, _ = mods++ class Block(torch.nn.Module):+ def __init__(self):+ super().__init__()+ for name, (o, i) in names_shapes.items():+ setattr(self, name, torch.nn.Linear(i, o, bias = False))++ cfg = LoraConfig(r = r, lora_alpha = 16, lora_dropout = 0.0, target_modules = list(names_shapes))+ torch.manual_seed(seed)+ dense = get_peft_model(Block().cuda().to(torch.bfloat16), cfg)+ torch.manual_seed(seed)+ quant = get_peft_model(Block().cuda().to(torch.bfloat16), cfg)+ for k, (name, (o, i)) in enumerate(names_shapes.items()):+ packed, scale, gs = _packed(o, i, seed + k)+ W = nvfp4.nvfp4_dequantize(packed, scale, gs, torch.bfloat16)+ d_layer = getattr(dense.base_model.model, name)+ q_layer = getattr(quant.base_model.model, name)+ d_layer.base_layer.weight.data.copy_(W)+ del q_layer.base_layer.weight+ packed.quant_state = nvfp4.NVFP4QuantState(scale, gs, (o, i), torch.bfloat16)+ q_layer.base_layer.weight = packed+ # Non-zero B: zero-initialised B hides dA bugs.+ for layer in (d_layer, q_layer):+ torch.manual_seed(100 + k)+ torch.nn.init.normal_(layer.lora_B["default"].weight, std = 0.02)+ torch.manual_seed(200 + k)+ torch.nn.init.normal_(layer.lora_A["default"].weight, std = 0.02)+ return quant.base_model.model, dense.base_model.model+++def _grads(model, names):+ return [getattr(model, n).lora_A["default"].weight.grad.float() for n in names] + [+ getattr(model, n).lora_B["default"].weight.grad.float() for n in names+ ]+++def _close(a, b, tol):+ a, b = a.float(), b.float()+ return float((a - b).norm() / b.norm().clamp(min = 1e-12)) < tol+++def _run(mods, names_shapes, fused, reference):+ q, d = _lora_model(mods, names_shapes)+ in_dim = next(iter(names_shapes.values()))[1]+ X = torch.randn(2, 24, in_dim, device = "cuda", dtype = torch.bfloat16)+ Xq, Xd = X.clone().requires_grad_(True), X.clone().requires_grad_(True)+ outs_q, outs_d = fused(q, Xq), reference(d, Xd)+ if not isinstance(outs_q, (tuple, list)):+ outs_q, outs_d = (outs_q,), (outs_d,)+ sum(o.float().square().mean() for o in outs_q).backward()+ sum(o.float().square().mean() for o in outs_d).backward()+ for oq, od in zip(outs_q, outs_d):+ assert oq.dtype == torch.bfloat16+ assert _close(oq, od, 1e-2)+ assert _close(Xq.grad, Xd.grad, 2e-2)+ for gq, gd in zip(_grads(q, names_shapes), _grads(d, names_shapes)):+ assert _close(gq, gd, 2e-2)+++def test_dequantize_matches_compressed_tensors(mods):+ ct = pytest.importorskip("compressed_tensors.compressors.nvfp4.base")+ _, nvfp4, _ = mods+ packed, scale, gs = _packed(96, 160, 1)+ from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme++ args = QuantizationArgs(num_bits = 4, type = "float", strategy = "tensor_group", group_size = 16)+ ref = ct.NVFP4PackedCompressor.decompress(+ {"weight_packed": packed, "weight_scale": scale, "weight_global_scale": gs},+ QuantizationScheme(targets = ["Linear"], weights = args),+ )["weight"]+ ours = nvfp4.nvfp4_dequantize(packed, scale, gs, ref.dtype)+ assert torch.equal(ours, ref)+ assert torch.equal(nvfp4.nvfp4_dequantize(packed.t(), scale, gs, ref.dtype), ref.t())+++def test_get_lora_parameters_returns_the_nvfp4_state(mods):+ _, nvfp4, utils = mods+ q, _ = _lora_model(mods, {"o_proj": (64, 96)})+ W, W_quant, A, B, S = utils.get_lora_parameters(q.o_proj)+ assert W.dtype == torch.uint8 and type(W_quant) is nvfp4.NVFP4QuantState+ assert A is not None and B is not None+++def test_lora_w_matches_dense(mods):+ fast_lora, _, _ = mods+ _run(+ mods,+ {"o_proj": (80, 96)},+ lambda m, X: fast_lora.apply_lora_o(m, X),+ lambda m, X: m.o_proj(X),+ )+++def test_lora_qkv_matches_dense(mods):+ fast_lora, _, _ = mods+ shapes = {"q_proj": (128, 96), "k_proj": (32, 96), "v_proj": (32, 96)}+ _run(+ mods,+ shapes,+ lambda m, X: fast_lora.apply_lora_qkv(m, X, inplace = False),+ lambda m, X: (m.q_proj(X), m.k_proj(X), m.v_proj(X)),+ )+++def test_lora_mlp_matches_dense(mods):+ fast_lora, _, _ = mods+ shapes = {"gate_proj": (160, 96), "up_proj": (160, 96), "down_proj": (96, 160)}++ def reference(m, X):+ return m.down_proj(torch.nn.functional.silu(m.gate_proj(X)) * m.up_proj(X))++ _run(mods, shapes, lambda m, X: fast_lora.apply_lora_mlp_swiglu(m, X, inplace = False), reference)+++def test_fast_linear_forward_decode(mods):+ _, _, utils = mods+ q, d = _lora_model(mods, {"o_proj": (80, 96)})+ X = torch.randn(1, 1, 96, device = "cuda", dtype = torch.bfloat16)+ with torch.no_grad():+ assert _close(utils.fast_linear_forward(q.o_proj, X), d.o_proj(X), 1e-2)+ X2 = torch.randn(3, 1, 96, device = "cuda", dtype = torch.bfloat16)+ assert _close(utils.fast_linear_forward(q.o_proj, X2), d.o_proj(X2), 1e-2)+++def test_bnb_4bit_path_is_unchanged(mods):+ bnb = pytest.importorskip("bitsandbytes")+ _, _, utils = mods+ layer = bnb.nn.Linear4bit(128, 64, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")+ dense = (torch.randn(64, 128) * 0.05).to(torch.bfloat16)+ layer.weight = bnb.nn.Params4bit(dense, requires_grad = False, quant_type = "nf4")+ layer = layer.cuda()+ W = layer.weight+ out = utils.fast_dequantize(W, W.quant_state)+ ref = bnb.functional.dequantize_4bit(W.data, W.quant_state)+ assert torch.equal(out, ref)diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 15cc1180a73..6fd5bb0b001 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -303,6 +303,7 @@ from .import_fixes import (fix_peft_transformers_tensor_parallel_import_compat,fix_peft_transformers_weight_conversion_import,patch_peft_weight_converter_compatibility,+ patch_peft_float8_adapter_upcast,fix_peft_stale_torchao_import_error,fix_peft_torchao_missing_tensor_subclass,patch_accelerate_recursively_apply,@@ -391,6 +392,7 @@ fix_accelerate_dtensor_check_without_torch_distributed()fix_peft_transformers_tensor_parallel_import_compat()fix_peft_transformers_weight_conversion_import()patch_peft_weight_converter_compatibility()+patch_peft_float8_adapter_upcast()# After peft is importable, so the already-bound is_torchao_available in peft.tuners.lora.torchao is# replaced too, not just import_utils'.fix_peft_stale_torchao_import_error()@@ -441,6 +443,7 @@ del fix_accelerate_dtensor_check_without_torch_distributeddel fix_peft_transformers_tensor_parallel_import_compatdel fix_peft_transformers_weight_conversion_importdel patch_peft_weight_converter_compatibility+del patch_peft_float8_adapter_upcastdel fix_peft_stale_torchao_import_errordel fix_peft_torchao_missing_tensor_subclassdel patch_accelerate_recursively_applydiff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 531408dab10..d59fed1c1d3 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -8149,6 +8149,53 @@ def _backfill_missing_conversion_symbols():return bool(added)+def patch_peft_float8_adapter_upcast():+ """PEFT < 0.19 builds adapters in an FP8 base weight's dtype; upcast them to float32 as PEFT >= 0.19 does."""+ try:+ import torch+ import peft.tuners.tuners_utils as tu+ except Exception:+ return+ original = getattr(tu, "cast_adapter_dtype", None)+ if original is None or getattr(original, "_unsloth_float8_upcast", False):+ return+ try:+ from peft.utils.constants import UPCAST_DTYPES # noqa: F401 (PEFT >= 0.19 already upcasts float8)+ return+ except ImportError:+ pass+ float8 = tuple(+ getattr(torch, n)+ for n in ("float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz")+ if hasattr(torch, n)+ )++ @functools.wraps(original)+ def cast_adapter_dtype(+ model,+ adapter_name,+ autocast_adapter_dtype = True,+ ):+ original(model, adapter_name = adapter_name, autocast_adapter_dtype = autocast_adapter_dtype)+ if not autocast_adapter_dtype:+ return+ for module in model.modules():+ if not isinstance(module, tu.BaseTunerLayer):+ continue+ for name in module.adapter_layer_names:+ layer = getattr(module, name, None)+ if not isinstance(layer, torch.nn.Module) or adapter_name not in layer:+ continue+ item = layer[adapter_name]+ params = item.parameters() if isinstance(item, torch.nn.Module) else (item,)+ for p in params:+ if p.dtype in float8:+ p.data = p.data.to(torch.float32)++ cast_adapter_dtype._unsloth_float8_upcast = True+ tu.cast_adapter_dtype = cast_adapter_dtype++def patch_peft_weight_converter_compatibility():"""Allow PEFT converter rebuilds on legacy converter constructors."""try:diff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.pyindex a58a3cdceed..55845a94fc7 100644--- a/unsloth/kernels/__init__.py+++ b/unsloth/kernels/__init__.py@@ -44,6 +44,7 @@ from .fast_lora import (from .fp8 import * # Patch FbgmemFP8Linear/FP8Linear forwards before model creation, so compiled non-fast-inference# models are covered too+from .nvfp4 import *from .utils import (fast_dequantize,fast_gemv,diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.pynew file mode 100644index 00000000000..42d88a079e4--- /dev/null+++ b/unsloth/kernels/nvfp4.py@@ -0,0 +1,239 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""W4A16 NVFP4: packed E2M1 weights (compressed-tensors `nvfp4-pack-quantized`) dequantized to 16 bit on the fly."""++import torch+import triton+import triton.language as tl++from .fp8 import _fp8_triton_device_context, _is_transposed_view, _opaque_under_compile++__all__ = [+ "NVFP4_GROUP_SIZE",+ "NVFP4QuantState",+ "nvfp4_dequantize",+ "NVFP4Linear_matmul",+ "nvfp4_linear",+]++NVFP4_GROUP_SIZE = 16+_E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)+++class NVFP4QuantState:+ __slots__ = ("scale", "global_scale", "shape", "dtype")++ def __init__(+ self,+ scale,+ global_scale,+ shape,+ dtype = torch.bfloat16,+ ):+ self.scale = scale+ self.global_scale = global_scale+ self.shape = tuple(shape)+ self.dtype = dtype+++def _nvfp4_dequantize_torch(packed, scale, global_scale, dtype):+ # Same arithmetic as compressed_tensors NVFP4PackedCompressor.decompress: scale / global_scale in fp32, times E2M1.+ rows, half = packed.shape+ lut = torch.tensor(_E2M1, dtype = torch.float32, device = packed.device)+ codes = torch.stack((packed & 0x0F, packed >> 4), dim = -1).reshape(rows, half * 2)+ values = lut[(codes & 0x07).long()] * torch.where((codes & 0x08).bool(), -1.0, 1.0)+ group_scale = scale.to(torch.float32) / global_scale.to(torch.float32).reshape(-1)[:1]+ group_scale = group_scale.repeat_interleave(NVFP4_GROUP_SIZE, dim = 1)[:, : half * 2]+ return (values * group_scale).to(dtype)+++def _e2m1_to_float(code):+ # Build the fp32 bits: mag >= 2 is (1 + m / 2) * 2^(e - 1), mag 1 is 0.5, mag 0 is 0; bit 3 is the sign (code 8 = -0.0).+ mag = code & 7+ bits = tl.where(+ mag >= 2, (((mag >> 1) + 126) << 23) | ((mag & 1) << 22), tl.where(mag == 1, 126 << 23, 0)+ )+ return (bits | ((code & 8) << 28)).to(tl.float32, bitcast = True)+++def _e4m3_to_float(b):+ # Triton has no float8e4nv before sm_89 (A100, T4), so the group scale arrives as raw bytes: sign, 4 exponent+ # bits (bias 7), 3 mantissa bits; exponent 0 is subnormal (m * 2^-9) and 0x7F / 0xFF are NaN, as in torch.+ exp = (b >> 3) & 15+ man = b & 7+ normal = (((exp + 120) << 23) | (man << 20)).to(tl.float32, bitcast = True)+ value = tl.where(exp == 0, man.to(tl.float32) * 0.001953125, normal)+ value = tl.where((b & 127) == 127, float("nan"), value)+ # Sign as a bit, not a negation: -x can lower to 0 - x, which turns code 0x80 (-0.0) into +0.0.+ return (value.to(tl.int32, bitcast = True) | ((b & 128) << 24)).to(tl.float32, bitcast = True)+++def _nvfp4_dequant_kernel(+ packed_ptr,+ scale_ptr,+ global_scale_ptr,+ out_ptr,+ rows,+ half_cols,+ scale_cols,+ BLOCK_R: tl.constexpr,+ BLOCK_C: tl.constexpr,+ SCALE_E4M3_BYTES: tl.constexpr,+ INT64_OFFSETS: tl.constexpr,+):+ r = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)+ if INT64_OFFSETS:+ r = r.to(tl.int64)+ c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)+ g = tl.program_id(1) * (BLOCK_C // 8) + tl.arange(0, BLOCK_C // 8)+ row_ok = r[:, None] < rows+ byte = tl.load(+ packed_ptr + r[:, None] * half_cols + c[None, :],+ mask = row_ok & (c[None, :] < half_cols),+ other = 0,+ ).to(tl.int32)+ scale = tl.load(+ scale_ptr + r[:, None] * scale_cols + g[None, :],+ mask = row_ok & (g[None, :] < scale_cols),+ other = 0.0,+ )+ scale = _e4m3_to_float(scale.to(tl.int32)) if SCALE_E4M3_BYTES else scale.to(tl.float32)+ group_scale = tl.math.div_rn(scale, tl.load(global_scale_ptr).to(tl.float32))+ # Low nibble is the even column: join + reshape interleaves (lo, hi) into contiguous output columns.+ values = tl.reshape(+ tl.join(_e2m1_to_float(byte & 15), _e2m1_to_float(byte >> 4)), (BLOCK_R, BLOCK_C // 8, 16)+ )+ values = tl.reshape(values * group_scale[:, :, None], (BLOCK_R, 2 * BLOCK_C))+ oc = tl.program_id(1) * (2 * BLOCK_C) + tl.arange(0, 2 * BLOCK_C)+ tl.store(+ out_ptr + r[:, None] * (2 * half_cols) + oc[None, :],+ values.to(out_ptr.dtype.element_ty),+ mask = row_ok & (oc[None, :] < 2 * half_cols),+ )+++@_opaque_under_compile(+ "nvfp4_dequantize_triton",+ lambda packed, scale, global_scale, dtype: packed.new_empty(+ (packed.shape[0], packed.shape[1] * 2), dtype = dtype+ ),+)+def _nvfp4_dequantize_triton(+ packed: torch.Tensor, scale: torch.Tensor, global_scale: torch.Tensor, dtype: torch.dtype+) -> torch.Tensor:+ rows, half = packed.shape+ out = torch.empty((rows, half * 2), dtype = dtype, device = packed.device)+ BLOCK_R, BLOCK_C = 8, 256+ e4m3 = scale.dtype == torch.float8_e4m3fn+ if e4m3:+ scale = scale.view(torch.uint8)+ with _fp8_triton_device_context(packed):+ _nvfp4_dequant_kernel[(triton.cdiv(rows, BLOCK_R), triton.cdiv(half, BLOCK_C))](+ packed,+ scale,+ global_scale,+ out,+ rows,+ half,+ scale.shape[1],+ BLOCK_R = BLOCK_R,+ BLOCK_C = BLOCK_C,+ SCALE_E4M3_BYTES = e4m3,+ # int32 offsets wrap once the output passes 2^31 elements; int64 only then, as it costs ~2%.+ INT64_OFFSETS = rows * half * 2 >= 2**31,+ num_warps = 4,+ )+ return out+++def nvfp4_dequantize(+ packed,+ scale,+ global_scale,+ dtype = torch.bfloat16,+):+ """Dense (out, in) weight of a packed (out, in / 2) NVFP4 tensor; a transposed view gives the transposed weight."""+ if _is_transposed_view(packed):+ return nvfp4_dequantize(packed.t(), scale, global_scale, dtype).t()+ if packed.shape[1] * 2 > scale.shape[1] * NVFP4_GROUP_SIZE:+ raise ValueError(+ f"Unsloth: NVFP4 weight {tuple(packed.shape)} needs {packed.shape[1] * 2 // NVFP4_GROUP_SIZE} "+ f"group scales per row, got {tuple(scale.shape)}."+ )+ if packed.device.type in ("cuda", "xpu"):+ return _nvfp4_dequantize_triton(+ packed.contiguous(), scale.contiguous(), global_scale.reshape(-1)[:1], dtype+ )+ return _nvfp4_dequantize_torch(packed, scale, global_scale, dtype)+++# Opaque to torch.compile as whole matmuls: if the dequantized weight were a graph value, AOTAutograd would save it for+# backward in every layer (3.6x the NVFP4 storage each), which is exactly the memory W4A16 exists to avoid.+@_opaque_under_compile(+ "nvfp4_matmul_fwd",+ lambda X, packed, scale, global_scale: X.new_empty(X.shape[:-1] + (packed.shape[0],)),+)+def _nvfp4_matmul_fwd(+ X: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, global_scale: torch.Tensor+) -> torch.Tensor:+ return torch.matmul(X, nvfp4_dequantize(packed, scale, global_scale, X.dtype).t())+++@_opaque_under_compile(+ "nvfp4_matmul_bwd",+ lambda grad, packed, scale, global_scale: grad.new_empty(+ grad.shape[:-1] + (packed.shape[1] * 2,)+ ),+)+def _nvfp4_matmul_bwd(+ grad: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, global_scale: torch.Tensor+) -> torch.Tensor:+ return torch.matmul(grad, nvfp4_dequantize(packed, scale, global_scale, grad.dtype))+++class NVFP4Linear_matmul(torch.autograd.Function):+ @staticmethod+ def forward(+ ctx,+ X,+ packed,+ scale,+ global_scale,+ bias = None,+ ):+ output = _nvfp4_matmul_fwd(X, packed, scale, global_scale)+ if bias is not None:+ output = output + bias+ # A float32 bias promotes; cast back only then (a no-op .to() aliases, zeroing dX under torch 2.11 compile).+ if output.dtype != X.dtype:+ output = output.to(X.dtype)+ ctx.save_for_backward(packed, scale, global_scale)+ ctx.bias_grad = bias is not None and bias.requires_grad+ ctx.dtype = X.dtype+ return output++ @staticmethod+ def backward(ctx, grad_output):+ packed, scale, global_scale = ctx.saved_tensors+ grad_X = None+ if ctx.needs_input_grad[0]:+ grad_X = _nvfp4_matmul_bwd(grad_output, packed, scale, global_scale)+ if grad_X.dtype != ctx.dtype:+ grad_X = grad_X.to(ctx.dtype)+ grad_bias = None+ if ctx.bias_grad:+ grad_bias = grad_output.reshape(-1, grad_output.shape[-1]).sum(0)+ return grad_X, None, None, None, grad_bias+++def nvfp4_linear(+ X,+ packed,+ scale,+ global_scale,+ bias = None,+):+ return NVFP4Linear_matmul.apply(X, packed, scale, global_scale, bias)diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 441bbc4c7fe..1c0d9578ea7 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -28,6 +28,7 @@ from ..device_type import ()from ..bnb_availability import native_kernels_readyfrom .fp8 import weight_dequant, fp8_linear, can_use_fp8_rowwise_gemv, fp8_rowwise_gemv+from .nvfp4 import NVFP4QuantState, nvfp4_dequantize, nvfp4_linearimport functools# torch.cuda.amp.custom_fwd is deprecated from 2.4.@@ -498,6 +499,10 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:return Wif W.dtype == torch.float8_e4m3fn:return weight_dequant(W, quant_state)+ if type(quant_state) is NVFP4QuantState:+ return nvfp4_dequantize(+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype+ )if type(quant_state) is not list:# New quant_state as a class, per TimDettmers/bitsandbytes#763.absmax = quant_state.absmax@@ -607,6 +612,10 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:return Wif W.dtype == torch.float8_e4m3fn:return weight_dequant(W, quant_state)+ if type(quant_state) is NVFP4QuantState:+ return nvfp4_dequantize(+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype+ )if type(quant_state) is not list:# New quant_state as a class, per TimDettmers/bitsandbytes#763.absmax = quant_state.absmax@@ -719,6 +728,10 @@ else:return Wif W.dtype == torch.float8_e4m3fn:return weight_dequant(W, quant_state)+ if type(quant_state) is NVFP4QuantState:+ return nvfp4_dequantize(+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype+ )if type(quant_state) is not list:# New quant_state as a class, per TimDettmers/bitsandbytes#763.# https://github.com/TimDettmers/bitsandbytes/pull/763/files@@ -1136,6 +1149,9 @@ def fast_linear_forward(if W_quant is None:out = torch_matmul(X, W.t(), out = out)+ elif type(W_quant) is NVFP4QuantState:+ # Bias is added once below.+ out = nvfp4_linear(X, W, W_quant.scale, W_quant.global_scale)elif _is_packed_state(W_quant):out = W_quant.matmul(X, W, out = out)elif W.dtype == torch.float8_e4m3fn:diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 58cdf31cc0e..1606805b291 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -1490,19 +1490,33 @@ def _zoo_peft_forward_keeps_fp8_inputs():)-def _route_compressed_tensors_fp8_to_unsloth(model):- # Opt-in: saves memory, but on-the-fly dequant trains slower than the decompressed bf16 model.- if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0") != "1":+def _route_compressed_tensors_fp8_to_unsloth(+ model,+ skip = (),+ default = "0",+):+ # Opt-in for FP8 checkpoints (on-the-fly dequant trains slower than bf16); default beside NVFP4, where the+ # decompressed FP8 layers would keep compressed-tensors' graph-breaking fake-quant forward.+ if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", default) != "1":return 0if getattr(getattr(model, "config", None), "quantization_config", None) is None:return 0if not _zoo_peft_forward_keeps_fp8_inputs():return 0- routable = []+ # The fused CE loss multiplies lm_head.weight directly, so a quantized lm_head is decompressed, never routed.+ get_output_embeddings = getattr(model, "get_output_embeddings", None)+ lm_head = get_output_embeddings() if callable(get_output_embeddings) else None+ routable, decompress = [], []for module in model.modules():+ # Modules another route already owns (NVFP4) do not count against the all-or-nothing check.+ if module in skip:+ continuescheme = getattr(module, "quantization_scheme", None)if scheme is None or getattr(scheme, "weights", None) is None:continue+ if module is lm_head:+ decompress.append(module)+ continueblock = Noneweight = getattr(module, "weight", None)on_gpu = isinstance(weight, torch.Tensor) and weight.device.type in ("cuda", "xpu")@@ -1515,7 +1529,18 @@ def _route_compressed_tensors_fp8_to_unsloth(model):if block is None:return 0routable.append((module, block))+ if not routable:+ return 0+ if decompress:+ try:+ with torch.inference_mode(False), torch.no_grad():+ for module in decompress:+ _decompress_like_a_full_load(module)+ _remove_same_device_compressed_tensors_offload(module)+ except Exception:+ return 0converted = 0+ compute_dtype = _fp8_dequant_target_dtype(model)for module, block in routable:scale = module.weight_scalescale_shape = tuple(scale.shape)@@ -1537,20 +1562,381 @@ def _route_compressed_tensors_fp8_to_unsloth(model):module.block_size = blockmodule.forward = _unsloth_compressed_tensors_fp8_forward.__get__(module)module._unsloth_compressed_tensors_fp8 = True+ module._unsloth_ct_compute_dtype = compute_dtypeconverted += 1if converted:- for owner in (model, getattr(model, "model", None), getattr(model, "base_model", None)):- hook = getattr(owner, "ct_decompress_hook", None)- if hook is not None:- hook.remove()- try:- delattr(owner, "ct_decompress_hook")- except AttributeError:- pass+ _remove_compressed_tensors_decompress_hook(model)+ _patch_peft_for_routed_compressed_tensors()model._unsloth_compressed_tensors_fp8 = convertedreturn converted+def _remove_compressed_tensors_decompress_hook(model):+ for owner in (model, getattr(model, "model", None), getattr(model, "base_model", None)):+ hook = getattr(owner, "ct_decompress_hook", None)+ if hook is not None:+ hook.remove()+ try:+ delattr(owner, "ct_decompress_hook")+ except AttributeError:+ pass+++def _nvfp4_scheme_supported(scheme):+ weights = getattr(scheme, "weights", None)+ if weights is None or getattr(scheme, "output_activations", None) is not None:+ return False+ enum = lambda v: str(getattr(v, "value", v)) if v is not None else None+ fmt = enum(getattr(scheme, "format", None))+ return (+ str(enum(getattr(weights, "type", None))) == "float"+ and getattr(weights, "num_bits", None) == 4+ and getattr(weights, "group_size", None) == 16+ and enum(getattr(weights, "strategy", None)) == "tensor_group"+ and not getattr(weights, "dynamic", False)+ # "static" is an alias of "weight": calibration order only, columns stored in place (no g_idx).+ and enum(getattr(weights, "actorder", None)) in (None, "weight", "static")+ and fmt in (None, "nvfp4-pack-quantized")+ )+++def _nvfp4_module_routable(module):+ packed = getattr(module, "weight_packed", None)+ scale = getattr(module, "weight_scale", None)+ global_scale = getattr(module, "weight_global_scale", None)+ return (+ isinstance(module, torch.nn.Linear)+ and isinstance(packed, torch.Tensor)+ and packed.dtype == torch.uint8+ and packed.dim() == 2+ and packed.device.type in ("cuda", "xpu")+ and isinstance(scale, torch.Tensor)+ and scale.dtype == torch.float8_e4m3fn+ and tuple(scale.shape) == (packed.shape[0], packed.shape[1] * 2 // 16)+ and isinstance(global_scale, torch.Tensor)+ and global_scale.numel() == 1+ and getattr(module, "weight_zero_point", None) is None+ and "weight" not in module._parameters+ and type(module) is torch.nn.Linear+ )+++def _unsloth_compressed_tensors_nvfp4_forward(self, input):+ from unsloth.kernels.nvfp4 import nvfp4_linear++ # W4A16: the checkpoint's NVFP4 activation quantization is not applied.+ out = nvfp4_linear(input, self.weight_packed, self.weight_scale, self.weight_global_scale)+ if self.bias is not None:+ out = out + self.bias.to(out.dtype)+ return out+++class _UnslothNVFP4Linear(torch.nn.Linear):+ """nn.Linear whose `weight` is the packed NVFP4 tensor carrying an NVFP4QuantState.++ PEFT and get_lora_parameters read `.weight`; the state_dict keeps only weight_packed, and the uint8 dtype keeps+ compiled LoRA forwards from casting activations to it. Built on each access so `.to()` / device moves stay valid.+ """++ @property+ def weight(self):+ from unsloth.kernels.nvfp4 import NVFP4QuantState++ packed = self.weight_packed+ packed.quant_state = NVFP4QuantState(+ self.weight_scale,+ self.weight_global_scale,+ (packed.shape[0], packed.shape[1] * 2),+ getattr(self, "_unsloth_nvfp4_dtype", torch.bfloat16),+ )+ return packed++ def _apply(+ self,+ fn,+ recurse = True,+ ):+ # model.to(dtype) / .half() may move the packed weight and its scales but never cast them: an fp16 global+ # scale overflows past 65504 and dequantizes the layer to zeros.+ quant = {}+ for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):+ param = self._parameters.get(name)+ if param is not None:+ quant[name] = self._parameters.pop(name)+ try:+ super()._apply(fn, recurse)+ finally:+ for name, param in quant.items():+ device = fn(param.data[:0]).device+ if param.device != device:+ param = torch.nn.Parameter(param.data.to(device), requires_grad = False)+ self._parameters[name] = param+ # The compute dtype follows the cast (model.half() dequantizes to fp16 from then on).+ dtype = getattr(self, "_unsloth_nvfp4_dtype", torch.bfloat16)+ self._unsloth_nvfp4_dtype = fn(torch.empty(0, dtype = dtype)).dtype+ return self++ def dequantize_(self):+ """Turn this layer into a plain nn.Linear holding the dense weight (PEFT merges add into `.weight`)."""+ self.__dict__.pop("forward", None)+ self.__class__ = torch.nn.Linear+ self._unsloth_compressed_tensors_nvfp4 = False+ _decompress_like_a_full_load(self)+ return self+++_CT_QUANT_PARAMS = (+ "weight_scale",+ "weight_global_scale",+ "weight_zero_point",+ "input_global_scale",+ "input_scale",+)+++def _decompress_like_a_full_load(module):+ """compressed-tensors' own per-module decompress, then a plain dense layer. The config still says compressed+ for the rest, so the layer joins its ignore list: save_pretrained then writes a checkpoint that reloads."""+ try:+ from compressed_tensors.compressors import decompress_module+ except ImportError:+ decompress_module = None # compressed-tensors < 0.13+ with torch.inference_mode(False), torch.no_grad():+ if decompress_module is not None:+ decompress_module(module)+ else:+ _dequantize_in_place_without_compressed_tensors(module)+ for name in _CT_QUANT_PARAMS:+ module._parameters.pop(name, None)+ for attr in ("quantization_scheme", "quantization_status"):+ module.__dict__.pop(attr, None)+ for p in module.parameters(recurse = False):+ p.requires_grad_(False)+ ct_config = getattr(getattr(module, "_unsloth_ct_config", None), "quantization_config", None)+ name = getattr(module, "_unsloth_ct_name", None)+ if ct_config is None or name is None:+ return+ if getattr(ct_config, "ignore", None) is None:+ ct_config.ignore = []+ if name not in ct_config.ignore:+ ct_config.ignore.append(name)+++def _dequantize_in_place_without_compressed_tensors(module):+ """Same dense weight from Unsloth's kernels (bit-identical to compressed-tensors for NVFP4)."""+ dtype = getattr(module, "_unsloth_nvfp4_dtype", None) or getattr(+ module, "_unsloth_ct_compute_dtype", torch.bfloat16+ )+ if isinstance(getattr(module, "weight_packed", None), torch.Tensor):+ from unsloth.kernels.nvfp4 import nvfp4_dequantize+ W = nvfp4_dequantize(+ module.weight_packed, module.weight_scale, module.weight_global_scale, dtype+ )+ module._parameters.pop("weight_packed", None)+ else:+ from unsloth.kernels.fp8 import weight_dequant+ W = weight_dequant(module.weight, module.weight_scale, dtype)+ module.weight = torch.nn.Parameter(W, requires_grad = False)+++def _tag_compressed_tensors_modules(model):+ config = getattr(getattr(model, "config", None), "quantization_config", None)+ for name, module in model.named_modules():+ if getattr(module, "quantization_scheme", None) is not None:+ module._unsloth_ct_name = name+ module._unsloth_ct_config = config+++def _routed_dense_weight(module):+ """Dense weight of an NVFP4- or FP8-routed linear, else None."""+ if isinstance(module, _UnslothNVFP4Linear):+ from unsloth.kernels.nvfp4 import nvfp4_dequantize+ return nvfp4_dequantize(+ module.weight_packed,+ module.weight_scale,+ module.weight_global_scale,+ getattr(module, "_unsloth_nvfp4_dtype", torch.bfloat16),+ )+ if getattr(module, "_unsloth_compressed_tensors_fp8", False):+ from unsloth.kernels.fp8 import weight_dequant+ return weight_dequant(+ module.weight,+ module.weight_scale,+ getattr(module, "_unsloth_ct_compute_dtype", torch.bfloat16),+ )+ return None+++def _dequantize_routed_linear_(module):+ """Turn a routed linear into a plain nn.Linear with its dense weight, for PEFT merges."""+ if isinstance(module, _UnslothNVFP4Linear):+ return module.dequantize_()+ if getattr(module, "_unsloth_compressed_tensors_fp8", False):+ module.__dict__.pop("forward", None)+ module._unsloth_compressed_tensors_fp8 = False+ _decompress_like_a_full_load(module)+ return module+++def _patch_peft_for_routed_compressed_tensors():+ """PEFT merges add a dense delta into base_layer.weight and DoRA reads it: give both the dense weight."""+ try:+ from peft.tuners.lora import layer as lora_layer+ except Exception:+ return+ merge = lora_layer.Linear.merge+ if not getattr(merge, "_unsloth_routed", False):++ @functools.wraps(merge)+ def patched_merge(self, *args, **kwargs):+ _dequantize_routed_linear_(self.get_base_layer())+ return merge(self, *args, **kwargs)++ patched_merge._unsloth_routed = True+ lora_layer.Linear.merge = patched_merge+ # PiSSA / OLoRA / CorDA / LoftQ / orthogonal / LoRA-GA read or rewrite the dense base weight (or its dtype).+ for init in (+ "pissa_init",+ "olora_init",+ "corda_init",+ "loftq_init",+ "orthogonal_init",+ "lora_ga_init",+ ):+ original = getattr(lora_layer.LoraLayer, init, None)+ if original is None or getattr(original, "_unsloth_routed", False):+ continue++ def patched_init(+ self,+ *args,+ _original = original,+ **kwargs,+ ):+ _dequantize_routed_linear_(self.get_base_layer())+ return _original(self, *args, **kwargs)++ patched_init = functools.wraps(original)(patched_init)+ patched_init._unsloth_routed = True+ setattr(lora_layer.LoraLayer, init, patched_init)+ # LoRA-GA / CorDA preprocessing reads each target's weight (or enables its grad) before get_peft_model.+ for module_name, attr in (("loraga", "get_target_modules"), ("corda", "target_modules")):+ try:+ owner = importlib.import_module("peft.tuners.lora." + module_name)+ except Exception:+ continue+ targets = getattr(owner, attr, None)+ if targets is None or getattr(targets, "_unsloth_routed", False):+ continue++ def patched_targets(+ *args,+ _targets = targets,+ **kwargs,+ ):+ for name, module in _targets(*args, **kwargs):+ yield name, _dequantize_routed_linear_(module)++ patched_targets = functools.wraps(targets)(patched_targets)+ patched_targets._unsloth_routed = True+ setattr(owner, attr, patched_targets)+ try:+ from peft.tuners.lora import dora+ except Exception:+ return+ dequantize = getattr(dora, "dequantize_module_weight", None)+ if dequantize is not None and not getattr(dequantize, "_unsloth_routed", False):++ @functools.wraps(dequantize)+ def patched_dequantize(module):+ W = _routed_dense_weight(module)+ return dequantize(module) if W is None else W++ patched_dequantize._unsloth_routed = True+ dora.dequantize_module_weight = patched_dequantize+ update_layer = dora.DoraLinearLayer.update_layer+ if not getattr(update_layer, "_unsloth_routed", False):++ @functools.wraps(update_layer)+ def patched_update_layer(self, *, base_layer, lora_A, lora_B, **kwargs):+ # PEFT has cast new adapters to an FP8 base's float8 dtype; DoRA multiplies them right away.+ for p in (lora_A, lora_B):+ if isinstance(p, torch.Tensor) and p.dtype in _FP8_DTYPES:+ p.data = p.data.to(torch.float32)+ return update_layer(self, base_layer = base_layer, lora_A = lora_A, lora_B = lora_B, **kwargs)++ patched_update_layer._unsloth_routed = True+ dora.DoraLinearLayer.update_layer = patched_update_layer+++def _route_compressed_tensors_nvfp4_to_unsloth(model):+ """Keep NVFP4 (nvfp4-pack-quantized) weights packed and run them W4A16; decompress or FP8-route the rest."""+ if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_NVFP4_KERNELS", "1") == "0":+ return 0+ if getattr(getattr(model, "config", None), "quantization_config", None) is None:+ return 0+ try:+ from compressed_tensors.compressors import decompress_module+ except Exception:+ decompress_module = None+ # The fused CE loss and decode read lm_head.weight directly, so an NVFP4 lm_head is decompressed too.+ get_output_embeddings = getattr(model, "get_output_embeddings", None)+ lm_head = get_output_embeddings() if callable(get_output_embeddings) else None+ nvfp4, others = [], []+ for module in model.modules():+ scheme = getattr(module, "quantization_scheme", None)+ if scheme is None or getattr(scheme, "weights", None) is None:+ continue+ if str(getattr(getattr(module, "quantization_status", None), "value", "")) != "compressed":+ continue+ if (+ module is not lm_head+ and _nvfp4_scheme_supported(scheme)+ and _nvfp4_module_routable(module)+ ):+ nvfp4.append(module)+ else:+ others.append(module)+ if not nvfp4:+ return 0+ if others and decompress_module is None:+ print(+ "Unsloth: this compressed-tensors has no per-module decompress; decompressing the NVFP4 layers too."+ )+ return 0+ compute_dtype = _fp8_dequant_target_dtype(model)+ # The FP8 kernels dequantize to bf16, so an fp16 model keeps its FP8 layers decompressed by default.+ fp8_routed = (+ _route_compressed_tensors_fp8_to_unsloth(+ model, skip = set(nvfp4), default = "1" if compute_dtype == torch.bfloat16 else "0"+ )+ if others+ else 0+ )+ if others and not fp8_routed:+ try:+ with torch.inference_mode(False), torch.no_grad():+ for module in others:+ _decompress_like_a_full_load(module)+ _remove_same_device_compressed_tensors_offload(module)+ except Exception as e:+ print(f"Unsloth: could not decompress the non-NVFP4 compressed-tensors layers ({e}).")+ return 0+ for module in nvfp4:+ for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):+ param = getattr(module, name, None)+ if isinstance(param, torch.Tensor):+ param.requires_grad_(False)+ module.__class__ = _UnslothNVFP4Linear+ module._unsloth_nvfp4_dtype = compute_dtype+ module.forward = _unsloth_compressed_tensors_nvfp4_forward.__get__(module)+ module._unsloth_compressed_tensors_nvfp4 = True+ _remove_compressed_tensors_decompress_hook(model)+ _patch_peft_for_routed_compressed_tensors()+ model._unsloth_compressed_tensors_nvfp4 = len(nvfp4)+ return len(nvfp4)++_FP8_DEQUANT_DTYPES = (torch.bfloat16, torch.float16, torch.float32)@@ -1936,7 +2322,9 @@ def _remove_same_device_compressed_tensors_offload(model):def _decompress_compressed_tensors_model(model):"""Decompress at load: PEFT skips the first-forward hook, and in `generate` it yields inference tensors."""- if getattr(model, "_unsloth_compressed_tensors_fp8", 0):+ if getattr(model, "_unsloth_compressed_tensors_fp8", 0) or getattr(+ model, "_unsloth_compressed_tensors_nvfp4", 0+ ):return Falsequant_config = getattr(getattr(model, "config", None), "quantization_config", None)if isinstance(quant_config, dict):@@ -1984,8 +2372,14 @@ def _decompress_compressed_tensors_model(model):def _prepare_compressed_tensors_model(model, full_finetuning = False):- # Routed FP8 weights are frozen, so full finetuning always takes the decompressed bf16 weights.- if full_finetuning or not _route_compressed_tensors_fp8_to_unsloth(model):+ # Routed FP8 / NVFP4 weights are frozen, so full finetuning always takes the decompressed bf16 weights.+ if full_finetuning:+ _decompress_compressed_tensors_model(model)+ return+ _tag_compressed_tensors_modules(model)+ if _route_compressed_tensors_nvfp4_to_unsloth(model):+ return+ if not _route_compressed_tensors_fp8_to_unsloth(model):_decompress_compressed_tensors_model(model)