Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Train LoRA on NVFP4 compressed-tensors checkpoints with the weights kept packed (W4A16) (#12185)

Daniel Han3cffce234b76
diff --git a/tests/_nvfp4_fixtures.py b/tests/_nvfp4_fixtures.py
new file mode 100644
index 00000000000..779e2c5a1ad
--- /dev/null
+++ b/tests/_nvfp4_fixtures.py
@@ -0,0 +1,374 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""Tiny compressed-tensors NVFP4 + FP8 mixed-precision checkpoints, laid out like unsloth/Qwen3.8-27B-NVFP4.
+
+MLPs are NVFP4 (weight_packed U8, weight_scale F8_E4M3 per 16 columns, weight_global_scale and input_global_scale F32[1]),
+attention and the last layer's MLP are FP8 per channel (weight F8_E4M3, weight_scale BF16 [out, 1]). Built without
+llmcompressor: scales are computed here and the tensors come from compressed-tensors' own compressors.
+
+ python tests/_nvfp4_fixtures.py OUT_DIR [--arch qwen3|qwen3_5]
+"""
+
+import argparse
+import copy
+import json
+import os
+import re
+import sys
+
+import torch
+
+QWEN3 = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+QWEN3_5 = "trl-internal-testing/tiny-Qwen3_5ForConditionalGeneration-NoThink"
+
+_FP8_ACT = {
+ "actorder": None,
+ "block_structure": None,
+ "dynamic": True,
+ "group_size": None,
+ "num_bits": 8,
+ "observer": None,
+ "observer_kwargs": {},
+ "scale_dtype": None,
+ "strategy": "token",
+ "symmetric": True,
+ "type": "float",
+ "zp_dtype": None,
+}
+_FP8_W = {
+ "actorder": None,
+ "block_structure": None,
+ "dynamic": False,
+ "group_size": None,
+ "num_bits": 8,
+ "observer": "memoryless_minmax",
+ "observer_kwargs": {},
+ "scale_dtype": None,
+ "strategy": "channel",
+ "symmetric": True,
+ "type": "float",
+ "zp_dtype": None,
+}
+_FP4 = {
+ "actorder": None,
+ "block_structure": None,
+ "dynamic": False,
+ "group_size": 16,
+ "num_bits": 4,
+ "observer_kwargs": {},
+ "scale_dtype": "torch.float8_e4m3fn",
+ "strategy": "tensor_group",
+ "symmetric": True,
+ "type": "float",
+ "zp_dtype": None,
+}
+
+
+def _quant_config(
+ num_layers,
+ lm_head,
+ attn_targets,
+ ignore,
+ lm_head_kind = "fp8",
+):
+ last = str(num_layers - 1)
+ fp8_targets = list(attn_targets) + [r"re:.*layers\.(" + last + r")\.mlp\.(gate|up|down)_proj$"]
+ fp4_targets = [r"re:.*mlp\.(gate|up|down)_proj$"]
+ if lm_head:
+ (fp8_targets if lm_head_kind == "fp8" else fp4_targets).append("re:.*lm_head")
+ fp4_act = dict(_FP4, dynamic = "local", observer = "static_minmax")
+ return {
+ "config_groups": {
+ "group_0": {
+ "format": "float-quantized",
+ "input_activations": _FP8_ACT,
+ "output_activations": None,
+ "targets": fp8_targets,
+ "weights": _FP8_W,
+ },
+ "group_1": {
+ "format": "nvfp4-pack-quantized",
+ "input_activations": fp4_act,
+ "output_activations": None,
+ "targets": fp4_targets,
+ "weights": dict(_FP4, actorder = "static"),
+ },
+ },
+ "format": "mixed-precision",
+ "global_compression_ratio": None,
+ "ignore": list(ignore),
+ "kv_cache_scheme": None,
+ "quant_method": "compressed-tensors",
+ "quantization_status": "compressed",
+ "sparsity_config": {},
+ "transform_config": {},
+ "version": "0.17.0",
+ }
+
+
+def _matches(name, targets):
+ return any(re.match(t[3:], name) if t.startswith("re:") else name == t for t in targets)
+
+
+def nvfp4_tensors(w):
+ """weight_packed, weight_scale (fp8), weight_global_scale for w (out, in), via compressed-tensors' compressor."""
+ from compressed_tensors.compressors import NVFP4PackedCompressor
+ from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
+
+ w = w.float()
+ out_f, in_f = w.shape
+ assert in_f % 16 == 0, in_f
+ global_scale = (448.0 * 6.0 / w.abs().amax().clamp(min = 1e-12)).reshape(1).float()
+ group_amax = w.reshape(out_f, in_f // 16, 16).abs().amax(-1)
+ # Rounded to fp8 values but kept float: the compressor divides by the global scale, then stores fp8.
+ scale = (global_scale * (group_amax / 6.0)).clamp(max = 448.0).to(torch.float8_e4m3fn).float()
+ scheme = QuantizationScheme(
+ targets = ["Linear"],
+ weights = QuantizationArgs(
+ **{k: v for k, v in _FP4.items() if v is not None and k not in ("observer_kwargs",)}
+ ),
+ )
+ out = NVFP4PackedCompressor.compress(
+ {"weight": w, "weight_scale": scale, "weight_global_scale": global_scale}, scheme
+ )
+ return out["weight_packed"], out["weight_scale"], global_scale
+
+
+def fp8_channel_tensors(w, scale_dtype = torch.bfloat16):
+ w = w.float()
+ scale = (w.abs().amax(1, keepdim = True) / 448.0).clamp(min = 1e-12)
+ q = (w / scale).clamp(-448, 448).to(torch.float8_e4m3fn)
+ return q, scale.to(scale_dtype)
+
+
+def dequant_nvfp4_reference(
+ packed,
+ scale,
+ global_scale,
+ dtype = torch.bfloat16,
+):
+ from compressed_tensors.compressors import NVFP4PackedCompressor
+ from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
+
+ scheme = QuantizationScheme(
+ targets = ["Linear"],
+ weights = QuantizationArgs(
+ **{k: v for k, v in _FP4.items() if v is not None and k not in ("observer_kwargs",)}
+ ),
+ )
+ out = NVFP4PackedCompressor.decompress(
+ {"weight_packed": packed, "weight_scale": scale, "weight_global_scale": global_scale},
+ scheme,
+ )
+ return out["weight"].to(dtype)
+
+
+def build(
+ out_dir,
+ arch = "qwen3",
+ seed = 3407,
+ lm_head_kind = "fp8",
+):
+ """Write a tiny mixed NVFP4/FP8 checkpoint to out_dir; returns {name: kind} for every quantized Linear."""
+ from safetensors.torch import save_file
+ from transformers import (
+ AutoConfig,
+ AutoModelForCausalLM,
+ AutoModelForImageTextToText,
+ AutoTokenizer,
+ )
+
+ torch.manual_seed(seed)
+ if arch == "qwen3":
+ repo = QWEN3
+ config = AutoConfig.from_pretrained(repo)
+ # hidden 8 cannot hold 16-column NVFP4 groups.
+ config.update(
+ dict(
+ hidden_size = 64,
+ intermediate_size = 128,
+ head_dim = 16,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ num_hidden_layers = 2,
+ )
+ )
+ config.torch_dtype = torch.bfloat16
+ model = AutoModelForCausalLM.from_config(config, dtype = torch.bfloat16)
+ attn = [r"re:.*self_attn\.(q|k|v|o)_proj$"]
+ ignore = ["model.layers.0.self_attn.o_proj"]
+ lm_head = True
+ elif arch == "qwen3_5":
+ repo = QWEN3_5
+ config = AutoConfig.from_pretrained(repo)
+ config.torch_dtype = torch.bfloat16
+ model = AutoModelForImageTextToText.from_config(config, dtype = torch.bfloat16)
+ attn = [
+ r"re:.*self_attn\.(q|k|v|o)_proj$",
+ r"re:.*linear_attn\.(in_proj_qkv|in_proj_z|out_proj)$",
+ ]
+ ignore = [
+ n
+ for n, m in model.named_modules()
+ if isinstance(m, torch.nn.Linear)
+ and (".visual." in f".{n}" or n.endswith(("in_proj_a", "in_proj_b")))
+ ]
+ lm_head = False # tied to the embeddings
+ else:
+ raise ValueError(arch)
+
+ num_layers = config.get_text_config().num_hidden_layers
+ qcfg = _quant_config(num_layers, lm_head, attn, ignore, lm_head_kind)
+ fp8_targets = qcfg["config_groups"]["group_0"]["targets"]
+ fp4_targets = qcfg["config_groups"]["group_1"]["targets"]
+
+ state = {k: v.detach().clone().contiguous() for k, v in model.state_dict().items()}
+ kinds = {}
+ for name, module in model.named_modules():
+ if not isinstance(module, torch.nn.Linear) or name in ignore:
+ continue
+ w = state.pop(name + ".weight", None)
+ if w is None:
+ continue
+ # Same precedence as the real checkpoint: the FP8 override wins over the broad MLP NVFP4 target.
+ if _matches(name, fp8_targets):
+ q, s = fp8_channel_tensors(w)
+ state[name + ".weight"] = q
+ state[name + ".weight_scale"] = s
+ kinds[name] = "fp8"
+ elif _matches(name, fp4_targets):
+ packed, scale, gscale = nvfp4_tensors(w)
+ state[name + ".weight_packed"] = packed
+ state[name + ".weight_scale"] = scale
+ state[name + ".weight_global_scale"] = gscale
+ state[name + ".input_global_scale"] = torch.tensor(
+ [448.0 * 6.0 / 4.0], dtype = torch.float32
+ )
+ kinds[name] = "nvfp4"
+ else:
+ state[name + ".weight"] = w
+ if getattr(config, "tie_word_embeddings", False) or getattr(
+ config.get_text_config(), "tie_word_embeddings", False
+ ):
+ state = {k: v for k, v in state.items() if not k.endswith("lm_head.weight")}
+
+ os.makedirs(out_dir, exist_ok = True)
+ save_file(state, os.path.join(out_dir, "model.safetensors"), metadata = {"format": "pt"})
+ cfg = copy.deepcopy(config.to_dict())
+ cfg["quantization_config"] = qcfg
+ cfg["torch_dtype"] = "bfloat16"
+ with open(os.path.join(out_dir, "config.json"), "w", encoding = "utf-8") as f:
+ json.dump(cfg, f, indent = 2)
+ AutoTokenizer.from_pretrained(repo).save_pretrained(out_dir)
+ try:
+ from transformers import AutoProcessor
+ AutoProcessor.from_pretrained(repo).save_pretrained(out_dir)
+ except Exception:
+ pass
+ with open(os.path.join(out_dir, "nvfp4_fixture_kinds.json"), "w", encoding = "utf-8") as f:
+ json.dump(kinds, f, indent = 1)
+ return kinds
+
+
+def run_lora_case(path, arch, api, out_dir):
+ """Load with Fast{LanguageModel,Model}, LoRA, 5 steps, save, reload; returns a JSON-able report. Run in a fresh process."""
+ dtype = getattr(torch, os.environ.get("NVFP4_CASE_DTYPE", "bfloat16"))
+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")
+ import unsloth
+ import unsloth.kernels.nvfp4 as nv
+ from peft import PeftModel
+
+ with open(os.path.join(path, "nvfp4_fixture_kinds.json"), encoding = "utf-8") as f:
+ kinds = json.load(f)
+ name = next(n for n, k in kinds.items() if k == "nvfp4")
+ seen = []
+ real = nv.nvfp4_linear
+
+ def recording(X, *args, **kwargs):
+ seen.append(str(X.dtype))
+ return real(X, *args, **kwargs)
+
+ nv.nvfp4_linear = recording
+ Fast = getattr(unsloth, api)
+ model, _ = Fast.from_pretrained(path, max_seq_length = 64, load_in_4bit = False, dtype = dtype)
+ model = Fast.get_peft_model(
+ model,
+ r = 8,
+ lora_alpha = 16,
+ lora_dropout = 0,
+ bias = "none",
+ random_state = 3407,
+ target_modules = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ],
+ )
+ base = next(
+ m
+ for n, m in model.named_modules()
+ if n.endswith((name, name + ".base_layer")) and hasattr(m, "weight_packed")
+ )
+ packed = base.weight_packed.clone()
+ lora_b = lambda: next(p for n, p in model.named_parameters() if "lora_B" in n).detach().clone()
+ b0 = lora_b()
+ params = [p for p in model.parameters() if p.requires_grad]
+ opt = torch.optim.AdamW(params, lr = 1e-3)
+ torch.manual_seed(0)
+ ids = torch.randint(0, 1000, (2, 32), device = "cuda")
+ losses = []
+ for _ in range(5):
+ loss = model(input_ids = ids, labels = ids).loss
+ loss.backward()
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ losses.append(loss.item())
+ model.eval()
+
+ # The first eval calls recompile after training and can run eager meanwhile; compiled and eager bf16 LoRA
+ # rounding differ by ~0.3%, so compare settled forwards (logits are bit-identical with TORCHDYNAMO_DISABLE=1).
+ def settled(m):
+ with torch.no_grad():
+ for _ in range(3):
+ m(input_ids = ids)
+ return m(input_ids = ids).logits.float()
+
+ want = settled(model)
+ model.save_pretrained(os.path.join(out_dir, "lora"))
+ fresh, _ = Fast.from_pretrained(path, max_seq_length = 64, load_in_4bit = False, dtype = dtype)
+ fresh = PeftModel.from_pretrained(fresh, os.path.join(out_dir, "lora"), torch_device = "cuda")
+ fresh.eval()
+ got = settled(fresh)
+ adapters = lambda m: {
+ k.replace(".default", ""): v for k, v in m.state_dict().items() if "lora_" in k
+ }
+ trained, reloaded = adapters(model), adapters(fresh)
+ return {
+ "losses": losses,
+ "input_dtypes": sorted(set(seen)),
+ "packed_unchanged": torch.equal(base.weight_packed, packed),
+ "packed_dtype": str(base.weight_packed.dtype),
+ "lora_changed": not torch.equal(lora_b(), b0),
+ "saved": sorted(os.listdir(os.path.join(out_dir, "lora"))),
+ "reload_max_abs": (got - want).abs().max().item(),
+ "adapters_equal": trained.keys() == reloaded.keys()
+ and all(torch.equal(trained[k], reloaded[k]) for k in trained),
+ "peak_gb": torch.cuda.max_memory_allocated() / 2**30,
+ }
+
+
+if __name__ == "__main__":
+ if len(sys.argv) > 1 and sys.argv[1] == "--lora-case":
+ report = run_lora_case(*sys.argv[2:6])
+ print("LORA_CASE " + json.dumps(report))
+ sys.exit(0)
+ ap = argparse.ArgumentParser()
+ ap.add_argument("out_dir")
+ ap.add_argument("--arch", default = "qwen3", choices = ["qwen3", "qwen3_5"])
+ a = ap.parse_args()
+ k = build(a.out_dir, a.arch)
+ print({v: sum(1 for x in k.values() if x == v) for v in set(k.values())})
diff --git a/tests/test_nvfp4_kernel.py b/tests/test_nvfp4_kernel.py
new file mode 100644
index 00000000000..f483bb5a4b2
--- /dev/null
+++ b/tests/test_nvfp4_kernel.py
@@ -0,0 +1,203 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Packed NVFP4 (compressed-tensors nvfp4-pack-quantized) must dequantize exactly as compressed-tensors does and train as W4A16.
+import os
+
+import pytest
+
+torch = pytest.importorskip("torch")
+ct_helpers = pytest.importorskip("compressed_tensors.compressors.nvfp4.helpers")
+ct_forward = pytest.importorskip("compressed_tensors.quantization.lifecycle.forward")
+
+CUDA = torch.cuda.is_available()
+needs_cuda = pytest.mark.skipif(not CUDA, reason = "needs CUDA")
+
+
[email protected](scope = "module")
+def N():
+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")
+ from unsloth.kernels import nvfp4
+ return nvfp4
+
+
+def _random_nvfp4(
+ rows,
+ cols,
+ device,
+ global_scale = 7.3,
+ seed = 0,
+):
+ g = torch.Generator(device = "cpu").manual_seed(seed)
+ packed = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, generator = g)
+ scale = (torch.rand(rows, cols // 16, generator = g) * 3 + 0.01).to(torch.float8_e4m3fn)
+ return packed.to(device), scale.to(device), torch.tensor([global_scale], device = device)
+
+
+def _reference(packed, scale, global_scale, dtype):
+ from compressed_tensors.quantization import QuantizationArgs
+
+ rows, half = packed.shape
+ x_q = ct_helpers.unpack_fp4_from_uint8(packed, rows, half * 2, dtype = dtype)
+ # The weight args NVFP4PackedCompressor.decompress passes; compressed-tensors >= 0.19 no longer infers them.
+ args = QuantizationArgs(num_bits = 4, type = "float", strategy = "tensor_group", group_size = 16)
+ return ct_forward.dequantize(
+ x_q = x_q, scale = scale.to(dtype), global_scale = global_scale, dtype = dtype, args = args
+ )
+
+
+def _devices():
+ return ["cpu", "cuda"] if CUDA else ["cpu"]
+
+
[email protected]("device", _devices())
+def test_every_code_and_sign(N, device):
+ # Byte b holds code b & 15 in the even column and b >> 4 in the odd one.
+ packed = torch.arange(256, dtype = torch.uint8).reshape(16, 16).to(device)
+ scale = torch.ones(16, 2, device = device).to(torch.float8_e4m3fn)
+ out = N.nvfp4_dequantize(packed, scale, torch.ones(1, device = device), torch.float32)
+ lut = [0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0]
+ codes = [(b & 15, b >> 4) for b in range(256)]
+ expect = torch.tensor(
+ [[(-1 if c & 8 else 1) * lut[c & 7] for c in pair] for pair in codes]
+ ).reshape(16, 32)
+ assert torch.equal(out.cpu(), expect)
+ assert torch.signbit(out.cpu()[0, 16]) # code 8 is -0.0, as in compressed-tensors
+
+
+@needs_cuda
+def test_every_e4m3_group_scale(N):
+ # Before sm_89 Triton has no fp8e4nv, so the kernel decodes the scale bytes itself: all 256 codes, incl.
+ # subnormals, -0.0 and NaN, must equal torch's own fp8 conversion.
+ codes = torch.arange(256, dtype = torch.uint8, device = "cuda").reshape(16, 16)
+ scale = codes.view(torch.float8_e4m3fn)
+ packed = torch.full((16, 128), 0x22, dtype = torch.uint8, device = "cuda") # every value 1.0
+ out = N.nvfp4_dequantize(packed, scale, torch.ones(1, device = "cuda"), torch.float32)
+ expect = scale.float().repeat_interleave(16, dim = 1)
+ assert torch.equal(out.isnan(), expect.isnan())
+ assert torch.equal(out.nan_to_num(), expect.nan_to_num())
+ finite = ~expect.isnan()
+ assert torch.equal(
+ out.signbit()[finite], expect.signbit()[finite]
+ ) # -0.0 (0x80) keeps its sign
+
+
[email protected]("device", _devices())
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("shape", [(64, 256), (48, 400), (257, 96)])
+def test_matches_compressed_tensors_bit_exact(N, device, dtype, shape):
+ packed, scale, gs = _random_nvfp4(*shape, device, global_scale = 0.37)
+ out = N.nvfp4_dequantize(packed, scale, gs, dtype)
+ assert out.dtype == dtype and out.shape == shape
+ assert torch.equal(out, _reference(packed, scale, gs, dtype))
+
+
+@needs_cuda
+def test_triton_equals_torch_reference(N):
+ packed, scale, gs = _random_nvfp4(320, 1024, "cuda")
+ tri = N.nvfp4_dequantize(packed, scale, gs)
+ ref = N._nvfp4_dequantize_torch(packed, scale, gs, torch.bfloat16)
+ assert torch.equal(tri, ref)
+
+
[email protected]("device", _devices())
+def test_transposed_view_is_the_transposed_weight(N, device):
+ packed, scale, gs = _random_nvfp4(96, 256, device)
+ dense = N.nvfp4_dequantize(packed, scale, gs)
+ assert torch.equal(N.nvfp4_dequantize(packed.t(), scale, gs), dense.t())
+
+
+def test_too_few_scales_is_an_error(N):
+ packed, scale, gs = _random_nvfp4(16, 64, "cpu")
+ with pytest.raises(ValueError):
+ N.nvfp4_dequantize(packed, scale[:, :2], gs)
+
+
+def _dense_reference_grads(W, X, dY, bias):
+ X = X.detach().clone().requires_grad_(True)
+ b = bias.detach().clone().requires_grad_(True) if bias is not None else None
+ out = X @ W.t()
+ if b is not None:
+ out = out + b
+ out.backward(dY)
+ return out.detach(), X.grad, None if b is None else b.grad
+
+
[email protected]("device", _devices())
[email protected]("with_bias", [False, True])
+def test_autograd_matches_the_dense_weight(N, device, with_bias):
+ dtype = torch.bfloat16 if device == "cuda" else torch.float32
+ packed, scale, gs = _random_nvfp4(192, 128, device)
+ W = N.nvfp4_dequantize(packed, scale, gs, dtype)
+ X = torch.randn(2, 5, 128, device = device, dtype = dtype, requires_grad = True)
+ dY = torch.randn(2, 5, 192, device = device, dtype = dtype)
+ bias = torch.randn(192, device = device, dtype = dtype, requires_grad = True) if with_bias else None
+ out = N.nvfp4_linear(X, packed, scale, gs, bias)
+ out.backward(dY)
+ ref_out, ref_dX, ref_db = _dense_reference_grads(W, X, dY, bias)
+ assert out.dtype == dtype
+ torch.testing.assert_close(out, ref_out)
+ torch.testing.assert_close(X.grad, ref_dX)
+ assert X.grad.abs().sum() > 0
+ if with_bias:
+ torch.testing.assert_close(bias.grad, ref_db)
+
+
+def test_float32_bias_keeps_the_activation_dtype(N):
+ packed, scale, gs = _random_nvfp4(32, 64, "cpu")
+ X = torch.randn(3, 64, dtype = torch.bfloat16)
+ out = N.nvfp4_linear(X, packed, scale, gs, torch.randn(32, dtype = torch.float32))
+ assert out.dtype == torch.bfloat16
+
+
+def test_backward_saves_only_the_packed_weight(N):
+ device = "cuda" if CUDA else "cpu"
+ packed, scale, gs = _random_nvfp4(256, 512, device)
+ X = torch.randn(4, 512, device = device, requires_grad = True)
+ saved = []
+
+ def pack_hook(t):
+ saved.append((tuple(t.shape), t.dtype))
+ return t
+
+ with torch.autograd.graph.saved_tensors_hooks(pack_hook, lambda t: t):
+ N.nvfp4_linear(X, packed, scale, gs)
+ dense = [s for s in saved if s[0] == (256, 512) or s[0] == (512, 256)]
+ assert dense == [], saved
+
+
+@needs_cuda
+def test_fullgraph_compile_has_no_breaks_and_matches_eager(N):
+ import torch._dynamo
+
+ packed, scale, gs = _random_nvfp4(384, 256, "cuda")
+ step = lambda X: N.nvfp4_linear(X, packed, scale, gs).float().square().mean()
+ # T4 (sm_75) trains in fp16: Inductor has no native bf16 there.
+ dtype = torch.bfloat16 if torch.cuda.get_device_capability() >= (8, 0) else torch.float16
+ X = torch.randn(64, 256, device = "cuda", dtype = dtype, requires_grad = True)
+ step(X).backward()
+ eager = X.grad.clone()
+ X.grad = None
+ torch._dynamo.reset()
+ assert torch._dynamo.explain(step)(X).graph_break_count == 0
+ torch._dynamo.reset()
+ compiled = torch.compile(step, fullgraph = True)
+ for rows in (16, 48, 64): # several shapes: dynamic-shape recompiles must stay correct
+ Xr = torch.randn(rows, 256, device = "cuda", dtype = dtype, requires_grad = True)
+ compiled(Xr).backward()
+ compiled(X).backward()
+ assert X.grad.abs().sum() > 0
+ torch.testing.assert_close(X.grad, eager)
+
+
+@needs_cuda
+def test_outputs_past_2_31_elements_use_64_bit_offsets(N):
+ # 131080 x 16384 outputs: int32 offsets of the last rows wrap negative.
+ rows, cols = 2**17 + 8, 16384
+ if torch.cuda.mem_get_info()[0] < 8 * 2**30:
+ pytest.skip("needs 8 GB free")
+ packed = torch.randint(0, 256, (rows, cols // 2), dtype = torch.uint8, device = "cuda")
+ scale = (torch.rand(rows, cols // 16, device = "cuda") + 0.5).to(torch.float8_e4m3fn)
+ gs = torch.tensor([2.0], device = "cuda")
+ W = N.nvfp4_dequantize(packed, scale, gs, torch.bfloat16)
+ tail = slice(rows - 16, rows)
+ want = N._nvfp4_dequantize_torch(packed[tail], scale[tail], gs, torch.bfloat16)
+ assert torch.equal(W[tail], want)
diff --git a/tests/test_nvfp4_loader.py b/tests/test_nvfp4_loader.py
new file mode 100644
index 00000000000..744678d6cd1
--- /dev/null
+++ b/tests/test_nvfp4_loader.py
@@ -0,0 +1,506 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+# NVFP4 compressed-tensors checkpoints (unsloth/Qwen3.8-27B-NVFP4 layout) stay packed and train W4A16 with LoRA.
+import os
+import sys
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("compressed_tensors")
+
+pytestmark = pytest.mark.skipif(
+ not torch.cuda.is_available(), reason = "NVFP4 routing needs CUDA weights"
+)
+
+sys.path.insert(0, os.path.dirname(__file__))
+import _nvfp4_fixtures as fx # noqa: E402
+
+
+class _Checkpoints(dict):
+ # Built on first use, so a transformers without qwen3_5 (4.57.x) still runs the qwen3 cases.
+ def __init__(self, tmp_path_factory):
+ super().__init__()
+ self.tmp_path_factory = tmp_path_factory
+
+ def __missing__(self, arch):
+ from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES
+
+ if arch not in CONFIG_MAPPING_NAMES:
+ pytest.skip(f"transformers has no {arch}")
+ d = self.tmp_path_factory.mktemp(arch)
+ self[arch] = (str(d), fx.build(str(d), arch))
+ return self[arch]
+
+
[email protected](scope = "module")
+def ckpt(tmp_path_factory):
+ return _Checkpoints(tmp_path_factory)
+
+
+def _load_raw(path, arch):
+ from transformers import AutoModelForCausalLM, AutoModelForImageTextToText
+ cls = AutoModelForCausalLM if arch == "qwen3" else AutoModelForImageTextToText
+ return cls.from_pretrained(path, device_map = "cuda", dtype = torch.bfloat16)
+
+
+def _module(model, name):
+ return dict(model.named_modules())[name]
+
+
[email protected]("arch", ["qwen3", "qwen3_5"])
+def test_fixture_decompresses_to_the_packed_weights(ckpt, arch):
+ path, kinds = ckpt[arch]
+ from safetensors import safe_open
+
+ f = safe_open(os.path.join(path, "model.safetensors"), "pt")
+ name = next(n for n, k in kinds.items() if k == "nvfp4")
+ packed, scale, gs = (
+ f.get_tensor(f"{name}.{s}")
+ for s in ("weight_packed", "weight_scale", "weight_global_scale")
+ )
+ from unsloth.kernels.nvfp4 import nvfp4_dequantize
+
+ ref = fx.dequant_nvfp4_reference(packed, scale, gs)
+ assert torch.equal(nvfp4_dequantize(packed.cuda(), scale.cuda(), gs.cuda()).cpu(), ref)
+ model = _load_raw(path, arch)
+ module = _module(model, name)
+ assert (
+ str(module.quantization_status.value) == "compressed"
+ and module.weight_packed.dtype == torch.uint8
+ )
+ # The broad MLP target overlaps the last layer's FP8 override; the override wins, as in the real checkpoint.
+ last = next(n for n, k in kinds.items() if k == "fp8" and ".mlp." in n)
+ assert _module(model, last).quantization_scheme.weights.num_bits == 8
+
+
[email protected]("arch", ["qwen3", "qwen3_5"])
+def test_nvfp4_layers_stay_packed_and_run_w4a16(ckpt, arch, monkeypatch):
+ from safetensors import safe_open
+ from unsloth.kernels.nvfp4 import nvfp4_dequantize
+ from unsloth.models import loader_utils
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ path, kinds = ckpt[arch]
+ model = _load_raw(path, arch)
+ loader_utils._prepare_compressed_tensors_model(model)
+ nvfp4 = [n for n, k in kinds.items() if k == "nvfp4"]
+ assert model._unsloth_compressed_tensors_nvfp4 == len(nvfp4)
+ assert getattr(model, "ct_decompress_hook", None) is None and not model._forward_pre_hooks
+ keys = set(safe_open(os.path.join(path, "model.safetensors"), "pt").keys())
+ # Same keys as the checkpoint, except that decompressed layers are dense and drop their scales
+ # (a tied lm_head may add its alias).
+ dense = [n for n, k in kinds.items() if k == "fp8"] + ["lm_head"]
+ keys -= {n + ".weight_scale" for n in dense}
+ assert set(model.state_dict()) - {"lm_head.weight"} == keys - {"lm_head.weight"}
+ assert not any(
+ n.endswith(".weight") and n[: -len(".weight")] in nvfp4 for n, _ in model.named_parameters()
+ )
+ for name, kind in kinds.items():
+ module = _module(model, name)
+ if kind == "fp8":
+ assert module.weight.dtype == torch.bfloat16 # decompressed per module
+ with torch.no_grad():
+ model(input_ids = torch.randint(0, 1000, (2, 8), device = "cuda"))
+ for name in nvfp4:
+ module = _module(model, name)
+ assert module.weight_packed.dtype == torch.uint8 and module.weight.dtype == torch.uint8
+ assert module.weight.quant_state.shape == (module.out_features, module.in_features)
+ x = torch.randn(3, module.in_features, device = "cuda", dtype = torch.bfloat16)
+ W = nvfp4_dequantize(
+ module.weight_packed, module.weight_scale, module.weight_global_scale, torch.bfloat16
+ )
+ with torch.no_grad():
+ # A16: exactly the dense matmul, no activation fake-quantization.
+ assert torch.equal(module(x), torch.nn.functional.linear(x, W))
+
+
+def test_dtype_casts_keep_the_nvfp4_storage_dtypes(ckpt, monkeypatch):
+ # model.to(fp16) must not cast the scales: an fp16 global scale overflows past 65504 and zeroes the layer.
+ from unsloth.models import loader_utils
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ module = _module(model, next(n for n, k in kinds.items() if k == "nvfp4"))
+ dtypes = {n: p.dtype for n, p in module.named_parameters()}
+ x = torch.randn(3, module.in_features, device = "cuda", dtype = torch.float16)
+ with torch.no_grad():
+ module.weight_global_scale.fill_(1e5)
+ want = module(x)
+ model.to(torch.float16)
+ assert {n: p.dtype for n, p in module.named_parameters()} == dtypes
+ assert torch.equal(module(x), want) and want.abs().max() > 0
+ # Fused LoRA paths dequantize through the quant state, which must follow the cast.
+ assert module.weight.quant_state.dtype == torch.float16
+
+
+def _fp8_route_available():
+ from unsloth.models import loader_utils
+ return loader_utils._zoo_peft_forward_keeps_fp8_inputs()
+
+
[email protected]("ct_helper", [True, False])
[email protected]("kind", ["nvfp4", "fp8"])
+def test_peft_merge_dequantizes_the_routed_base(ckpt, kind, ct_helper, monkeypatch):
+ # merge_and_unload / merge_adapter / merged_4bit add the dense delta into base_layer.weight.
+ from peft import LoraConfig, get_peft_model
+ from unsloth.models import loader_utils
+
+ if kind == "fp8" and not _fp8_route_available():
+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1" if kind == "fp8" else "0")
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ name = next(
+ n
+ for n, k in kinds.items()
+ if k == kind and ("layers.0." if kind == "nvfp4" else "layers.1.") in n
+ )
+ model = get_peft_model(model, LoraConfig(r = 4, target_modules = [name.split(".")[-1]]))
+ lora = _module(model, "base_model.model." + name)
+ torch.nn.init.normal_(lora.lora_B["default"].weight, std = 0.02)
+ want = loader_utils._routed_dense_weight(lora.get_base_layer())
+ want += lora.get_delta_weight("default")
+ if not ct_helper:
+ # compressed-tensors < 0.13 has no module-level decompress_module.
+ import compressed_tensors.compressors as ct_compressors
+ monkeypatch.delattr(ct_compressors, "decompress_module")
+ merged = model.merge_and_unload()
+ layer = _module(merged, name)
+ assert type(layer) is torch.nn.Linear and layer.weight.dtype == torch.bfloat16
+ assert torch.equal(layer.weight, want)
+
+
[email protected]("kind", ["nvfp4", "fp8"])
+def test_dora_reads_the_dense_routed_weight(ckpt, kind, monkeypatch):
+ from peft import LoraConfig, get_peft_model
+ from unsloth.models import loader_utils
+
+ if kind == "fp8" and not _fp8_route_available():
+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1" if kind == "fp8" else "0")
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ name = next(
+ n
+ for n, k in kinds.items()
+ if k == kind and ("layers.0." if kind == "nvfp4" else "layers.1.") in n
+ )
+ model = get_peft_model(
+ model, LoraConfig(r = 4, target_modules = [name.split(".")[-1]], use_dora = True)
+ )
+ lora = _module(model, "base_model.model." + name)
+ W = loader_utils._routed_dense_weight(lora.get_base_layer()).float()
+ magnitude = lora.lora_magnitude_vector["default"].weight
+ delta = lora.get_delta_weight("default").float()
+ assert torch.allclose(magnitude.float(), (W + delta).norm(dim = 1), rtol = 1e-2)
+ out = model(input_ids = torch.randint(0, 1000, (1, 8), device = "cuda")).logits
+ assert torch.isfinite(out).all()
+
+
[email protected]("merge", [False, True])
+def test_saved_checkpoint_reloads_with_plain_transformers(ckpt, merge, tmp_path, monkeypatch):
+ # save_pretrained (after merge_and_unload or not): per-module decompressed layers such as lm_head and the
+ # merged ones are dense, so they must join the ignore list, or the reload reads them as FP8 / packed.
+ from peft import LoraConfig, get_peft_model
+ from transformers import AutoModelForCausalLM
+ from unsloth.models import loader_utils
+
+ if not _fp8_route_available():
+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "1")
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ if merge:
+ model = get_peft_model(model, LoraConfig(r = 4, target_modules = ["gate_proj", "q_proj"]))
+ for n, p in model.named_parameters():
+ if "lora_B" in n:
+ torch.nn.init.normal_(p, std = 0.02)
+ model = model.merge_and_unload()
+ ids = torch.randint(0, 1000, (1, 16), device = "cuda")
+ with torch.no_grad():
+ want = model(input_ids = ids).logits.float()
+ model.save_pretrained(str(tmp_path / "out"))
+ reloaded = AutoModelForCausalLM.from_pretrained(
+ str(tmp_path / "out"), device_map = "cuda", dtype = torch.bfloat16
+ )
+ with torch.no_grad():
+ got = reloaded(input_ids = ids).logits.float()
+ # Plain compressed-tensors fake-quantizes activations (W4A4 / W8A8), so only close, not equal;
+ # a layer read in the wrong format gives logits near zero.
+ assert (got - want).abs().max() < 0.1 * want.abs().max()
+
+
[email protected]("init", ["pissa_niter_2", "olora", "loftq", "orthogonal"])
+def test_svd_style_lora_inits_see_the_dense_base(ckpt, init, monkeypatch):
+ # PiSSA / OLoRA rewrite base_layer.weight from its SVD / QR and orthogonal init builds adapters in its dtype,
+ # so the packed base goes dense first.
+ from peft import LoraConfig, get_peft_model
+ from unsloth.models import loader_utils
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ name = next(n for n, k in kinds.items() if k == "nvfp4")
+ extra = {}
+ if init == "loftq":
+ pytest.importorskip("scipy")
+ from peft import LoftQConfig
+ extra["loftq_config"] = LoftQConfig(loftq_bits = 4, loftq_iter = 1)
+ model = get_peft_model(
+ model,
+ LoraConfig(r = 4, target_modules = [name.split(".")[-1]], init_lora_weights = init, **extra),
+ )
+ base = _module(model, "base_model.model." + name).get_base_layer()
+ assert type(base) is torch.nn.Linear and base.weight.dtype == torch.bfloat16
+ out = model(input_ids = torch.randint(0, 1000, (1, 8), device = "cuda")).logits
+ assert torch.isfinite(out).all()
+
+
+def test_lora_ga_preprocess_sees_the_dense_base(ckpt, monkeypatch):
+ # preprocess_loraga sets requires_grad on each target weight before get_peft_model runs.
+ peft = pytest.importorskip("peft")
+ if not hasattr(peft, "preprocess_loraga"):
+ pytest.skip("PEFT without LoRA-GA")
+ from peft import LoraConfig, LoraGAConfig, get_peft_model, preprocess_loraga
+ from unsloth.models import loader_utils
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ name = next(n for n, k in kinds.items() if k == "nvfp4")
+ config = LoraConfig(
+ r = 4,
+ target_modules = [name.split(".")[-1]],
+ init_lora_weights = "lora_ga",
+ lora_ga_config = LoraGAConfig(),
+ )
+ ids = torch.randint(0, 1000, (1, 8), device = "cuda")
+
+ def train_step():
+ model(input_ids = ids, labels = ids).loss.backward()
+
+ preprocess_loraga(model, config, train_step)
+ assert _module(model, name).weight.dtype == torch.bfloat16
+ model = get_peft_model(model, config)
+ out = model(input_ids = ids).logits
+ assert torch.isfinite(out).all()
+
+
+def test_corda_preprocess_sees_the_dense_base(ckpt, monkeypatch):
+ # preprocess_corda multiplies each target weight by its input covariance before get_peft_model runs.
+ corda = pytest.importorskip("peft.tuners.lora.corda")
+ from peft import LoraConfig, get_peft_model
+ from peft.tuners.lora.config import CordaConfig
+ from unsloth.models import loader_utils
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ name = next(n for n, k in kinds.items() if k == "nvfp4")
+ config = LoraConfig(
+ r = 4,
+ target_modules = [name.split(".")[-1]],
+ init_lora_weights = "corda",
+ corda_config = CordaConfig(),
+ )
+ ids = torch.randint(0, 1000, (1, 64), device = "cuda")
+
+ def run_model():
+ # CorDA's covariance hook squeezes a batch of one.
+ for _ in range(4):
+ model(input_ids = torch.randint(0, 1000, (1, 64), device = "cuda"))
+
+ corda.preprocess_corda(model, config, run_model = run_model)
+ assert _module(model, name).weight.dtype == torch.bfloat16
+ model = get_peft_model(model, config)
+ out = model(input_ids = ids).logits
+ assert torch.isfinite(out).all()
+
+
+def test_decompressed_layers_join_a_missing_ignore_list(ckpt, monkeypatch):
+ from unsloth.models import loader_utils
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ path, _ = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ model.config.quantization_config.quantization_config.ignore = None
+ loader_utils._prepare_compressed_tensors_model(model)
+ assert "lm_head" in model.config.quantization_config.quantization_config.ignore
+
+
+def test_nvfp4_lm_head_is_decompressed(tmp_path, monkeypatch):
+ # Decode and the fused CE loss read lm_head.weight directly, so it cannot stay packed.
+ from unsloth.models import loader_utils
+
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0")
+ kinds = fx.build(str(tmp_path), "qwen3", lm_head_kind = "nvfp4")
+ assert kinds["lm_head"] == "nvfp4"
+ model = _load_raw(str(tmp_path), "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ head = model.get_output_embeddings()
+ assert head.weight.dtype == torch.bfloat16
+ assert tuple(head.weight.shape) == (model.config.vocab_size, model.config.hidden_size)
+ assert model._unsloth_compressed_tensors_nvfp4 == len(kinds) - 1 - sum(
+ k == "fp8" for k in kinds.values()
+ )
+
+
+def test_opt_out_keeps_the_full_decompress(ckpt, monkeypatch):
+ from unsloth.models import loader_utils
+
+ path, kinds = ckpt["qwen3"]
+ monkeypatch.setenv("UNSLOTH_COMPRESSED_TENSORS_NVFP4_KERNELS", "0")
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ name = next(n for n, k in kinds.items() if k == "nvfp4")
+ assert _module(model, name).weight.dtype == torch.bfloat16
+ assert not getattr(model, "_unsloth_compressed_tensors_nvfp4", 0)
+
+
+def test_full_finetuning_decompresses_everything(ckpt):
+ from unsloth.models import loader_utils
+
+ path, kinds = ckpt["qwen3"]
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model, full_finetuning = True)
+ assert all(_module(model, n).weight.dtype == torch.bfloat16 for n in kinds)
+
+
+def test_fp8_group_routes_with_the_fp8_kernels_by_default(ckpt, monkeypatch):
+ from unsloth.models import loader_utils
+
+ path, kinds = ckpt["qwen3"]
+ monkeypatch.delenv("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", raising = False)
+ monkeypatch.setattr(loader_utils, "_zoo_peft_forward_keeps_fp8_inputs", lambda: True)
+ model = _load_raw(path, "qwen3")
+ loader_utils._prepare_compressed_tensors_model(model)
+ lm_head = model.get_output_embeddings()
+ for name, kind in kinds.items():
+ module = _module(model, name)
+ if module is lm_head:
+ # The fused CE loss reads lm_head.weight directly, so it is decompressed, not routed.
+ assert module.weight.dtype == torch.bfloat16
+ assert not getattr(module, "_unsloth_compressed_tensors_fp8", False)
+ elif kind == "fp8":
+ assert (
+ module.weight.dtype == torch.float8_e4m3fn
+ and module._unsloth_compressed_tensors_fp8
+ )
+ else:
+ assert module._unsloth_compressed_tensors_nvfp4
+
+
+ "arch, api, fp8_kernels",
+ [
+ ("qwen3", "FastLanguageModel", False),
+ ("qwen3", "FastModel", False),
+ ("qwen3_5", "FastModel", False),
+ ("qwen3_5", "FastLanguageModel", False),
+ # The qwen3 fixture has an untied FP8 lm_head, which the FP8 route once handed to the fused CE loss.
+ ("qwen3", "FastModel", True),
+ ],
+)
+def test_lora_trains_on_the_packed_base_and_reloads(ckpt, arch, api, fp8_kernels, tmp_path):
+ _check_lora_case(ckpt, arch, api, fp8_kernels, tmp_path)
+
+
[email protected]("api", ["FastLanguageModel", "FastModel"])
+def test_lora_trains_in_float16(ckpt, api, tmp_path):
+ # T4 / V100 load in fp16: the NVFP4 quant state must dequantize to fp16, and the bf16-only FP8 kernels stay off.
+ _check_lora_case(ckpt, "qwen3", api, None, tmp_path, dtype = "float16")
+
+
+def _check_lora_case(
+ ckpt,
+ arch,
+ api,
+ fp8_kernels,
+ tmp_path,
+ dtype = "bfloat16",
+):
+ import json
+ import subprocess
+
+ if fp8_kernels:
+ from unsloth.models import loader_utils
+ if not loader_utils._zoo_peft_forward_keeps_fp8_inputs():
+ pytest.skip("installed unsloth_zoo predates the FP8 kernel route")
+ path, _ = ckpt[arch]
+ root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ # A fresh process per case: patches and the compiled module cache from one loader leak into the next.
+ env = dict(
+ os.environ,
+ PYTHONPATH = root + os.pathsep + os.environ.get("PYTHONPATH", ""),
+ UNSLOTH_COMPILE_LOCATION = str(tmp_path / "compiled"),
+ UNSLOTH_IS_PRESENT = "1",
+ NVFP4_CASE_DTYPE = dtype,
+ )
+ if fp8_kernels is not None:
+ env["UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS"] = "1" if fp8_kernels else "0"
+ else:
+ env.pop("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", None)
+ run = subprocess.run(
+ [
+ sys.executable,
+ os.path.join(os.path.dirname(__file__), "_nvfp4_fixtures.py"),
+ "--lora-case",
+ path,
+ arch,
+ api,
+ str(tmp_path),
+ ],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 1200,
+ )
+ lines = [l for l in run.stdout.splitlines() if l.startswith("LORA_CASE ")]
+ assert run.returncode == 0 and lines, run.stderr[-3000:]
+ r = json.loads(lines[-1][len("LORA_CASE ") :])
+ assert all(torch.isfinite(torch.tensor(r["losses"]))) and r["losses"][-1] < r["losses"][0]
+ # The packed base is never cast into, and activations reach it in a float dtype (W4A16, not uint8).
+ assert r["input_dtypes"] and set(r["input_dtypes"]) <= {
+ "torch.bfloat16",
+ "torch.float16",
+ "torch.float32",
+ }
+ assert r["packed_unchanged"] and r["packed_dtype"] == "torch.uint8" and r["lora_changed"]
+ assert "adapter_model.safetensors" in r["saved"]
+ assert r["adapters_equal"]
+ # FastLanguageModel trains through fused LoRA kernels; a plain PEFT reload differs by ~0.04 on bf16 main too.
+ if api == "FastModel":
+ assert r["reload_max_abs"] <= 1e-2
+
+
+def test_float8_lora_adapters_are_upcast_on_every_peft():
+ # PEFT < 0.19 gives an FP8 base float8 LoRA weights, on get_peft_model and on PeftModel.from_pretrained;
+ # the FP8 route beside NVFP4 relies on Unsloth's upcast.
+ import unsloth # noqa: F401
+ from peft import LoraConfig, get_peft_model
+
+ base = torch.nn.Sequential(torch.nn.Linear(32, 32, bias = False)).cuda()
+ base[0].weight.data = base[0].weight.data.to(torch.float8_e4m3fn)
+ model = get_peft_model(base, LoraConfig(r = 4, target_modules = ["0"]))
+ dtypes = {n: p.dtype for n, p in model.named_parameters()}
+ assert all(d == torch.float32 for n, d in dtypes.items() if "lora_" in n), dtypes
+ assert model.base_model.model[0].base_layer.weight.dtype == torch.float8_e4m3fn
+
+ import tempfile
+ from peft import PeftModel
+
+ with tempfile.TemporaryDirectory() as d:
+ model.save_pretrained(d)
+ fresh = torch.nn.Sequential(torch.nn.Linear(32, 32, bias = False)).cuda()
+ fresh[0].weight.data = fresh[0].weight.data.to(torch.float8_e4m3fn)
+ fresh = PeftModel.from_pretrained(fresh, d, torch_device = "cuda")
+ assert all(p.dtype == torch.float32 for n, p in fresh.named_parameters() if "lora_" in n)
diff --git a/tests/test_nvfp4_lora.py b/tests/test_nvfp4_lora.py
new file mode 100644
index 00000000000..a5260646ae1
--- /dev/null
+++ b/tests/test_nvfp4_lora.py
@@ -0,0 +1,175 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+# Packed NVFP4 bases (plain `weight` attribute = weight_packed with an NVFP4QuantState) through Unsloth's fused LoRA kernels.
+import os
+
+import pytest
+
+torch = pytest.importorskip("torch")
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+
+
[email protected](scope = "module")
+def mods():
+ os.environ.setdefault("UNSLOTH_IS_PRESENT", "1")
+ from unsloth.kernels import fast_lora, nvfp4, utils
+ return fast_lora, nvfp4, utils
+
+
+def _packed(out_features, in_features, seed):
+ g = torch.Generator(device = "cuda").manual_seed(seed)
+ packed = torch.randint(
+ 0, 256, (out_features, in_features // 2), generator = g, device = "cuda", dtype = torch.uint8
+ )
+ # Asymmetric group scales so a wrong group axis or orientation shows up.
+ scale = torch.rand(out_features, in_features // 16, generator = g, device = "cuda") * 3 + 0.25
+ scale = scale * torch.linspace(0.5, 2.0, in_features // 16, device = "cuda")
+ return packed, scale.to(torch.float8_e4m3fn), torch.tensor([37.5], device = "cuda")
+
+
+def _lora_model(
+ mods,
+ names_shapes,
+ r = 8,
+ seed = 0,
+):
+ """PEFT LoRA over nn.Linears whose base weights are swapped for packed NVFP4; returns (nvfp4 model, dense twin)."""
+ from peft import LoraConfig, get_peft_model
+
+ _, nvfp4, _ = mods
+
+ class Block(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ for name, (o, i) in names_shapes.items():
+ setattr(self, name, torch.nn.Linear(i, o, bias = False))
+
+ cfg = LoraConfig(r = r, lora_alpha = 16, lora_dropout = 0.0, target_modules = list(names_shapes))
+ torch.manual_seed(seed)
+ dense = get_peft_model(Block().cuda().to(torch.bfloat16), cfg)
+ torch.manual_seed(seed)
+ quant = get_peft_model(Block().cuda().to(torch.bfloat16), cfg)
+ for k, (name, (o, i)) in enumerate(names_shapes.items()):
+ packed, scale, gs = _packed(o, i, seed + k)
+ W = nvfp4.nvfp4_dequantize(packed, scale, gs, torch.bfloat16)
+ d_layer = getattr(dense.base_model.model, name)
+ q_layer = getattr(quant.base_model.model, name)
+ d_layer.base_layer.weight.data.copy_(W)
+ del q_layer.base_layer.weight
+ packed.quant_state = nvfp4.NVFP4QuantState(scale, gs, (o, i), torch.bfloat16)
+ q_layer.base_layer.weight = packed
+ # Non-zero B: zero-initialised B hides dA bugs.
+ for layer in (d_layer, q_layer):
+ torch.manual_seed(100 + k)
+ torch.nn.init.normal_(layer.lora_B["default"].weight, std = 0.02)
+ torch.manual_seed(200 + k)
+ torch.nn.init.normal_(layer.lora_A["default"].weight, std = 0.02)
+ return quant.base_model.model, dense.base_model.model
+
+
+def _grads(model, names):
+ return [getattr(model, n).lora_A["default"].weight.grad.float() for n in names] + [
+ getattr(model, n).lora_B["default"].weight.grad.float() for n in names
+ ]
+
+
+def _close(a, b, tol):
+ a, b = a.float(), b.float()
+ return float((a - b).norm() / b.norm().clamp(min = 1e-12)) < tol
+
+
+def _run(mods, names_shapes, fused, reference):
+ q, d = _lora_model(mods, names_shapes)
+ in_dim = next(iter(names_shapes.values()))[1]
+ X = torch.randn(2, 24, in_dim, device = "cuda", dtype = torch.bfloat16)
+ Xq, Xd = X.clone().requires_grad_(True), X.clone().requires_grad_(True)
+ outs_q, outs_d = fused(q, Xq), reference(d, Xd)
+ if not isinstance(outs_q, (tuple, list)):
+ outs_q, outs_d = (outs_q,), (outs_d,)
+ sum(o.float().square().mean() for o in outs_q).backward()
+ sum(o.float().square().mean() for o in outs_d).backward()
+ for oq, od in zip(outs_q, outs_d):
+ assert oq.dtype == torch.bfloat16
+ assert _close(oq, od, 1e-2)
+ assert _close(Xq.grad, Xd.grad, 2e-2)
+ for gq, gd in zip(_grads(q, names_shapes), _grads(d, names_shapes)):
+ assert _close(gq, gd, 2e-2)
+
+
+def test_dequantize_matches_compressed_tensors(mods):
+ ct = pytest.importorskip("compressed_tensors.compressors.nvfp4.base")
+ _, nvfp4, _ = mods
+ packed, scale, gs = _packed(96, 160, 1)
+ from compressed_tensors.quantization import QuantizationArgs, QuantizationScheme
+
+ args = QuantizationArgs(num_bits = 4, type = "float", strategy = "tensor_group", group_size = 16)
+ ref = ct.NVFP4PackedCompressor.decompress(
+ {"weight_packed": packed, "weight_scale": scale, "weight_global_scale": gs},
+ QuantizationScheme(targets = ["Linear"], weights = args),
+ )["weight"]
+ ours = nvfp4.nvfp4_dequantize(packed, scale, gs, ref.dtype)
+ assert torch.equal(ours, ref)
+ assert torch.equal(nvfp4.nvfp4_dequantize(packed.t(), scale, gs, ref.dtype), ref.t())
+
+
+def test_get_lora_parameters_returns_the_nvfp4_state(mods):
+ _, nvfp4, utils = mods
+ q, _ = _lora_model(mods, {"o_proj": (64, 96)})
+ W, W_quant, A, B, S = utils.get_lora_parameters(q.o_proj)
+ assert W.dtype == torch.uint8 and type(W_quant) is nvfp4.NVFP4QuantState
+ assert A is not None and B is not None
+
+
+def test_lora_w_matches_dense(mods):
+ fast_lora, _, _ = mods
+ _run(
+ mods,
+ {"o_proj": (80, 96)},
+ lambda m, X: fast_lora.apply_lora_o(m, X),
+ lambda m, X: m.o_proj(X),
+ )
+
+
+def test_lora_qkv_matches_dense(mods):
+ fast_lora, _, _ = mods
+ shapes = {"q_proj": (128, 96), "k_proj": (32, 96), "v_proj": (32, 96)}
+ _run(
+ mods,
+ shapes,
+ lambda m, X: fast_lora.apply_lora_qkv(m, X, inplace = False),
+ lambda m, X: (m.q_proj(X), m.k_proj(X), m.v_proj(X)),
+ )
+
+
+def test_lora_mlp_matches_dense(mods):
+ fast_lora, _, _ = mods
+ shapes = {"gate_proj": (160, 96), "up_proj": (160, 96), "down_proj": (96, 160)}
+
+ def reference(m, X):
+ return m.down_proj(torch.nn.functional.silu(m.gate_proj(X)) * m.up_proj(X))
+
+ _run(mods, shapes, lambda m, X: fast_lora.apply_lora_mlp_swiglu(m, X, inplace = False), reference)
+
+
+def test_fast_linear_forward_decode(mods):
+ _, _, utils = mods
+ q, d = _lora_model(mods, {"o_proj": (80, 96)})
+ X = torch.randn(1, 1, 96, device = "cuda", dtype = torch.bfloat16)
+ with torch.no_grad():
+ assert _close(utils.fast_linear_forward(q.o_proj, X), d.o_proj(X), 1e-2)
+ X2 = torch.randn(3, 1, 96, device = "cuda", dtype = torch.bfloat16)
+ assert _close(utils.fast_linear_forward(q.o_proj, X2), d.o_proj(X2), 1e-2)
+
+
+def test_bnb_4bit_path_is_unchanged(mods):
+ bnb = pytest.importorskip("bitsandbytes")
+ _, _, utils = mods
+ layer = bnb.nn.Linear4bit(128, 64, bias = False, compute_dtype = torch.bfloat16, quant_type = "nf4")
+ dense = (torch.randn(64, 128) * 0.05).to(torch.bfloat16)
+ layer.weight = bnb.nn.Params4bit(dense, requires_grad = False, quant_type = "nf4")
+ layer = layer.cuda()
+ W = layer.weight
+ out = utils.fast_dequantize(W, W.quant_state)
+ ref = bnb.functional.dequantize_4bit(W.data, W.quant_state)
+ assert torch.equal(out, ref)
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 15cc1180a73..6fd5bb0b001 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -303,6 +303,7 @@ from .import_fixes import (
fix_peft_transformers_tensor_parallel_import_compat,
fix_peft_transformers_weight_conversion_import,
patch_peft_weight_converter_compatibility,
+ patch_peft_float8_adapter_upcast,
fix_peft_stale_torchao_import_error,
fix_peft_torchao_missing_tensor_subclass,
patch_accelerate_recursively_apply,
@@ -391,6 +392,7 @@ fix_accelerate_dtensor_check_without_torch_distributed()
fix_peft_transformers_tensor_parallel_import_compat()
fix_peft_transformers_weight_conversion_import()
patch_peft_weight_converter_compatibility()
+patch_peft_float8_adapter_upcast()
# After peft is importable, so the already-bound is_torchao_available in peft.tuners.lora.torchao is
# replaced too, not just import_utils'.
fix_peft_stale_torchao_import_error()
@@ -441,6 +443,7 @@ del fix_accelerate_dtensor_check_without_torch_distributed
del fix_peft_transformers_tensor_parallel_import_compat
del fix_peft_transformers_weight_conversion_import
del patch_peft_weight_converter_compatibility
+del patch_peft_float8_adapter_upcast
del fix_peft_stale_torchao_import_error
del fix_peft_torchao_missing_tensor_subclass
del patch_accelerate_recursively_apply
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 531408dab10..d59fed1c1d3 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -8149,6 +8149,53 @@ def _backfill_missing_conversion_symbols():
return bool(added)
+def patch_peft_float8_adapter_upcast():
+ """PEFT < 0.19 builds adapters in an FP8 base weight's dtype; upcast them to float32 as PEFT >= 0.19 does."""
+ try:
+ import torch
+ import peft.tuners.tuners_utils as tu
+ except Exception:
+ return
+ original = getattr(tu, "cast_adapter_dtype", None)
+ if original is None or getattr(original, "_unsloth_float8_upcast", False):
+ return
+ try:
+ from peft.utils.constants import UPCAST_DTYPES # noqa: F401 (PEFT >= 0.19 already upcasts float8)
+ return
+ except ImportError:
+ pass
+ float8 = tuple(
+ getattr(torch, n)
+ for n in ("float8_e4m3fn", "float8_e4m3fnuz", "float8_e5m2", "float8_e5m2fnuz")
+ if hasattr(torch, n)
+ )
+
+ @functools.wraps(original)
+ def cast_adapter_dtype(
+ model,
+ adapter_name,
+ autocast_adapter_dtype = True,
+ ):
+ original(model, adapter_name = adapter_name, autocast_adapter_dtype = autocast_adapter_dtype)
+ if not autocast_adapter_dtype:
+ return
+ for module in model.modules():
+ if not isinstance(module, tu.BaseTunerLayer):
+ continue
+ for name in module.adapter_layer_names:
+ layer = getattr(module, name, None)
+ if not isinstance(layer, torch.nn.Module) or adapter_name not in layer:
+ continue
+ item = layer[adapter_name]
+ params = item.parameters() if isinstance(item, torch.nn.Module) else (item,)
+ for p in params:
+ if p.dtype in float8:
+ p.data = p.data.to(torch.float32)
+
+ cast_adapter_dtype._unsloth_float8_upcast = True
+ tu.cast_adapter_dtype = cast_adapter_dtype
+
+
def patch_peft_weight_converter_compatibility():
"""Allow PEFT converter rebuilds on legacy converter constructors."""
try:
diff --git a/unsloth/kernels/__init__.py b/unsloth/kernels/__init__.py
index a58a3cdceed..55845a94fc7 100644
--- a/unsloth/kernels/__init__.py
+++ b/unsloth/kernels/__init__.py
@@ -44,6 +44,7 @@ from .fast_lora import (
from .fp8 import * # Patch FbgmemFP8Linear/FP8Linear forwards before model creation, so compiled non-fast-inference
# models are covered too
+from .nvfp4 import *
from .utils import (
fast_dequantize,
fast_gemv,
diff --git a/unsloth/kernels/nvfp4.py b/unsloth/kernels/nvfp4.py
new file mode 100644
index 00000000000..42d88a079e4
--- /dev/null
+++ b/unsloth/kernels/nvfp4.py
@@ -0,0 +1,239 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""W4A16 NVFP4: packed E2M1 weights (compressed-tensors `nvfp4-pack-quantized`) dequantized to 16 bit on the fly."""
+
+import torch
+import triton
+import triton.language as tl
+
+from .fp8 import _fp8_triton_device_context, _is_transposed_view, _opaque_under_compile
+
+__all__ = [
+ "NVFP4_GROUP_SIZE",
+ "NVFP4QuantState",
+ "nvfp4_dequantize",
+ "NVFP4Linear_matmul",
+ "nvfp4_linear",
+]
+
+NVFP4_GROUP_SIZE = 16
+_E2M1 = (0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0)
+
+
+class NVFP4QuantState:
+ __slots__ = ("scale", "global_scale", "shape", "dtype")
+
+ def __init__(
+ self,
+ scale,
+ global_scale,
+ shape,
+ dtype = torch.bfloat16,
+ ):
+ self.scale = scale
+ self.global_scale = global_scale
+ self.shape = tuple(shape)
+ self.dtype = dtype
+
+
+def _nvfp4_dequantize_torch(packed, scale, global_scale, dtype):
+ # Same arithmetic as compressed_tensors NVFP4PackedCompressor.decompress: scale / global_scale in fp32, times E2M1.
+ rows, half = packed.shape
+ lut = torch.tensor(_E2M1, dtype = torch.float32, device = packed.device)
+ codes = torch.stack((packed & 0x0F, packed >> 4), dim = -1).reshape(rows, half * 2)
+ values = lut[(codes & 0x07).long()] * torch.where((codes & 0x08).bool(), -1.0, 1.0)
+ group_scale = scale.to(torch.float32) / global_scale.to(torch.float32).reshape(-1)[:1]
+ group_scale = group_scale.repeat_interleave(NVFP4_GROUP_SIZE, dim = 1)[:, : half * 2]
+ return (values * group_scale).to(dtype)
+
+
+def _e2m1_to_float(code):
+ # Build the fp32 bits: mag >= 2 is (1 + m / 2) * 2^(e - 1), mag 1 is 0.5, mag 0 is 0; bit 3 is the sign (code 8 = -0.0).
+ mag = code & 7
+ bits = tl.where(
+ mag >= 2, (((mag >> 1) + 126) << 23) | ((mag & 1) << 22), tl.where(mag == 1, 126 << 23, 0)
+ )
+ return (bits | ((code & 8) << 28)).to(tl.float32, bitcast = True)
+
+
+def _e4m3_to_float(b):
+ # Triton has no float8e4nv before sm_89 (A100, T4), so the group scale arrives as raw bytes: sign, 4 exponent
+ # bits (bias 7), 3 mantissa bits; exponent 0 is subnormal (m * 2^-9) and 0x7F / 0xFF are NaN, as in torch.
+ exp = (b >> 3) & 15
+ man = b & 7
+ normal = (((exp + 120) << 23) | (man << 20)).to(tl.float32, bitcast = True)
+ value = tl.where(exp == 0, man.to(tl.float32) * 0.001953125, normal)
+ value = tl.where((b & 127) == 127, float("nan"), value)
+ # Sign as a bit, not a negation: -x can lower to 0 - x, which turns code 0x80 (-0.0) into +0.0.
+ return (value.to(tl.int32, bitcast = True) | ((b & 128) << 24)).to(tl.float32, bitcast = True)
+
+
+def _nvfp4_dequant_kernel(
+ packed_ptr,
+ scale_ptr,
+ global_scale_ptr,
+ out_ptr,
+ rows,
+ half_cols,
+ scale_cols,
+ BLOCK_R: tl.constexpr,
+ BLOCK_C: tl.constexpr,
+ SCALE_E4M3_BYTES: tl.constexpr,
+ INT64_OFFSETS: tl.constexpr,
+):
+ r = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)
+ if INT64_OFFSETS:
+ r = r.to(tl.int64)
+ c = tl.program_id(1) * BLOCK_C + tl.arange(0, BLOCK_C)
+ g = tl.program_id(1) * (BLOCK_C // 8) + tl.arange(0, BLOCK_C // 8)
+ row_ok = r[:, None] < rows
+ byte = tl.load(
+ packed_ptr + r[:, None] * half_cols + c[None, :],
+ mask = row_ok & (c[None, :] < half_cols),
+ other = 0,
+ ).to(tl.int32)
+ scale = tl.load(
+ scale_ptr + r[:, None] * scale_cols + g[None, :],
+ mask = row_ok & (g[None, :] < scale_cols),
+ other = 0.0,
+ )
+ scale = _e4m3_to_float(scale.to(tl.int32)) if SCALE_E4M3_BYTES else scale.to(tl.float32)
+ group_scale = tl.math.div_rn(scale, tl.load(global_scale_ptr).to(tl.float32))
+ # Low nibble is the even column: join + reshape interleaves (lo, hi) into contiguous output columns.
+ values = tl.reshape(
+ tl.join(_e2m1_to_float(byte & 15), _e2m1_to_float(byte >> 4)), (BLOCK_R, BLOCK_C // 8, 16)
+ )
+ values = tl.reshape(values * group_scale[:, :, None], (BLOCK_R, 2 * BLOCK_C))
+ oc = tl.program_id(1) * (2 * BLOCK_C) + tl.arange(0, 2 * BLOCK_C)
+ tl.store(
+ out_ptr + r[:, None] * (2 * half_cols) + oc[None, :],
+ values.to(out_ptr.dtype.element_ty),
+ mask = row_ok & (oc[None, :] < 2 * half_cols),
+ )
+
+
+@_opaque_under_compile(
+ "nvfp4_dequantize_triton",
+ lambda packed, scale, global_scale, dtype: packed.new_empty(
+ (packed.shape[0], packed.shape[1] * 2), dtype = dtype
+ ),
+)
+def _nvfp4_dequantize_triton(
+ packed: torch.Tensor, scale: torch.Tensor, global_scale: torch.Tensor, dtype: torch.dtype
+) -> torch.Tensor:
+ rows, half = packed.shape
+ out = torch.empty((rows, half * 2), dtype = dtype, device = packed.device)
+ BLOCK_R, BLOCK_C = 8, 256
+ e4m3 = scale.dtype == torch.float8_e4m3fn
+ if e4m3:
+ scale = scale.view(torch.uint8)
+ with _fp8_triton_device_context(packed):
+ _nvfp4_dequant_kernel[(triton.cdiv(rows, BLOCK_R), triton.cdiv(half, BLOCK_C))](
+ packed,
+ scale,
+ global_scale,
+ out,
+ rows,
+ half,
+ scale.shape[1],
+ BLOCK_R = BLOCK_R,
+ BLOCK_C = BLOCK_C,
+ SCALE_E4M3_BYTES = e4m3,
+ # int32 offsets wrap once the output passes 2^31 elements; int64 only then, as it costs ~2%.
+ INT64_OFFSETS = rows * half * 2 >= 2**31,
+ num_warps = 4,
+ )
+ return out
+
+
+def nvfp4_dequantize(
+ packed,
+ scale,
+ global_scale,
+ dtype = torch.bfloat16,
+):
+ """Dense (out, in) weight of a packed (out, in / 2) NVFP4 tensor; a transposed view gives the transposed weight."""
+ if _is_transposed_view(packed):
+ return nvfp4_dequantize(packed.t(), scale, global_scale, dtype).t()
+ if packed.shape[1] * 2 > scale.shape[1] * NVFP4_GROUP_SIZE:
+ raise ValueError(
+ f"Unsloth: NVFP4 weight {tuple(packed.shape)} needs {packed.shape[1] * 2 // NVFP4_GROUP_SIZE} "
+ f"group scales per row, got {tuple(scale.shape)}."
+ )
+ if packed.device.type in ("cuda", "xpu"):
+ return _nvfp4_dequantize_triton(
+ packed.contiguous(), scale.contiguous(), global_scale.reshape(-1)[:1], dtype
+ )
+ return _nvfp4_dequantize_torch(packed, scale, global_scale, dtype)
+
+
+# Opaque to torch.compile as whole matmuls: if the dequantized weight were a graph value, AOTAutograd would save it for
+# backward in every layer (3.6x the NVFP4 storage each), which is exactly the memory W4A16 exists to avoid.
+@_opaque_under_compile(
+ "nvfp4_matmul_fwd",
+ lambda X, packed, scale, global_scale: X.new_empty(X.shape[:-1] + (packed.shape[0],)),
+)
+def _nvfp4_matmul_fwd(
+ X: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, global_scale: torch.Tensor
+) -> torch.Tensor:
+ return torch.matmul(X, nvfp4_dequantize(packed, scale, global_scale, X.dtype).t())
+
+
+@_opaque_under_compile(
+ "nvfp4_matmul_bwd",
+ lambda grad, packed, scale, global_scale: grad.new_empty(
+ grad.shape[:-1] + (packed.shape[1] * 2,)
+ ),
+)
+def _nvfp4_matmul_bwd(
+ grad: torch.Tensor, packed: torch.Tensor, scale: torch.Tensor, global_scale: torch.Tensor
+) -> torch.Tensor:
+ return torch.matmul(grad, nvfp4_dequantize(packed, scale, global_scale, grad.dtype))
+
+
+class NVFP4Linear_matmul(torch.autograd.Function):
+ @staticmethod
+ def forward(
+ ctx,
+ X,
+ packed,
+ scale,
+ global_scale,
+ bias = None,
+ ):
+ output = _nvfp4_matmul_fwd(X, packed, scale, global_scale)
+ if bias is not None:
+ output = output + bias
+ # A float32 bias promotes; cast back only then (a no-op .to() aliases, zeroing dX under torch 2.11 compile).
+ if output.dtype != X.dtype:
+ output = output.to(X.dtype)
+ ctx.save_for_backward(packed, scale, global_scale)
+ ctx.bias_grad = bias is not None and bias.requires_grad
+ ctx.dtype = X.dtype
+ return output
+
+ @staticmethod
+ def backward(ctx, grad_output):
+ packed, scale, global_scale = ctx.saved_tensors
+ grad_X = None
+ if ctx.needs_input_grad[0]:
+ grad_X = _nvfp4_matmul_bwd(grad_output, packed, scale, global_scale)
+ if grad_X.dtype != ctx.dtype:
+ grad_X = grad_X.to(ctx.dtype)
+ grad_bias = None
+ if ctx.bias_grad:
+ grad_bias = grad_output.reshape(-1, grad_output.shape[-1]).sum(0)
+ return grad_X, None, None, None, grad_bias
+
+
+def nvfp4_linear(
+ X,
+ packed,
+ scale,
+ global_scale,
+ bias = None,
+):
+ return NVFP4Linear_matmul.apply(X, packed, scale, global_scale, bias)
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 441bbc4c7fe..1c0d9578ea7 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -28,6 +28,7 @@ from ..device_type import (
)
from ..bnb_availability import native_kernels_ready
from .fp8 import weight_dequant, fp8_linear, can_use_fp8_rowwise_gemv, fp8_rowwise_gemv
+from .nvfp4 import NVFP4QuantState, nvfp4_dequantize, nvfp4_linear
import functools
# torch.cuda.amp.custom_fwd is deprecated from 2.4.
@@ -498,6 +499,10 @@ if DEVICE_TYPE == "xpu" and HAS_XPU_STREAM:
return W
if W.dtype == torch.float8_e4m3fn:
return weight_dequant(W, quant_state)
+ if type(quant_state) is NVFP4QuantState:
+ return nvfp4_dequantize(
+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype
+ )
if type(quant_state) is not list:
# New quant_state as a class, per TimDettmers/bitsandbytes#763.
absmax = quant_state.absmax
@@ -607,6 +612,10 @@ elif DEVICE_TYPE in ("cuda", "hip") and HAS_CUDA_STREAM:
return W
if W.dtype == torch.float8_e4m3fn:
return weight_dequant(W, quant_state)
+ if type(quant_state) is NVFP4QuantState:
+ return nvfp4_dequantize(
+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype
+ )
if type(quant_state) is not list:
# New quant_state as a class, per TimDettmers/bitsandbytes#763.
absmax = quant_state.absmax
@@ -719,6 +728,10 @@ else:
return W
if W.dtype == torch.float8_e4m3fn:
return weight_dequant(W, quant_state)
+ if type(quant_state) is NVFP4QuantState:
+ return nvfp4_dequantize(
+ W, quant_state.scale, quant_state.global_scale, quant_state.dtype
+ )
if type(quant_state) is not list:
# New quant_state as a class, per TimDettmers/bitsandbytes#763.
# https://github.com/TimDettmers/bitsandbytes/pull/763/files
@@ -1136,6 +1149,9 @@ def fast_linear_forward(
if W_quant is None:
out = torch_matmul(X, W.t(), out = out)
+ elif type(W_quant) is NVFP4QuantState:
+ # Bias is added once below.
+ out = nvfp4_linear(X, W, W_quant.scale, W_quant.global_scale)
elif _is_packed_state(W_quant):
out = W_quant.matmul(X, W, out = out)
elif W.dtype == torch.float8_e4m3fn:
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 58cdf31cc0e..1606805b291 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -1490,19 +1490,33 @@ def _zoo_peft_forward_keeps_fp8_inputs():
)
-def _route_compressed_tensors_fp8_to_unsloth(model):
- # Opt-in: saves memory, but on-the-fly dequant trains slower than the decompressed bf16 model.
- if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", "0") != "1":
+def _route_compressed_tensors_fp8_to_unsloth(
+ model,
+ skip = (),
+ default = "0",
+):
+ # Opt-in for FP8 checkpoints (on-the-fly dequant trains slower than bf16); default beside NVFP4, where the
+ # decompressed FP8 layers would keep compressed-tensors' graph-breaking fake-quant forward.
+ if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_FP8_KERNELS", default) != "1":
return 0
if getattr(getattr(model, "config", None), "quantization_config", None) is None:
return 0
if not _zoo_peft_forward_keeps_fp8_inputs():
return 0
- routable = []
+ # The fused CE loss multiplies lm_head.weight directly, so a quantized lm_head is decompressed, never routed.
+ get_output_embeddings = getattr(model, "get_output_embeddings", None)
+ lm_head = get_output_embeddings() if callable(get_output_embeddings) else None
+ routable, decompress = [], []
for module in model.modules():
+ # Modules another route already owns (NVFP4) do not count against the all-or-nothing check.
+ if module in skip:
+ continue
scheme = getattr(module, "quantization_scheme", None)
if scheme is None or getattr(scheme, "weights", None) is None:
continue
+ if module is lm_head:
+ decompress.append(module)
+ continue
block = None
weight = getattr(module, "weight", None)
on_gpu = isinstance(weight, torch.Tensor) and weight.device.type in ("cuda", "xpu")
@@ -1515,7 +1529,18 @@ def _route_compressed_tensors_fp8_to_unsloth(model):
if block is None:
return 0
routable.append((module, block))
+ if not routable:
+ return 0
+ if decompress:
+ try:
+ with torch.inference_mode(False), torch.no_grad():
+ for module in decompress:
+ _decompress_like_a_full_load(module)
+ _remove_same_device_compressed_tensors_offload(module)
+ except Exception:
+ return 0
converted = 0
+ compute_dtype = _fp8_dequant_target_dtype(model)
for module, block in routable:
scale = module.weight_scale
scale_shape = tuple(scale.shape)
@@ -1537,20 +1562,381 @@ def _route_compressed_tensors_fp8_to_unsloth(model):
module.block_size = block
module.forward = _unsloth_compressed_tensors_fp8_forward.__get__(module)
module._unsloth_compressed_tensors_fp8 = True
+ module._unsloth_ct_compute_dtype = compute_dtype
converted += 1
if converted:
- for owner in (model, getattr(model, "model", None), getattr(model, "base_model", None)):
- hook = getattr(owner, "ct_decompress_hook", None)
- if hook is not None:
- hook.remove()
- try:
- delattr(owner, "ct_decompress_hook")
- except AttributeError:
- pass
+ _remove_compressed_tensors_decompress_hook(model)
+ _patch_peft_for_routed_compressed_tensors()
model._unsloth_compressed_tensors_fp8 = converted
return converted
+def _remove_compressed_tensors_decompress_hook(model):
+ for owner in (model, getattr(model, "model", None), getattr(model, "base_model", None)):
+ hook = getattr(owner, "ct_decompress_hook", None)
+ if hook is not None:
+ hook.remove()
+ try:
+ delattr(owner, "ct_decompress_hook")
+ except AttributeError:
+ pass
+
+
+def _nvfp4_scheme_supported(scheme):
+ weights = getattr(scheme, "weights", None)
+ if weights is None or getattr(scheme, "output_activations", None) is not None:
+ return False
+ enum = lambda v: str(getattr(v, "value", v)) if v is not None else None
+ fmt = enum(getattr(scheme, "format", None))
+ return (
+ str(enum(getattr(weights, "type", None))) == "float"
+ and getattr(weights, "num_bits", None) == 4
+ and getattr(weights, "group_size", None) == 16
+ and enum(getattr(weights, "strategy", None)) == "tensor_group"
+ and not getattr(weights, "dynamic", False)
+ # "static" is an alias of "weight": calibration order only, columns stored in place (no g_idx).
+ and enum(getattr(weights, "actorder", None)) in (None, "weight", "static")
+ and fmt in (None, "nvfp4-pack-quantized")
+ )
+
+
+def _nvfp4_module_routable(module):
+ packed = getattr(module, "weight_packed", None)
+ scale = getattr(module, "weight_scale", None)
+ global_scale = getattr(module, "weight_global_scale", None)
+ return (
+ isinstance(module, torch.nn.Linear)
+ and isinstance(packed, torch.Tensor)
+ and packed.dtype == torch.uint8
+ and packed.dim() == 2
+ and packed.device.type in ("cuda", "xpu")
+ and isinstance(scale, torch.Tensor)
+ and scale.dtype == torch.float8_e4m3fn
+ and tuple(scale.shape) == (packed.shape[0], packed.shape[1] * 2 // 16)
+ and isinstance(global_scale, torch.Tensor)
+ and global_scale.numel() == 1
+ and getattr(module, "weight_zero_point", None) is None
+ and "weight" not in module._parameters
+ and type(module) is torch.nn.Linear
+ )
+
+
+def _unsloth_compressed_tensors_nvfp4_forward(self, input):
+ from unsloth.kernels.nvfp4 import nvfp4_linear
+
+ # W4A16: the checkpoint's NVFP4 activation quantization is not applied.
+ out = nvfp4_linear(input, self.weight_packed, self.weight_scale, self.weight_global_scale)
+ if self.bias is not None:
+ out = out + self.bias.to(out.dtype)
+ return out
+
+
+class _UnslothNVFP4Linear(torch.nn.Linear):
+ """nn.Linear whose `weight` is the packed NVFP4 tensor carrying an NVFP4QuantState.
+
+ PEFT and get_lora_parameters read `.weight`; the state_dict keeps only weight_packed, and the uint8 dtype keeps
+ compiled LoRA forwards from casting activations to it. Built on each access so `.to()` / device moves stay valid.
+ """
+
+ @property
+ def weight(self):
+ from unsloth.kernels.nvfp4 import NVFP4QuantState
+
+ packed = self.weight_packed
+ packed.quant_state = NVFP4QuantState(
+ self.weight_scale,
+ self.weight_global_scale,
+ (packed.shape[0], packed.shape[1] * 2),
+ getattr(self, "_unsloth_nvfp4_dtype", torch.bfloat16),
+ )
+ return packed
+
+ def _apply(
+ self,
+ fn,
+ recurse = True,
+ ):
+ # model.to(dtype) / .half() may move the packed weight and its scales but never cast them: an fp16 global
+ # scale overflows past 65504 and dequantizes the layer to zeros.
+ quant = {}
+ for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):
+ param = self._parameters.get(name)
+ if param is not None:
+ quant[name] = self._parameters.pop(name)
+ try:
+ super()._apply(fn, recurse)
+ finally:
+ for name, param in quant.items():
+ device = fn(param.data[:0]).device
+ if param.device != device:
+ param = torch.nn.Parameter(param.data.to(device), requires_grad = False)
+ self._parameters[name] = param
+ # The compute dtype follows the cast (model.half() dequantizes to fp16 from then on).
+ dtype = getattr(self, "_unsloth_nvfp4_dtype", torch.bfloat16)
+ self._unsloth_nvfp4_dtype = fn(torch.empty(0, dtype = dtype)).dtype
+ return self
+
+ def dequantize_(self):
+ """Turn this layer into a plain nn.Linear holding the dense weight (PEFT merges add into `.weight`)."""
+ self.__dict__.pop("forward", None)
+ self.__class__ = torch.nn.Linear
+ self._unsloth_compressed_tensors_nvfp4 = False
+ _decompress_like_a_full_load(self)
+ return self
+
+
+_CT_QUANT_PARAMS = (
+ "weight_scale",
+ "weight_global_scale",
+ "weight_zero_point",
+ "input_global_scale",
+ "input_scale",
+)
+
+
+def _decompress_like_a_full_load(module):
+ """compressed-tensors' own per-module decompress, then a plain dense layer. The config still says compressed
+ for the rest, so the layer joins its ignore list: save_pretrained then writes a checkpoint that reloads."""
+ try:
+ from compressed_tensors.compressors import decompress_module
+ except ImportError:
+ decompress_module = None # compressed-tensors < 0.13
+ with torch.inference_mode(False), torch.no_grad():
+ if decompress_module is not None:
+ decompress_module(module)
+ else:
+ _dequantize_in_place_without_compressed_tensors(module)
+ for name in _CT_QUANT_PARAMS:
+ module._parameters.pop(name, None)
+ for attr in ("quantization_scheme", "quantization_status"):
+ module.__dict__.pop(attr, None)
+ for p in module.parameters(recurse = False):
+ p.requires_grad_(False)
+ ct_config = getattr(getattr(module, "_unsloth_ct_config", None), "quantization_config", None)
+ name = getattr(module, "_unsloth_ct_name", None)
+ if ct_config is None or name is None:
+ return
+ if getattr(ct_config, "ignore", None) is None:
+ ct_config.ignore = []
+ if name not in ct_config.ignore:
+ ct_config.ignore.append(name)
+
+
+def _dequantize_in_place_without_compressed_tensors(module):
+ """Same dense weight from Unsloth's kernels (bit-identical to compressed-tensors for NVFP4)."""
+ dtype = getattr(module, "_unsloth_nvfp4_dtype", None) or getattr(
+ module, "_unsloth_ct_compute_dtype", torch.bfloat16
+ )
+ if isinstance(getattr(module, "weight_packed", None), torch.Tensor):
+ from unsloth.kernels.nvfp4 import nvfp4_dequantize
+ W = nvfp4_dequantize(
+ module.weight_packed, module.weight_scale, module.weight_global_scale, dtype
+ )
+ module._parameters.pop("weight_packed", None)
+ else:
+ from unsloth.kernels.fp8 import weight_dequant
+ W = weight_dequant(module.weight, module.weight_scale, dtype)
+ module.weight = torch.nn.Parameter(W, requires_grad = False)
+
+
+def _tag_compressed_tensors_modules(model):
+ config = getattr(getattr(model, "config", None), "quantization_config", None)
+ for name, module in model.named_modules():
+ if getattr(module, "quantization_scheme", None) is not None:
+ module._unsloth_ct_name = name
+ module._unsloth_ct_config = config
+
+
+def _routed_dense_weight(module):
+ """Dense weight of an NVFP4- or FP8-routed linear, else None."""
+ if isinstance(module, _UnslothNVFP4Linear):
+ from unsloth.kernels.nvfp4 import nvfp4_dequantize
+ return nvfp4_dequantize(
+ module.weight_packed,
+ module.weight_scale,
+ module.weight_global_scale,
+ getattr(module, "_unsloth_nvfp4_dtype", torch.bfloat16),
+ )
+ if getattr(module, "_unsloth_compressed_tensors_fp8", False):
+ from unsloth.kernels.fp8 import weight_dequant
+ return weight_dequant(
+ module.weight,
+ module.weight_scale,
+ getattr(module, "_unsloth_ct_compute_dtype", torch.bfloat16),
+ )
+ return None
+
+
+def _dequantize_routed_linear_(module):
+ """Turn a routed linear into a plain nn.Linear with its dense weight, for PEFT merges."""
+ if isinstance(module, _UnslothNVFP4Linear):
+ return module.dequantize_()
+ if getattr(module, "_unsloth_compressed_tensors_fp8", False):
+ module.__dict__.pop("forward", None)
+ module._unsloth_compressed_tensors_fp8 = False
+ _decompress_like_a_full_load(module)
+ return module
+
+
+def _patch_peft_for_routed_compressed_tensors():
+ """PEFT merges add a dense delta into base_layer.weight and DoRA reads it: give both the dense weight."""
+ try:
+ from peft.tuners.lora import layer as lora_layer
+ except Exception:
+ return
+ merge = lora_layer.Linear.merge
+ if not getattr(merge, "_unsloth_routed", False):
+
+ @functools.wraps(merge)
+ def patched_merge(self, *args, **kwargs):
+ _dequantize_routed_linear_(self.get_base_layer())
+ return merge(self, *args, **kwargs)
+
+ patched_merge._unsloth_routed = True
+ lora_layer.Linear.merge = patched_merge
+ # PiSSA / OLoRA / CorDA / LoftQ / orthogonal / LoRA-GA read or rewrite the dense base weight (or its dtype).
+ for init in (
+ "pissa_init",
+ "olora_init",
+ "corda_init",
+ "loftq_init",
+ "orthogonal_init",
+ "lora_ga_init",
+ ):
+ original = getattr(lora_layer.LoraLayer, init, None)
+ if original is None or getattr(original, "_unsloth_routed", False):
+ continue
+
+ def patched_init(
+ self,
+ *args,
+ _original = original,
+ **kwargs,
+ ):
+ _dequantize_routed_linear_(self.get_base_layer())
+ return _original(self, *args, **kwargs)
+
+ patched_init = functools.wraps(original)(patched_init)
+ patched_init._unsloth_routed = True
+ setattr(lora_layer.LoraLayer, init, patched_init)
+ # LoRA-GA / CorDA preprocessing reads each target's weight (or enables its grad) before get_peft_model.
+ for module_name, attr in (("loraga", "get_target_modules"), ("corda", "target_modules")):
+ try:
+ owner = importlib.import_module("peft.tuners.lora." + module_name)
+ except Exception:
+ continue
+ targets = getattr(owner, attr, None)
+ if targets is None or getattr(targets, "_unsloth_routed", False):
+ continue
+
+ def patched_targets(
+ *args,
+ _targets = targets,
+ **kwargs,
+ ):
+ for name, module in _targets(*args, **kwargs):
+ yield name, _dequantize_routed_linear_(module)
+
+ patched_targets = functools.wraps(targets)(patched_targets)
+ patched_targets._unsloth_routed = True
+ setattr(owner, attr, patched_targets)
+ try:
+ from peft.tuners.lora import dora
+ except Exception:
+ return
+ dequantize = getattr(dora, "dequantize_module_weight", None)
+ if dequantize is not None and not getattr(dequantize, "_unsloth_routed", False):
+
+ @functools.wraps(dequantize)
+ def patched_dequantize(module):
+ W = _routed_dense_weight(module)
+ return dequantize(module) if W is None else W
+
+ patched_dequantize._unsloth_routed = True
+ dora.dequantize_module_weight = patched_dequantize
+ update_layer = dora.DoraLinearLayer.update_layer
+ if not getattr(update_layer, "_unsloth_routed", False):
+
+ @functools.wraps(update_layer)
+ def patched_update_layer(self, *, base_layer, lora_A, lora_B, **kwargs):
+ # PEFT has cast new adapters to an FP8 base's float8 dtype; DoRA multiplies them right away.
+ for p in (lora_A, lora_B):
+ if isinstance(p, torch.Tensor) and p.dtype in _FP8_DTYPES:
+ p.data = p.data.to(torch.float32)
+ return update_layer(self, base_layer = base_layer, lora_A = lora_A, lora_B = lora_B, **kwargs)
+
+ patched_update_layer._unsloth_routed = True
+ dora.DoraLinearLayer.update_layer = patched_update_layer
+
+
+def _route_compressed_tensors_nvfp4_to_unsloth(model):
+ """Keep NVFP4 (nvfp4-pack-quantized) weights packed and run them W4A16; decompress or FP8-route the rest."""
+ if os.environ.get("UNSLOTH_COMPRESSED_TENSORS_NVFP4_KERNELS", "1") == "0":
+ return 0
+ if getattr(getattr(model, "config", None), "quantization_config", None) is None:
+ return 0
+ try:
+ from compressed_tensors.compressors import decompress_module
+ except Exception:
+ decompress_module = None
+ # The fused CE loss and decode read lm_head.weight directly, so an NVFP4 lm_head is decompressed too.
+ get_output_embeddings = getattr(model, "get_output_embeddings", None)
+ lm_head = get_output_embeddings() if callable(get_output_embeddings) else None
+ nvfp4, others = [], []
+ for module in model.modules():
+ scheme = getattr(module, "quantization_scheme", None)
+ if scheme is None or getattr(scheme, "weights", None) is None:
+ continue
+ if str(getattr(getattr(module, "quantization_status", None), "value", "")) != "compressed":
+ continue
+ if (
+ module is not lm_head
+ and _nvfp4_scheme_supported(scheme)
+ and _nvfp4_module_routable(module)
+ ):
+ nvfp4.append(module)
+ else:
+ others.append(module)
+ if not nvfp4:
+ return 0
+ if others and decompress_module is None:
+ print(
+ "Unsloth: this compressed-tensors has no per-module decompress; decompressing the NVFP4 layers too."
+ )
+ return 0
+ compute_dtype = _fp8_dequant_target_dtype(model)
+ # The FP8 kernels dequantize to bf16, so an fp16 model keeps its FP8 layers decompressed by default.
+ fp8_routed = (
+ _route_compressed_tensors_fp8_to_unsloth(
+ model, skip = set(nvfp4), default = "1" if compute_dtype == torch.bfloat16 else "0"
+ )
+ if others
+ else 0
+ )
+ if others and not fp8_routed:
+ try:
+ with torch.inference_mode(False), torch.no_grad():
+ for module in others:
+ _decompress_like_a_full_load(module)
+ _remove_same_device_compressed_tensors_offload(module)
+ except Exception as e:
+ print(f"Unsloth: could not decompress the non-NVFP4 compressed-tensors layers ({e}).")
+ return 0
+ for module in nvfp4:
+ for name in ("weight_packed", "weight_scale", "weight_global_scale", "input_global_scale"):
+ param = getattr(module, name, None)
+ if isinstance(param, torch.Tensor):
+ param.requires_grad_(False)
+ module.__class__ = _UnslothNVFP4Linear
+ module._unsloth_nvfp4_dtype = compute_dtype
+ module.forward = _unsloth_compressed_tensors_nvfp4_forward.__get__(module)
+ module._unsloth_compressed_tensors_nvfp4 = True
+ _remove_compressed_tensors_decompress_hook(model)
+ _patch_peft_for_routed_compressed_tensors()
+ model._unsloth_compressed_tensors_nvfp4 = len(nvfp4)
+ return len(nvfp4)
+
+
_FP8_DEQUANT_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
@@ -1936,7 +2322,9 @@ def _remove_same_device_compressed_tensors_offload(model):
def _decompress_compressed_tensors_model(model):
"""Decompress at load: PEFT skips the first-forward hook, and in `generate` it yields inference tensors."""
- if getattr(model, "_unsloth_compressed_tensors_fp8", 0):
+ if getattr(model, "_unsloth_compressed_tensors_fp8", 0) or getattr(
+ model, "_unsloth_compressed_tensors_nvfp4", 0
+ ):
return False
quant_config = getattr(getattr(model, "config", None), "quantization_config", None)
if isinstance(quant_config, dict):
@@ -1984,8 +2372,14 @@ def _decompress_compressed_tensors_model(model):
def _prepare_compressed_tensors_model(model, full_finetuning = False):
- # Routed FP8 weights are frozen, so full finetuning always takes the decompressed bf16 weights.
- if full_finetuning or not _route_compressed_tensors_fp8_to_unsloth(model):
+ # Routed FP8 / NVFP4 weights are frozen, so full finetuning always takes the decompressed bf16 weights.
+ if full_finetuning:
+ _decompress_compressed_tensors_model(model)
+ return
+ _tag_compressed_tensors_modules(model)
+ if _route_compressed_tensors_nvfp4_to_unsloth(model):
+ return
+ if not _route_compressed_tensors_fp8_to_unsloth(model):
_decompress_compressed_tensors_model(model)