Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Add explicit FP8 frozen-base LoRA training (#16423)
Daniel Hanee3fa40a7c84
diff --git a/scripts/exec_literals_baseline.json b/scripts/exec_literals_baseline.jsonindex b30ad5b0e42..00d02fd1abb 100644--- a/scripts/exec_literals_baseline.json+++ b/scripts/exec_literals_baseline.json@@ -149,6 +149,13 @@"count": 1,"reason": "Installs a patched attention `__init__` generated by this file from a fixed template, and binds it by the name that template defines."},+ {+ "file": "unsloth/models/float8_training.py",+ "sink": "compile",+ "digest": "13108d7353b8c426",+ "count": 1,+ "reason": "Compiles a fixed forwarding-contract template using function and argument identifiers read from installed library code objects. The resulting code object is only fingerprinted to recognize a generated proxy; this helper never executes it or accepts model, prompt, or user strings."+ },{"file": "unsloth/models/gemma.py","sink": "eval",diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.jsonindex 563ebfa2f23..f861a5592be 100644--- a/scripts/risky_loaders_baseline.json+++ b/scripts/risky_loaders_baseline.json@@ -10,6 +10,14 @@"unsloth-cli.py"],"entries": [+ {+ "file": "unsloth/models/float8_training.py",+ "rule": "dynamic-import",+ "sink": "spec_from_file_location",+ "digest": "78b864cf6acb1555",+ "count": 1,+ "reason": "Standalone import fallback loads only unsloth/kernels/fp8_lora.py beside this shipped module, from __file__; no model metadata or caller input selects the module name or path."+ },{"file": "scripts/prequant_probe.py","rule": "unsafe-deserialize",diff --git a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py b/tests/python/test_get_lora_parameters_bias_fp8_block_size.pyindex 322fa0f29c4..8209b0781fe 100644--- a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py+++ b/tests/python/test_get_lora_parameters_bias_fp8_block_size.py@@ -2,11 +2,10 @@ import astfrom pathlib import Path-_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state", "_fp8_lora_state")def _load_function(name):- # Extract without importing unsloth, which needs a GPU.source = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"tree = ast.parse(source.read_text(encoding = "utf-8"))funcs = [@@ -42,7 +41,6 @@ def _make_disabled_block_fp8_proj(block_size):def test_bias_variant_propagates_fp8_block_size_on_disabled_path():- # fp8 kernels read weight_scale.block_size, so the real block size must survive this path.get_lora_parameters_bias = _load_function("get_lora_parameters_bias")proj, weight_scale = _make_disabled_block_fp8_proj([64, 128])@@ -67,7 +65,6 @@ def _make_decompressed_merged_proj():def test_bias_variant_keeps_none_quant_state_for_decompressed_layer():- # No quant state here; fast_linear_forward needs W_quant None to fall back to matmul.get_lora_parameters_bias = _load_function("get_lora_parameters_bias")W, W_quant = get_lora_parameters_bias(_make_decompressed_merged_proj())[:2]diff --git a/tests/python/test_get_lora_parameters_fp8_block_size.py b/tests/python/test_get_lora_parameters_fp8_block_size.pyindex 62d5dbc5512..b881bcd7cb9 100644--- a/tests/python/test_get_lora_parameters_fp8_block_size.py+++ b/tests/python/test_get_lora_parameters_fp8_block_size.py@@ -2,11 +2,10 @@ import astfrom pathlib import Path-_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state", "_fp8_lora_state")def _load_function(name):- # Extract without importing unsloth, which needs a GPU.source = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"tree = ast.parse(source.read_text(encoding = "utf-8"))funcs = [@@ -46,7 +45,6 @@ def _make_disabled_block_fp8_proj(block_size):def test_propagates_fp8_block_size_on_disabled_path():- # fp8 kernels read weight_scale.block_size, so the real block size must survive this path.get_lora_parameters = _load_function("get_lora_parameters")proj, weight_scale = _make_disabled_block_fp8_proj([64, 128])@@ -70,7 +68,6 @@ def _make_decompressed_merged_proj():def test_keeps_none_quant_state_for_decompressed_layer():- # No quant state here; fast_lora needs W_quant None to fall back to a plain matmul.get_lora_parameters = _load_function("get_lora_parameters")W, W_quant = get_lora_parameters(_make_decompressed_merged_proj())[:2]diff --git a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py b/tests/python/test_get_lora_parameters_fsdp2_quant_state.pyindex a083a07d5d2..333d2c6d0d4 100644--- a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py+++ b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py@@ -11,7 +11,7 @@ from pathlib import Pathimport torch_SOURCE = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"-_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state", "_fp8_lora_state")def _load_function(name, fsdp2_qlora_prepared = True):diff --git a/tests/test_block_compile.py b/tests/test_block_compile.pyindex 521ba55821a..8607413cebe 100644--- a/tests/test_block_compile.py+++ b/tests/test_block_compile.py@@ -328,6 +328,57 @@ def _install_blocks(M, model, monkeypatch):return inner[email protected]("fp8_first", [False, True])+def test_fp8_precision_options_are_per_instance_and_reused(monkeypatch, fp8_first):+ import copy++ M = _load()++ class PrecisionDecoderLayer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.proj = torch.nn.Linear(8, 8)++ def forward(self, x):+ return self.proj(x)++ plain, fp8 = PrecisionDecoderLayer(), PrecisionDecoderLayer()+ fp8.proj._unsloth_fp8_lora = object()+ options = []++ def compile(fn, **kwargs):+ precision = kwargs["options"].get("emulate_precision_casts", False)+ options.append(precision)++ def run(*args, **kw):+ return fn(*args, **kw) + int(precision)++ return run++ monkeypatch.setattr(torch, "compile", compile)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)+ monkeypatch.setattr(M, "_layers", lambda model: [(model, True)])+ monkeypatch.setattr(M, "_torch_supported", lambda: True)+ monkeypatch.setattr(M, "_zero3_partitioned", lambda *args: False)+ original = PrecisionDecoderLayer.forward+ for layer in (fp8, plain) if fp8_first else (plain, fp8):+ M.patch_block_compile(layer)+ x = torch.randn(2, 4, 8)+ torch.testing.assert_close(plain(x), original(plain, x))+ torch.testing.assert_close(fp8(x), original(fp8, x) + 1)+ assert "forward" not in plain.__dict__+ assert not M._COMPILE_OPTIONS.get("emulate_precision_casts", False)+ assert options == [False, True]+ forward = fp8.forward+ M.patch_block_compile(fp8)+ assert fp8.forward is forward and options == [False, True]+ twin = copy.deepcopy(fp8)+ assert twin.forward.__self__ is twin+ torch.testing.assert_close(twin(x), original(twin, x) + 1)+ fp8.eval()+ torch.testing.assert_close(fp8(x), original(fp8, x))++def _outer_loss(model,ids,@@ -886,6 +937,10 @@ def test_sm_count_is_read_outside_the_trace(monkeypatch):fla.get_multiprocessor_count = get_multiprocessor_countmonkeypatch.setattr(torch.cuda, "device_count", lambda: 1)+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)+ monkeypatch.setattr(+ torch._C, "_cuda_init", lambda: pytest.fail("CPU test initialized CUDA"), raising = False+ )M._route_sm_count({fla.__name__: fla})out = torch.compile(lambda x: x * fla.get_multiprocessor_count(0), fullgraph = True, backend = "eager"diff --git a/tests/test_float8_training.py b/tests/test_float8_training.pyindex 617617ebec8..e9e54a4d180 100644--- a/tests/test_float8_training.py+++ b/tests/test_float8_training.py@@ -240,12 +240,14 @@ def test_refuses_when_scaled_mm_probe_fails(monkeypatch):f8t._check_hardware(torch.device("cuda", 0), _emulated_config("rowwise"))-def test_loader_rejects_float8_training_for_lora():+def test_loader_float8_training_for_lora_needs_a_16bit_base():from unsloth import FastLanguageModel, FastModelfor loader in (FastLanguageModel, FastModel):- with pytest.raises(ValueError, match = "full_finetuning = True"):+ with pytest.raises(ValueError, match = "load_in_4bit = False"):loader.from_pretrained(- "trl-internal-testing/tiny-Qwen3ForCausalLM", float8_training = True+ "trl-internal-testing/tiny-Qwen3ForCausalLM",+ load_in_4bit = True,+ float8_training = True,)diff --git a/tests/test_for_training_generation_flag.py b/tests/test_for_training_generation_flag.pyindex 9481a5bbb76..d7fc5e85c18 100644--- a/tests/test_for_training_generation_flag.py+++ b/tests/test_for_training_generation_flag.py@@ -94,6 +94,7 @@ def _for_training(module, class_name):namespace = _Namespace(os = os, inspect = inspect, torch = _torch_or_stub(), functools = functools)# No tensor parallel world in these tests (for_training calls the TP guard first since #15368).namespace["_tensor_parallel_for_training"] = lambda model: None+ namespace["enable_fp8_lora"] = lambda model: []exec(compile(ast.fix_missing_locations(compiled), str(path), "exec"), namespace)return namespace["for_training"]diff --git a/tests/test_fp8_lora.py b/tests/test_fp8_lora.pynew file mode 100644index 00000000000..2aa8f50d63d--- /dev/null+++ b/tests/test_fp8_lora.py@@ -0,0 +1,1915 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""FP8 LoRA on a frozen 16-bit base (float8_training with LoRA, kernels/fp8_lora.py), on CPU: torch's CPU+_scaled_mm runs the same fp8 GEMMs, so numerics, caching, the LoRA kernels' backward routing, refusals and+torch.compile tracing are all checked without a GPU."""++from __future__ import annotations++import pytest+import torch+import torch.nn.functional as F++from unsloth import precision as P+from unsloth.kernels import fp8_lora as FL+from unsloth.kernels import fast_lora as FLK+from unsloth.models import float8_training as f8t+from real_accelerator import has_real_cuda++pytestmark = pytest.mark.skipif(+ not hasattr(torch, "float8_e4m3fn") or not hasattr(torch, "_scaled_mm"),+ reason = "needs torch float8 dtypes",+)+++def _cpu_scaled_mm_works(rowwise = False):+ if not hasattr(torch, "float8_e4m3fn") or not hasattr(torch, "_scaled_mm"):+ return False+ try:+ a = torch.zeros(16, 16).to(torch.float8_e4m3fn)+ scale_a = torch.ones((16, 1) if rowwise else (), dtype = torch.float32)+ scale_b = torch.ones((1, 16) if rowwise else (), dtype = torch.float32)+ torch._scaled_mm(a, a.t(), scale_a = scale_a, scale_b = scale_b, out_dtype = torch.bfloat16)+ return True+ except (NotImplementedError, RuntimeError):+ return False+++needs_cpu_scaled_mm = pytest.mark.skipif(+ not _cpu_scaled_mm_works(), reason = "this torch build has no CPU torch._scaled_mm kernel"+)++FP8_FWD_TOL = 0.06+FP8_DGRAD_TOL = 0.09+++ not has_real_cuda()+ or torch.version.hip is not None+ or torch.cuda.get_device_capability() < (8, 9),+ reason = "requires NVIDIA compute capability 8.9+ for FP8 CUDA cast kernels",+)[email protected](FL.triton is None, reason = "requires Triton for the CUDA cast kernels")[email protected]("dtype", [torch.bfloat16, torch.float16])[email protected]("fp8_dtype", [torch.float8_e4m3fn, torch.float8_e5m2])[email protected]("rowwise", [False, True])+def test_cuda_casts_match_native_rounding(dtype, fp8_dtype, rowwise):+ limit = torch.finfo(fp8_dtype).max+ for seed in range(16):+ generator = torch.Generator(device = "cuda").manual_seed(seed)+ x = torch.randn(32, 256, device = "cuda", dtype = dtype, generator = generator)+ if seed == 0:+ x[0].fill_(-0.0)+ x[1].fill_(torch.finfo(dtype).tiny)+ x[2, 0] = torch.finfo(dtype).max+ amax = x.float().abs().amax(dim = 1, keepdim = True) if rowwise else x.float().abs().amax()+ amax = amax.clamp_min(1e-12)+ expected = (x.float() * (limit / amax)).clamp(-limit, limit).to(fp8_dtype)+ quantize = FL.quantize_rowwise if rowwise else FL.quantize_tensorwise+ actual, inverse = quantize(x, fp8_dtype)+ assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8))+ assert torch.equal(inverse, amax / limit)+++def _rel(a, b):+ a, b = a.float(), b.float()+ return ((a - b).norm() / b.norm()).item()+++def _cos(a, b):+ return F.cosine_similarity(a.flatten().float(), b.flatten().float(), dim = 0).item()++[email protected](autouse = True)+def _stats():+ FL.reset_fp8_lora_stats()+ yield+++def _weight(+ N = 384,+ K = 256,+ dtype = torch.bfloat16,+ seed = 0,+):+ g = torch.Generator().manual_seed(seed)+ return (torch.randn(N, K, generator = g) * K**-0.5).to(dtype)++[email protected]("dtype", [torch.bfloat16, torch.float16])+@needs_cpu_scaled_mm+def test_fwd_and_dgrad_within_fp8_error(dtype):+ W = _weight(dtype = dtype)+ x = torch.randn(64, 256).to(dtype)+ dy = torch.randn(64, 384).to(dtype)+ state = FL.Fp8LoRABaseState(W)+ y = state.matmul(x, W)+ dx = state.dgrad(dy, W)+ ref_y = x.float() @ W.float().t()+ ref_dx = dy.float() @ W.float()+ assert y.dtype == dtype and dx.dtype == dtype+ e_fwd, e_dgrad = _rel(y, ref_y), _rel(dx, ref_dx)+ assert 0.01 < e_fwd < FP8_FWD_TOL, e_fwd+ assert 0.01 < e_dgrad < FP8_DGRAD_TOL, e_dgrad+ assert torch.equal(state.matmul(dy, W.t()), dx)+ assert FL.FP8_LORA_STATS["fwd"] == 1 and FL.FP8_LORA_STATS["dgrad"] == 2+++@needs_cpu_scaled_mm+def test_scaled_mm_counted_and_fp16_never_written_by_the_fp8_gemm(monkeypatch):+ seen = []+ real = torch._scaled_mm++ def spy(*args, **kwargs):+ seen.append(kwargs.get("out_dtype"))+ return real(*args, **kwargs)++ W = _weight(dtype = torch.float16)+ state = FL.Fp8LoRABaseState(W)+ monkeypatch.setattr(torch, "_scaled_mm", spy)+ with P.count_lowp_gemms() as counts:+ y = state.matmul(torch.randn(32, 256).half(), W)+ state.dgrad(torch.randn(32, 384).half(), W)+ assert counts["_scaled_mm"] == 2+ assert seen == [torch.float32, torch.float32]+ assert y.dtype == torch.float16 and torch.isfinite(y).all()+++@needs_cpu_scaled_mm+def test_large_fp16_values_stay_finite():+ W = torch.full((32, 64), 0.5, dtype = torch.float16)+ x = torch.full((16, 64), 1800.0, dtype = torch.float16) # each output 57600 < 65504+ y = FL.Fp8LoRABaseState(W).matmul(x, W)+ assert torch.isfinite(y).all()+ assert abs(y.float().mean().item() - 57600.0) / 57600.0 < 0.07+++@needs_cpu_scaled_mm+def test_fp16_results_at_the_fp16_limit_stay_finite():+ W = torch.full((32, 64), 0.5, dtype = torch.float16)+ x = torch.full((16, 64), 2047.0, dtype = torch.float16) # each output 65504, the fp16 maximum+ state = FL.Fp8LoRABaseState(W)+ assert torch.isfinite(state.matmul(x, W)).all()+ dy = torch.full((16, 32), 4094.0, dtype = torch.float16) # each dgrad output 65504+ assert torch.isfinite(state.dgrad(dy, W)).all()+++@needs_cpu_scaled_mm+def test_nan_propagates_for_the_grad_scaler():+ W = _weight()+ state = FL.Fp8LoRABaseState(W)+ dy = torch.randn(32, 384).bfloat16()+ dy[3, 5] = float("nan")+ assert torch.isnan(state.dgrad(dy, W)).any()+++def test_decode_rows_use_the_16bit_weight():+ W = _weight()+ state = FL.Fp8LoRABaseState(W)+ x = torch.randn(1, 1, 256).bfloat16()+ torch.testing.assert_close(state.matmul(x, W), x @ W.t())+ assert FL.FP8_LORA_STATS["fwd"] == 0 and FL.FP8_LORA_STATS["fwd_16bit"] == 1+++@needs_cpu_scaled_mm+def test_quantized_once_and_requantized_after_an_in_place_change(monkeypatch):+ monkeypatch.setattr(FL, "_CACHE_T", True)+ W = _weight()+ state = FL.Fp8LoRABaseState(W)+ x = torch.randn(32, 256).bfloat16()+ for _ in range(5):+ state.refresh(W)+ state.matmul(x, W)+ assert state.quantizations == 1 and FL.FP8_LORA_STATS["quantize"] == 1+ w8 = state.w8+ assert torch.equal(state.w8t.view(torch.uint8), w8.t().contiguous().view(torch.uint8))+ with torch.no_grad():+ W.mul_(2.0)+ state.refresh(W)+ assert state.quantizations == 2+ assert _rel(state.matmul(x, W), x.float() @ W.float().t()) < FP8_FWD_TOL+++class _Proj(torch.nn.Module):+ """A PEFT lora.Linear stand-in: base_layer + lora_A / lora_B ModuleDicts."""++ def __init__(+ self,+ K,+ N,+ r = 8,+ seed = 0,+ ):+ super().__init__()+ g = torch.Generator().manual_seed(seed)+ self.base_layer = torch.nn.Linear(K, N, bias = False, dtype = torch.bfloat16)+ with torch.no_grad():+ self.base_layer.weight.copy_(torch.randn(N, K, generator = g) * K**-0.5)+ self.base_layer.weight.requires_grad_(False)+ self.lora_A = torch.nn.ModuleDict(+ {"default": torch.nn.Linear(K, r, bias = False, dtype = torch.bfloat16)}+ )+ self.lora_B = torch.nn.ModuleDict(+ {"default": torch.nn.Linear(r, N, bias = False, dtype = torch.bfloat16)}+ )+ with torch.no_grad():+ self.lora_A["default"].weight.copy_(torch.randn(r, K, generator = g) * 0.05)+ self.lora_B["default"].weight.copy_(torch.randn(N, r, generator = g) * 0.05)+ self.scaling = {"default": 2.0}+ self.disable_adapters = False+ self.merged = False+ self.active_adapters = ["default"]+++def _silu_fwd(e, g):+ return F.silu(e.float()).to(e.dtype) * g+++def _silu_bwd(DW, e, g):+ ef = e.float()+ s = torch.sigmoid(ef)+ f = ef * s+ h = (f * g.float()).to(DW.dtype)+ df = (DW.float() * f).to(DW.dtype)+ de = (DW.float() * g.float() * (s * (1 + ef * (1 - s)))).to(DW.dtype)+ return h, df, de+++def _lora_params(projs):+ return [p.lora_A["default"].weight for p in projs] + [p.lora_B["default"].weight for p in projs]+++def _run_block(kind, fp8):+ torch.manual_seed(0)+ H, I = 256, 512+ if kind == "mlp":+ projs = [_Proj(H, I, seed = 1), _Proj(H, I, seed = 2), _Proj(I, H, seed = 3)]+ elif kind == "qkv":+ projs = [_Proj(H, H, seed = 1), _Proj(H, 128, seed = 2), _Proj(H, 128, seed = 3)]+ else:+ projs = [_Proj(H, H, seed = 4)]+ if fp8:+ for p in projs:+ FL.attach_fp8_lora_base(p.base_layer)+ X = (torch.randn(2, 32, H) * 0.5).bfloat16().requires_grad_(True)+ if kind == "mlp":+ args = []+ for p in projs:+ args += list(FLK.get_lora_parameters(p))+ out = FLK.LoRA_MLP.apply(X, *args, _silu_fwd, _silu_bwd, False)+ outs = (out,)+ elif kind == "qkv":+ args = []+ for p in projs:+ args += list(FLK.get_lora_parameters(p))+ outs = FLK.LoRA_QKV.apply(X, *args, False)+ else:+ outs = (FLK.LoRA_W.apply(X, *FLK.get_lora_parameters(projs[0])),)+ g = torch.Generator().manual_seed(9)+ loss = sum((o.float() * torch.randn(o.shape, generator = g)).sum() for o in outs)+ loss.backward()+ return [o.detach() for o in outs], X.grad, [p.grad for p in _lora_params(projs)], projs++[email protected]("kind", ["mlp", "qkv", "o"])+@needs_cpu_scaled_mm+def test_lora_kernels_run_fp8_and_keep_lora_grads_16bit(kind):+ outs_ref, dX_ref, grads_ref, _ = _run_block(kind, fp8 = False)+ FL.reset_fp8_lora_stats()+ outs, dX, grads, projs = _run_block(kind, fp8 = True)+ n = len(projs)+ assert FL.FP8_LORA_STATS["fwd"] == n, FL.FP8_LORA_STATS+ assert FL.FP8_LORA_STATS["dgrad"] == n, FL.FP8_LORA_STATS+ assert FL.FP8_LORA_STATS["fwd_16bit"] == 0 and FL.FP8_LORA_STATS["dgrad_16bit"] == 0+ for o, r in zip(outs, outs_ref):+ assert _rel(o, r) < FP8_FWD_TOL * 1.5+ assert _rel(dX, dX_ref) < FP8_DGRAD_TOL * 1.5+ for g, r, p in zip(grads, grads_ref, _lora_params(projs)):+ assert g is not None and g.dtype == p.dtype == torch.bfloat16+ assert _cos(g, r) > 0.99, _cos(g, r)+ for p in projs:+ assert p.base_layer.weight.grad is None+++@needs_cpu_scaled_mm+def test_peft_path_base_forward_and_backward():+ base = torch.nn.Linear(256, 384, bias = True, dtype = torch.bfloat16)+ base.requires_grad_(False)+ ref = torch.nn.Linear(256, 384, bias = True, dtype = torch.bfloat16)+ ref.load_state_dict(base.state_dict())+ ref.requires_grad_(False)+ FL.attach_fp8_lora_base(base)+ x = torch.randn(2, 16, 256).bfloat16()+ xa, xb = x.clone().requires_grad_(True), x.clone().requires_grad_(True)+ ya, yb = base(xa), ref(xb)+ dy = torch.randn_like(ya)+ ya.backward(dy)+ yb.backward(dy)+ assert _rel(ya, yb) < FP8_FWD_TOL and _rel(xa.grad, xb.grad) < FP8_DGRAD_TOL+ assert FL.FP8_LORA_STATS["fwd"] == 1 and FL.FP8_LORA_STATS["dgrad"] == 1+ assert set(base.state_dict()) == {"weight", "bias"}+ assert base.weight.dtype == torch.bfloat16+++def _tiny_qwen3_peft():+ from transformers import Qwen3Config, Qwen3ForCausalLM+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(0)+ config = Qwen3Config(+ vocab_size = 256,+ hidden_size = 128,+ intermediate_size = 256,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 32,+ )+ model = Qwen3ForCausalLM(config).to(torch.bfloat16)+ lora = LoraConfig(+ r = 8,+ lora_alpha = 16,+ target_modules = [+ "q_proj",+ "k_proj",+ "v_proj",+ "o_proj",+ "gate_proj",+ "up_proj",+ "down_proj",+ ],+ init_lora_weights = False,+ )+ return get_peft_model(model, lora)+++def _loss_and_grads(model):+ torch.manual_seed(1)+ ids = torch.randint(0, 256, (2, 32))+ out = model(input_ids = ids, labels = ids)+ out.loss.backward()+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.requires_grad}+ model.zero_grad(set_to_none = True)+ return out.loss.item(), grads+++@needs_cpu_scaled_mm+def test_enable_fp8_lora_on_a_peft_model():+ model = _tiny_qwen3_peft()+ loss_ref, grads_ref = _loss_and_grads(model)+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ converted = f8t.enable_fp8_lora(model, allow_cpu = True)+ assert len(converted) == 2 * 7+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["modules"] == 14+ loss, grads = _loss_and_grads(model)+ assert FL.FP8_LORA_STATS["fwd"] == 14 and FL.FP8_LORA_STATS["dgrad"] == 11+ assert abs(loss - loss_ref) / loss_ref < 0.01+ assert grads.keys() == grads_ref.keys()+ for name, g in grads.items():+ assert g.dtype == grads_ref[name].dtype+ assert _cos(g, grads_ref[name]) > 0.98, (name, _cos(g, grads_ref[name]))+ _loss_and_grads(model)+ assert FL.FP8_LORA_STATS["quantize"] == 14++[email protected]("flag", [None, False, True])[email protected]("private_namespace", [False, True])+def test_fp8_loss_ignores_model_output_flags_only_on_its_head(monkeypatch, flag, private_namespace):+ import copy+ import types+ from unsloth_zoo.fused_losses import cross_entropy_loss as ce+ from unsloth_zoo.fused_losses import forward_adapter++ monkeypatch.setattr(ce, "_CHUNKED_CE", True)+ monkeypatch.setattr(ce, "_FUSED_CE_COMPILE_DISABLED", False)+ monkeypatch.setattr(ce, "TARGET_GB", 0)+ monkeypatch.setattr(ce, "N_CHUNKS", 0)+ monkeypatch.setattr(ce._chunked_logprob, "_FUSED_LINEAR", True)+ routes = []++ def observe_loss(**kwargs):+ extras = {+ k: kwargs[k] for k in ("output_attentions", "output_hidden_states") if k in kwargs+ }+ routes.append(ce._use_chunked_ce(extras, None))+ return kwargs["hidden_states"].sum() * 0++ monkeypatch.setattr(forward_adapter, "unsloth_fused_ce_loss", observe_loss)+ model = _tiny_qwen3_peft()+ owner = model.get_base_model()+ ordinary = _tiny_qwen3_peft().get_base_model()+ hidden = torch.zeros(2, 3, 128)+ labels = torch.zeros(2, 3, dtype = torch.long)++ def dispatch(self, hidden, labels):+ return unsloth_fused_lm_head_loss(+ hidden,+ self.get_output_embeddings(),+ labels,+ output_attentions = flag,+ output_hidden_states = flag,+ )++ original = forward_adapter.unsloth_fused_lm_head_loss+ namespace = {"__builtins__": __builtins__} if private_namespace else dispatch.__globals__+ monkeypatch.setitem(namespace, "unsloth_fused_lm_head_loss", original)+ function = types.FunctionType(dispatch.__code__, namespace, closure = dispatch.__closure__)+ attribute = "_old_forward" if private_namespace else "forward"+ setattr(owner, attribute, types.MethodType(function, owner))+ setattr(ordinary, attribute, types.MethodType(function, ordinary))++ def run(target):+ getattr(target, attribute)(hidden, labels)+ return routes[-1]++ assert not run(owner)+ f8t.mark_fp8_lora(owner, f8t.FP8_LORA_RECIPE)+ assert f8t.enable_fp8_lora(model, allow_cpu = True)+ assert run(owner)+ assert namespace["unsloth_fused_lm_head_loss"] is original+ assert forward_adapter.unsloth_fused_lm_head_loss is original+ assert getattr(ordinary, attribute).__func__ is function+ assert not run(ordinary)+ clone = copy.deepcopy(model)+ assert run(clone.get_base_model())+ f8t.restore_fp8_lora_for_inference(clone)+ assert not run(clone.get_base_model())+ assert run(owner)+ f8t.restore_fp8_lora_for_inference(model)+ assert not run(owner)++[email protected]("decorated", [False, True])[email protected]("private_namespace", [False, True])+def test_fp8_loss_route_traces_fullgraph_without_changing_loss_or_input_gradient(+ monkeypatch, decorated, private_namespace+):+ import types+ from transformers.loss.loss_utils import ForCausalLMLoss+ from unsloth_zoo.fused_losses import forward_adapter++ def cpu_reference(hidden_states, lm_head_weight, lm_head_bias, labels, n_items, **kwargs):+ assert "output_attentions" not in kwargs+ assert "output_hidden_states" not in kwargs+ return ForCausalLMLoss(+ logits = F.linear(hidden_states, lm_head_weight, lm_head_bias),+ labels = labels,+ vocab_size = lm_head_weight.shape[0],+ num_items_in_batch = n_items,+ )++ monkeypatch.setattr(forward_adapter, "unsloth_fused_ce_loss", cpu_reference)+ model = _tiny_qwen3_peft()+ owner = model.get_base_model()+ head = model.get_output_embeddings()+ labels = torch.tensor([[1, 2, 3, -100], [-100, 3, 2, 1]])+ x = torch.randn(2, 4, 128, dtype = torch.bfloat16, requires_grad = True)++ def routed(self, hidden):+ return unsloth_fused_lm_head_loss(+ hidden,+ self.get_output_embeddings(),+ labels,+ num_items_in_batch = 5,+ output_attentions = None,+ output_hidden_states = None,+ )++ namespace = {"__builtins__": __builtins__} if private_namespace else routed.__globals__+ monkeypatch.setitem(+ namespace, "unsloth_fused_lm_head_loss", forward_adapter.unsloth_fused_lm_head_loss+ )+ function = types.FunctionType(routed.__code__, namespace, closure = routed.__closure__)+ if decorated:+ from transformers.utils.generic import can_return_tuple+ function = can_return_tuple(function)+ owner.forward = types.MethodType(function, owner)+ f8t.mark_fp8_lora(owner, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ expected = ForCausalLMLoss(+ logits = head(x),+ labels = labels,+ vocab_size = 256,+ num_items_in_batch = 5,+ )+ (expected_grad,) = torch.autograd.grad(expected, x)+ torch._dynamo.reset()+ compiled = torch.compile(owner, backend = "aot_eager", fullgraph = True)+ actual = compiled(x)+ (actual_grad,) = torch.autograd.grad(actual, x)+ assert torch.equal(actual, expected)+ assert torch.equal(actual_grad, expected_grad)+ if decorated and private_namespace:+ from torch._dynamo.utils import counters++ before = counters["stats"]["unique_graphs"]+ compiled(x)+ stable = counters["stats"]["unique_graphs"] - before+ f8t.restore_fp8_lora_for_inference(model)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ before = counters["stats"]["unique_graphs"]+ compiled(x)+ rebound = counters["stats"]["unique_graphs"] - before+ print("LOSS_REBIND_GRAPHS", stable, rebound)+ assert stable == rebound == 0++[email protected]("return_dict", [False, True])+def test_fp8_loss_route_preserves_installed_transformers_decorators(monkeypatch, return_dict):+ from transformers.loss.loss_utils import ForCausalLMLoss+ from unsloth_zoo.fused_losses import forward_adapter, forward_install++ model = _tiny_qwen3_peft()+ owner = model.get_base_model()+ cls = type(owner)+ original = cls.forward+ monkeypatch.setattr(cls, "forward", original)+ for name in ("_PATCHED", "_FAILED", "_UNMATCHED"):+ monkeypatch.setattr(forward_install, name, dict(getattr(forward_install, name)))+ monkeypatch.setenv("UNSLOTH_FUSED_FORWARD", "1")+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "0")+ assert forward_install.install_for_class(cls), forward_install.audit()+ installed = cls.forward+ calls = []++ def cpu_reference(hidden_states, lm_head_weight, lm_head_bias, labels, n_items, **kwargs):+ calls.append(kwargs)+ assert "output_attentions" not in kwargs+ assert "output_hidden_states" not in kwargs+ return ForCausalLMLoss(+ logits = F.linear(hidden_states, lm_head_weight, lm_head_bias),+ labels = labels,+ vocab_size = lm_head_weight.shape[0],+ num_items_in_batch = n_items,+ )++ monkeypatch.setattr(forward_adapter, "unsloth_fused_ce_loss", cpu_reference)+ monkeypatch.setattr(FL, "_MIN_ROWS", 1024)+ owner.loss_function = forward_install._STOCK_CAUSAL_LM_LOSS+ f8t.mark_fp8_lora(owner, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ assert owner.__dict__.get("_unsloth_fp8_lora_loss_forwards")+ ids = torch.tensor([[1, 3, 2, 5]])+ result = owner(+ input_ids = ids,+ labels = ids,+ use_cache = False,+ return_dict = return_dict,+ output_attentions = False,+ output_hidden_states = False,+ )+ assert calls+ assert isinstance(result, tuple) is (not return_dict)+ loss = result.loss if return_dict else result[0]+ assert torch.isfinite(loss)+ assert cls.forward is installed+ assert FL.FP8_LORA_STATS["fwd"] == 0+ assert FL.FP8_LORA_STATS["fwd_16bit"] > 0+ f8t.restore_fp8_lora_for_inference(model)+ assert owner.forward.__func__ is installed+++def test_no_request_no_conversion_and_strict_zero_engagement(monkeypatch):+ model = _tiny_qwen3_peft()+ assert f8t.enable_fp8_lora(model) == []+ for module in model.modules():+ assert "_unsloth_fp8_lora" not in module.__dict__+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")+ with pytest.raises(P.EngagementError):+ f8t.enable_fp8_lora(model, recipe = f8t.FP8_LORA_RECIPE)+++def test_eligibility_reasons():+ assert FL.eligible_base(torch.nn.Linear(100, 64).bfloat16().requires_grad_(False)) is not None+ assert "trainable" in FL.eligible_base(torch.nn.Linear(64, 64).bfloat16())+ assert "dtype" in FL.eligible_base(torch.nn.Linear(64, 64).requires_grad_(False))+ assert FL.eligible_base(torch.nn.Linear(64, 64).bfloat16().requires_grad_(False)) is None+ sub = torch.nn.Linear(64, 64).bfloat16().requires_grad_(False)++ class _Sub(torch.Tensor):+ pass++ sub.weight = torch.nn.Parameter(sub.weight.detach().as_subclass(_Sub), requires_grad = False)+ assert "_Sub" in FL.eligible_base(sub)+++@needs_cpu_scaled_mm+def test_compile_aot_eager_fullgraph():+ torch._dynamo.reset()+ base = torch.nn.Linear(256, 384, bias = False, dtype = torch.bfloat16).requires_grad_(False)+ FL.attach_fp8_lora_base(base)+ proj = _Proj(256, 384)+ FL.attach_fp8_lora_base(proj.base_layer)++ def step(x):+ y = base(x)+ W, W_quant, A, B, S = FLK.get_lora_parameters(proj)+ z = FLK.LoRA_W.apply(y[..., :256].contiguous(), W, W_quant, A, B, S)+ return z.float().square().mean()++ x = torch.randn(2, 16, 256).bfloat16().requires_grad_(True)+ eager = step(x)+ eager.backward()+ g_eager = x.grad.clone()+ x.grad = None+ FL.reset_fp8_lora_stats()+ compiled = torch.compile(step, backend = "aot_eager", fullgraph = True)+ out = compiled(x)+ out.backward()+ torch.testing.assert_close(out, eager)+ torch.testing.assert_close(x.grad, g_eager)+ assert FL.FP8_LORA_STATS["fwd"] == 2 and FL.FP8_LORA_STATS["dgrad"] == 2, FL.FP8_LORA_STATS+ explain = torch._dynamo.explain(step)(x)+ assert explain.graph_break_count == 0+++def test_resolve_refusals_and_names():+ R = f8t.FP8_LORA_RECIPE+ with pytest.raises(P.PrecisionRefused, match = "fp8 tensor cores"):+ f8t.resolve_fp8_lora("fp8_lora", gpu = "sm75")+ with pytest.raises(P.PrecisionRefused, match = "fp8 tensor cores"):+ f8t.resolve_fp8_lora("fp8_lora", gpu = "sm80")+ with pytest.raises(P.PrecisionRefused, match = "ROCm"):+ f8t.resolve_fp8_lora("fp8_lora", gpu = "rocm")+ with pytest.raises(P.PrecisionRefused, match = "no CUDA device"):+ f8t.resolve_fp8_lora("fp8_lora", gpu = None)+ with pytest.raises(P.PrecisionRefused, match = "load_in_4bit = False"):+ f8t.resolve_fp8_lora("fp8_lora", load_in_4bit = True, gpu = "sm89")+ with pytest.raises(P.PrecisionRefused, match = "qat_scheme"):+ f8t.resolve_fp8_lora("fp8_lora", qat_scheme = "fp8-fp8", gpu = "sm89")+ with pytest.raises(P.PrecisionRefused, match = "fp32"):+ f8t.resolve_fp8_lora("fp8_lora", dtype = torch.float32, gpu = "sm89")+ with pytest.raises(ValueError, match = "full finetuning recipe"):+ f8t.resolve_fp8_lora("rowwise", gpu = "sm89")+ assert f8t.resolve_fp8_lora("fp8_lora", gpu = "sm89", dtype = torch.bfloat16) == R+ assert f8t.resolve_fp8_lora("fp8_lora", gpu = "sm120", dtype = torch.float16) == "fp8_lora_rowwise"+ assert FL.scaling_for_class("sm120") == "row" and FL.scaling_for_class("sm89") == "tensor"+ assert f8t.resolve_fp8_lora(False) is None and f8t.resolve_fp8_lora(None) is None+ gate = P.get_recipe(R).gate("sm89").status+ if gate != "named":+ with pytest.raises(P.PrecisionRefused, match = "not been measured faster"):+ f8t.resolve_fp8_lora(True, gpu = "sm89")++[email protected]("gpu", [None, "sm75", "rocm"])+def test_loader_refuses_before_any_load(monkeypatch, gpu):+ from unsloth.models import loader+ monkeypatch.setattr(P, "current_gpu_class", lambda: gpu)+ for cls in (loader.FastLanguageModel, loader.FastModel):+ with pytest.raises(P.PrecisionRefused, match = "load_in_4bit = False"):+ cls.from_pretrained(+ model_name = "unsloth/does-not-exist", load_in_4bit = True, float8_training = "fp8_lora"+ )+ with pytest.raises(P.PrecisionRefused):+ cls.from_pretrained(+ model_name = "unsloth/does-not-exist",+ load_in_4bit = False,+ float8_training = "fp8_lora",+ )+++def test_registry_row():+ r = P.get_recipe(f8t.FP8_LORA_RECIPE)+ assert r.kind == "frozen_base" and r.modes == ("lora",)+ assert r.fwd.a.dtype == "e4m3" and r.fwd.b.granularity == "tensor"+ assert r.dgrad.a.dtype == "e5m2" and not r.wgrad.low_precision+ assert "fp32" not in r.input_dtypes+ assert r.gate("sm75").status == "refused" and r.gate("sm80").status == "refused"+++def rowwise(monkeypatch):+ if not _cpu_scaled_mm_works(rowwise = True):+ pytest.skip("this torch build has no CPU rowwise torch._scaled_mm kernel")+ monkeypatch.setattr(FL, "_SCALING", "row")+ monkeypatch.setattr(FL, "_DGRAD_DTYPE", FL.E4M3)+ yield+++@needs_cpu_scaled_mm+def test_rowwise_numerics_and_outlier_tokens(rowwise):+ W = _weight()+ x = torch.randn(64, 256).bfloat16()+ x[0] *= 1e5+ dy = torch.randn(64, 384).bfloat16()+ state = FL.Fp8LoRABaseState(W)+ assert state.w_inv_fwd.shape == (1, 384) and state.w_inv_dgrad.shape == (1, 256)+ y = state.matmul(x, W)+ ref = x.float() @ W.float().t()+ rest = _rel(y[1:], ref[1:])+ assert rest < FP8_FWD_TOL, rest+ FL._SCALING = "tensor"+ tensorwise = _rel(FL.Fp8LoRABaseState(W).matmul(x, W)[1:], ref[1:])+ FL._SCALING = "row"+ assert tensorwise > 2 * rest, (tensorwise, rest)+ dx = state.dgrad(dy, W)+ assert _rel(dx, dy.float() @ W.float()) < FP8_FWD_TOL+++@needs_cpu_scaled_mm+def test_rowwise_lora_kernels_and_peft_path(rowwise):+ outs_ref, dX_ref, grads_ref, _ = _run_block("mlp", fp8 = False)+ FL.reset_fp8_lora_stats()+ outs, dX, grads, projs = _run_block("mlp", fp8 = True)+ assert FL.FP8_LORA_STATS["fwd"] == 3 and FL.FP8_LORA_STATS["dgrad"] == 3+ assert _rel(outs[0], outs_ref[0]) < FP8_FWD_TOL * 1.5 and _rel(dX, dX_ref) < FP8_FWD_TOL * 1.5+ for g, r in zip(grads, grads_ref):+ assert _cos(g, r) > 0.99+ base = torch.nn.Linear(256, 384, bias = False, dtype = torch.bfloat16).requires_grad_(False)+ FL.attach_fp8_lora_base(base)+ x = torch.randn(2, 8, 256).bfloat16().requires_grad_(True)+ base(x).float().sum().backward()+ assert x.grad is not None and torch.isfinite(x.grad).all()+++def test_loaded_adapter_is_converted_when_marked(monkeypatch):+ model = _tiny_qwen3_peft()+ calls = []+ real = f8t.enable_fp8_lora+ monkeypatch.setattr(+ f8t, "enable_fp8_lora", lambda m, r = None, **k: calls.append(r) or real(m, r, allow_cpu = True)+ )+ f8t.mark_fp8_lora(model, f8t.FP8_LORA_RECIPE)+ assert calls == [f8t.FP8_LORA_RECIPE]+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["modules"] == 14+++@needs_cpu_scaled_mm+def test_default_no_transposed_cache_matches_cached(monkeypatch):+ W = _weight()+ dy = torch.randn(32, 384).bfloat16()+ lean = FL.Fp8LoRABaseState(W)+ assert lean.w8t is None+ monkeypatch.setattr(FL, "_CACHE_T", True)+ cached = FL.Fp8LoRABaseState(W)+ assert cached.w8t is not None+ torch.testing.assert_close(lean.dgrad(dy, W), cached.dgrad(dy, W), rtol = 0, atol = 0)+++@needs_cpu_scaled_mm+def test_compiled_graph_has_no_fp8_transpose_outside_the_ops():+ torch._dynamo.reset()+ base = torch.nn.Linear(256, 384, bias = False, dtype = torch.bfloat16).requires_grad_(False)+ FL.attach_fp8_lora_base(base)+ proj = _Proj(256, 384)+ FL.attach_fp8_lora_base(proj.base_layer)+ graphs = []++ def backend(gm, example_inputs):+ from torch._functorch.aot_autograd import aot_module_simplified+ def capture(g, _):+ graphs.append(g)+ return g++ return aot_module_simplified(gm, example_inputs, fw_compiler = capture, bw_compiler = capture)++ def step(x):+ y = base(x)+ W, W_quant, A, B, S = FLK.get_lora_parameters(proj)+ return FLK.LoRA_W.apply(y[..., :256].contiguous(), W, W_quant, A, B, S).float().sum()++ x = torch.randn(2, 16, 256).bfloat16().requires_grad_(True)+ torch.compile(step, backend = backend, fullgraph = True)(x).backward()+ assert len(graphs) >= 2+ fp8 = (torch.float8_e4m3fn, torch.float8_e5m2)+ targets = []+ for g in graphs:+ for node in g.graph.nodes:+ val = node.meta.get("val")+ if node.op == "call_function" and isinstance(val, torch.Tensor) and val.dtype in fp8:+ targets.append(str(node.target))+ assert targets == [], targets+ names = {str(n.target) for g in graphs for n in g.graph.nodes if n.op == "call_function"}+ assert any("fp8_lora_fwd" in t for t in names) and any(+ "fp8_lora_dgrad" in t for t in names+ ), names+++def offload_on_cpu(monkeypatch):+ monkeypatch.setattr(FL, "_OFFLOAD_CPU_WEIGHTS", True)+ monkeypatch.setattr(FL, "_OFFLOAD_MASTER", True)+ monkeypatch.delenv("WORLD_SIZE", raising = False)+++def _frozen_linear(+ K = 256,+ N = 384,+ bias = False,+ seed = 0,+):+ torch.manual_seed(seed)+ lin = torch.nn.Linear(K, N, bias = bias, dtype = torch.bfloat16)+ return lin.requires_grad_(False)+++def test_host_master_leaves_only_fp8_bytes_on_the_compute_device(offload_on_cpu):+ lin, ref = _frozen_linear(), _frozen_linear()+ W0, param = lin.weight.detach().clone(), lin.weight+ state = FL.attach_fp8_lora_base(lin)+ ref_state = FL.attach_fp8_lora_base(ref, offload = False)+ assert state.offloaded and not ref_state.offloaded+ assert lin.weight is param+ assert torch.equal(lin.weight, W0)+ numel = 256 * 384+ assert state.resident_bytes(lin) == {+ "fp8": numel + 8,+ "master_device": 0,+ "master_host": 2 * numel,+ }+ assert ref_state.resident_bytes(ref) == {+ "fp8": numel + 8,+ "master_device": 2 * numel,+ "master_host": 0,+ }+ assert FL.fp8_lora_memory(torch.nn.Sequential(lin, ref))["total"] == {+ "fp8": 2 * (numel + 8),+ "master_device": 2 * numel,+ "master_host": 2 * numel,+ }+++@needs_cpu_scaled_mm+def test_host_master_gives_the_same_fp8_results(offload_on_cpu):+ lin, ref = _frozen_linear(bias = True), _frozen_linear(bias = True)+ FL.attach_fp8_lora_base(lin)+ FL.attach_fp8_lora_base(ref, offload = False)+ x = torch.randn(2, 16, 256).bfloat16()+ xa, xb = x.clone().requires_grad_(True), x.clone().requires_grad_(True)+ ya, yb = lin(xa), ref(xb)+ dy = torch.randn_like(ya)+ ya.backward(dy)+ yb.backward(dy)+ assert torch.equal(ya, yb) and torch.equal(xa.grad, xb.grad)+ assert FL.FP8_LORA_STATS["fwd"] == 2 and FL.FP8_LORA_STATS["dgrad"] == 2+ assert FL.FP8_LORA_STATS["quantize"] == 2+++def test_host_master_decode_rows_use_the_dequantized_fp8_weight(offload_on_cpu):+ lin = _frozen_linear()+ W0 = lin.weight.detach().clone()+ state = FL.attach_fp8_lora_base(lin)+ x = torch.randn(1, 1, 256).bfloat16()+ dense = (state.w8.float() * state.w_inv).bfloat16()+ torch.testing.assert_close(state.matmul(x, lin.weight), x @ dense.t())+ torch.testing.assert_close(lin(x), x @ dense.t())+ dy = torch.randn(1, 1, 384).bfloat16()+ torch.testing.assert_close(state.matmul(dy, lin.weight.t()), dy @ dense)+ assert _rel(lin(x), x @ W0.t()) < FP8_FWD_TOL+ assert FL.FP8_LORA_STATS["fwd"] == 0+++@needs_cpu_scaled_mm[email protected]("backend", ["llama", "vision"])+def test_generation_restores_exact_base_then_training_reenables_fp8(+ offload_on_cpu, monkeypatch, backend+):+ from unsloth.models import llama, vision++ owner = llama if backend == "llama" else vision+ api = owner.FastLlamaModel if backend == "llama" else owner.FastBaseModel+ monkeypatch.setenv("UNSLOTH_RETURN_HIDDEN_STATES", "0")+ monkeypatch.setattr(+ owner, "enable_fp8_lora", lambda model: f8t.enable_fp8_lora(model, allow_cpu = True)+ )+ model = _tiny_qwen3_peft().eval()+ ids = torch.randint(0, 256, (1, 32))+ with torch.no_grad():+ expected = model(ids).logits+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ assert FL.fp8_lora_memory(model)["layers"]+ api.for_inference(model)+ with torch.no_grad():+ actual = model(ids).logits+ assert torch.equal(actual, expected)+ assert not FL.fp8_lora_memory(model)["layers"]+ api.for_training(model, use_gradient_checkpointing = False)+ assert len(FL.fp8_lora_memory(model)["layers"]) == 14+ FL.reset_fp8_lora_stats()+ _, grads = _loss_and_grads(model)+ assert FL.FP8_LORA_STATS["fwd"] == 14+ assert any(torch.count_nonzero(grad) for grad in grads.values())+++def test_enabling_an_offloaded_base_again_keeps_engagement(offload_on_cpu, monkeypatch):+ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ expected = f8t.enable_fp8_lora(model, allow_cpu = True)+ states = [+ module.__dict__["_unsloth_fp8_lora"]+ for module in model.modules()+ if "_unsloth_fp8_lora" in module.__dict__+ ]+ for state in states:+ assert state.offloaded+ state.device = torch.device("cuda:0")+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9))+ monkeypatch.setattr(torch.version, "hip", None)+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")+ assert f8t.enable_fp8_lora(model) == expected+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["modules"] == len(expected)+ assert all(state.offloaded and state.quantizations == 1 for state in states)+++def test_host_master_dequantize_is_exact(offload_on_cpu):+ lin = _frozen_linear()+ W0 = lin.weight.detach().clone()+ state = FL.attach_fp8_lora_base(lin)+ from unsloth.kernels.utils import fast_dequantize++ assert torch.equal(fast_dequantize(lin.weight, state), W0)+ assert torch.equal(fast_dequantize(lin.weight.t(), state), W0.t())+++def test_repeated_inference_does_not_rescan_restored_layers(offload_on_cpu, monkeypatch):+ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ f8t.restore_fp8_lora_for_inference(model)+ calls = []+ original = model.modules++ def modules():+ calls.append(True)+ return original()++ monkeypatch.setattr(model, "modules", modules)+ f8t.restore_fp8_lora_for_inference(model)+ assert calls == []+++@needs_cpu_scaled_mm+def test_module_moves_keep_or_restore_the_host_master(offload_on_cpu):+ lin = _frozen_linear(bias = True)+ state = FL.attach_fp8_lora_base(lin)+ w8 = state.w8+ lin.to(state.device)+ lin.to(torch.bfloat16)+ assert state.offloaded and state.w8 is w8+ W0 = lin.weight.detach().clone()+ lin.half()+ assert not state.offloaded and lin.weight.dtype == torch.float16+ assert torch.equal(lin.weight, W0.half())+ x = torch.randn(32, 256).half()+ y = lin(x)+ assert (+ y.dtype == torch.float16+ and _rel(y, x.float() @ W0.float().t() + lin.bias.float()) < FP8_FWD_TOL+ )+++def test_no_host_master_for_16bit_dgrad_or_distributed(offload_on_cpu, monkeypatch):+ monkeypatch.setattr(FL, "_DGRAD_DTYPE", None)+ assert not FL.offload_supported()+ monkeypatch.setattr(FL, "_DGRAD_DTYPE", "auto")+ assert FL.offload_supported()+ monkeypatch.setenv("WORLD_SIZE", "2")+ assert not FL.offload_supported()+ monkeypatch.delenv("WORLD_SIZE")+ monkeypatch.setattr(FL, "_OFFLOAD_MASTER", False)+ assert not FL.offload_supported()+++@needs_cpu_scaled_mm+def test_save_and_merge_exact_with_host_master(offload_on_cpu):+ import copy++ model = _tiny_qwen3_peft()+ ref = copy.deepcopy(model)+ frozen = {n: p.detach().clone() for n, p in model.named_parameters() if not p.requires_grad}+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ converted = f8t.enable_fp8_lora(model, allow_cpu = True)+ assert len(converted) == 14+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["detail"]["master_offloaded"] == 14+ memory = FL.fp8_lora_memory(model)+ assert len(memory["layers"]) == 14 and memory["total"]["master_device"] == 0+ opt = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr = 0.1)+ for _ in range(3):+ _loss_and_grads_keep(model)+ opt.step()+ opt.zero_grad(set_to_none = True)+ sd = model.state_dict()+ for name, value in frozen.items():+ assert torch.equal(sd[name], value), name+ from safetensors.torch import load, save++ base = {n: sd[n].contiguous() for n in frozen if "base_layer" in n}+ assert len(base) == 14+ loaded = load(save(base))+ assert all(torch.equal(loaded[n], frozen[n]) for n in base)+ ref.load_state_dict({n: p for n, p in sd.items() if "lora_" in n}, strict = False)+ from unsloth.save import _merge_lora++ mods, ref_mods = dict(model.named_modules()), dict(ref.named_modules())+ for name in converted:+ W, _ = _merge_lora(mods[name], name)+ W_ref, _ = _merge_lora(ref_mods[name], name)+ assert W.dtype == W_ref.dtype and torch.equal(W, W_ref), name+ merged = model.merge_and_unload().state_dict()+ merged_ref = ref.merge_and_unload().state_dict()+ assert merged.keys() == merged_ref.keys()+ for name in merged_ref:+ assert torch.equal(merged[name], merged_ref[name]), name+ assert all(+ not m.__dict__["_unsloth_fp8_lora"].offloaded+ for m in model.modules()+ if "_unsloth_fp8_lora" in m.__dict__+ )++[email protected]("method", ["merge_and_unload", "unload"])+def test_unloaded_model_matches_its_reloaded_dense_weights(offload_on_cpu, method):+ import copy++ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ assert len(f8t.enable_fp8_lora(model, allow_cpu = True)) == 14+ clone = copy.deepcopy(model)+ merged = getattr(clone, method)().eval()+ reloaded = type(merged)(copy.deepcopy(merged.config)).to(torch.bfloat16).eval()+ reloaded.load_state_dict(merged.state_dict())+ ids = torch.randint(0, 256, (2, 32))+ FL.reset_fp8_lora_stats()+ with torch.no_grad():+ actual = merged(input_ids = ids).logits+ expected = reloaded(input_ids = ids).logits+ assert torch.equal(actual, expected), (actual - expected).abs().max().item()+ assert FL.FP8_LORA_STATS["fwd"] == 0+ assert not FL.fp8_lora_memory(merged)["layers"]+ assert not merged.__dict__.get("_unsloth_fp8_lora_loss_forwards")+ assert len(FL.fp8_lora_memory(model)["layers"]) == 14+++def _loss_and_grads_keep(model):+ torch.manual_seed(1)+ ids = torch.randint(0, 256, (2, 32))+ model(input_ids = ids, labels = ids).loss.backward()+++def test_square_tile_transpose_kernel_byte_exact_under_the_triton_interpreter():+ triton = pytest.importorskip("triton")+ import os+ import subprocess+ import sys+ import textwrap++ path = os.path.abspath(FL.__file__)+ code = textwrap.dedent(+ f"""+ import importlib.util, torch+ spec = importlib.util.spec_from_file_location("fp8_lora_standalone", {path!r})+ FL = importlib.util.module_from_spec(spec); spec.loader.exec_module(FL)+ assert FL.triton is not None+ for R, C in ((64, 64), (128, 192), (100, 37), (1, 300), (65, 129), (256, 64)):+ x = torch.randint(0, 255, (R, C), dtype = torch.uint8).view(torch.float8_e4m3fn)+ y = FL.fp8_transpose(x, use_triton = True)+ assert y.shape == (C, R) and y.is_contiguous() and y.dtype == x.dtype+ assert torch.equal(y.view(torch.uint8), x.t().contiguous().view(torch.uint8)), (R, C)+ print("TRANSPOSE_OK")+ """+ )+ env = dict(os.environ, TRITON_INTERPRET = "1", CUDA_VISIBLE_DEVICES = "")+ p = subprocess.run(+ [sys.executable, "-c", code], capture_output = True, text = True, env = env, timeout = 600+ )+ assert "TRANSPOSE_OK" in p.stdout, p.stdout[-2000:] + p.stderr[-4000:]+ import inspect++ src = inspect.getsource(FL.fp8_transpose)+ assert "_TRANSPOSE_BLOCK), triton.cdiv(C, _TRANSPOSE_BLOCK)" in src+++def test_merge_of_a_deepcopy_merges_the_copy_only(offload_on_cpu):+ import copy++ model = _tiny_qwen3_peft()+ plain = copy.deepcopy(model)+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ twin = copy.deepcopy(model)+ name = "base_model.model.model.layers.0.mlp.down_proj"+ layer, twin_layer = dict(model.named_modules())[name], dict(twin.named_modules())[name]+ before = layer.base_layer.weight.detach().clone()+ twin_layer.merge()+ assert twin_layer.merged and not layer.merged+ assert torch.equal(layer.base_layer.weight, before)+ assert layer.base_layer.__dict__["_unsloth_fp8_lora"].offloaded+ plain_layer = dict(plain.named_modules())[name]+ plain_layer.merge()+ assert torch.equal(twin_layer.base_layer.weight, plain_layer.base_layer.weight)+++@needs_cpu_scaled_mm+def test_compiled_fp8_lora_ops_are_counted():+ torch._dynamo.reset()+ W = _weight()+ state = FL.Fp8LoRABaseState(W)++ def run(x, dy):+ return state.matmul(x, W), state.dgrad(dy, W)++ compiled = torch.compile(run, backend = "aot_eager", fullgraph = True)+ x, dy = torch.randn(32, 256).bfloat16(), torch.randn(32, 384).bfloat16()+ compiled(x, dy)+ with P.count_lowp_gemms() as counts:+ compiled(x, dy)+ assert counts["unsloth::fp8_lora_fwd"] == 1 and counts["unsloth::fp8_lora_dgrad"] == 1, dict(+ counts+ )+++@needs_cpu_scaled_mm+def test_compiled_fused_fp8_lora_ops_are_counted(monkeypatch):+ torch._dynamo.reset()+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")+ W = _weight()+ state = FL.Fp8LoRABaseState(W)+ A = torch.randn(16, 256).bfloat16()+ Bt = torch.randn(16, 384).bfloat16()++ def run(x, dy):+ return state.lora_first(x, W, x @ A.t(), Bt, 1.0), state.dgrad_accum(+ dy, W, torch.zeros_like(x)+ )++ compiled = torch.compile(run, backend = "aot_eager", fullgraph = True)+ x, dy = torch.randn(32, 256).bfloat16(), torch.randn(32, 384).bfloat16()+ compiled(x, dy)+ with P.count_lowp_gemms() as counts:+ compiled(x, dy)+ P.require("fp8_lora", sum(counts.values()))+ assert counts["unsloth::fp8_lora_first"] == counts["unsloth::fp8_lora_accum"] == 1+ assert counts["_scaled_mm"] == 0+++@needs_cpu_scaled_mm[email protected]("compiled_first", [False, True])+def test_enable_fp8_refreshes_only_an_existing_block_compiler(monkeypatch, compiled_first):+ from unsloth.models import _block_compile++ model = _tiny_qwen3_peft()+ model._unsloth_block_compile_installed = compiled_first+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ calls = []++ def refresh(model):+ assert FL.fp8_lora_memory(model)["layers"]+ calls.append(model)++ monkeypatch.setattr(_block_compile, "patch_block_compile", refresh)+ assert f8t.enable_fp8_lora(model, allow_cpu = True)+ assert calls == ([model] if compiled_first else [])+++@needs_cpu_scaled_mm+def test_merge_requantizes_the_fp8_copy(offload_on_cpu):+ import copy++ model = _tiny_qwen3_peft()+ ref = copy.deepcopy(model)+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ ids = torch.randint(0, 256, (2, 32))+ with torch.no_grad():+ before = model(input_ids = ids).logits.float()+ model.merge_adapter()+ ref.merge_adapter()+ merged = model(input_ids = ids).logits.float()+ want = ref(input_ids = ids).logits.float()+ assert _rel(merged, want) < 0.3 and _rel(merged, before) < 0.3+++def test_dora_layers_keep_the_master_on_the_device(offload_on_cpu):+ from transformers import Qwen3Config, Qwen3ForCausalLM+ from peft import LoraConfig, get_peft_model++ torch.manual_seed(0)+ config = Qwen3Config(+ vocab_size = 256,+ hidden_size = 128,+ intermediate_size = 256,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 32,+ )+ model = get_peft_model(+ Qwen3ForCausalLM(config).to(torch.bfloat16),+ LoraConfig(r = 8, target_modules = ["q_proj", "v_proj"], use_dora = True),+ )+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ assert len(f8t.enable_fp8_lora(model, allow_cpu = True)) == 2+ states = [+ m.base_layer.__dict__["_unsloth_fp8_lora"] for m in model.modules() if hasattr(m, "lora_A")+ ]+ assert states and not any(s.offloaded for s in states)+++def test_an_added_adapter_lands_on_the_compute_device(offload_on_cpu, monkeypatch):+ from peft import LoraConfig++ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ layers = [m for m in model.modules() if hasattr(m, "lora_A")]+ compute = torch.device("meta") # stands in for cuda; the master stays on the CPU host+ for m in layers:+ m.base_layer.__dict__["_unsloth_fp8_lora"].device = compute+ moved = []+ real_to = torch.nn.Module.to++ def spy(self, *args, **kwargs):+ if args and args[0] == compute:+ moved.append(self)+ return self+ return real_to(self, *args, **kwargs)++ monkeypatch.setattr(torch.nn.Module, "to", spy)+ monkeypatch.setattr(FL.Fp8LoRABaseState, "refresh", lambda self, weight: self)+ model.add_adapter("second", LoraConfig(r = 4, target_modules = ["q_proj"]))+ added = [m for m in layers if "second" in m.lora_A]+ assert added and all(any(x is m.lora_A["second"] for x in moved) for m in added)+ monkeypatch.setattr(torch.nn.Module, "to", real_to)+ for m in layers:+ m.base_layer.__dict__["_unsloth_fp8_lora"].device = torch.device("cpu")+ model.add_adapter("third", LoraConfig(r = 4, target_modules = ["v_proj"], use_dora = True))+ dora = [m for m in layers if "third" in m.lora_A]+ assert dora and not any(m.base_layer.__dict__["_unsloth_fp8_lora"].offloaded for m in dora)+++def test_low_cpu_mem_adapters_stay_on_meta_for_the_loader(offload_on_cpu):+ from peft import LoraConfig++ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ model.add_adapter("second", LoraConfig(r = 4, target_modules = ["q_proj"]), low_cpu_mem_usage = True)+ added = [m for m in model.modules() if hasattr(m, "lora_A") and "second" in m.lora_A]+ assert added and all(m.lora_A["second"].weight.device.type == "meta" for m in added)+++def test_a_dora_adapter_restores_the_master_before_placement(offload_on_cpu, monkeypatch):+ from peft import LoraConfig+ from peft.tuners.lora import LoraLayer++ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ seen = []+ real = LoraLayer._move_adapter_to_device_of_base_layer++ def spy(self, adapter_name, *args, **kwargs):+ if adapter_name == "dora":+ seen.append(self.base_layer.__dict__["_unsloth_fp8_lora"].offloaded)+ return real(self, adapter_name, *args, **kwargs)++ monkeypatch.setattr(LoraLayer, "_move_adapter_to_device_of_base_layer", spy)+ model.add_adapter("dora", LoraConfig(r = 4, target_modules = ["v_proj"], use_dora = True))+ assert seen and not any(seen)+++@needs_cpu_scaled_mm+def test_load_state_dict_requantizes_for_compiled_calls():+ torch._dynamo.reset()+ lin = _frozen_linear()+ FL.attach_fp8_lora_base(lin, offload = False)+ compiled = torch.compile(lin, backend = "aot_eager", fullgraph = True)+ x = torch.randn(32, 256).bfloat16()+ assert compiled(x).abs().sum() > 0+ lin.load_state_dict({"weight": torch.zeros_like(lin.weight)})+ assert compiled(x).abs().sum() == 0+++def test_a_dtype_move_requantizes_for_compiled_calls():+ lin = _frozen_linear()+ state = FL.attach_fp8_lora_base(lin, offload = False)+ lin.half()+ assert state.dtype == torch.float16 and state._key == state._weight_key(lin.weight)+++def test_a_pissa_adapter_requantizes_the_rewritten_base(offload_on_cpu):+ from peft import LoraConfig++ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ model.add_adapter(+ "pissa",+ LoraConfig(+ r = 8, lora_alpha = 16, target_modules = ["q_proj"], init_lora_weights = "pissa_niter_1"+ ),+ )+ layers = [m for m in model.modules() if hasattr(m, "lora_A") and "pissa" in m.lora_A]+ assert layers+ for m in layers:+ state = m.base_layer.__dict__["_unsloth_fp8_lora"]+ assert state._key == state._weight_key(m.base_layer.weight)+++def test_mamba_raw_weight_projections_keep_the_master(offload_on_cpu):+ from peft import LoraConfig, get_peft_model++ class MambaMixer(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.in_proj = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)+ self.out_proj = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)++ class Model(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.mixer = MambaMixer()++ model = get_peft_model(Model(), LoraConfig(r = 4, target_modules = ["in_proj", "out_proj"]))+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ assert len(f8t.enable_fp8_lora(model, allow_cpu = True)) == 2+ mixer = model.base_model.model.mixer+ assert not mixer.out_proj.base_layer.__dict__["_unsloth_fp8_lora"].offloaded+ assert mixer.in_proj.base_layer.__dict__["_unsloth_fp8_lora"].offloaded+++@needs_cpu_scaled_mm+def test_legacy_scaled_mm_tuple_return(monkeypatch):+ real = torch._scaled_mm+ monkeypatch.setattr(torch, "_scaled_mm", lambda *a, **k: (real(*a, **k), torch.zeros(())))+ W = _weight()+ state = FL.Fp8LoRABaseState(W)+ y = state.matmul(torch.randn(32, 256).bfloat16(), W)+ dx = state.dgrad(torch.randn(32, 384).bfloat16(), W)+ assert torch.is_tensor(y) and torch.is_tensor(dx)++[email protected]("arch", ["falcon_h1", "wavlm"])+def test_parents_reading_a_child_weight_keep_its_master(offload_on_cpu, arch):+ if arch == "falcon_h1":+ mod = pytest.importorskip("transformers.models.falcon_h1.modeling_falcon_h1")+ parent_cls, leaf = mod.FalconH1Mixer, "out_proj"+ else:+ mod = pytest.importorskip("transformers.models.wavlm.modeling_wavlm")+ parent_cls, leaf = mod.WavLMAttention, "q_proj"+ parent = parent_cls.__new__(parent_cls)+ torch.nn.Module.__init__(parent)+ setattr(parent, leaf, torch.nn.Linear(4, 4))+ model = torch.nn.Module()+ model.block = parent+ assert f8t._mamba_raw_weight_leaf(model, f"block.{leaf}")+++@needs_cpu_scaled_mm+def test_refresh_under_inference_mode_builds_trainable_fp8_tensors():+ lin = _frozen_linear()+ state = FL.attach_fp8_lora_base(lin, offload = False)+ with torch.inference_mode():+ state._key = None+ state.refresh(lin.weight)+ assert not state.w8.is_inference()+ x = torch.randn(32, 256).bfloat16().requires_grad_(True)+ lin(x).float().sum().backward()+ assert x.grad is not None+++def test_grpo_out_of_place_merge_leaves_the_fp8_copy_alone(offload_on_cpu):+ from unsloth.models.rl_replacements import _unsloth_vllm_peft_merge_out_of_place++ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ layer = next(m for m in model.modules() if hasattr(m, "lora_A") and hasattr(m, "base_layer"))+ state = layer.base_layer.__dict__["_unsloth_fp8_lora"]+ w8, key, offloaded = state.w8, state._key, state.offloaded+ merged = _unsloth_vllm_peft_merge_out_of_place(layer.base_layer, [layer])+ assert merged+ assert state.w8 is w8 and state._key == key and state.offloaded == offloaded+++def test_a_base_weight_shared_by_two_lora_layers_stays_16bit(offload_on_cpu):+ from peft import LoraConfig, get_peft_model++ class Tied(torch.nn.Module):+ def __init__(self):+ super().__init__()+ self.a = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)+ self.b = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)+ self.b.weight = self.a.weight++ model = get_peft_model(+ Tied(), LoraConfig(r = 4, target_modules = ["a", "b"], init_lora_weights = False)+ )+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ assert f8t.enable_fp8_lora(model, allow_cpu = True) == []+ inner = model.base_model.model+ model.merge_adapter()+ x = torch.randn(32, 64).bfloat16()+ with torch.no_grad():+ torch.testing.assert_close(inner.a(x), inner.b(x))+++def test_block_swapped_layers_are_not_converted(offload_on_cpu):+ import types++ model = _tiny_qwen3_peft()+ layers = model.base_model.model.model.layers+ layers._unsloth_block_swap = types.SimpleNamespace(indices = [1])+ for m in layers[1].modules():+ if isinstance(m, torch.nn.Linear) and not m.weight.requires_grad:+ m.weight = torch.nn.Parameter(torch.empty(0, dtype = m.weight.dtype), requires_grad = False)+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ converted = f8t.enable_fp8_lora(model, allow_cpu = True)+ assert converted and all(".layers.0." in name for name in converted)+++def test_sharded_or_preloaded_weights_are_not_eligible():+ linear = torch.nn.Linear(64, 64, bias = False).bfloat16().requires_grad_(False)+ linear.weight = torch.nn.Parameter(linear.weight.data[:32].clone(), requires_grad = False)+ assert "shard" in FL.eligible_base(linear)+ full = torch.nn.Linear(64, 64, bias = False).bfloat16().requires_grad_(False)+ full.weight.__dict__["_unsloth_fsdp2_preload"] = object()+ assert "shard" in FL.eligible_base(full)+++def test_get_peft_model_converts_after_block_swap():+ import inspect+ from unsloth.models.llama import FastLlamaModel++ get_src = inspect.getsource(FastLlamaModel.get_peft_model)+ assert "_defer_fp8_lora = True" in get_src+ swap = get_src.index("install_block_swap(")+ assert get_src.index("enable_fp8_lora(model)", swap) > swap+ patch_src = inspect.getsource(FastLlamaModel.patch_peft_model)+ assert "if not _defer_fp8_lora:" in patch_src+++def test_column_major_weights_are_not_eligible():+ linear = torch.nn.Linear(64, 128, bias = False).bfloat16().requires_grad_(False)+ linear.weight = torch.nn.Parameter(torch.randn(64, 128).bfloat16().t(), requires_grad = False)+ assert "contiguous" in FL.eligible_base(linear)++[email protected]("variant", ["bdlora", "velora"])+def test_a_variant_adapter_restores_the_master_before_placement(+ offload_on_cpu, monkeypatch, variant+):+ import peft+ from peft import LoraConfig+ from peft.tuners.lora import LoraLayer++ if variant == "bdlora":+ if not hasattr(peft, "BdLoraConfig"):+ pytest.skip("this PEFT has no BD-LoRA")+ config = LoraConfig(+ r = 4,+ target_modules = ["q_proj"],+ use_bdlora = peft.BdLoraConfig(nblocks = 2, target_modules_bd_a = ["q_proj"]),+ )+ else:+ if not hasattr(peft, "VeloraConfig"):+ pytest.skip("this PEFT has no VeLoRA")+ config = LoraConfig(+ r = 4,+ target_modules = ["q_proj"],+ velora_config = peft.VeloraConfig(num_groups = 2, init_type = "random"),+ )+ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ seen = []+ real = LoraLayer._move_adapter_to_device_of_base_layer++ def spy(self, adapter_name, *args, **kwargs):+ if adapter_name == variant:+ seen.append(self.base_layer.__dict__["_unsloth_fp8_lora"].offloaded)+ return real(self, adapter_name, *args, **kwargs)++ monkeypatch.setattr(LoraLayer, "_move_adapter_to_device_of_base_layer", spy)+ try:+ model.add_adapter(variant, config)+ except (TypeError, ValueError) as exc:+ pytest.skip(f"this PEFT cannot build {variant} here: {exc}")+ assert seen and not any(seen)+++@needs_cpu_scaled_mm[email protected]("error_type", [RuntimeError, ValueError])[email protected]("outputs", ["bf16", "bf16 and fp16"])+def test_rowwise_fp16_without_float32_out_stays_finite(rowwise, monkeypatch, error_type, outputs):+ real = torch._scaled_mm++ def refuse_fp32_rowwise(+ a,+ b,+ *args,+ scale_a = None,+ out_dtype = None,+ **kwargs,+ ):+ if out_dtype == torch.float32 and scale_a is not None and scale_a.numel() > 1:+ raise error_type(+ f"Only {outputs} high precision output types are supported for row-wise scaling."+ )+ return real(a, b, *args, scale_a = scale_a, out_dtype = out_dtype, **kwargs)++ monkeypatch.setattr(torch, "_scaled_mm", refuse_fp32_rowwise)+ monkeypatch.setattr(FL, "_ROWWISE_BF16_ONLY", set())+ W = torch.full((32, 64), 0.5, dtype = torch.float16)+ state = FL.Fp8LoRABaseState(W)+ assert state.w_inv_fwd.dim() == 2+ x = torch.full((16, 64), 2047.0, dtype = torch.float16)+ assert torch.isfinite(state.matmul(x, W)).all()+ assert torch.isfinite(state.matmul(x, W)).all()+ dy = torch.full((16, 32), 4094.0, dtype = torch.float16)+ assert torch.isfinite(state.dgrad(dy, W)).all()+ Wb = W.bfloat16()+ y = FL.Fp8LoRABaseState(Wb).matmul(torch.randn(16, 64).bfloat16(), Wb)+ assert y.dtype == torch.bfloat16+++ "error",+ [+ torch.OutOfMemoryError("allocation failed"),+ RuntimeError("CUDA error: illegal memory access"),+ ],+)+def test_rowwise_failure_does_not_change_the_scaling_recipe(monkeypatch, error):+ monkeypatch.setattr(FL, "_ROWWISE_BF16_ONLY", set())+ calls = []++ def fail_once(*args):+ calls.append(args)+ if len(calls) == 1:+ raise error+ return torch.zeros(16, 16)++ monkeypatch.setattr(FL, "_scaled_mm", fail_once)+ x = torch.zeros(16, 16)+ with pytest.raises(type(error), match = str(error)):+ FL._rowwise_scaled_mm(x, x.t(), torch.ones(16, 1), torch.ones(1, 16), torch.float16, True)+ assert len(calls) == 1+ assert FL._ROWWISE_BF16_ONLY == set()+++def test_vllm_server_merge_of_a_host_master_lands_on_the_adapter_device():+ from types import SimpleNamespace++ from unsloth.models.rl_replacements import _unsloth_vllm_merged_weight++ lora_A = torch.nn.Linear(64, 4, bias = False, device = "meta")+ lora_B = torch.nn.Linear(4, 32, bias = False, device = "meta")+ module = SimpleNamespace(+ active_adapters = ["default"],+ lora_A = {"default": lora_A},+ lora_B = {"default": lora_B},+ scaling = {"default": 2.0},+ )+ weight = torch.nn.Parameter(torch.randn(32, 64).bfloat16(), requires_grad = False)+ merged = _unsloth_vllm_merged_weight(module, weight)+ assert merged.device.type == "meta" and merged.dtype == torch.bfloat16+ lora_A.to_empty(device = "cpu")+ lora_B.to_empty(device = "cpu")+ module.lora_A["default"], module.lora_B["default"] = lora_A, lora_B+ assert _unsloth_vllm_merged_weight(module, weight).device.type == "cpu"+++@needs_cpu_scaled_mm+def test_a_merge_under_inference_mode_leaves_a_trainable_master(offload_on_cpu, monkeypatch):+ real_to = torch.Tensor.to++ def allocating_to(self, *args, **kwargs):+ if torch.is_inference_mode_enabled():+ kwargs["copy"] = True+ return real_to(self, *args, **kwargs)++ model = _tiny_qwen3_peft()+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)+ f8t.enable_fp8_lora(model, allow_cpu = True)+ monkeypatch.setattr(torch.Tensor, "to", allocating_to)+ with torch.inference_mode():+ model.merge_adapter()+ model.unmerge_adapter()+ monkeypatch.setattr(torch.Tensor, "to", real_to)+ ids = torch.randint(0, 256, (2, 16))+ model(input_ids = ids, labels = ids).loss.backward()+ assert any(p.grad is not None for n, p in model.named_parameters() if "lora_" in n)+++def test_text_diffusion_refuses_fp8_lora_instead_of_dropping_it():+ import inspect++ from unsloth.models.loader import FastModel++ src = inspect.getsource(FastModel.from_pretrained)+ body = src[src.index("def _dispatch_diffusion():") :]+ body = body[: body.index("FastDiffusionModel.from_pretrained(")]+ assert "if fp8_lora is not None:" in body and "NotImplementedError" in body+++def test_vllm_server_sync_of_a_host_master_without_an_active_delta_lands_on_the_device():+ from types import SimpleNamespace++ from unsloth.models.rl_replacements import _unsloth_vllm_merged_weight++ base = torch.nn.Linear(64, 32, bias = False).bfloat16().requires_grad_(False)+ base.__dict__["_unsloth_fp8_lora"] = SimpleNamespace(device = torch.device("meta"))+ module = SimpleNamespace(+ base_layer = base, active_adapters = ["other"], lora_A = {}, lora_B = {}, scaling = {}+ )+ assert _unsloth_vllm_merged_weight(module, base.weight).device.type == "meta"+ del base.__dict__["_unsloth_fp8_lora"]+ assert _unsloth_vllm_merged_weight(module, base.weight).device.type == "cpu"+++def test_fast_language_model_forwards_fp8_lora_to_fastmodel():+ import inspect++ from unsloth.models.loader import FastLanguageModel++ src = inspect.getsource(FastLanguageModel.from_pretrained)+ assert '**({"float8_training": fp8_request} if fp8_lora is not None else {})' in src+++@needs_cpu_scaled_mm+def test_fp8_lora_accumulates_small_adapter_before_base_rounding(monkeypatch):+ monkeypatch.setattr(FL, "_SCALING", "tensor")+ monkeypatch.setattr(FL, "_MIN_ROWS", 16)+ x = torch.ones(16, 16, dtype = torch.bfloat16)+ w = torch.zeros(16, 16, dtype = torch.bfloat16)+ w[:, 0], w[:, 1] = 1, 1 / 512+ a = torch.zeros(1, 16, dtype = torch.bfloat16)+ a[0, 0] = 1 / 256+ b = torch.ones(16, 1, dtype = torch.bfloat16)+ state = FL.Fp8LoRABaseState(w)+ delta = torch.empty_like(x).addmm_(x @ a.t(), b.t(), beta = 0)+ reference = delta.addmm_(x, w.t())+ actual = FLK.matmul_lora(x, w, state, a, b, 1)+ old_order = FLK.matmul_lora(x, w, state, a, b, 1, fused = False)+ assert torch.equal(reference, torch.full_like(x, 1 + 1 / 128))+ assert torch.equal(actual, reference)+ assert not torch.equal(old_order, reference)++[email protected]("projection", ["gate_proj", "up_proj", "down_proj"])+def test_biased_native_mlp_keeps_its_forward(projection):+ from peft import LoraConfig, get_peft_model+ from transformers import LlamaConfig, LlamaForCausalLM++ model = get_peft_model(+ LlamaForCausalLM(+ LlamaConfig(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ mlp_bias = True,+ )+ ),+ LoraConfig(r = 4, target_modules = ["gate_proj", "up_proj", "down_proj"]),+ )+ mlp = model.get_base_model().model.layers[0].mlp+ for name in ("gate_proj", "up_proj", "down_proj"):+ base = getattr(mlp, name).base_layer+ if name != projection:+ base.bias = None+ else:+ base.bias.data.fill_(0.25)+ base.__dict__["_unsloth_fp8_lora"] = FL.Fp8LoRABaseState(base.weight)+ original = mlp.forward+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ assert mlp.forward == original+ x = torch.randn(2, 16, 32)+ torch.testing.assert_close(mlp(x), original(x), rtol = 0, atol = 0)+++ "weight_dtype, input_dtype",+ [+ (torch.bfloat16, torch.float32),+ (torch.float16, torch.bfloat16),+ (torch.bfloat16, torch.float16),+ ],+)+def test_mixed_native_norm_uses_its_original_equation(monkeypatch, weight_dtype, input_dtype):+ from unsloth.kernels import rms_layernorm++ def unsupported(*args, **kwargs):+ pytest.fail("mixed-dtype normalization entered the fused kernel")++ monkeypatch.setattr(rms_layernorm, "fast_rms_layernorm", unsupported)+ model = _tiny_qwen3_peft()+ norm = model.get_base_model().model.norm.to(weight_dtype)+ original = norm.forward+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ x = torch.randn(2, 16, norm.weight.numel()).to(input_dtype).requires_grad_(True)+ expected = original(x)+ for call in (norm, torch.compile(norm, backend = "aot_eager", fullgraph = True)):+ actual = call(x)+ assert actual.dtype == expected.dtype+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)++[email protected]("dtype", [torch.bfloat16, torch.float16])+def test_fp32_native_norm_keeps_its_promoted_output(dtype):+ model = _tiny_qwen3_peft()+ norm = model.get_base_model().model.norm.float()+ norm.weight.data.fill_(1.001)+ original = norm.forward+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ assert norm.forward == original+ x = torch.randn(2, 16, norm.weight.numel()).to(dtype)+ normalized = x.float() * torch.rsqrt(+ x.float().square().mean(-1, keepdim = True) + norm.variance_epsilon+ )+ expected = norm.weight * normalized.to(dtype)+ actual = norm(x)+ assert actual.dtype == expected.dtype == torch.float32+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)++[email protected]("dtype", [torch.bfloat16, torch.float32])+def test_norm_upcast_after_binding_keeps_the_native_equation(monkeypatch, dtype):+ from unsloth.kernels import rms_layernorm++ def unsupported(*args, **kwargs):+ pytest.fail("FP32 normalization entered the fused kernel")++ monkeypatch.setattr(rms_layernorm, "fast_rms_layernorm", unsupported)+ model = _tiny_qwen3_peft()+ norm = model.get_base_model().model.norm+ original = norm.forward+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ norm.float()+ x = torch.randn(2, 16, norm.weight.numel()).to(dtype)+ actual = norm(x)+ assert actual.dtype == torch.float32+ torch.testing.assert_close(actual, original(x), rtol = 0, atol = 0)+++ "mode", ["full", "gelu", "partial", "custom", "custom_class", "private_norm", "generated"]+)[email protected]("layout", ["3d", "2d", "strided_3d", "strided_2d"])+def test_native_operator_binding_preserves_keywords_and_restores_instances(+ monkeypatch, mode, layout+):+ import types+ from peft import LoraConfig, get_peft_model+ from transformers import Qwen3Config, Qwen3ForCausalLM+ from transformers.models.qwen3.modeling_qwen3 import Qwen3MLP, Qwen3RMSNorm+ from unsloth.kernels import rms_layernorm++ model = get_peft_model(+ Qwen3ForCausalLM(+ Qwen3Config(+ vocab_size = 64,+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ )+ ).to(torch.bfloat16),+ LoraConfig(r = 4, target_modules = ["gate_proj", "up_proj", "down_proj"]),+ )+ owner = model.get_base_model()+ mlp = owner.model.layers[0].mlp+ norm = owner.model.norm+ original_mlp, original_norm = Qwen3MLP.forward, Qwen3RMSNorm.forward+ for layer in owner.model.layers:+ for part in ("gate_proj", "up_proj", "down_proj"):+ base = getattr(layer.mlp, part).base_layer+ base.__dict__["_unsloth_fp8_lora"] = FL.Fp8LoRABaseState(base.weight)+ if mode == "gelu":+ mlp.act_fn = torch.nn.GELU()+ elif mode == "partial":+ mlp.down_proj.base_layer.__dict__.pop("_unsloth_fp8_lora")+ elif mode in ("custom", "custom_class"):++ def residual(self, x):+ return original_mlp(self, x) + x++ if mode == "custom":+ mlp.forward = types.MethodType(residual, mlp)+ else:+ residual.__module__ = Qwen3MLP.__module__+ monkeypatch.setattr(Qwen3MLP, "forward", residual)+ elif mode == "private_norm":+ namespace = {k: v for k, v in original_norm.__globals__.items() if k != "__name__"}+ private = types.FunctionType(original_norm.__code__, namespace)+ assert private.__module__ is None+ norm.forward = types.MethodType(private, norm)+ elif mode == "generated":+ from unsloth_zoo.temporary_patches.utils import torch_compile_with_fallback+ for cls, original, argument in (+ (Qwen3MLP, original_mlp, "x"),+ (Qwen3RMSNorm, original_norm, "hidden_states"),+ ):+ helper = cls.__name__ + "_forward"+ namespace = {+ "__name__": "unsloth_compiled_module_test",+ helper: torch_compile_with_fallback(backend = "aot_eager")(original),+ }+ exec(+ f"def forward(self, {argument}):\n return {helper}(self, {argument}={argument})\n",+ namespace,+ )+ monkeypatch.setattr(cls, "__module__", namespace["__name__"])+ monkeypatch.setattr(cls, "forward", namespace["forward"])+ installed_class_forward = Qwen3MLP.forward+ installed_norm_forward = Qwen3RMSNorm.forward+ saved_override = mlp.__dict__.get("forward")+ saved_norm_override = norm.__dict__.get("forward")+ x = torch.randn(2, 3, 64)[..., ::2] if layout.startswith("strided") else torch.randn(2, 3, 32)+ if layout.endswith("2d"):+ x = x.reshape(6, 32)+ x = x.to(torch.bfloat16).requires_grad_(True)+ expected_mlp, expected_norm = mlp(x = x), norm(hidden_states = x)+ calls = {"norm": 0, "mlp": 0}++ def norm_spy(+ layernorm,+ X,+ gemma = False,+ save_output = False,+ reuse_grad = False,+ ):+ if not torch.compiler.is_compiling():+ calls["norm"] += 1+ return original_norm(layernorm, X)++ def mlp_spy(+ self,+ X,+ inplace = True,+ ):+ assert X.ndim == 3 and X.is_contiguous()+ if not torch.compiler.is_compiling():+ calls["mlp"] += 1+ return original_mlp(self, X)++ monkeypatch.setattr(rms_layernorm, "fast_rms_layernorm", norm_spy)+ monkeypatch.setattr(FLK, "apply_lora_mlp_swiglu", mlp_spy)+ monkeypatch.setenv("UNSLOTH_FP8_LORA_FUSED_OPS", "0")+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ assert "_unsloth_fp8_lora_kernel_forwards" not in owner.__dict__+ monkeypatch.setenv("UNSLOTH_FP8_LORA_FUSED_OPS", "1")+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ installed = owner.__dict__["_unsloth_fp8_lora_kernel_forwards"]+ assert installed+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ assert owner.__dict__["_unsloth_fp8_lora_kernel_forwards"] is installed+ assert torch.equal(mlp(x = x), expected_mlp)+ assert torch.equal(norm(hidden_states = x), expected_norm)+ assert calls == {"norm": 1, "mlp": int(mode in ("full", "private_norm", "generated"))}+ assert (+ Qwen3MLP.forward is installed_class_forward+ and Qwen3RMSNorm.forward is installed_norm_forward+ )+ if mode == "private_norm" and layout == "strided_2d":+ reference_grad = torch.autograd.grad(expected_mlp.sum() + expected_norm.sum(), x)[0]+ compiled_mlp = torch.compile(mlp, backend = "aot_eager", fullgraph = True)+ compiled_norm = torch.compile(norm, backend = "aot_eager", fullgraph = True)+ actual_mlp, actual_norm = compiled_mlp(x = x), compiled_norm(hidden_states = x)+ torch.testing.assert_close(actual_mlp, expected_mlp)+ torch.testing.assert_close(actual_norm, expected_norm)+ actual_grad = torch.autograd.grad(actual_mlp.sum() + actual_norm.sum(), x)[0]+ torch.testing.assert_close(actual_grad, reference_grad)+ assert calls == {"norm": 1, "mlp": 1}+ from torch._dynamo.utils import counters++ before = counters["stats"]["unique_graphs"]+ compiled_mlp(x = x)+ compiled_norm(hidden_states = x)+ stable = counters["stats"]["unique_graphs"] - before+ f8t.restore_fp8_lora_for_inference(model)+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)+ before = counters["stats"]["unique_graphs"]+ compiled_mlp(x = x)+ compiled_norm(hidden_states = x)+ rebound = counters["stats"]["unique_graphs"] - before+ print("OPERATOR_REBIND_GRAPHS", stable, rebound)+ assert stable == rebound == 0++ def call(module, value):+ return module(value)++ shared = torch.compile(call, backend = "aot_eager", fullgraph = True)+ shared(norm, x)+ shared(mlp, x)+ before = counters["stats"]["unique_graphs"]+ other_norm = owner.model.layers[0].input_layernorm+ other_mlp = owner.model.layers[1].mlp+ torch.testing.assert_close(shared(other_norm, x), original_norm(other_norm, x))+ torch.testing.assert_close(shared(other_mlp, x), original_mlp(other_mlp, x))+ shared_growth = counters["stats"]["unique_graphs"] - before+ print("OPERATOR_SHARED_GRAPHS", shared_growth)+ assert shared_growth == 0+ f8t.restore_fp8_lora_for_inference(model)+ assert mlp.__dict__.get("forward") is saved_override+ assert norm.__dict__.get("forward") is saved_norm_override+ assert torch.equal(mlp(x = x), expected_mlp)+ assert torch.equal(norm(hidden_states = x), expected_norm)+ assert calls == {"norm": 1, "mlp": int(mode in ("full", "private_norm", "generated"))}diff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.pyindex 77bfc7ad51c..c6f14c3c77b 100644--- a/tests/test_lora_fused_passes.py+++ b/tests/test_lora_fused_passes.py@@ -1094,6 +1094,35 @@ def test_decode_keeps_a_subclass_adapter(which, bsz, monkeypatch):_same(got, want, True)[email protected]("bsz", [1, 3])[email protected]("fp8", [False, True])+def test_decode_packed_base_keeps_its_matmul(bsz, fp8):+ _, block = _model("cpu", torch.float32)+ proj = block.o_proj+ calls = []++ class Packed:+ _unsloth_packed_weight_state = True+ _unsloth_fp8_dgrad = fp8++ def matmul(+ self,+ X,+ W,+ out = None,+ ):+ calls.append(X.shape)+ return torch.matmul(X, W.t(), out = out)++ proj.base_layer.weight.quant_state = Packed()+ X = torch.randn(bsz, 1, H)+ with torch.inference_mode():+ expected = proj(X)+ actual = kernel_utils.fast_linear_forward(proj, X)+ torch.testing.assert_close(actual, expected)+ assert calls == [X.shape]++@pytest.mark.parametrize("which", ["A", "B"])def test_the_traced_no_grad_accumulate_keeps_a_subclass_result(which, monkeypatch):# The opaque op returns nothing, so a subclass operand takes the in-place method and keeps its type.diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.pyindex 2cc8ba82f7e..36fd4625573 100644--- a/tests/test_return_logits_survives_for_training.py+++ b/tests/test_return_logits_survives_for_training.py@@ -77,7 +77,11 @@ def _fast_base_model():helpers = {n.name: n for n in utils.body if isinstance(n, ast.FunctionDef)}# Helpers that import a sibling module (a relative import has no package in this namespace): stubbed with what# they return for these fakes. A model split by hand (apply_pipeline_parallel) keeps no UNSLOTH_PP_SIZE replica.- stubbed = {"_pipeline_keeps_replica": lambda model: False}+ stubbed = {+ "_pipeline_keeps_replica": lambda model: False,+ "enable_fp8_lora": lambda model: [],+ "restore_fp8_lora_for_inference": lambda model: None,+ }lifted = {n.name for n in wanted} | set(stubbed)todo = list(wanted)while todo:@@ -174,6 +178,7 @@ def _fast_llama_for_training():arm_gradient_checkpointing = lambda model, checkpointing = None: False,set_module_gradient_checkpointing = lambda module, flag: None,_tensor_parallel_for_training = lambda model: None, # tp_replica.py: no tp model here+ enable_fp8_lora = lambda model: [],)module = ast.fix_missing_locations(ast.Module(body = [fn], type_ignores = []))exec(compile(module, str(MODELS / "llama.py"), "exec"), namespace)diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.pyindex c3036909935..b61333c1261 100644--- a/unsloth/kernels/fast_lora.py+++ b/unsloth/kernels/fast_lora.py@@ -29,6 +29,7 @@ from .utils import (torch_amp_custom_bwd,)from . import rms_layernorm as _rms_layernorm+from .fp8_lora import has_fp8_dgrad_is_compiling = torch.compiler.is_compiling@@ -669,11 +670,17 @@ class LoRA_MLP(torch.autograd.Function):gateS,X if ctx.inplace and not _is_compiling() else None,)- upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)- addmm_(dX, df, upW.t())+ if has_fp8_dgrad(upW_quant):+ dX = upW_quant.dgrad_accum(df, upW, dX)+ else:+ upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)+ addmm_(dX, df, upW.t())del upW- gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)- addmm_(dX, de, gateW.t())+ if has_fp8_dgrad(gateW_quant):+ dX = gateW_quant.dgrad_accum(de, gateW, dX)+ else:+ gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)+ addmm_(dX, de, gateW.t())del gateWd_gateBias = d_upBias = Noned_downBias = _lora_bias_grad(dY, downS) if ctx.biases[2] else None@@ -732,15 +739,21 @@ class LoRA_MLP(torch.autograd.Function):d_downBias = _lora_bias_grad(dY, downS) if down_has_bias else None# dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.- upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)- # Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.- dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)- del upW+ if has_fp8_dgrad(upW_quant):+ dX = upW_quant.dgrad(df, upW)+ else:+ upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)+ # Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.+ dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)+ del upW_add_lora_dX(dX, up_dB, upA, upS, up_mask, up_p)- gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)- addmm_(dX, de, gateW.t())- del gateW+ if has_fp8_dgrad(gateW_quant):+ dX.add_(gateW_quant.dgrad(de, gateW))+ else:+ gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)+ addmm_(dX, de, gateW.t())+ del gateW_add_lora_dX(dX, gate_dB, gateA, gateS, gate_mask, gate_p)return (@@ -1109,12 +1122,15 @@ class LoRA_QKV(torch.autograd.Function):X if ctx.inplace and not _is_compiling() else None,)for W_, W_quant_, dY_ in ((QW, QW_quant, dQ), (KW, KW_quant, dK), (VW, VW_quant, dV)):- W_ = (- W_.t()- if W_quant_ is None- else fast_dequantize(W_.t(), W_quant_, dtype = dY_.dtype)- )- addmm_(dX, dY_, W_.t())+ if has_fp8_dgrad(W_quant_):+ dX = W_quant_.dgrad_accum(dY_, W_, dX)+ else:+ W_ = (+ W_.t()+ if W_quant_ is None+ else fast_dequantize(W_.t(), W_quant_, dtype = dY_.dtype)+ )+ addmm_(dX, dY_, W_.t())del W_return (dX.view(batch, seq_len, hd),@@ -1175,19 +1191,28 @@ class LoRA_QKV(torch.autograd.Function):d_VBias = _lora_bias_grad(dV, VS) if V_has_bias else None# Combine the per-projection derivatives into dX.- QW = fast_dequantize(QW.t(), QW_quant, dtype = dQ.dtype)- dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)- del QW+ if has_fp8_dgrad(QW_quant):+ dX = QW_quant.dgrad(dQ, QW)+ else:+ QW = fast_dequantize(QW.t(), QW_quant, dtype = dQ.dtype)+ dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)+ del QW_add_lora_dX(dX, q_dB, QA, QS, Q_mask, Q_p)- KW = fast_dequantize(KW.t(), KW_quant, dtype = dK.dtype)- addmm_(dX, dK, KW.t())- del KW+ if has_fp8_dgrad(KW_quant):+ dX.add_(KW_quant.dgrad(dK, KW))+ else:+ KW = fast_dequantize(KW.t(), KW_quant, dtype = dK.dtype)+ addmm_(dX, dK, KW.t())+ del KW_add_lora_dX(dX, k_dB, KA, KS, K_mask, K_p)- VW = fast_dequantize(VW.t(), VW_quant, dtype = dV.dtype)- addmm_(dX, dV, VW.t())- del VW+ if has_fp8_dgrad(VW_quant):+ dX.add_(VW_quant.dgrad(dV, VW))+ else:+ VW = fast_dequantize(VW.t(), VW_quant, dtype = dV.dtype)+ addmm_(dX, dV, VW.t())+ del VW_add_lora_dX(dX, v_dB, VA, VS, V_mask, V_p)return (@@ -1365,12 +1390,18 @@ class LoRA_W(torch.autograd.Function):dX = y_dB.new_empty((y_dB.shape[0], A.shape[0])).addmm_(y_dB, A.t(), beta = 0, alpha = S)- W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)- addmm_(dX, dY, W.t())+ if has_fp8_dgrad(W_quant):+ dX = W_quant.dgrad_accum(dY, W, dX)+ else:+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)+ addmm_(dX, dY, W.t())del Welse:- W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)- dX = dY @ W.t()+ if has_fp8_dgrad(W_quant):+ dX = W_quant.dgrad(dY, W)+ else:+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)+ dX = dY @ W.t()del W_add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)diff --git a/unsloth/kernels/fp8_lora.py b/unsloth/kernels/fp8_lora.pynew file mode 100644index 00000000000..3f565965663--- /dev/null+++ b/unsloth/kernels/fp8_lora.py@@ -0,0 +1,967 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""FP8 frozen-base GEMMs with 16-bit LoRA adapters and an exact master for saving and merging.++cuBLAS needs a K-major dgrad operand; transpose inside the opaque op, or cache it with+UNSLOTH_FP8_LORA_CACHE_T=1. Offloaded masters remain the same Parameter on the host;+decode reconstructs the FP8 weight, while save and merge restore the exact master.+"""++from __future__ import annotations++import contextlib+import functools+import os+from typing import Optional++import torch++__all__ = [+ "FP8_LORA_STATS",+ "Fp8LoRABaseState",+ "fp8_lora_fwd",+ "fp8_lora_dgrad",+ "quantize_tensorwise",+ "quantize_rowwise",+ "attach_fp8_lora_base",+ "detach_fp8_lora_base",+ "fp8_lora_memory",+ "fp8_transpose",+ "restore_fp8_lora_master",+ "has_fp8_dgrad",+ "reset_fp8_lora_stats",+]++E4M3 = torch.float8_e4m3fn+E5M2 = torch.float8_e5m2+_FP8_MAX = {E4M3: 448.0, E5M2: 57344.0}+FP8_LORA_STATS = {"fwd": 0, "dgrad": 0, "dgrad_16bit": 0, "fwd_16bit": 0, "quantize": 0}+++def reset_fp8_lora_stats():+ for key in FP8_LORA_STATS:+ FP8_LORA_STATS[key] = 0+++_SCALING = os.environ.get("UNSLOTH_FP8_LORA_SCALING", "auto").strip().lower()+_SCALING = {"rowwise": "row", "tensorwise": "tensor"}.get(_SCALING, _SCALING)+if _SCALING not in ("auto", "row", "tensor"):+ _SCALING = "auto"+++def scaling_for_class(gpu_class) -> str:+ """'row' or 'tensor' for a precision-registry GPU class ('sm89', 'sm120', ...); the env var wins."""+ if _SCALING != "auto":+ return _SCALING+ return "row" if gpu_class == "sm120" else "tensor"+++def _scaling_for(device):+ gpu_class = None+ try:+ if device.type == "cuda" and torch.version.hip is None:+ gpu_class = "sm120" if torch.cuda.get_device_capability(device)[0] >= 12 else "other"+ except Exception:+ pass+ return scaling_for_class(gpu_class)+++def _env_dgrad():+ value = os.environ.get("UNSLOTH_FP8_LORA_DGRAD", "auto").strip().lower()+ if value in ("0", "false", "off", "no", "bf16", "16bit"):+ return None+ return {"e4m3": E4M3, "e5m2": E5M2}.get(value, "auto")+++_DGRAD_DTYPE = _env_dgrad()+_FWD_FAST_ACCUM = os.environ.get("UNSLOTH_FP8_LORA_FWD_FAST_ACCUM", "1") != "0"+_DGRAD_FAST_ACCUM = os.environ.get("UNSLOTH_FP8_LORA_DGRAD_FAST_ACCUM", "0") == "1"+_CACHE_T = os.environ.get("UNSLOTH_FP8_LORA_CACHE_T", "0") == "1"+_OFFLOAD_MASTER = os.environ.get("UNSLOTH_FP8_LORA_OFFLOAD_MASTER", "1") != "0"+_OFFLOAD_CPU_WEIGHTS = False+_MIN_ROWS = int(os.environ.get("UNSLOTH_FP8_LORA_MIN_ROWS", "16"))++try:+ import triton+ import triton.language as tl+except Exception:+ triton = None+_fast_launch = None+if triton is not None:+ try:+ from .triton_launch import launch as _fast_launch, tag_compile_cache+ except ImportError:+ pass+ else:+ tag_compile_cache(__file__)+++def _launch(+ kernel,+ grid,+ args,+ n_tensors,+ constexprs,+ device_index,+ enable_fp_fusion = True,+ **options,+):+ if _fast_launch is not None and not _compiling():+ return _fast_launch(+ kernel,+ grid,+ args,+ n_tensors,+ constexprs,+ device_index,+ enable_fp_fusion = enable_fp_fusion,+ **options,+ )+ kernel[grid](*args, **constexprs, enable_fp_fusion = enable_fp_fusion, **options)+++# Read once: device_count() inside a compiled region graph-breaks.+_MULTI_GPU = torch.cuda.is_available() and torch.cuda.device_count() > 1+++if triton is not None:++ @triton.jit+ def _cast_fp8_rn(x, IS_E5M2: tl.constexpr):+ # Triton 3.8 rounds through fp16 on sm89; native f32 conversion avoids double rounding.+ if IS_E5M2:+ packed = tl.inline_asm_elementwise(+ "cvt.rn.satfinite.e5m2x2.f32 $0, $1, $1;",+ constraints = "=h,f",+ args = [x],+ dtype = tl.uint16,+ is_pure = True,+ pack = 1,+ )+ return packed.to(tl.uint8).to(tl.float8e5, bitcast = True)+ else:+ packed = tl.inline_asm_elementwise(+ "cvt.rn.satfinite.e4m3x2.f32 $0, $1, $1;",+ constraints = "=h,f",+ args = [x],+ dtype = tl.uint16,+ is_pure = True,+ pack = 1,+ )+ return packed.to(tl.uint8).to(tl.float8e4nv, bitcast = True)++ @triton.jit+ def _fp8_tensorwise_cast_kernel(+ x_ptr,+ y_ptr,+ amax_ptr,+ inv_ptr,+ numel,+ FP8_MAX: tl.constexpr,+ IS_E5M2: tl.constexpr,+ BLOCK: tl.constexpr,+ ):+ pid = tl.program_id(0).to(tl.int64)+ offs = pid * BLOCK + tl.arange(0, BLOCK)+ mask = offs < numel+ amax = tl.load(amax_ptr).to(tl.float32)+ # NaN / inf amax propagates into the output, so a GradScaler still sees the overflow.+ amax = tl.maximum(amax, 1e-12, propagate_nan = tl.PropagateNan.ALL)+ # Match PyTorch's reciprocal-then-multiply rounding order (_tensor.py::__rdiv__).+ scale = tl.div_rn(1.0, amax) * FP8_MAX+ x = tl.load(x_ptr + offs, mask = mask, other = 0.0).to(tl.float32)+ y = tl.maximum(x * scale, -FP8_MAX, propagate_nan = tl.PropagateNan.ALL)+ y = tl.minimum(y, FP8_MAX, propagate_nan = tl.PropagateNan.ALL)+ tl.store(y_ptr + offs, _cast_fp8_rn(y, IS_E5M2), mask = mask)+ if pid == 0:+ tl.store(inv_ptr, amax * (1.0 / FP8_MAX))+++def quantize_tensorwise(x: torch.Tensor, fp8_dtype = E4M3):+ """x -> (fp8 x with x's shape, contiguous; fp32 0-d dequant scale amax / fp8_max)."""+ fp8_max = _FP8_MAX[fp8_dtype]+ x = x.contiguous()+ amax = torch.linalg.vector_norm(x, float("inf"), dtype = torch.float32)+ if triton is not None and x.is_cuda and x.numel() > 0:+ y = torch.empty(x.shape, dtype = fp8_dtype, device = x.device)+ inv = torch.empty((), dtype = torch.float32, device = x.device)+ BLOCK = 2048+ grid = (triton.cdiv(x.numel(), BLOCK),)+ with torch.cuda.device(x.device) if _MULTI_GPU else contextlib.nullcontext():+ _launch(+ _fp8_tensorwise_cast_kernel,+ grid,+ (x, y, amax, inv, x.numel()),+ 4,+ {"FP8_MAX": fp8_max, "IS_E5M2": fp8_dtype is E5M2, "BLOCK": BLOCK},+ x.device.index,+ num_warps = 4,+ )+ return y, inv+ amax = amax.clamp(min = 1e-12) if torch.isfinite(amax) else amax+ y = (x.float() * (fp8_max / amax)).clamp(-fp8_max, fp8_max).to(fp8_dtype)+ return y, (amax / fp8_max).reshape(())+++if triton is not None:++ @triton.jit+ def _fp8_rowwise_cast_kernel(+ x_ptr,+ y_ptr,+ inv_ptr,+ K,+ stride_xm,+ FP8_MAX: tl.constexpr,+ IS_E5M2: tl.constexpr,+ BLOCK_K: tl.constexpr,+ ONE_BLOCK: tl.constexpr,+ ):+ row = tl.program_id(0).to(tl.int64)+ x_row = x_ptr + row * stride_xm+ y_row = y_ptr + row * K+ offs = tl.arange(0, BLOCK_K)+ if ONE_BLOCK:+ x = tl.load(x_row + offs, mask = offs < K, other = 0.0).to(tl.float32)+ amax = tl.max(tl.abs(x), axis = 0)+ nans = tl.sum((x != x).to(tl.int32), axis = 0)+ else:+ acc = tl.zeros((BLOCK_K,), dtype = tl.float32)+ cnt = tl.zeros((BLOCK_K,), dtype = tl.int32)+ for k in range(0, K, BLOCK_K):+ xk = tl.load(x_row + k + offs, mask = k + offs < K, other = 0.0).to(tl.float32)+ acc = tl.maximum(acc, tl.abs(xk))+ cnt += (xk != xk).to(tl.int32)+ amax = tl.max(acc, axis = 0)+ nans = tl.sum(cnt, axis = 0)+ # tl.max drops NaN: put it back so a GradScaler still sees the overflow.+ amax = tl.where(nans > 0, float("nan"), amax)+ amax = tl.maximum(amax, 1e-12, propagate_nan = tl.PropagateNan.ALL)+ scale = tl.div_rn(1.0, amax) * FP8_MAX+ if ONE_BLOCK:+ y = tl.maximum(x * scale, -FP8_MAX, propagate_nan = tl.PropagateNan.ALL)+ y = tl.minimum(y, FP8_MAX, propagate_nan = tl.PropagateNan.ALL)+ tl.store(y_row + offs, _cast_fp8_rn(y, IS_E5M2), mask = offs < K)+ else:+ for k in range(0, K, BLOCK_K):+ mask = k + offs < K+ xk = tl.load(x_row + k + offs, mask = mask, other = 0.0).to(tl.float32)+ y = tl.maximum(xk * scale, -FP8_MAX, propagate_nan = tl.PropagateNan.ALL)+ y = tl.minimum(y, FP8_MAX, propagate_nan = tl.PropagateNan.ALL)+ tl.store(y_row + k + offs, _cast_fp8_rn(y, IS_E5M2), mask = mask)+ tl.store(inv_ptr + row, amax * (1.0 / FP8_MAX))+++def quantize_rowwise(x: torch.Tensor, fp8_dtype = E4M3):+ """x [M, K] -> (fp8 x [M, K] contiguous; fp32 [M, 1] dequant scales, one per row)."""+ fp8_max = _FP8_MAX[fp8_dtype]+ if x.stride(-1) != 1:+ x = x.contiguous()+ M, K = x.shape+ if triton is not None and x.is_cuda and x.numel() > 0:+ y = torch.empty((M, K), dtype = fp8_dtype, device = x.device)+ inv = torch.empty((M, 1), dtype = torch.float32, device = x.device)+ one = K <= 8192+ BLOCK_K = triton.next_power_of_2(K) if one else 2048+ with torch.cuda.device(x.device) if _MULTI_GPU else contextlib.nullcontext():+ _launch(+ _fp8_rowwise_cast_kernel,+ (M,),+ (x, y, inv, K, x.stride(0)),+ 3,+ {+ "FP8_MAX": fp8_max,+ "IS_E5M2": fp8_dtype is E5M2,+ "BLOCK_K": BLOCK_K,+ "ONE_BLOCK": one,+ },+ x.device.index,+ num_warps = 8 if BLOCK_K >= 4096 else 4,+ )+ return y, inv+ xf = x.float()+ amax = xf.abs().amax(dim = 1, keepdim = True)+ amax = torch.where(torch.isnan(xf).any(1, keepdim = True), torch.nan, amax).clamp(min = 1e-12)+ y = (xf * (fp8_max / amax)).clamp(-fp8_max, fp8_max).to(fp8_dtype)+ return y, amax / fp8_max+++if triton is not None:++ @triton.jit+ def _transpose_u8_kernel(x_ptr, y_ptr, R, C, BLOCK: tl.constexpr):+ # Square tiles on purpose: non-square 2D fp8 transposed stores write corrupt bytes on sm89 (L4, triton 3.x).+ rows = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)+ cols = tl.program_id(1).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)+ mask = (rows[:, None] < R) & (cols[None, :] < C)+ tile = tl.load(x_ptr + rows[:, None] * C + cols[None, :], mask = mask)+ tl.store(y_ptr + cols[None, :] * R + rows[:, None], tile, mask = mask)+++_TRANSPOSE_BLOCK = 64+++def fp8_transpose(w8: torch.Tensor, use_triton: Optional[bool] = None) -> torch.Tensor:+ """w8 [R, C] (contiguous, 1 byte per element) -> contiguous [C, R], byte for byte."""+ if use_triton is None:+ use_triton = triton is not None and w8.is_cuda+ if not use_triton or w8.dim() != 2 or w8.numel() == 0:+ return w8.t().contiguous()+ w8 = w8.contiguous()+ R, C = w8.shape+ out = torch.empty((C, R), dtype = w8.dtype, device = w8.device)+ grid = (triton.cdiv(R, _TRANSPOSE_BLOCK), triton.cdiv(C, _TRANSPOSE_BLOCK))+ with torch.cuda.device(w8.device) if (_MULTI_GPU and w8.is_cuda) else contextlib.nullcontext():+ _launch(+ _transpose_u8_kernel,+ grid,+ (w8.view(torch.uint8), out.view(torch.uint8), R, C),+ 2,+ {"BLOCK": _TRANSPOSE_BLOCK},+ w8.device.index,+ num_warps = 4,+ )+ return out+++def _out_dtype(dtype):+ return torch.bfloat16 if dtype == torch.bfloat16 else torch.float32+++_ROWWISE_BF16_ONLY = set()+++def _rowwise_ok(dtype, device) -> bool:+ return dtype != torch.float16 or device not in _ROWWISE_BF16_ONLY+++def _rowwise_scaled_mm(a, b, scale_a, scale_b, dtype, fast_accum):+ """None for fp16 when this device's rowwise GEMM refuses float32 out: bf16 would round 65504 to an inf."""+ out_dtype = _out_dtype(dtype)+ if out_dtype == torch.float32 and a.device not in _ROWWISE_BF16_ONLY:+ try:+ return _scaled_mm(a, b, scale_a, scale_b, out_dtype, fast_accum)+ except (RuntimeError, ValueError) as exc:+ if str(exc) not in (+ "Only bf16 high precision output types are supported for row-wise scaling.",+ "Only bf16 and fp16 high precision output types are supported for row-wise scaling.",+ ):+ raise+ _ROWWISE_BF16_ONLY.add(a.device)+ if dtype == torch.float16:+ return None+ return _scaled_mm(a, b, scale_a, scale_b, torch.bfloat16, fast_accum)+++def _weight_scale(w_inv):+ return w_inv.reshape(-1)[0] if w_inv.dim() == 2 else w_inv+++def _scaled_mm(a, b, scale_a, scale_b, out_dtype, fast_accum):+ out = torch._scaled_mm(+ a,+ b,+ scale_a = scale_a,+ scale_b = scale_b,+ out_dtype = out_dtype,+ use_fast_accum = fast_accum,+ )+ return out[0] if isinstance(out, tuple) else out+++# Module level: a torch.library.Library deregisters its ops once collected.+_library = None+++def _opaque_under_compile(name, fake):+ def decorator(fn):+ global _library+ substitute_in_graph = getattr(torch._dynamo, "substitute_in_graph", None)+ if substitute_in_graph is None or not hasattr(torch.library, "infer_schema"):+ return fn+ try:+ if _library is None:+ _library = torch.library.Library("unsloth", "FRAGMENT")+ _library.define(name + torch.library.infer_schema(fn, mutates_args = ()))+ _library.impl(name, fn, "CompositeExplicitAutograd")+ torch.library.register_fake(f"unsloth::{name}", fake, lib = _library)+ substitute_in_graph(fn)(+ functools.wraps(fn)(+ lambda *args, **kwargs: getattr(torch.ops.unsloth, name)(*args, **kwargs)+ )+ )+ except (RuntimeError, TypeError, ValueError): # registered by an earlier import+ pass+ return fn++ return decorator+++def _fwd_fake(x, w8, w_inv):+ return x.new_empty((x.shape[0], w8.shape[0]))+++@_opaque_under_compile("fp8_lora_fwd", _fwd_fake)+def fp8_lora_fwd(x: torch.Tensor, w8: torch.Tensor, w_inv: torch.Tensor) -> torch.Tensor:+ """x [M, K] (16-bit) @ dequant(w8 [N, K]).T -> [M, N] in x.dtype. w_inv: 0-d (tensorwise activations) or+ [1, N] (rowwise activations: the weight's tensorwise scale per output column)."""+ FP8_LORA_STATS["fwd"] += 1+ out = None+ if w_inv.dim() == 2 and _rowwise_ok(x.dtype, x.device):+ x8, x_inv = quantize_rowwise(x, E4M3)+ out = _rowwise_scaled_mm(x8, w8.t(), x_inv, w_inv, x.dtype, _FWD_FAST_ACCUM)+ if out is None:+ x8, x_inv = quantize_tensorwise(x, E4M3)+ out = _scaled_mm(+ x8, w8.t(), x_inv, _weight_scale(w_inv), _out_dtype(x.dtype), _FWD_FAST_ACCUM+ )+ return out if out.dtype == x.dtype else out.to(x.dtype)+++def _dgrad_fake(dy, w8, w_inv, w_is_t):+ return dy.new_empty((dy.shape[0], w8.shape[0] if w_is_t else w8.shape[1]))+++@_opaque_under_compile("fp8_lora_dgrad", _dgrad_fake)+def fp8_lora_dgrad(+ dy: torch.Tensor, w8: torch.Tensor, w_inv: torch.Tensor, w_is_t: bool+) -> torch.Tensor:+ """dy [M, N] (16-bit) @ dequant(W [N, K]) -> dx [M, K] in dy.dtype. w8 is the cached K-major copy [K, N]+ (w_is_t) or the forward copy [N, K], transposed here by the square-tile kernel inside the op, never by an+ Inductor-generated fp8 transpose (those write corrupt bytes for non-square tile configs on sm89)."""+ FP8_LORA_STATS["dgrad"] += 1+ w8t = w8 if w_is_t else fp8_transpose(w8)+ out = None+ if w_inv.dim() == 2 and _rowwise_ok(dy.dtype, dy.device):+ dy8, dy_inv = quantize_rowwise(dy, E4M3)+ out = _rowwise_scaled_mm(dy8, w8t.t(), dy_inv, w_inv, dy.dtype, _DGRAD_FAST_ACCUM)+ if out is None:+ dy8, dy_inv = quantize_tensorwise(+ dy, E5M2 if _DGRAD_DTYPE in (None, "auto") else _DGRAD_DTYPE+ )+ out = _scaled_mm(+ dy8, w8t.t(), dy_inv, _weight_scale(w_inv), _out_dtype(dy.dtype), _DGRAD_FAST_ACCUM+ )+ return out if out.dtype == dy.dtype else out.to(dy.dtype)+++if triton is not None:++ @triton.jit(do_not_specialize = ["N", "numel", "raw_s0", "raw_s1", "delta_s0", "delta_s1"])+ def _fp8_accum_epilogue_kernel(+ raw_ptr,+ x_inv_ptr,+ w_inv_ptr,+ delta_ptr,+ out_ptr,+ N,+ numel,+ raw_s0,+ raw_s1,+ delta_s0,+ delta_s1,+ ROW_SCALE: tl.constexpr,+ BLOCK: tl.constexpr,+ ):+ offset = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)+ row, col = offset // N, offset % N+ mask = offset < numel+ raw = tl.load(raw_ptr + row * raw_s0 + col * raw_s1, mask, other = 0)+ sx = tl.load(x_inv_ptr + tl.where(ROW_SCALE, row, 0), mask, other = 0)+ sw = tl.load(w_inv_ptr + tl.where(ROW_SCALE, col, 0), mask, other = 0)+ delta = tl.load(delta_ptr + row * delta_s0 + col * delta_s1, mask, other = 0).to(tl.float32)+ out = tl.inline_asm_elementwise(+ "{ .reg .f32 v; mul.rn.f32 v, $1, $2; mul.rn.f32 v, v, $3; add.rn.f32 $0, v, $4; }",+ constraints = "=f,f,f,f,f",+ args = (raw, sx, sw, delta),+ dtype = tl.float32,+ is_pure = True,+ pack = 1,+ )+ tl.store(out_ptr + offset, out, mask)+++def _fp8_accum_epilogue(raw, x_inv, w_inv, delta):+ if triton is None or not raw.is_cuda or torch.version.hip is not None:+ return (raw * x_inv * w_inv + delta.float()).to(delta.dtype)+ out = torch.empty(raw.shape, device = raw.device, dtype = delta.dtype)+ with torch.cuda.device(raw.device) if _MULTI_GPU else contextlib.nullcontext():+ _launch(+ _fp8_accum_epilogue_kernel,+ ((raw.numel() + 1023) // 1024,),+ (+ raw,+ x_inv,+ w_inv,+ delta,+ out,+ raw.shape[1],+ raw.numel(),+ *raw.stride(),+ *delta.stride(),+ ),+ 5,+ {"ROW_SCALE": w_inv.dim() == 2, "BLOCK": 1024},+ raw.device.index,+ num_warps = 4,+ )+ return out+++def _accum_fake(x, w8, w_inv, delta, unit_scale, backward, transpose_weight):+ return delta.new_empty(delta.shape)+++@_opaque_under_compile("fp8_lora_accum", _accum_fake)+def fp8_lora_accum(+ x: torch.Tensor,+ w8: torch.Tensor,+ w_inv: torch.Tensor,+ delta: torch.Tensor,+ unit_scale: torch.Tensor,+ backward: bool,+ transpose_weight: bool,+) -> torch.Tensor:+ FP8_LORA_STATS["dgrad" if backward else "fwd"] += 1+ if transpose_weight:+ w8 = fp8_transpose(w8)+ if w_inv.dim() == 2:+ x8, x_inv = quantize_rowwise(x, E4M3)+ else:+ dtype = (+ E5M2+ if backward and _DGRAD_DTYPE in (None, "auto")+ else (_DGRAD_DTYPE if backward else E4M3)+ )+ x8, x_inv = quantize_tensorwise(x, dtype)+ raw = _scaled_mm(+ x8,+ w8.t(),+ unit_scale,+ unit_scale,+ torch.float32,+ _DGRAD_FAST_ACCUM if backward else _FWD_FAST_ACCUM,+ )+ return _fp8_accum_epilogue(raw, x_inv, w_inv, delta)+++def _first_fake(x, w8, w_inv, xa, bt, alpha, unit_scale, backward, transpose_weight):+ return x.new_empty((x.shape[0], bt.shape[1]))+++@_opaque_under_compile("fp8_lora_first", _first_fake)+def fp8_lora_first(+ x: torch.Tensor,+ w8: torch.Tensor,+ w_inv: torch.Tensor,+ xa: torch.Tensor,+ bt: torch.Tensor,+ alpha: float,+ unit_scale: torch.Tensor,+ backward: bool,+ transpose_weight: bool,+) -> torch.Tensor:+ delta = x.new_empty((x.shape[0], bt.shape[1]))+ delta.addmm_(xa, bt, alpha = alpha, beta = 0)+ return fp8_lora_accum(x, w8, w_inv, delta, unit_scale, backward, transpose_weight)+++def _is_transposed(W):+ return W.dim() == 2 and W.stride(0) == 1 and W.stride(1) != 1+++def _compiling():+ try:+ return torch.compiler.is_compiling()+ except Exception:+ return False+++class Fp8LoRABaseState:+ """Quant state of a frozen LoRA base in Unsloth's LoRA kernels (a packed weight state): ``W`` is the+ 16-bit weight [out, in] (forward) or its ``.t()`` view (input gradient)."""++ _unsloth_packed_weight_state = True+ _unsloth_fp8_dgrad = True++ def __init__(self, weight: torch.Tensor):+ self.dtype = weight.dtype+ self.device = weight.device # where the fp8 copy and the GEMMs live+ self.w8 = self.w8t = self.w_inv = self.w_inv_fwd = self.w_inv_dgrad = self.unit_scale = None+ self.scaling = "tensor"+ self.offloaded = False # True: the module's 16-bit weight is the host copy+ self._key = None+ self.quantizations = 0+ self.refresh(weight)++ @staticmethod+ def _weight_key(weight):+ return (+ weight.data_ptr(),+ weight._version,+ weight.device,+ weight.dtype,+ tuple(weight.shape),+ )++ def refresh(self, weight: torch.Tensor):+ """Quantize `weight` unless the cache already holds it (same storage, version, device)."""+ if _compiling():+ return self+ key = self._weight_key(weight)+ if key == self._key:+ return self+ with torch.inference_mode(False), torch.no_grad():+ w = weight.detach()+ if self.offloaded:+ w = w.to(self.device)+ else:+ self.device = w.device+ self.w8, self.w_inv = quantize_tensorwise(w, E4M3)+ self.unit_scale = torch.ones((), device = w.device, dtype = torch.float32)+ self.w8t = fp8_transpose(self.w8) if _CACHE_T else None+ self.scaling = _scaling_for(w.device)+ if self.scaling == "row":+ self.w_inv_fwd = self.w_inv.reshape(1, 1).expand(1, w.shape[0]).contiguous()+ self.w_inv_dgrad = self.w_inv.reshape(1, 1).expand(1, w.shape[1]).contiguous()+ else:+ self.w_inv_fwd = self.w_inv_dgrad = self.w_inv+ self.dtype = weight.dtype+ self._key = key+ self.quantizations += 1+ FP8_LORA_STATS["quantize"] += 1+ return self++ def _dgrad_weight(self):+ return (self.w8t, True) if self.w8t is not None else (self.w8, False)++ def offload(self, module):+ """Move module.weight (the 16-bit master) to host RAM, keeping the Parameter object."""+ weight = module.weight+ if self.offloaded or (weight.device.type == "cpu" and not _OFFLOAD_CPU_WEIGHTS):+ return False+ self.refresh(weight)+ host = None+ if weight.is_cuda:+ try:+ host = torch.empty(weight.shape, dtype = weight.dtype, device = "cpu", pin_memory = True)+ except (+ RuntimeError+ ): # pinned host memory exhausted: pageable works, restores are slower+ host = None+ if host is None:+ host = torch.empty(weight.shape, dtype = weight.dtype, device = "cpu")+ host.copy_(weight.detach())+ weight.data = host+ self.offloaded = True+ self._key = self._weight_key(weight)+ return True++ def restore(self, module):+ if not self.offloaded:+ return False+ weight = module.weight+ with torch.inference_mode(False), torch.no_grad():+ weight.data = weight.data.to(self.device, copy = True)+ self.offloaded = False+ self._key = self._weight_key(weight)+ return True++ def _dense(self, W):+ """A 16-bit operand on the compute device for the non-fp8 fallbacks: W itself, or (master on the host)+ the dequantized fp8 weight in W's orientation, which is what the fp8 forward uses anyway."""+ if W is not None and not self.offloaded and W.device == self.device:+ return W+ dense = self.w8.to(torch.float32) * self.w_inv+ dense = dense.to(self.dtype if W is None else W.dtype)+ return dense.t() if W is not None and _is_transposed(W) else dense++ def resident_bytes(self, module = None):+ fp8 = sum(+ t.numel() * t.element_size()+ for t in {+ id(t): t+ for t in (+ self.w8,+ self.w8t,+ self.w_inv,+ self.w_inv_fwd,+ self.w_inv_dgrad,+ self.unit_scale,+ )+ if t is not None+ }.values()+ )+ out = {"fp8": fp8, "master_device": 0, "master_host": 0}+ if module is not None:+ w = module.weight+ nbytes = w.numel() * w.element_size()+ if self.offloaded:+ out["master_host"] = nbytes+ elif w.device == self.device:+ out["master_device"] = nbytes+ return out++ def dequantize(+ self,+ W = None,+ dtype = None,+ ):+ if W is not None and W.device != self.device:+ return W.to(self.device)+ return W++ def _use_fp8(self, X):+ return X.dtype in (torch.bfloat16, torch.float16, torch.float32) and (+ X.numel() // max(X.shape[-1], 1) >= _MIN_ROWS+ )++ def matmul(+ self,+ X,+ W,+ out = None,+ ):+ """``X @ W.t()``: the forward for the stored W, the input gradient for its transposed view."""+ shape = X.shape+ X2 = X.reshape(-1, shape[-1])+ if not self._use_fp8(X2):+ FP8_LORA_STATS["fwd_16bit"] += 1+ result = torch.matmul(X2, self._dense(W).t().to(X2.dtype))+ elif _is_transposed(W):+ result = self.dgrad(X2, W.t())+ else:+ result = fp8_lora_fwd(X2, self.w8, self.w_inv_fwd)+ result = result.view(*shape[:-1], result.shape[-1])+ return result if out is None else out.copy_(result)++ def lora_first(+ self,+ X,+ W,+ XA,+ Bt,+ alpha,+ out = None,+ ):+ backward = _is_transposed(W)+ if not self._use_fp8(X) or (backward and _DGRAD_DTYPE is None):+ delta = X.new_empty((X.shape[0], Bt.shape[1]))+ delta.addmm_(XA, Bt, alpha = alpha, beta = 0)+ result = delta.addmm_(X, self._dense(W).t().to(X.dtype))+ else:+ w8 = self._dgrad_weight()[0] if backward else self.w8+ inv = self.w_inv_dgrad if backward else self.w_inv_fwd+ result = fp8_lora_first(+ X, w8, inv, XA, Bt, alpha, self.unit_scale, backward, backward and self.w8t is None+ )+ return result if out is None else out.copy_(result)++ def dgrad_accum(self, dY, W, delta):+ if _DGRAD_DTYPE is None or not self._use_fp8(dY):+ return delta.addmm_(dY, self._dense(W).to(dY.dtype))+ w8, transposed = self._dgrad_weight()+ return fp8_lora_accum(+ dY, w8, self.w_inv_dgrad, delta, self.unit_scale, True, not transposed+ )++ def dgrad(+ self,+ dY,+ W = None,+ ):+ """dY [..., out] @ W -> [..., in]: the base layer's input gradient."""+ shape = dY.shape+ dY2 = dY.reshape(-1, shape[-1])+ if _DGRAD_DTYPE is None or not self._use_fp8(dY2):+ FP8_LORA_STATS["dgrad_16bit"] += 1+ if W is None and not self.offloaded:+ raise RuntimeError("Unsloth: 16-bit dgrad needs the base weight")+ result = torch.matmul(dY2, self._dense(W).to(dY2.dtype))+ else:+ w8, w_is_t = self._dgrad_weight()+ result = fp8_lora_dgrad(dY2, w8, self.w_inv_dgrad, w_is_t)+ return result.view(*shape[:-1], result.shape[-1])+++def has_fp8_dgrad(W_quant) -> bool:+ return getattr(type(W_quant), "_unsloth_fp8_dgrad", False)+++class _Fp8FrozenLinear(torch.autograd.Function):+ @staticmethod+ def forward(ctx, x, weight, w8, w8t, w_inv_fwd, w_inv_dgrad):+ ctx.w_is_t = w8t is not None+ ctx.save_for_backward(+ weight,+ w8t if w8t is not None else w8,+ w_inv_fwd if w_inv_dgrad is None else w_inv_dgrad,+ )+ shape = x.shape+ x2 = x.reshape(-1, shape[-1])+ y = fp8_lora_fwd(x2, w8, w_inv_fwd)+ return y.view(*shape[:-1], y.shape[-1])++ @staticmethod+ def backward(ctx, dy):+ weight, w8t, w_inv = ctx.saved_tensors+ shape = dy.shape+ dy2 = dy.reshape(-1, shape[-1])+ if _DGRAD_DTYPE is None:+ FP8_LORA_STATS["dgrad_16bit"] += 1+ dx = torch.matmul(dy2, weight.to(dy2.dtype))+ else:+ dx = fp8_lora_dgrad(dy2, w8t, w_inv, ctx.w_is_t)+ return dx.view(*shape[:-1], dx.shape[-1]), None, None, None, None, None+++def _fp8_base_forward(self, x):+ state = self.__dict__.get("_unsloth_fp8_lora")+ weight = self.weight+ if state is None or weight.requires_grad or not state._use_fp8(x.reshape(-1, x.shape[-1])):+ FP8_LORA_STATS["fwd_16bit"] += 1+ if state is not None and state.offloaded:+ if weight.requires_grad: # unfrozen: train the exact weight+ state.restore(self)+ else:+ weight = state._dense(weight)+ return torch.nn.functional.linear(x, weight.to(x.dtype), self.bias)+ state.refresh(weight)+ # Tensorwise: one scale serves both GEMMs (Dynamo rejects the same tensor passed twice to apply).+ dgrad_scale = None if state.w_inv_dgrad is state.w_inv_fwd else state.w_inv_dgrad+ y = _Fp8FrozenLinear.apply(x, weight, state.w8, state.w8t, state.w_inv_fwd, dgrad_scale)+ if self.bias is not None:+ y = y + self.bias.to(y.dtype)+ return y+++def eligible_base(module) -> Optional[str]:+ if type(module) is not torch.nn.Linear:+ return f"{type(module).__name__} is not a plain nn.Linear"+ weight = module.weight+ if type(weight) not in (torch.Tensor, torch.nn.Parameter):+ return f"weight is a {type(weight).__name__}"+ if getattr(weight, "_unsloth_fsdp2_preload", None) is not None or tuple(weight.shape) != (+ module.out_features,+ module.in_features,+ ):+ return f"weight is a {tuple(weight.shape)} shard of {module.out_features} x {module.in_features}"+ if not weight.is_contiguous():+ return "weight is not contiguous"+ if weight.dtype not in (torch.bfloat16, torch.float16):+ return f"weight dtype {weight.dtype}"+ if weight.requires_grad:+ return "weight is trainable"+ if module.in_features % 16 or module.out_features % 16:+ return f"{module.in_features} x {module.out_features} is not a multiple of 16"+ if getattr(module, "weight_fake_quantizer", None) is not None:+ return "QAT fake-quantized"+ return None+++def offload_supported() -> bool:+ """The host master needs a quantized-weight dgrad and one process owning the weights."""+ if not _OFFLOAD_MASTER or _DGRAD_DTYPE is None:+ return False+ try:+ if int(os.environ.get("WORLD_SIZE", "1")) > 1:+ return False+ import torch.distributed as dist+ if dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1:+ return False+ except Exception:+ return False+ return True+++def _fp8_base_apply(+ self,+ fn,+ recurse = True,+):+ state = self.__dict__.get("_unsloth_fp8_lora")+ if state is not None and state.offloaded:+ probe = fn(torch.empty(0, dtype = state.dtype, device = state.device))+ if probe.device == state.device and probe.dtype == state.dtype:+ weight = self._parameters.pop("weight")+ try:+ return torch.nn.Module._apply(self, fn, recurse)+ finally:+ self._parameters["weight"] = weight+ state.restore(self)+ out = torch.nn.Module._apply(self, fn, recurse)+ if state is not None:+ state._key = None+ state.refresh(self.weight)+ return out+++def attach_fp8_lora_base(module, offload: Optional[bool] = None) -> Fp8LoRABaseState:+ """Quantize the frozen base `module` (nn.Linear) once and route its GEMMs through fp8. offload: move the+ 16-bit master to host RAM (None: offload_supported())."""+ import types++ state = module.__dict__.get("_unsloth_fp8_lora")+ if state is None:+ state = Fp8LoRABaseState(module.weight)+ module.__dict__["_unsloth_fp8_lora"] = state+ module.forward = types.MethodType(_fp8_base_forward, module)+ module._apply = types.MethodType(_fp8_base_apply, module)+ module.__dict__["_unsloth_fp8_lora_load_hook"] = module.register_load_state_dict_post_hook(+ _fp8_requantize_after_load+ )+ if offload is None:+ offload = offload_supported()+ if offload:+ state.offload(module)+ return state+++def _fp8_requantize_after_load(module, incompatible_keys):+ state = module.__dict__.get("_unsloth_fp8_lora")+ if state is not None:+ state._key = None+ state.refresh(module.weight)+++def restore_fp8_lora_master(module) -> bool:+ """Bring a base layer's exact 16-bit weight back to its compute device (PEFT merge, export)."""+ state = module.__dict__.get("_unsloth_fp8_lora")+ return state.restore(module) if state is not None else False+++def detach_fp8_lora_base(module):+ restore_fp8_lora_master(module)+ module.__dict__.pop("_unsloth_fp8_lora", None)+ hook = module.__dict__.pop("_unsloth_fp8_lora_load_hook", None)+ if hook is not None:+ hook.remove()+ module.__dict__.pop("forward", None)+ module.__dict__.pop("_apply", None)+++def fp8_lora_memory(model) -> dict:+ """fp8 / device-master / host-master bytes per converted base layer and in total."""+ layers, total = {}, {"fp8": 0, "master_device": 0, "master_host": 0}+ for name, module in model.named_modules():+ state = module.__dict__.get("_unsloth_fp8_lora")+ if state is None:+ continue+ layers[name] = state.resident_bytes(module)+ for key in total:+ total[key] += layers[name][key]+ return {"layers": layers, "total": total}diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.pyindex 69030e330da..e9e66f1c249 100644--- a/unsloth/kernels/utils.py+++ b/unsloth/kernels/utils.py@@ -484,6 +484,14 @@ def _has_packed_qweight(proj):)+def _fp8_lora_state(base_layer, W):+ """The fp8 frozen-base state (kernels/fp8_lora.py, float8_training with LoRA), refreshed if W changed."""+ state = base_layer.__dict__.get("_unsloth_fp8_lora")+ if state is None or W.requires_grad:+ return None+ return state.refresh(W)++def _is_packed_state(quant_state):return getattr(type(quant_state), "_unsloth_packed_weight_state", False)@@ -527,6 +535,8 @@ def get_lora_parameters(proj):if packed is not None:W_quant = packed[1]+ elif W_quant is None:+ W_quant = _fp8_lora_state(base_layer, W)if getattr(proj, "disable_adapters", True) or proj.merged:return W, W_quant, None, None, None@@ -590,6 +600,8 @@ def get_lora_parameters_bias(proj):if packed is not None:W_quant = packed[1]+ elif W_quant is None:+ W_quant = _fp8_lora_state(base_layer, W)if getattr(proj, "disable_adapters", True) or proj.merged:return W, W_quant, None, None, None, base_layer.bias@@ -2024,6 +2036,8 @@ def _lora_fused_ok(W, W_quant, dtype):return Falseif not _is_compiling() and W.layout != torch.strided:return False+ if getattr(type(W_quant), "_unsloth_fp8_dgrad", False):+ return W_quant.dtype == dtypeif type(W_quant) is Int4QuantState or _is_packed_state(W_quant):return Falseif W_quant is None:@@ -2081,22 +2095,7 @@ def matmul_lora(if XA is not None and out is not None and _shares_storage(out, XA):XA = None- if isinstance(W, Float8Tensor):- assert W.ndim == 2- if W.block_size[0] == W.shape[0] and W.block_size[1] == 1:- # Rowwise scaling becomes colwise after transpose, so this detects the backward pass.- W = W.dequantize()- else:- W = W.contiguous()- out = torch_matmul(X, W.t(), out = out)- elif W.dtype == torch.float8_e4m3fn:- out = fp8_linear(X, W, W_quant)- elif type(W_quant) is Int4QuantState and W.stride(-1) == 1:- # Forward orientation: few rows use the fused kernel, the rest decode once + cuBLAS.- out = _int4_matmul(X, W, W_quant, out = out)- elif _is_packed_state(W_quant):- out = W_quant.matmul(X, W, out = out)- elif (+ lora_first = (A is not Noneand fused# Compiled inside torch.func transforms or native forward-mode AD: the op has no forward-mode or@@ -2119,11 +2118,31 @@ def matmul_lora(and not _shares_storage(out, W, X, A, B, XA)))+ )+ if isinstance(W, Float8Tensor):+ assert W.ndim == 2+ if W.block_size[0] == W.shape[0] and W.block_size[1] == 1:+ # Rowwise scaling becomes colwise after transpose, so this detects the backward pass.+ W = W.dequantize()+ else:+ W = W.contiguous()+ out = torch_matmul(X, W.t(), out = out)+ elif W.dtype == torch.float8_e4m3fn:+ out = fp8_linear(X, W, W_quant)+ elif type(W_quant) is Int4QuantState and W.stride(-1) == 1:+ # Forward orientation: few rows use the fused kernel, the rest decode once + cuBLAS.+ out = _int4_matmul(X, W, W_quant, out = out)+ elif _is_packed_state(W_quant) and not (+ lora_first and getattr(type(W_quant), "_unsloth_fp8_dgrad", False)):+ out = W_quant.matmul(X, W, out = out)+ elif lora_first:if XA is None:XA = torch_matmul(X, A.t() if A.dtype is dtype else A.t().to(dtype))B = B.t() if B.dtype is dtype else B.t().to(dtype)- if out is not None:+ if getattr(type(W_quant), "_unsloth_fp8_dgrad", False):+ out = W_quant.lora_first(X, W, XA, B, s, out = out)+ elif out is not None:# Eager only: a traced caller buffer cannot be checked for aliasing (_shares_storage). Dequantize# first: the caller's buffer may share storage with the quant state (absmax, code, ...).if W_quant is not None:diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.pyindex b46911975e2..ff6b617964a 100644--- a/unsloth/models/_block_compile.py+++ b/unsloth/models/_block_compile.py@@ -1406,8 +1406,10 @@ def _wrap(cls,compile_options,kinds = 1,+ *,+ install = True,):- original = cls.forward+ original = getattr(cls.forward, "_unsloth_block_original", cls.forward)# Host-read packed lengths / PrefixGrouper segments stay stock.unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")try:@@ -1662,7 +1664,9 @@ def _wrap(forward._unsloth_block_original = originalforward._unsloth_block_state = statesetattr(forward, _MARK, True)- cls.forward = forward+ if install:+ cls.forward = forward+ return forward_TOWER_SEGMENTS = (@@ -1971,10 +1975,28 @@ def patch_block_compile(model):continue_wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)wrapped.append(cls.__name__)+ for layer in compiled_layers:+ if not any(m.__dict__.get("_unsloth_fp8_lora") is not None for m in layer.modules()):+ continue+ default = type(layer).forward+ precise = getattr(default, "_unsloth_fp8_block_forward", None)+ if precise is None:+ # FP8 dgrad amplifies elided BF16 rounding; other instances of this class keep their options.+ precise = _wrap(+ type(layer),+ {**_COMPILE_OPTIONS, "emulate_precision_casts": True},+ len(kinds.get(type(layer), ())) or 1,+ install = False,+ )+ default._unsloth_fp8_block_forward = precise+ if getattr(layer.forward, "__func__", None) is not precise:+ layer.forward = types.MethodType(precise, layer)+ layer.__dict__.pop("_unsloth_block_checked", None)if _COMPILE_OPTIONS.get("triton.cudagraphs") or (_CUDAGRAPH_MODE == "auto" and _auto_platform_reason() is None):_watch_hidden_states(model)+ model.__dict__["_unsloth_block_compile_installed"] = Trueif _CUDAGRAPH_MODE == "auto":# A new model decides again: a launch-bound model loaded first must not turn graphs on for# a GPU-bound one. Graphs that went off for shape churn stay off.diff --git a/unsloth/models/float8_training.py b/unsloth/models/float8_training.pyindex 1cf9916d977..4b6742ece7f 100644--- a/unsloth/models/float8_training.py+++ b/unsloth/models/float8_training.py@@ -6,6 +6,8 @@ torchao.float8.Float8LinearConfig); with native FSDP2 convert before sharding.""from __future__ import annotations+from functools import lru_cache+import torch@@ -49,6 +51,10 @@ __all__ = ["resolve_float8_training","float8_training_filter","convert_model_to_float8_training",+ "resolve_fp8_lora",+ "mark_fp8_lora",+ "enable_fp8_lora",+ "restore_fp8_lora_for_inference",]# Names and gates come from the precision registry (unsloth/precision/recipes.py).@@ -408,3 +414,609 @@ def convert_model_to_float8_training(except Exception:passreturn converted+++FP8_LORA_RECIPE = "fp8_lora_tensorwise"+_FP8_LORA_BY_NAME = ("tensorwise",)+_FP8_LORA_EXPLICIT = ("fp8_lora", FP8_LORA_RECIPE)+++def resolve_fp8_lora(+ float8_training,+ *,+ dtype = None,+ load_in_4bit = False,+ load_in_8bit = False,+ load_in_fp8 = False,+ qat_scheme = None,+ gpu = "current",+):+ """`float8_training` with LoRA / QLoRA -> FP8_LORA_RECIPE or None; refuses (PrecisionRefused, a ValueError)+ before anything loads. `gpu` is a registry GPU class ('sm89', ...); 'current' reads the CUDA device."""+ if float8_training is None or float8_training is False:+ return None+ refused = _precision.PrecisionRefused+ if float8_training is True:+ explicit = False+ elif isinstance(float8_training, str) and float8_training.strip().lower() in (+ _FP8_LORA_BY_NAME + _FP8_LORA_EXPLICIT+ ):+ explicit = float8_training.strip().lower() in _FP8_LORA_EXPLICIT+ else:+ raise ValueError(+ f"Unsloth: float8_training = {float8_training!r} is a full finetuning recipe "+ "(`full_finetuning = True`). With LoRA, float8_training = 'fp8_lora' (or True where it is "+ "measured faster) runs the frozen base's forward and input gradient in fp8."+ )+ if load_in_4bit or load_in_8bit or (load_in_fp8 not in (False, None)):+ raise refused(+ "Unsloth: float8_training with LoRA quantizes a 16-bit base to fp8 itself; pass "+ "load_in_4bit = False (and no load_in_8bit / load_in_fp8). QLoRA bases are already 4 / 8 bit."+ )+ if qat_scheme is not None:+ raise refused(+ "Unsloth: float8_training with LoRA and qat_scheme cannot be combined: QAT fake-quantizes the "+ "base weight that fp8 LoRA would quantize."+ )+ if gpu == "current":+ gpu = _precision.current_gpu_class()+ if gpu is None:+ raise refused(+ "Unsloth: float8_training with LoRA needs an NVIDIA GPU with compute capability 8.9+ "+ "(L4, RTX 40 / 50, RTX PRO 6000, H100, B200); no CUDA device is visible."+ )+ recipe = fp8_lora_recipe_for(gpu)+ gate = _check_precision_recipe(recipe, gpu = gpu, input_dtype = dtype, mode = "lora")+ if not explicit and gate.status != "named":+ raise refused(+ f"Unsloth: float8_training = {float8_training!r} with LoRA has not been measured faster than "+ f"bfloat16 LoRA on {gpu} ({gate.status}), so it is not enabled by name there. Pass "+ "float8_training = 'fp8_lora' to run the fp8 frozen base anyway."+ )+ if gate.status == "unprobed":+ print(f"Unsloth: fp8 LoRA has not been tested on {gpu}; running it as requested.")+ return recipe+++def fp8_lora_recipe_for(gpu):+ """The frozen-base recipe the kernels run on this GPU class: per-token scales on sm120, tensorwise+ elsewhere (UNSLOTH_FP8_LORA_SCALING overrides; kernels/fp8_lora.scaling_for_class is the one rule)."""+ scaling = _load_fp8_lora_kernels().scaling_for_class(gpu)+ return "fp8_lora_rowwise" if scaling == "row" else FP8_LORA_RECIPE+++def mark_fp8_lora(model, recipe):+ """Remember the request on the loaded model; get_peft_model converts the LoRA bases. A model that is+ already a PeftModel (an adapter repo was loaded, so get_peft_model is skipped) is converted now."""+ if recipe is not None and model is not None:+ try:+ model._unsloth_fp8_lora_recipe = recipe+ except Exception:+ pass+ if getattr(model, "peft_config", None):+ enable_fp8_lora(model, recipe)+ return model+++def _fp8_lora_request(model):+ stack, seen = [model], set()+ while stack and len(seen) < 8:+ m = stack.pop()+ if m is None or id(m) in seen:+ continue+ seen.add(id(m))+ recipe = getattr(m, "_unsloth_fp8_lora_recipe", None)+ if isinstance(recipe, str):+ return recipe+ stack.extend(getattr(m, attr, None) for attr in ("base_model", "model"))+ return None+++def _load_fp8_lora_kernels():+ try:+ from ..kernels import fp8_lora+ return fp8_lora+ except ImportError:+ import importlib.util+ import os+ import sys++ name = "_unsloth_fp8_lora_standalone"+ if name in sys.modules:+ return sys.modules[name]+ path = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "kernels", "fp8_lora.py"+ )+ spec = importlib.util.spec_from_file_location(name, path)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return module+++def _merge_with_master(+ self,+ *args,+ _unsloth_method = "merge",+ **kwargs,+):+ base = getattr(self, "base_layer", None)+ if base is not None:+ _load_fp8_lora_kernels().restore_fp8_lora_master(base)+ out = getattr(type(self), _unsloth_method)(self, *args, **kwargs)+ state = base.__dict__.get("_unsloth_fp8_lora") if base is not None else None+ if state is not None:+ state._key = None+ state.refresh(base.weight)+ return out+++def _unload_fp8_lora(self, *args, **kwargs):+ restore_fp8_lora_for_inference(self.model)+ return type(self)._unload_and_optionally_merge(self, *args, **kwargs)+++def _has_variant(lora_module, adapter_name = None) -> bool:+ """Variants may read the master directly or initialize tensors on its device."""+ dora = getattr(lora_module, "use_dora", {}) or {}+ variants = getattr(lora_module, "lora_variant", {}) or {}+ if adapter_name is not None:+ return bool(dora.get(adapter_name, False)) or adapter_name in variants+ return any(dora.values()) or bool(variants)+++def _move_adapter_with_master(self, adapter_name, *args, **kwargs):+ """Place adapters on the compute device; variants also need their master there."""+ base = getattr(self, "base_layer", None)+ state = base.__dict__.get("_unsloth_fp8_lora") if base is not None else None+ if state is not None and state.offloaded and _has_variant(self, adapter_name):+ _load_fp8_lora_kernels().restore_fp8_lora_master(base)+ out = type(self)._move_adapter_to_device_of_base_layer(self, adapter_name, *args, **kwargs)+ if state is None:+ return out+ state._key = None+ state.refresh(base.weight)+ if not state.offloaded:+ return out+ for name in tuple(getattr(self, "adapter_layer_names", ())) + tuple(+ getattr(self, "other_param_names", ())+ ):+ layer = getattr(self, name, None)+ if (+ not isinstance(layer, (torch.nn.ModuleDict, torch.nn.ParameterDict))+ or adapter_name not in layer+ ):+ continue+ entry = layer[adapter_name]+ tensors = list(entry.parameters()) if isinstance(entry, torch.nn.Module) else [entry]+ if any(t.device.type == "meta" for t in tensors):+ # low_cpu_mem_usage: PEFT leaves meta parameters for the checkpoint loader to materialize.+ continue+ if isinstance(entry, torch.nn.Module):+ layer[adapter_name] = entry.to(state.device)+ else:+ layer[adapter_name] = torch.nn.Parameter(+ entry.data.to(state.device), requires_grad = entry.requires_grad+ )+ return out+++def _restore_master_before_merge(lora_module):+ import functools+ import types++ for method in ("merge", "unmerge"):+ if method in lora_module.__dict__ or not callable(getattr(type(lora_module), method, None)):+ continue+ fn = functools.partial(_merge_with_master, _unsloth_method = method)+ object.__setattr__(lora_module, method, types.MethodType(fn, lora_module))+ move = "_move_adapter_to_device_of_base_layer"+ if move not in lora_module.__dict__ and callable(getattr(type(lora_module), move, None)):+ object.__setattr__(+ lora_module, move, types.MethodType(_move_adapter_with_master, lora_module)+ )+++_RAW_READS = {}+++def _reads_raw_weight(parent, leaf) -> bool:+ """Direct weight readers bypass the patched forward, so their masters must stay resident."""+ import inspect+ import re++ key = (type(parent), leaf)+ if key not in _RAW_READS:+ try:+ source = inspect.getsource(type(parent))+ except (OSError, TypeError):+ source = ""+ _RAW_READS[key] = re.search(rf"\b{re.escape(leaf)}\.weight\b", source) is not None+ return _RAW_READS[key]+++def _mamba_raw_weight_leaf(model, name) -> bool:+ parent_name, _, leaf = name.rpartition(".")+ try:+ parent = model.get_submodule(parent_name) if parent_name else model+ except AttributeError:+ return False+ if _reads_raw_weight(parent, leaf):+ return True+ try:+ from ._mamba_lora import _RAW_WEIGHT_LEAVES, _is_mamba_mixer+ except Exception:+ return False+ # Unsloth's fused Mamba LoRA path builds W + delta from these leaves of any Mamba-style mixer.+ return leaf in _RAW_WEIGHT_LEAVES and _is_mamba_mixer(parent)+++def _block_swapped_modules(model) -> set:+ out = set()+ for module in model.modules():+ swapper = module.__dict__.get("_unsloth_block_swap")+ if swapper is None or not isinstance(module, torch.nn.ModuleList):+ continue+ for index in getattr(swapper, "indices", ()) or ():+ if 0 <= index < len(module):+ out.update(id(m) for m in module[index].modules())+ return out+++def _fp8_lora_owner(model):+ getter = getattr(model, "get_base_model", None)+ return getter() if getattr(model, "peft_config", None) and callable(getter) else model+++@lru_cache(maxsize = 128)+def _fp8_lora_operator_code(code, names):+ return code.replace(co_varnames = names + code.co_varnames[2:])+++@lru_cache(maxsize = 128)+def _fp8_lora_proxy_code(name, receiver, value):+ source = f"def forward({receiver}, {value}):\n return {name}({receiver}, {value}={value})\n"+ return compile(source, "<Unsloth FP8 forwarding contract>", "exec").co_consts[0]+++def _prepare_fp8_lora_operators(model, allow_cpu = False):+ import inspect+ import os+ import types+ from transformers.activations import ACT2FN+ from ..kernels.fast_lora import apply_lora_mlp_swiglu+ from ..kernels.rms_layernorm import fast_rms_layernorm+ from unsloth_zoo.temporary_patches.common import unwrap_already_compiled++ owner = _fp8_lora_owner(model)+ if os.environ.get("UNSLOTH_FP8_LORA_FUSED_OPS", "1") == "0" or owner.__dict__.get(+ "_unsloth_fp8_lora_kernel_forwards"+ ):+ return++ def fingerprint(function):+ if isinstance(function, types.CodeType):+ code = function+ elif isinstance(function, types.FunctionType):+ code = function.__code__+ else:+ return None+ return (+ code.co_code,+ code.co_consts,+ code.co_names,+ code.co_argcount,+ code.co_kwonlyargcount,+ code.co_posonlyargcount,+ code.co_flags & (inspect.CO_VARARGS | inspect.CO_VARKEYWORDS),+ )++ def fused_mlp(self, X):+ X = X.contiguous()+ if X.ndim == 2:+ return apply_lora_mlp_swiglu(self, X.unsqueeze(0), inplace = False).squeeze(0)+ return apply_lora_mlp_swiglu(self, X, inplace = False)++ def rms_equation(self, x):+ dtype = x.dtype+ x = x.to(torch.float32)+ variance = x.pow(2).mean(-1, keepdim = True)+ x = x * torch.rsqrt(variance + self.variance_epsilon)+ return self.weight * x.to(dtype)++ def fused_rms(self, X):+ if X.dtype not in (torch.float16, torch.bfloat16) or X.dtype != self.weight.dtype:+ return rms_equation(self, X)+ return fast_rms_layernorm(self, X)++ def mlp_equation(self, x):+ output = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))+ return output++ def mlp_expression(self, x):+ return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))++ templates = (+ (rms_equation, fused_rms),+ (mlp_equation, fused_mlp),+ (mlp_expression, fused_mlp),+ )+ cache = owner.__dict__.setdefault("_unsloth_fp8_lora_kernel_cache", {})+ forwards = []+ for module in model.modules():+ if not type(module).__module__.startswith(+ ("transformers.models.", "unsloth_compiled_module_")+ ):+ continue+ for name in ("forward", "_old_forward"):+ method = getattr(module, name, None)+ function = getattr(method, "__func__", None)+ if function is None or function.__code__.co_argcount != 2:+ continue+ equation = function+ if type(module).__module__.startswith("unsloth_compiled_module_"):+ name_forward = type(module).__name__ + "_forward"+ if function.__code__.co_names != (name_forward,):+ continue+ proxy = _fp8_lora_proxy_code(name_forward, *function.__code__.co_varnames[:2])+ if fingerprint(function) != fingerprint(proxy):+ continue+ equation = unwrap_already_compiled(function.__globals__.get(name_forward))+ for template, kernel in templates:+ # Match the installed native equation, including constants, before replacing it.+ if fingerprint(equation) != fingerprint(template):+ continue+ if function.__code__.co_argcount != 2:+ continue+ if kernel is fused_mlp:+ activation = getattr(module, "act_fn", None)+ if type(activation) not in (type(ACT2FN["silu"]), torch.nn.SiLU):+ continue+ if (+ getattr(activation, "act", torch.nn.functional.silu)+ is not torch.nn.functional.silu+ ):+ continue+ projs = [+ getattr(module, part, None)+ for part in ("gate_proj", "up_proj", "down_proj")+ ]+ if any(+ getattr(getattr(proj, "base_layer", None), "bias", None) is not None+ for proj in projs+ ):+ continue+ states = [+ getattr(getattr(proj, "base_layer", None), "__dict__", {}).get(+ "_unsloth_fp8_lora"+ )+ for proj in projs+ ]+ if not all(+ state is not None and (allow_cpu or state.device.type == "cuda")+ for state in states+ ):+ continue+ else:+ weight = getattr(module, "weight", None)+ if equation.__globals__.get("torch") is not torch or weight is None:+ continue+ if weight.dtype not in (torch.float16, torch.bfloat16):+ continue+ if not allow_cpu and weight.device.type != "cuda":+ continue+ operator = apply_lora_mlp_swiglu if kernel is fused_mlp else fast_rms_layernorm+ key = (id(module), name, function, operator, kernel.__code__)+ replacement = cache.get(key)+ if replacement is None:+ # Preserve native keyword names without a Python wrapper in each block.+ code = _fp8_lora_operator_code(+ kernel.__code__, function.__code__.co_varnames[:2]+ )+ replacement = types.FunctionType(+ code,+ kernel.__globals__,+ function.__name__,+ kernel.__defaults__,+ kernel.__closure__,+ )+ replacement = types.MethodType(replacement, method.__self__)+ # Reuse the binding across GRPO inference/training cycles to retain compiled graphs.+ cache[key] = replacement+ forwards.append((module, name, module.__dict__.get(name)))+ setattr(module, name, replacement)+ break+ owner.__dict__["_unsloth_fp8_lora_kernel_forwards"] = tuple(forwards)+++def _prepare_fp8_lora_loss(model):+ import functools+ import types++ owner = _fp8_lora_owner(model)+ if owner.__dict__.get("_unsloth_fp8_lora_loss_forwards"):+ return+ getter = getattr(owner, "get_output_embeddings", None)+ if not callable(getter) or not isinstance(getter(), torch.nn.Module):+ return+ from unsloth_zoo.fused_losses import forward_adapter++ original = forward_adapter.unsloth_fused_lm_head_loss++ @functools.wraps(original)+ def routed(hidden_states, lm_head, labels, *args, **kwargs):+ # HF's fixed_cross_entropy ignores these flags; they must not select another loss.+ kwargs.pop("output_attentions", None)+ kwargs.pop("output_hidden_states", None)+ return original(hidden_states, lm_head, labels, *args, **kwargs)++ memo = {}++ def isolate(function):+ if not isinstance(function, types.FunctionType):+ return function+ if function in memo:+ return memo[function]+ memo[function] = function+ inner = getattr(function, "__wrapped__", None)+ changed_inner = isolate(inner)+ closure = function.__closure__+ # Decorators call their closure; changing __wrapped__ alone does not reroute them.+ if changed_inner is not inner and closure:+ closure = tuple(+ types.CellType(changed_inner) if cell.cell_contents is inner else cell+ for cell in closure+ )+ direct = (+ "unsloth_fused_lm_head_loss" in function.__code__.co_names+ and function.__globals__.get("unsloth_fused_lm_head_loss") is original+ )+ if not direct and closure == function.__closure__:+ return function+ namespace = function.__globals__+ if direct:+ namespace = {**namespace, "unsloth_fused_lm_head_loss": routed}+ # Older Dynamo resolves named scopes through the original module instead of this copy.+ namespace.pop("__name__", None)+ replacement = types.FunctionType(+ function.__code__,+ namespace,+ function.__name__,+ function.__defaults__,+ closure,+ )+ replacement.__dict__.update(function.__dict__)+ for attribute in (+ "__kwdefaults__",+ "__annotations__",+ "__qualname__",+ "__module__",+ "__doc__",+ ):+ setattr(replacement, attribute, getattr(function, attribute))+ if changed_inner is not inner:+ replacement.__wrapped__ = changed_inner+ memo[function] = replacement+ return replacement++ forwards = []+ for module in model.modules():+ for name in ("forward", "_old_forward"):+ method = getattr(module, name, None)+ function = getattr(method, "__func__", method)+ replacement = isolate(function)+ if replacement is function:+ continue+ forwards.append((module, name, module.__dict__.get(name)))+ if isinstance(method, types.MethodType):+ replacement = types.MethodType(replacement, method.__self__)+ setattr(module, name, replacement)+ owner.__dict__["_unsloth_fp8_lora_loss_forwards"] = tuple(forwards)+++def restore_fp8_lora_for_inference(model):+ owner = _fp8_lora_owner(model)+ for module, name, method in owner.__dict__.pop("_unsloth_fp8_lora_kernel_forwards", ()):+ if method is None:+ module.__dict__.pop(name, None)+ else:+ setattr(module, name, method)+ for module, name, method in owner.__dict__.pop("_unsloth_fp8_lora_loss_forwards", ()):+ if method is None:+ module.__dict__.pop(name, None)+ else:+ setattr(module, name, method)+ bases = owner.__dict__.get("_unsloth_fp8_lora_bases", ())+ if not bases:+ return+ kernels = _load_fp8_lora_kernels()+ for module in bases:+ kernels.detach_fp8_lora_base(module)+ owner.__dict__["_unsloth_fp8_lora_bases"] = ()+++def enable_fp8_lora(+ model,+ recipe = None,+ allow_cpu = False,+):+ """Attach the fp8 frozen-base state to every LoRA base nn.Linear (frozen, 16-bit, dims % 16, not a+ high-precision layer). Returns the converted names. A no-op unless the model was loaded with it."""+ recipe = recipe or _fp8_lora_request(model)+ if recipe is None:+ return []+ kernels = _load_fp8_lora_kernels()+ policy = _precision.get_recipe(recipe).hp_layers+ try:+ is_hp = _precision.hp_filter(model, policy)+ except Exception:+ is_hp = lambda module, fqn: False+ converted, skipped = [], {}+ uses = {}+ for _, p in model.named_parameters(remove_duplicate = False):+ uses[id(p)] = uses.get(id(p), 0) + 1+ swapped = _block_swapped_modules(model)+ offload = kernels.offload_supported()+ offloaded = 0+ for name, module in model.named_modules():+ base = getattr(module, "base_layer", None)+ if base is None or not hasattr(module, "lora_A") or not isinstance(base, torch.nn.Module):+ continue+ if is_hp(base, name + ".base_layer") or is_hp(module, name):+ skipped[name] = "high precision layer"+ continue+ why = kernels.eligible_base(base)+ if why is None and not allow_cpu:+ state = base.__dict__.get("_unsloth_fp8_lora")+ device = state.device if state is not None and state.offloaded else base.weight.device+ if device.type != "cuda" or torch.version.hip is not None:+ why = f"weight on {device}"+ elif torch.cuda.get_device_capability(device) < (8, 9):+ why = "compute capability below 8.9"+ if why is None and id(base) in swapped:+ why = "block-swapped layer"+ if why is None and uses.get(id(base.weight), 0) > 1:+ why = "weight shared with another module"+ if why is not None:+ skipped[name] = why+ continue+ reads_raw = _has_variant(module) or _mamba_raw_weight_leaf(model, name)+ state = kernels.attach_fp8_lora_base(base, offload = offload and not reads_raw)+ offloaded += int(state.offloaded)+ _restore_master_before_merge(module)+ converted.append(name)+ if offloaded and torch.cuda.is_available():+ torch.cuda.empty_cache()+ _fp8_lora_owner(model).__dict__["_unsloth_fp8_lora_bases"] = tuple(+ module for module in model.modules() if "_unsloth_fp8_lora" in module.__dict__+ )+ detail = {"skipped": len(skipped)} if skipped else {}+ if offloaded:+ detail["master_offloaded"] = offloaded+ _record_engagement(model, recipe, len(converted), detail = detail or None)+ if not _require_engagement(recipe, len(converted), "no frozen 16-bit LoRA base Linear layers"):+ print(+ "Unsloth: float8_training with LoRA found no eligible base layers"+ + (f" ({next(iter(skipped.values()))})" if skipped else "")+ + "; training stays in 16 bit."+ )+ else:+ print(f"Unsloth: fp8 LoRA base on {len(converted)} Linear layers.")+ if converted:+ _prepare_fp8_lora_loss(model)+ _prepare_fp8_lora_operators(model)+ import types+ from peft.tuners.tuners_utils import BaseTuner++ tuner = model if isinstance(model, BaseTuner) else getattr(model, "base_model", None)+ unload = "_unload_and_optionally_merge"+ if (+ isinstance(tuner, BaseTuner)+ and unload not in tuner.__dict__+ and callable(getattr(type(tuner), unload, None))+ ):+ object.__setattr__(tuner, unload, types.MethodType(_unload_fp8_lora, tuner))+ if converted and getattr(model, "_unsloth_block_compile_installed", False):+ from ._block_compile import patch_block_compile+ patch_block_compile(model)+ return converteddiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex cc25b3c52eb..57405afb428 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -88,6 +88,7 @@ from .loader_utils import (warn_if_bitsandbytes_quantized_nothing,)from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storage+from .float8_training import enable_fp8_lora, restore_fp8_lora_for_inferencefrom ..utils.packing import (get_packed_info_from_kwargs,mask_packed_boundary_labels,@@ -5228,7 +5229,9 @@ class FastLlamaModel:model._saved_temp_tokenizer = _saved_temp_tokenizer- model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)+ model = FastLlamaModel.patch_peft_model(+ model, use_gradient_checkpointing, _defer_fp8_lora = True+ )offload_embedding_if_tight(model)install_block_swap(model,@@ -5236,6 +5239,7 @@ class FastLlamaModel:prefetch_depth = prefetch_depth,use_gradient_checkpointing = use_gradient_checkpointing,)+ enable_fp8_lora(model)skip_checkpointing(model, checkpoint_skip_layers)if ensure_weight_tying:@@ -5336,7 +5340,11 @@ class FastLlamaModel:return model@staticmethod- def patch_peft_model(model, use_gradient_checkpointing = "unsloth"):+ def patch_peft_model(+ model,+ use_gradient_checkpointing = "unsloth",+ _defer_fp8_lora = False,+ ):# Persist the GC mode: for_inference() clears it every GRPO step and TrainingArguments defaults# it to False, which would silently disable it at train time.model._unsloth_gradient_checkpointing = use_gradient_checkpointing@@ -5565,6 +5573,8 @@ class FastLlamaModel:m.for_training = functools.partial(FastBaseModel.for_training, m)m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model+ if not _defer_fp8_lora:+ enable_fp8_lora(model)_unsloth_install_pretrain_detector(model)_legacy_block_compile(model)return model@@ -5576,6 +5586,7 @@ class FastLlamaModel:"Unsloth: I think you're passing a tokenizer, not the model to for_inference!")_tensor_parallel_for_inference(model)+ restore_fp8_lora_for_inference(model)def _for_inference(m):if hasattr(m, "gradient_checkpointing"):@@ -5648,6 +5659,7 @@ class FastLlamaModel:m = m.model_for_training(m)apply_training_mode()+ enable_fp8_lora(model)# Since transformers 4.53, this must be turned on explicitly.for module in checkpointing:set_module_gradient_checkpointing(module, use_gradient_checkpointing)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 0e6a48accea..dbf53df547f 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -36,7 +36,12 @@ from peft import PeftConfig, PeftModelfrom .grouped_linear_lora import register_grouped_linear_lora_for_adapterfrom .fp8_to_nf4 import track_explicit_4bit_requestfrom .fsdp2_load import sharded_load_env as fsdp2_sharded_load_env-from .float8_training import convert_model_to_float8_training, resolve_float8_training+from .float8_training import (+ convert_model_to_float8_training,+ mark_fp8_lora,+ resolve_float8_training,+ resolve_fp8_lora,+)from .mistral_format import (MistralFormatRedirect,is_mistral_format_view,@@ -1341,10 +1346,16 @@ class FastLanguageModel(FastLlamaModel):device_map = distributed_device_map# @_offline_aware_load already forced offline when needed; delegations inherit it.+ fp8_lora = None+ fp8_request = kwargs.get("float8_training", None)if kwargs.get("float8_training", None) not in (None, False) and not full_finetuning:- raise ValueError(- "Unsloth: `float8_training` is for `full_finetuning = True` only; LoRA / QLoRA keep "- "their base weights frozen, so there is no fp8 training step to take."+ fp8_lora = resolve_fp8_lora(+ kwargs.pop("float8_training"),+ dtype = dtype,+ load_in_4bit = load_in_4bit,+ load_in_8bit = load_in_8bit,+ load_in_fp8 = load_in_fp8,+ qat_scheme = qat_scheme,)if kwargs.get("float8_training", None) not in (None, False) and qat_scheme is not None:raise ValueError(@@ -1926,6 +1937,7 @@ class FastLanguageModel(FastLlamaModel):unsloth_tiled_mlp = unsloth_tiled_mlp,text_only = text_only,*args,+ **({"float8_training": fp8_request} if fp8_lora is not None else {}),**kwargs,)return _mark_requested_float32(delegated, user_float32), tokenizer@@ -2168,6 +2180,7 @@ class FastLanguageModel(FastLlamaModel):model = _mark_forced_float32(model, False)# Full finetuning delegates to FastModel above, so this path is always LoRA.model = _mark_full_finetuning(model, False)+ model = mark_fp8_lora(model, fp8_lora)return _mark_requested_float32(model, user_float32), tokenizer@@ -2281,12 +2294,17 @@ class FastModel(FastBaseModel):):rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)user_config = kwargs.pop("config", None)- float8_training = resolve_float8_training(kwargs.pop("float8_training", None), dtype = dtype)- if float8_training is not None and not full_finetuning:- raise ValueError(- "Unsloth: `float8_training` is for `full_finetuning = True` only; LoRA / QLoRA keep "- "their base weights frozen, so there is no fp8 training step to take."+ fp8_lora = None+ if not full_finetuning and kwargs.get("float8_training", None) not in (None, False):+ fp8_lora = resolve_fp8_lora(+ kwargs.pop("float8_training"),+ dtype = dtype,+ load_in_4bit = load_in_4bit,+ load_in_8bit = load_in_8bit,+ load_in_fp8 = load_in_fp8,+ qat_scheme = qat_scheme,)+ float8_training = resolve_float8_training(kwargs.pop("float8_training", None), dtype = dtype)if float8_training is not None and qat_scheme is not None:raise ValueError("Unsloth: `float8_training` and `qat_scheme` cannot be combined. QAT swaps the Linear "@@ -2556,6 +2574,12 @@ class FastModel(FastBaseModel):# Text-diffusion slow-path dispatch, factored so the normal route and the legacy-config fallback share one call site.def _dispatch_diffusion():+ if fp8_lora is not None:+ raise NotImplementedError(+ f"Unsloth: float8_training = {fp8_lora!r} (fp8 LoRA) is not supported for text-diffusion "+ "models yet. Instead: drop float8_training for LoRA, or float8_training with "+ "full_finetuning = True."+ )model, tokenizer = FastDiffusionModel.from_pretrained(model_name = model_name,max_seq_length = max_seq_length,@@ -3672,6 +3696,7 @@ class FastModel(FastBaseModel):print(f"Unsloth: float8 training on {len(converted)} Linear layers.")model = _mark_forced_float32(model, do_forced_float32)model = _mark_full_finetuning(model, full_finetuning)+ model = mark_fp8_lora(model, fp8_lora)return _mark_requested_float32(model, user_float32), tokenizerdiff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex a62541c5885..82db7d57c25 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -3319,11 +3319,16 @@ def _unsloth_vllm_merged_weight(module, weight):delta = module.get_delta_weight(adapter).float()else:continue- delta = delta.to(weight.device).reshape(weight.shape)- merged = delta if merged is None else merged + delta+ device = weight.device if weight.device.type != "cpu" else delta.device+ delta = delta.to(device).reshape(weight.shape)+ merged = delta if merged is None else merged + delta.to(merged.device)if merged is None:+ base = getattr(module, "base_layer", None)+ state = base.__dict__.get("_unsloth_fp8_lora") if base is not None else None+ if state is not None and weight.device != state.device:+ return weight.data.to(state.device)return weight.data- return (weight.float() + merged).to(weight.dtype)+ return (weight.to(merged.device).float() + merged).to(weight.dtype)def _unsloth_vllm_tuner_groups(model):@@ -3347,6 +3352,8 @@ def _unsloth_vllm_peft_merge_out_of_place(base, layers):the base tensors and return {id(param): merged tensor}; the live parameters keep their exact bytes."""params = list(base.parameters(recurse = False))originals = [param.data for param in params]+ fp8 = base.__dict__.get("_unsloth_fp8_lora")+ fp8_saved = dict(fp8.__dict__) if fp8 is not None else Nonestates = [(layer, list(getattr(layer, "merged_adapters", [])), dict(getattr(layer, "_caches", {})))for layer in layers@@ -3369,6 +3376,9 @@ def _unsloth_vllm_peft_merge_out_of_place(base, layers):module.training = trainingfor param, data in zip(params, originals):param.data = data+ if fp8 is not None:+ fp8.__dict__.clear()+ fp8.__dict__.update(fp8_saved)for layer, merged_adapters, caches in states:if hasattr(layer, "merged_adapters"):layer.merged_adapters[:] = merged_adaptersdiff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex fe0b3a7d094..70539820213 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -283,6 +283,7 @@ from ._utils import ()from ._utils import *from ._utils import _ignore_block_swap_params_for_ddp+from .float8_training import enable_fp8_lora, restore_fp8_lora_for_inferencefrom ._utils import _tensor_parallel_for_inference, _tensor_parallel_for_trainingfrom ._uma_safetensors import is_integrated_unified_memory_gpufrom ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimate@@ -5818,6 +5819,7 @@ class FastBaseModel:m.for_training = functools.partial(FastBaseModel.for_training, m)m.for_inference = functools.partial(FastBaseModel.for_inference, m)m = m.model+ enable_fp8_lora(model)# Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would otherwise poison (see prepare_for_training_mode)._unsloth_install_pretrain_detector(model)model = _exclude_rope_inv_freq_from_ddp(model)@@ -6013,6 +6015,7 @@ class FastBaseModel:"Unsloth: I think you're passing a tokenizer, not the model to for_inference!")_tensor_parallel_for_inference(model)+ restore_fp8_lora_for_inference(model)def _for_inference(m):if hasattr(m, "gradient_checkpointing"):@@ -6091,6 +6094,7 @@ class FastBaseModel:m = m.model_for_training(m)apply_training_mode()+ enable_fp8_lora(model)# Since transformers 4.53, this must be turned on explicitly.for module in checkpointing:set_module_gradient_checkpointing(module, use_gradient_checkpointing)diff --git a/unsloth/precision/engage.py b/unsloth/precision/engage.pyindex ea891206763..a5567c89733 100644--- a/unsloth/precision/engage.py+++ b/unsloth/precision/engage.py@@ -85,6 +85,10 @@ LOWP_GEMM_CUSTOM_OPS = ("unsloth::fp8_block_matmul_triton","unsloth::fp8_block_matmul_torchao","unsloth::fp8_block_small_m_linear",+ "unsloth::fp8_lora_fwd",+ "unsloth::fp8_lora_dgrad",+ "unsloth::fp8_lora_first",+ "unsloth::fp8_lora_accum","unsloth_zoo::mxfp4_ogs_moe","fbgemm::f8f8bf16","fbgemm::f8f8bf16_tensorwise",diff --git a/unsloth/precision/recipes.py b/unsloth/precision/recipes.pyindex 07d99d80fe6..b56c6628e8b 100644--- a/unsloth/precision/recipes.py+++ b/unsloth/precision/recipes.py@@ -512,6 +512,83 @@ _add())+_FP8_LORA_RUNS = (+ "Qwen3 LoRA r=16, seq 2048, bs 2, 200 steps on alpaca-cleaned, eval every 50 steps on 32 held-out "+ "rows, 2 seeds, vs bf16 LoRA in the same Colab VM"+)+_add(+ Recipe(+ name = "fp8_lora_tensorwise",+ kind = "frozen_base",+ modes = ("lora",),+ fwd = GemmPolicy(_op(E4M3, "tensor"), _op(E4M3, "tensor")),+ dgrad = GemmPolicy(_op(E5M2, "tensor"), _op(E4M3, "tensor")),+ hp_layers = _HP_FP8,+ input_dtypes = ("bf16", "fp16"),+ grad_scaler = "per_dtype",+ compile = "opaque",+ backend = "unsloth.kernels.fp8_lora (float8_training with LoRA on a 16-bit base)",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ "sm89": Gate(+ "explicit",+ evidence = "L4, 16-bit master in host RAM (default): step 1.15x bf16 at Qwen3-0.6B (A/A 1.0%), "+ "1.30x at 1.7B (A/A 0.05%); peak memory -11% / -33% vs bf16 LoRA. Eval loss (fp8 forward) "+ "+0.53-0.64% at 0.6B, +0.39-0.55% at 1.7B (bf16 seed band 0.03%); the trained adapter on the "+ "exact 16-bit base +0.07-0.17%. Explicit, not named: the fp8-forward eval gap passes "+ "jobs/compare.py (rtol 5%) but not the 0.5% bar at 0.6B. Saved and merged weights bitwise equal "+ "to the checkpoint. " + _FP8_LORA_RUNS,+ ),+ "sm90": Gate("unprobed", "H100 is not available to test on"),+ "sm100": Gate("unprobed"),+ "sm120": Gate(+ "explicit",+ evidence = "RTX PRO 6000: 1.08-1.10x at Qwen3-0.6B, 1.17x at 1.7B; rowwise "+ "(fp8_lora_rowwise) is faster there. " + _FP8_LORA_RUNS,+ ),+ "rocm": Gate("refused", "ROCm fp8 (fnuz) GEMMs are not wired for the frozen base"),+ },+ notes = "base GEMMs fp8, LoRA branch 16-bit (Megatron-Bridge TEFusedLoRALinear split); weight "+ "quantized once; the fp8 GEMM writes bf16 for fp16 activations",+ )+)+_add(+ Recipe(+ name = "fp8_lora_rowwise",+ kind = "frozen_base",+ modes = ("lora",),+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "tensor")),+ dgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "tensor")),+ hp_layers = _HP_FP8,+ input_dtypes = ("bf16", "fp16"),+ grad_scaler = "per_dtype",+ compile = "opaque",+ backend = "unsloth.kernels.fp8_lora (UNSLOTH_FP8_LORA_SCALING=row; the default on sm120)",+ gates = {+ "sm75": _NO_FP8_HW,+ "sm80": _NO_FP8_HW,+ "sm89": Gate(+ "explicit",+ evidence = "L4: rowwise _scaled_mm 0.55-0.90x bf16 at op level, no faster than bf16 LoRA "+ "end to end; use fp8_lora_tensorwise there. " + _FP8_LORA_RUNS,+ ),+ "sm90": Gate("unprobed", "H100 is not available to test on"),+ "sm100": Gate("unprobed"),+ "sm120": Gate(+ "explicit",+ evidence = "RTX PRO 6000, 16-bit master in host RAM (default): step 1.16x bf16 at Qwen3-0.6B "+ "(A/A 0.02%), 1.22x at 1.7B (A/A 0.17%); peak memory -12% / -33% vs bf16 LoRA. Eval loss "+ "(fp8 forward) +0.48-0.56% at 0.6B, +0.32-0.54% at 1.7B (bf16 seed band 0.02%); the trained "+ "adapter on the exact 16-bit base +0.06-0.12%. Explicit, not named: the fp8-forward eval gap "+ "passes jobs/compare.py (rtol 5%) but not the 0.5% bar at 0.6B. " + _FP8_LORA_RUNS,+ ),+ "rocm": Gate("refused", "ROCm fp8 (fnuz) GEMMs are not wired for the frozen base"),+ },+ notes = "per-token activation / gradient scales, the weight's tensorwise scale broadcast per column",+ )+)+def get_recipe(name: str) -> Recipe:try: