Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Add explicit FP8 frozen-base LoRA training (#16423)

Daniel Hanee3fa40a7c84
diff --git a/scripts/exec_literals_baseline.json b/scripts/exec_literals_baseline.json
index b30ad5b0e42..00d02fd1abb 100644
--- a/scripts/exec_literals_baseline.json
+++ b/scripts/exec_literals_baseline.json
@@ -149,6 +149,13 @@
"count": 1,
"reason": "Installs a patched attention `__init__` generated by this file from a fixed template, and binds it by the name that template defines."
},
+ {
+ "file": "unsloth/models/float8_training.py",
+ "sink": "compile",
+ "digest": "13108d7353b8c426",
+ "count": 1,
+ "reason": "Compiles a fixed forwarding-contract template using function and argument identifiers read from installed library code objects. The resulting code object is only fingerprinted to recognize a generated proxy; this helper never executes it or accepts model, prompt, or user strings."
+ },
{
"file": "unsloth/models/gemma.py",
"sink": "eval",
diff --git a/scripts/risky_loaders_baseline.json b/scripts/risky_loaders_baseline.json
index 563ebfa2f23..f861a5592be 100644
--- a/scripts/risky_loaders_baseline.json
+++ b/scripts/risky_loaders_baseline.json
@@ -10,6 +10,14 @@
"unsloth-cli.py"
],
"entries": [
+ {
+ "file": "unsloth/models/float8_training.py",
+ "rule": "dynamic-import",
+ "sink": "spec_from_file_location",
+ "digest": "78b864cf6acb1555",
+ "count": 1,
+ "reason": "Standalone import fallback loads only unsloth/kernels/fp8_lora.py beside this shipped module, from __file__; no model metadata or caller input selects the module name or path."
+ },
{
"file": "scripts/prequant_probe.py",
"rule": "unsafe-deserialize",
diff --git a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py b/tests/python/test_get_lora_parameters_bias_fp8_block_size.py
index 322fa0f29c4..8209b0781fe 100644
--- a/tests/python/test_get_lora_parameters_bias_fp8_block_size.py
+++ b/tests/python/test_get_lora_parameters_bias_fp8_block_size.py
@@ -2,11 +2,10 @@ import ast
from pathlib import Path
-_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")
+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state", "_fp8_lora_state")
def _load_function(name):
- # Extract without importing unsloth, which needs a GPU.
source = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"
tree = ast.parse(source.read_text(encoding = "utf-8"))
funcs = [
@@ -42,7 +41,6 @@ def _make_disabled_block_fp8_proj(block_size):
def test_bias_variant_propagates_fp8_block_size_on_disabled_path():
- # fp8 kernels read weight_scale.block_size, so the real block size must survive this path.
get_lora_parameters_bias = _load_function("get_lora_parameters_bias")
proj, weight_scale = _make_disabled_block_fp8_proj([64, 128])
@@ -67,7 +65,6 @@ def _make_decompressed_merged_proj():
def test_bias_variant_keeps_none_quant_state_for_decompressed_layer():
- # No quant state here; fast_linear_forward needs W_quant None to fall back to matmul.
get_lora_parameters_bias = _load_function("get_lora_parameters_bias")
W, W_quant = get_lora_parameters_bias(_make_decompressed_merged_proj())[:2]
diff --git a/tests/python/test_get_lora_parameters_fp8_block_size.py b/tests/python/test_get_lora_parameters_fp8_block_size.py
index 62d5dbc5512..b881bcd7cb9 100644
--- a/tests/python/test_get_lora_parameters_fp8_block_size.py
+++ b/tests/python/test_get_lora_parameters_fp8_block_size.py
@@ -2,11 +2,10 @@ import ast
from pathlib import Path
-_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")
+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state", "_fp8_lora_state")
def _load_function(name):
- # Extract without importing unsloth, which needs a GPU.
source = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"
tree = ast.parse(source.read_text(encoding = "utf-8"))
funcs = [
@@ -46,7 +45,6 @@ def _make_disabled_block_fp8_proj(block_size):
def test_propagates_fp8_block_size_on_disabled_path():
- # fp8 kernels read weight_scale.block_size, so the real block size must survive this path.
get_lora_parameters = _load_function("get_lora_parameters")
proj, weight_scale = _make_disabled_block_fp8_proj([64, 128])
@@ -70,7 +68,6 @@ def _make_decompressed_merged_proj():
def test_keeps_none_quant_state_for_decompressed_layer():
- # No quant state here; fast_lora needs W_quant None to fall back to a plain matmul.
get_lora_parameters = _load_function("get_lora_parameters")
W, W_quant = get_lora_parameters(_make_decompressed_merged_proj())[:2]
diff --git a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
index a083a07d5d2..333d2c6d0d4 100644
--- a/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
+++ b/tests/python/test_get_lora_parameters_fsdp2_quant_state.py
@@ -11,7 +11,7 @@ from pathlib import Path
import torch
_SOURCE = Path(__file__).parents[2] / "unsloth" / "kernels" / "utils.py"
-_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state")
+_HELPERS = ("_packed_base", "_is_packed_state", "_base_quant_state", "_fp8_lora_state")
def _load_function(name, fsdp2_qlora_prepared = True):
diff --git a/tests/test_block_compile.py b/tests/test_block_compile.py
index 521ba55821a..8607413cebe 100644
--- a/tests/test_block_compile.py
+++ b/tests/test_block_compile.py
@@ -328,6 +328,57 @@ def _install_blocks(M, model, monkeypatch):
return inner
[email protected]("fp8_first", [False, True])
+def test_fp8_precision_options_are_per_instance_and_reused(monkeypatch, fp8_first):
+ import copy
+
+ M = _load()
+
+ class PrecisionDecoderLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.proj = torch.nn.Linear(8, 8)
+
+ def forward(self, x):
+ return self.proj(x)
+
+ plain, fp8 = PrecisionDecoderLayer(), PrecisionDecoderLayer()
+ fp8.proj._unsloth_fp8_lora = object()
+ options = []
+
+ def compile(fn, **kwargs):
+ precision = kwargs["options"].get("emulate_precision_casts", False)
+ options.append(precision)
+
+ def run(*args, **kw):
+ return fn(*args, **kw) + int(precision)
+
+ return run
+
+ monkeypatch.setattr(torch, "compile", compile)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: True)
+ monkeypatch.setattr(M, "_layers", lambda model: [(model, True)])
+ monkeypatch.setattr(M, "_torch_supported", lambda: True)
+ monkeypatch.setattr(M, "_zero3_partitioned", lambda *args: False)
+ original = PrecisionDecoderLayer.forward
+ for layer in (fp8, plain) if fp8_first else (plain, fp8):
+ M.patch_block_compile(layer)
+ x = torch.randn(2, 4, 8)
+ torch.testing.assert_close(plain(x), original(plain, x))
+ torch.testing.assert_close(fp8(x), original(fp8, x) + 1)
+ assert "forward" not in plain.__dict__
+ assert not M._COMPILE_OPTIONS.get("emulate_precision_casts", False)
+ assert options == [False, True]
+ forward = fp8.forward
+ M.patch_block_compile(fp8)
+ assert fp8.forward is forward and options == [False, True]
+ twin = copy.deepcopy(fp8)
+ assert twin.forward.__self__ is twin
+ torch.testing.assert_close(twin(x), original(twin, x) + 1)
+ fp8.eval()
+ torch.testing.assert_close(fp8(x), original(fp8, x))
+
+
def _outer_loss(
model,
ids,
@@ -886,6 +937,10 @@ def test_sm_count_is_read_outside_the_trace(monkeypatch):
fla.get_multiprocessor_count = get_multiprocessor_count
monkeypatch.setattr(torch.cuda, "device_count", lambda: 1)
+ monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
+ monkeypatch.setattr(
+ torch._C, "_cuda_init", lambda: pytest.fail("CPU test initialized CUDA"), raising = False
+ )
M._route_sm_count({fla.__name__: fla})
out = torch.compile(
lambda x: x * fla.get_multiprocessor_count(0), fullgraph = True, backend = "eager"
diff --git a/tests/test_float8_training.py b/tests/test_float8_training.py
index 617617ebec8..e9e54a4d180 100644
--- a/tests/test_float8_training.py
+++ b/tests/test_float8_training.py
@@ -240,12 +240,14 @@ def test_refuses_when_scaled_mm_probe_fails(monkeypatch):
f8t._check_hardware(torch.device("cuda", 0), _emulated_config("rowwise"))
-def test_loader_rejects_float8_training_for_lora():
+def test_loader_float8_training_for_lora_needs_a_16bit_base():
from unsloth import FastLanguageModel, FastModel
for loader in (FastLanguageModel, FastModel):
- with pytest.raises(ValueError, match = "full_finetuning = True"):
+ with pytest.raises(ValueError, match = "load_in_4bit = False"):
loader.from_pretrained(
- "trl-internal-testing/tiny-Qwen3ForCausalLM", float8_training = True
+ "trl-internal-testing/tiny-Qwen3ForCausalLM",
+ load_in_4bit = True,
+ float8_training = True,
)
diff --git a/tests/test_for_training_generation_flag.py b/tests/test_for_training_generation_flag.py
index 9481a5bbb76..d7fc5e85c18 100644
--- a/tests/test_for_training_generation_flag.py
+++ b/tests/test_for_training_generation_flag.py
@@ -94,6 +94,7 @@ def _for_training(module, class_name):
namespace = _Namespace(os = os, inspect = inspect, torch = _torch_or_stub(), functools = functools)
# No tensor parallel world in these tests (for_training calls the TP guard first since #15368).
namespace["_tensor_parallel_for_training"] = lambda model: None
+ namespace["enable_fp8_lora"] = lambda model: []
exec(compile(ast.fix_missing_locations(compiled), str(path), "exec"), namespace)
return namespace["for_training"]
diff --git a/tests/test_fp8_lora.py b/tests/test_fp8_lora.py
new file mode 100644
index 00000000000..2aa8f50d63d
--- /dev/null
+++ b/tests/test_fp8_lora.py
@@ -0,0 +1,1915 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""FP8 LoRA on a frozen 16-bit base (float8_training with LoRA, kernels/fp8_lora.py), on CPU: torch's CPU
+_scaled_mm runs the same fp8 GEMMs, so numerics, caching, the LoRA kernels' backward routing, refusals and
+torch.compile tracing are all checked without a GPU."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from unsloth import precision as P
+from unsloth.kernels import fp8_lora as FL
+from unsloth.kernels import fast_lora as FLK
+from unsloth.models import float8_training as f8t
+from real_accelerator import has_real_cuda
+
+pytestmark = pytest.mark.skipif(
+ not hasattr(torch, "float8_e4m3fn") or not hasattr(torch, "_scaled_mm"),
+ reason = "needs torch float8 dtypes",
+)
+
+
+def _cpu_scaled_mm_works(rowwise = False):
+ if not hasattr(torch, "float8_e4m3fn") or not hasattr(torch, "_scaled_mm"):
+ return False
+ try:
+ a = torch.zeros(16, 16).to(torch.float8_e4m3fn)
+ scale_a = torch.ones((16, 1) if rowwise else (), dtype = torch.float32)
+ scale_b = torch.ones((1, 16) if rowwise else (), dtype = torch.float32)
+ torch._scaled_mm(a, a.t(), scale_a = scale_a, scale_b = scale_b, out_dtype = torch.bfloat16)
+ return True
+ except (NotImplementedError, RuntimeError):
+ return False
+
+
+needs_cpu_scaled_mm = pytest.mark.skipif(
+ not _cpu_scaled_mm_works(), reason = "this torch build has no CPU torch._scaled_mm kernel"
+)
+
+FP8_FWD_TOL = 0.06
+FP8_DGRAD_TOL = 0.09
+
+
+ not has_real_cuda()
+ or torch.version.hip is not None
+ or torch.cuda.get_device_capability() < (8, 9),
+ reason = "requires NVIDIA compute capability 8.9+ for FP8 CUDA cast kernels",
+)
[email protected](FL.triton is None, reason = "requires Triton for the CUDA cast kernels")
[email protected]("dtype", [torch.bfloat16, torch.float16])
[email protected]("fp8_dtype", [torch.float8_e4m3fn, torch.float8_e5m2])
[email protected]("rowwise", [False, True])
+def test_cuda_casts_match_native_rounding(dtype, fp8_dtype, rowwise):
+ limit = torch.finfo(fp8_dtype).max
+ for seed in range(16):
+ generator = torch.Generator(device = "cuda").manual_seed(seed)
+ x = torch.randn(32, 256, device = "cuda", dtype = dtype, generator = generator)
+ if seed == 0:
+ x[0].fill_(-0.0)
+ x[1].fill_(torch.finfo(dtype).tiny)
+ x[2, 0] = torch.finfo(dtype).max
+ amax = x.float().abs().amax(dim = 1, keepdim = True) if rowwise else x.float().abs().amax()
+ amax = amax.clamp_min(1e-12)
+ expected = (x.float() * (limit / amax)).clamp(-limit, limit).to(fp8_dtype)
+ quantize = FL.quantize_rowwise if rowwise else FL.quantize_tensorwise
+ actual, inverse = quantize(x, fp8_dtype)
+ assert torch.equal(actual.view(torch.uint8), expected.view(torch.uint8))
+ assert torch.equal(inverse, amax / limit)
+
+
+def _rel(a, b):
+ a, b = a.float(), b.float()
+ return ((a - b).norm() / b.norm()).item()
+
+
+def _cos(a, b):
+ return F.cosine_similarity(a.flatten().float(), b.flatten().float(), dim = 0).item()
+
+
[email protected](autouse = True)
+def _stats():
+ FL.reset_fp8_lora_stats()
+ yield
+
+
+def _weight(
+ N = 384,
+ K = 256,
+ dtype = torch.bfloat16,
+ seed = 0,
+):
+ g = torch.Generator().manual_seed(seed)
+ return (torch.randn(N, K, generator = g) * K**-0.5).to(dtype)
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
+@needs_cpu_scaled_mm
+def test_fwd_and_dgrad_within_fp8_error(dtype):
+ W = _weight(dtype = dtype)
+ x = torch.randn(64, 256).to(dtype)
+ dy = torch.randn(64, 384).to(dtype)
+ state = FL.Fp8LoRABaseState(W)
+ y = state.matmul(x, W)
+ dx = state.dgrad(dy, W)
+ ref_y = x.float() @ W.float().t()
+ ref_dx = dy.float() @ W.float()
+ assert y.dtype == dtype and dx.dtype == dtype
+ e_fwd, e_dgrad = _rel(y, ref_y), _rel(dx, ref_dx)
+ assert 0.01 < e_fwd < FP8_FWD_TOL, e_fwd
+ assert 0.01 < e_dgrad < FP8_DGRAD_TOL, e_dgrad
+ assert torch.equal(state.matmul(dy, W.t()), dx)
+ assert FL.FP8_LORA_STATS["fwd"] == 1 and FL.FP8_LORA_STATS["dgrad"] == 2
+
+
+@needs_cpu_scaled_mm
+def test_scaled_mm_counted_and_fp16_never_written_by_the_fp8_gemm(monkeypatch):
+ seen = []
+ real = torch._scaled_mm
+
+ def spy(*args, **kwargs):
+ seen.append(kwargs.get("out_dtype"))
+ return real(*args, **kwargs)
+
+ W = _weight(dtype = torch.float16)
+ state = FL.Fp8LoRABaseState(W)
+ monkeypatch.setattr(torch, "_scaled_mm", spy)
+ with P.count_lowp_gemms() as counts:
+ y = state.matmul(torch.randn(32, 256).half(), W)
+ state.dgrad(torch.randn(32, 384).half(), W)
+ assert counts["_scaled_mm"] == 2
+ assert seen == [torch.float32, torch.float32]
+ assert y.dtype == torch.float16 and torch.isfinite(y).all()
+
+
+@needs_cpu_scaled_mm
+def test_large_fp16_values_stay_finite():
+ W = torch.full((32, 64), 0.5, dtype = torch.float16)
+ x = torch.full((16, 64), 1800.0, dtype = torch.float16) # each output 57600 < 65504
+ y = FL.Fp8LoRABaseState(W).matmul(x, W)
+ assert torch.isfinite(y).all()
+ assert abs(y.float().mean().item() - 57600.0) / 57600.0 < 0.07
+
+
+@needs_cpu_scaled_mm
+def test_fp16_results_at_the_fp16_limit_stay_finite():
+ W = torch.full((32, 64), 0.5, dtype = torch.float16)
+ x = torch.full((16, 64), 2047.0, dtype = torch.float16) # each output 65504, the fp16 maximum
+ state = FL.Fp8LoRABaseState(W)
+ assert torch.isfinite(state.matmul(x, W)).all()
+ dy = torch.full((16, 32), 4094.0, dtype = torch.float16) # each dgrad output 65504
+ assert torch.isfinite(state.dgrad(dy, W)).all()
+
+
+@needs_cpu_scaled_mm
+def test_nan_propagates_for_the_grad_scaler():
+ W = _weight()
+ state = FL.Fp8LoRABaseState(W)
+ dy = torch.randn(32, 384).bfloat16()
+ dy[3, 5] = float("nan")
+ assert torch.isnan(state.dgrad(dy, W)).any()
+
+
+def test_decode_rows_use_the_16bit_weight():
+ W = _weight()
+ state = FL.Fp8LoRABaseState(W)
+ x = torch.randn(1, 1, 256).bfloat16()
+ torch.testing.assert_close(state.matmul(x, W), x @ W.t())
+ assert FL.FP8_LORA_STATS["fwd"] == 0 and FL.FP8_LORA_STATS["fwd_16bit"] == 1
+
+
+@needs_cpu_scaled_mm
+def test_quantized_once_and_requantized_after_an_in_place_change(monkeypatch):
+ monkeypatch.setattr(FL, "_CACHE_T", True)
+ W = _weight()
+ state = FL.Fp8LoRABaseState(W)
+ x = torch.randn(32, 256).bfloat16()
+ for _ in range(5):
+ state.refresh(W)
+ state.matmul(x, W)
+ assert state.quantizations == 1 and FL.FP8_LORA_STATS["quantize"] == 1
+ w8 = state.w8
+ assert torch.equal(state.w8t.view(torch.uint8), w8.t().contiguous().view(torch.uint8))
+ with torch.no_grad():
+ W.mul_(2.0)
+ state.refresh(W)
+ assert state.quantizations == 2
+ assert _rel(state.matmul(x, W), x.float() @ W.float().t()) < FP8_FWD_TOL
+
+
+class _Proj(torch.nn.Module):
+ """A PEFT lora.Linear stand-in: base_layer + lora_A / lora_B ModuleDicts."""
+
+ def __init__(
+ self,
+ K,
+ N,
+ r = 8,
+ seed = 0,
+ ):
+ super().__init__()
+ g = torch.Generator().manual_seed(seed)
+ self.base_layer = torch.nn.Linear(K, N, bias = False, dtype = torch.bfloat16)
+ with torch.no_grad():
+ self.base_layer.weight.copy_(torch.randn(N, K, generator = g) * K**-0.5)
+ self.base_layer.weight.requires_grad_(False)
+ self.lora_A = torch.nn.ModuleDict(
+ {"default": torch.nn.Linear(K, r, bias = False, dtype = torch.bfloat16)}
+ )
+ self.lora_B = torch.nn.ModuleDict(
+ {"default": torch.nn.Linear(r, N, bias = False, dtype = torch.bfloat16)}
+ )
+ with torch.no_grad():
+ self.lora_A["default"].weight.copy_(torch.randn(r, K, generator = g) * 0.05)
+ self.lora_B["default"].weight.copy_(torch.randn(N, r, generator = g) * 0.05)
+ self.scaling = {"default": 2.0}
+ self.disable_adapters = False
+ self.merged = False
+ self.active_adapters = ["default"]
+
+
+def _silu_fwd(e, g):
+ return F.silu(e.float()).to(e.dtype) * g
+
+
+def _silu_bwd(DW, e, g):
+ ef = e.float()
+ s = torch.sigmoid(ef)
+ f = ef * s
+ h = (f * g.float()).to(DW.dtype)
+ df = (DW.float() * f).to(DW.dtype)
+ de = (DW.float() * g.float() * (s * (1 + ef * (1 - s)))).to(DW.dtype)
+ return h, df, de
+
+
+def _lora_params(projs):
+ return [p.lora_A["default"].weight for p in projs] + [p.lora_B["default"].weight for p in projs]
+
+
+def _run_block(kind, fp8):
+ torch.manual_seed(0)
+ H, I = 256, 512
+ if kind == "mlp":
+ projs = [_Proj(H, I, seed = 1), _Proj(H, I, seed = 2), _Proj(I, H, seed = 3)]
+ elif kind == "qkv":
+ projs = [_Proj(H, H, seed = 1), _Proj(H, 128, seed = 2), _Proj(H, 128, seed = 3)]
+ else:
+ projs = [_Proj(H, H, seed = 4)]
+ if fp8:
+ for p in projs:
+ FL.attach_fp8_lora_base(p.base_layer)
+ X = (torch.randn(2, 32, H) * 0.5).bfloat16().requires_grad_(True)
+ if kind == "mlp":
+ args = []
+ for p in projs:
+ args += list(FLK.get_lora_parameters(p))
+ out = FLK.LoRA_MLP.apply(X, *args, _silu_fwd, _silu_bwd, False)
+ outs = (out,)
+ elif kind == "qkv":
+ args = []
+ for p in projs:
+ args += list(FLK.get_lora_parameters(p))
+ outs = FLK.LoRA_QKV.apply(X, *args, False)
+ else:
+ outs = (FLK.LoRA_W.apply(X, *FLK.get_lora_parameters(projs[0])),)
+ g = torch.Generator().manual_seed(9)
+ loss = sum((o.float() * torch.randn(o.shape, generator = g)).sum() for o in outs)
+ loss.backward()
+ return [o.detach() for o in outs], X.grad, [p.grad for p in _lora_params(projs)], projs
+
+
[email protected]("kind", ["mlp", "qkv", "o"])
+@needs_cpu_scaled_mm
+def test_lora_kernels_run_fp8_and_keep_lora_grads_16bit(kind):
+ outs_ref, dX_ref, grads_ref, _ = _run_block(kind, fp8 = False)
+ FL.reset_fp8_lora_stats()
+ outs, dX, grads, projs = _run_block(kind, fp8 = True)
+ n = len(projs)
+ assert FL.FP8_LORA_STATS["fwd"] == n, FL.FP8_LORA_STATS
+ assert FL.FP8_LORA_STATS["dgrad"] == n, FL.FP8_LORA_STATS
+ assert FL.FP8_LORA_STATS["fwd_16bit"] == 0 and FL.FP8_LORA_STATS["dgrad_16bit"] == 0
+ for o, r in zip(outs, outs_ref):
+ assert _rel(o, r) < FP8_FWD_TOL * 1.5
+ assert _rel(dX, dX_ref) < FP8_DGRAD_TOL * 1.5
+ for g, r, p in zip(grads, grads_ref, _lora_params(projs)):
+ assert g is not None and g.dtype == p.dtype == torch.bfloat16
+ assert _cos(g, r) > 0.99, _cos(g, r)
+ for p in projs:
+ assert p.base_layer.weight.grad is None
+
+
+@needs_cpu_scaled_mm
+def test_peft_path_base_forward_and_backward():
+ base = torch.nn.Linear(256, 384, bias = True, dtype = torch.bfloat16)
+ base.requires_grad_(False)
+ ref = torch.nn.Linear(256, 384, bias = True, dtype = torch.bfloat16)
+ ref.load_state_dict(base.state_dict())
+ ref.requires_grad_(False)
+ FL.attach_fp8_lora_base(base)
+ x = torch.randn(2, 16, 256).bfloat16()
+ xa, xb = x.clone().requires_grad_(True), x.clone().requires_grad_(True)
+ ya, yb = base(xa), ref(xb)
+ dy = torch.randn_like(ya)
+ ya.backward(dy)
+ yb.backward(dy)
+ assert _rel(ya, yb) < FP8_FWD_TOL and _rel(xa.grad, xb.grad) < FP8_DGRAD_TOL
+ assert FL.FP8_LORA_STATS["fwd"] == 1 and FL.FP8_LORA_STATS["dgrad"] == 1
+ assert set(base.state_dict()) == {"weight", "bias"}
+ assert base.weight.dtype == torch.bfloat16
+
+
+def _tiny_qwen3_peft():
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(0)
+ config = Qwen3Config(
+ vocab_size = 256,
+ hidden_size = 128,
+ intermediate_size = 256,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 32,
+ )
+ model = Qwen3ForCausalLM(config).to(torch.bfloat16)
+ lora = LoraConfig(
+ r = 8,
+ lora_alpha = 16,
+ target_modules = [
+ "q_proj",
+ "k_proj",
+ "v_proj",
+ "o_proj",
+ "gate_proj",
+ "up_proj",
+ "down_proj",
+ ],
+ init_lora_weights = False,
+ )
+ return get_peft_model(model, lora)
+
+
+def _loss_and_grads(model):
+ torch.manual_seed(1)
+ ids = torch.randint(0, 256, (2, 32))
+ out = model(input_ids = ids, labels = ids)
+ out.loss.backward()
+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.requires_grad}
+ model.zero_grad(set_to_none = True)
+ return out.loss.item(), grads
+
+
+@needs_cpu_scaled_mm
+def test_enable_fp8_lora_on_a_peft_model():
+ model = _tiny_qwen3_peft()
+ loss_ref, grads_ref = _loss_and_grads(model)
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ converted = f8t.enable_fp8_lora(model, allow_cpu = True)
+ assert len(converted) == 2 * 7
+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["modules"] == 14
+ loss, grads = _loss_and_grads(model)
+ assert FL.FP8_LORA_STATS["fwd"] == 14 and FL.FP8_LORA_STATS["dgrad"] == 11
+ assert abs(loss - loss_ref) / loss_ref < 0.01
+ assert grads.keys() == grads_ref.keys()
+ for name, g in grads.items():
+ assert g.dtype == grads_ref[name].dtype
+ assert _cos(g, grads_ref[name]) > 0.98, (name, _cos(g, grads_ref[name]))
+ _loss_and_grads(model)
+ assert FL.FP8_LORA_STATS["quantize"] == 14
+
+
[email protected]("flag", [None, False, True])
[email protected]("private_namespace", [False, True])
+def test_fp8_loss_ignores_model_output_flags_only_on_its_head(monkeypatch, flag, private_namespace):
+ import copy
+ import types
+ from unsloth_zoo.fused_losses import cross_entropy_loss as ce
+ from unsloth_zoo.fused_losses import forward_adapter
+
+ monkeypatch.setattr(ce, "_CHUNKED_CE", True)
+ monkeypatch.setattr(ce, "_FUSED_CE_COMPILE_DISABLED", False)
+ monkeypatch.setattr(ce, "TARGET_GB", 0)
+ monkeypatch.setattr(ce, "N_CHUNKS", 0)
+ monkeypatch.setattr(ce._chunked_logprob, "_FUSED_LINEAR", True)
+ routes = []
+
+ def observe_loss(**kwargs):
+ extras = {
+ k: kwargs[k] for k in ("output_attentions", "output_hidden_states") if k in kwargs
+ }
+ routes.append(ce._use_chunked_ce(extras, None))
+ return kwargs["hidden_states"].sum() * 0
+
+ monkeypatch.setattr(forward_adapter, "unsloth_fused_ce_loss", observe_loss)
+ model = _tiny_qwen3_peft()
+ owner = model.get_base_model()
+ ordinary = _tiny_qwen3_peft().get_base_model()
+ hidden = torch.zeros(2, 3, 128)
+ labels = torch.zeros(2, 3, dtype = torch.long)
+
+ def dispatch(self, hidden, labels):
+ return unsloth_fused_lm_head_loss(
+ hidden,
+ self.get_output_embeddings(),
+ labels,
+ output_attentions = flag,
+ output_hidden_states = flag,
+ )
+
+ original = forward_adapter.unsloth_fused_lm_head_loss
+ namespace = {"__builtins__": __builtins__} if private_namespace else dispatch.__globals__
+ monkeypatch.setitem(namespace, "unsloth_fused_lm_head_loss", original)
+ function = types.FunctionType(dispatch.__code__, namespace, closure = dispatch.__closure__)
+ attribute = "_old_forward" if private_namespace else "forward"
+ setattr(owner, attribute, types.MethodType(function, owner))
+ setattr(ordinary, attribute, types.MethodType(function, ordinary))
+
+ def run(target):
+ getattr(target, attribute)(hidden, labels)
+ return routes[-1]
+
+ assert not run(owner)
+ f8t.mark_fp8_lora(owner, f8t.FP8_LORA_RECIPE)
+ assert f8t.enable_fp8_lora(model, allow_cpu = True)
+ assert run(owner)
+ assert namespace["unsloth_fused_lm_head_loss"] is original
+ assert forward_adapter.unsloth_fused_lm_head_loss is original
+ assert getattr(ordinary, attribute).__func__ is function
+ assert not run(ordinary)
+ clone = copy.deepcopy(model)
+ assert run(clone.get_base_model())
+ f8t.restore_fp8_lora_for_inference(clone)
+ assert not run(clone.get_base_model())
+ assert run(owner)
+ f8t.restore_fp8_lora_for_inference(model)
+ assert not run(owner)
+
+
[email protected]("decorated", [False, True])
[email protected]("private_namespace", [False, True])
+def test_fp8_loss_route_traces_fullgraph_without_changing_loss_or_input_gradient(
+ monkeypatch, decorated, private_namespace
+):
+ import types
+ from transformers.loss.loss_utils import ForCausalLMLoss
+ from unsloth_zoo.fused_losses import forward_adapter
+
+ def cpu_reference(hidden_states, lm_head_weight, lm_head_bias, labels, n_items, **kwargs):
+ assert "output_attentions" not in kwargs
+ assert "output_hidden_states" not in kwargs
+ return ForCausalLMLoss(
+ logits = F.linear(hidden_states, lm_head_weight, lm_head_bias),
+ labels = labels,
+ vocab_size = lm_head_weight.shape[0],
+ num_items_in_batch = n_items,
+ )
+
+ monkeypatch.setattr(forward_adapter, "unsloth_fused_ce_loss", cpu_reference)
+ model = _tiny_qwen3_peft()
+ owner = model.get_base_model()
+ head = model.get_output_embeddings()
+ labels = torch.tensor([[1, 2, 3, -100], [-100, 3, 2, 1]])
+ x = torch.randn(2, 4, 128, dtype = torch.bfloat16, requires_grad = True)
+
+ def routed(self, hidden):
+ return unsloth_fused_lm_head_loss(
+ hidden,
+ self.get_output_embeddings(),
+ labels,
+ num_items_in_batch = 5,
+ output_attentions = None,
+ output_hidden_states = None,
+ )
+
+ namespace = {"__builtins__": __builtins__} if private_namespace else routed.__globals__
+ monkeypatch.setitem(
+ namespace, "unsloth_fused_lm_head_loss", forward_adapter.unsloth_fused_lm_head_loss
+ )
+ function = types.FunctionType(routed.__code__, namespace, closure = routed.__closure__)
+ if decorated:
+ from transformers.utils.generic import can_return_tuple
+ function = can_return_tuple(function)
+ owner.forward = types.MethodType(function, owner)
+ f8t.mark_fp8_lora(owner, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ expected = ForCausalLMLoss(
+ logits = head(x),
+ labels = labels,
+ vocab_size = 256,
+ num_items_in_batch = 5,
+ )
+ (expected_grad,) = torch.autograd.grad(expected, x)
+ torch._dynamo.reset()
+ compiled = torch.compile(owner, backend = "aot_eager", fullgraph = True)
+ actual = compiled(x)
+ (actual_grad,) = torch.autograd.grad(actual, x)
+ assert torch.equal(actual, expected)
+ assert torch.equal(actual_grad, expected_grad)
+ if decorated and private_namespace:
+ from torch._dynamo.utils import counters
+
+ before = counters["stats"]["unique_graphs"]
+ compiled(x)
+ stable = counters["stats"]["unique_graphs"] - before
+ f8t.restore_fp8_lora_for_inference(model)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ before = counters["stats"]["unique_graphs"]
+ compiled(x)
+ rebound = counters["stats"]["unique_graphs"] - before
+ print("LOSS_REBIND_GRAPHS", stable, rebound)
+ assert stable == rebound == 0
+
+
[email protected]("return_dict", [False, True])
+def test_fp8_loss_route_preserves_installed_transformers_decorators(monkeypatch, return_dict):
+ from transformers.loss.loss_utils import ForCausalLMLoss
+ from unsloth_zoo.fused_losses import forward_adapter, forward_install
+
+ model = _tiny_qwen3_peft()
+ owner = model.get_base_model()
+ cls = type(owner)
+ original = cls.forward
+ monkeypatch.setattr(cls, "forward", original)
+ for name in ("_PATCHED", "_FAILED", "_UNMATCHED"):
+ monkeypatch.setattr(forward_install, name, dict(getattr(forward_install, name)))
+ monkeypatch.setenv("UNSLOTH_FUSED_FORWARD", "1")
+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "0")
+ assert forward_install.install_for_class(cls), forward_install.audit()
+ installed = cls.forward
+ calls = []
+
+ def cpu_reference(hidden_states, lm_head_weight, lm_head_bias, labels, n_items, **kwargs):
+ calls.append(kwargs)
+ assert "output_attentions" not in kwargs
+ assert "output_hidden_states" not in kwargs
+ return ForCausalLMLoss(
+ logits = F.linear(hidden_states, lm_head_weight, lm_head_bias),
+ labels = labels,
+ vocab_size = lm_head_weight.shape[0],
+ num_items_in_batch = n_items,
+ )
+
+ monkeypatch.setattr(forward_adapter, "unsloth_fused_ce_loss", cpu_reference)
+ monkeypatch.setattr(FL, "_MIN_ROWS", 1024)
+ owner.loss_function = forward_install._STOCK_CAUSAL_LM_LOSS
+ f8t.mark_fp8_lora(owner, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ assert owner.__dict__.get("_unsloth_fp8_lora_loss_forwards")
+ ids = torch.tensor([[1, 3, 2, 5]])
+ result = owner(
+ input_ids = ids,
+ labels = ids,
+ use_cache = False,
+ return_dict = return_dict,
+ output_attentions = False,
+ output_hidden_states = False,
+ )
+ assert calls
+ assert isinstance(result, tuple) is (not return_dict)
+ loss = result.loss if return_dict else result[0]
+ assert torch.isfinite(loss)
+ assert cls.forward is installed
+ assert FL.FP8_LORA_STATS["fwd"] == 0
+ assert FL.FP8_LORA_STATS["fwd_16bit"] > 0
+ f8t.restore_fp8_lora_for_inference(model)
+ assert owner.forward.__func__ is installed
+
+
+def test_no_request_no_conversion_and_strict_zero_engagement(monkeypatch):
+ model = _tiny_qwen3_peft()
+ assert f8t.enable_fp8_lora(model) == []
+ for module in model.modules():
+ assert "_unsloth_fp8_lora" not in module.__dict__
+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")
+ with pytest.raises(P.EngagementError):
+ f8t.enable_fp8_lora(model, recipe = f8t.FP8_LORA_RECIPE)
+
+
+def test_eligibility_reasons():
+ assert FL.eligible_base(torch.nn.Linear(100, 64).bfloat16().requires_grad_(False)) is not None
+ assert "trainable" in FL.eligible_base(torch.nn.Linear(64, 64).bfloat16())
+ assert "dtype" in FL.eligible_base(torch.nn.Linear(64, 64).requires_grad_(False))
+ assert FL.eligible_base(torch.nn.Linear(64, 64).bfloat16().requires_grad_(False)) is None
+ sub = torch.nn.Linear(64, 64).bfloat16().requires_grad_(False)
+
+ class _Sub(torch.Tensor):
+ pass
+
+ sub.weight = torch.nn.Parameter(sub.weight.detach().as_subclass(_Sub), requires_grad = False)
+ assert "_Sub" in FL.eligible_base(sub)
+
+
+@needs_cpu_scaled_mm
+def test_compile_aot_eager_fullgraph():
+ torch._dynamo.reset()
+ base = torch.nn.Linear(256, 384, bias = False, dtype = torch.bfloat16).requires_grad_(False)
+ FL.attach_fp8_lora_base(base)
+ proj = _Proj(256, 384)
+ FL.attach_fp8_lora_base(proj.base_layer)
+
+ def step(x):
+ y = base(x)
+ W, W_quant, A, B, S = FLK.get_lora_parameters(proj)
+ z = FLK.LoRA_W.apply(y[..., :256].contiguous(), W, W_quant, A, B, S)
+ return z.float().square().mean()
+
+ x = torch.randn(2, 16, 256).bfloat16().requires_grad_(True)
+ eager = step(x)
+ eager.backward()
+ g_eager = x.grad.clone()
+ x.grad = None
+ FL.reset_fp8_lora_stats()
+ compiled = torch.compile(step, backend = "aot_eager", fullgraph = True)
+ out = compiled(x)
+ out.backward()
+ torch.testing.assert_close(out, eager)
+ torch.testing.assert_close(x.grad, g_eager)
+ assert FL.FP8_LORA_STATS["fwd"] == 2 and FL.FP8_LORA_STATS["dgrad"] == 2, FL.FP8_LORA_STATS
+ explain = torch._dynamo.explain(step)(x)
+ assert explain.graph_break_count == 0
+
+
+def test_resolve_refusals_and_names():
+ R = f8t.FP8_LORA_RECIPE
+ with pytest.raises(P.PrecisionRefused, match = "fp8 tensor cores"):
+ f8t.resolve_fp8_lora("fp8_lora", gpu = "sm75")
+ with pytest.raises(P.PrecisionRefused, match = "fp8 tensor cores"):
+ f8t.resolve_fp8_lora("fp8_lora", gpu = "sm80")
+ with pytest.raises(P.PrecisionRefused, match = "ROCm"):
+ f8t.resolve_fp8_lora("fp8_lora", gpu = "rocm")
+ with pytest.raises(P.PrecisionRefused, match = "no CUDA device"):
+ f8t.resolve_fp8_lora("fp8_lora", gpu = None)
+ with pytest.raises(P.PrecisionRefused, match = "load_in_4bit = False"):
+ f8t.resolve_fp8_lora("fp8_lora", load_in_4bit = True, gpu = "sm89")
+ with pytest.raises(P.PrecisionRefused, match = "qat_scheme"):
+ f8t.resolve_fp8_lora("fp8_lora", qat_scheme = "fp8-fp8", gpu = "sm89")
+ with pytest.raises(P.PrecisionRefused, match = "fp32"):
+ f8t.resolve_fp8_lora("fp8_lora", dtype = torch.float32, gpu = "sm89")
+ with pytest.raises(ValueError, match = "full finetuning recipe"):
+ f8t.resolve_fp8_lora("rowwise", gpu = "sm89")
+ assert f8t.resolve_fp8_lora("fp8_lora", gpu = "sm89", dtype = torch.bfloat16) == R
+ assert f8t.resolve_fp8_lora("fp8_lora", gpu = "sm120", dtype = torch.float16) == "fp8_lora_rowwise"
+ assert FL.scaling_for_class("sm120") == "row" and FL.scaling_for_class("sm89") == "tensor"
+ assert f8t.resolve_fp8_lora(False) is None and f8t.resolve_fp8_lora(None) is None
+ gate = P.get_recipe(R).gate("sm89").status
+ if gate != "named":
+ with pytest.raises(P.PrecisionRefused, match = "not been measured faster"):
+ f8t.resolve_fp8_lora(True, gpu = "sm89")
+
+
[email protected]("gpu", [None, "sm75", "rocm"])
+def test_loader_refuses_before_any_load(monkeypatch, gpu):
+ from unsloth.models import loader
+ monkeypatch.setattr(P, "current_gpu_class", lambda: gpu)
+ for cls in (loader.FastLanguageModel, loader.FastModel):
+ with pytest.raises(P.PrecisionRefused, match = "load_in_4bit = False"):
+ cls.from_pretrained(
+ model_name = "unsloth/does-not-exist", load_in_4bit = True, float8_training = "fp8_lora"
+ )
+ with pytest.raises(P.PrecisionRefused):
+ cls.from_pretrained(
+ model_name = "unsloth/does-not-exist",
+ load_in_4bit = False,
+ float8_training = "fp8_lora",
+ )
+
+
+def test_registry_row():
+ r = P.get_recipe(f8t.FP8_LORA_RECIPE)
+ assert r.kind == "frozen_base" and r.modes == ("lora",)
+ assert r.fwd.a.dtype == "e4m3" and r.fwd.b.granularity == "tensor"
+ assert r.dgrad.a.dtype == "e5m2" and not r.wgrad.low_precision
+ assert "fp32" not in r.input_dtypes
+ assert r.gate("sm75").status == "refused" and r.gate("sm80").status == "refused"
+
+
+def rowwise(monkeypatch):
+ if not _cpu_scaled_mm_works(rowwise = True):
+ pytest.skip("this torch build has no CPU rowwise torch._scaled_mm kernel")
+ monkeypatch.setattr(FL, "_SCALING", "row")
+ monkeypatch.setattr(FL, "_DGRAD_DTYPE", FL.E4M3)
+ yield
+
+
+@needs_cpu_scaled_mm
+def test_rowwise_numerics_and_outlier_tokens(rowwise):
+ W = _weight()
+ x = torch.randn(64, 256).bfloat16()
+ x[0] *= 1e5
+ dy = torch.randn(64, 384).bfloat16()
+ state = FL.Fp8LoRABaseState(W)
+ assert state.w_inv_fwd.shape == (1, 384) and state.w_inv_dgrad.shape == (1, 256)
+ y = state.matmul(x, W)
+ ref = x.float() @ W.float().t()
+ rest = _rel(y[1:], ref[1:])
+ assert rest < FP8_FWD_TOL, rest
+ FL._SCALING = "tensor"
+ tensorwise = _rel(FL.Fp8LoRABaseState(W).matmul(x, W)[1:], ref[1:])
+ FL._SCALING = "row"
+ assert tensorwise > 2 * rest, (tensorwise, rest)
+ dx = state.dgrad(dy, W)
+ assert _rel(dx, dy.float() @ W.float()) < FP8_FWD_TOL
+
+
+@needs_cpu_scaled_mm
+def test_rowwise_lora_kernels_and_peft_path(rowwise):
+ outs_ref, dX_ref, grads_ref, _ = _run_block("mlp", fp8 = False)
+ FL.reset_fp8_lora_stats()
+ outs, dX, grads, projs = _run_block("mlp", fp8 = True)
+ assert FL.FP8_LORA_STATS["fwd"] == 3 and FL.FP8_LORA_STATS["dgrad"] == 3
+ assert _rel(outs[0], outs_ref[0]) < FP8_FWD_TOL * 1.5 and _rel(dX, dX_ref) < FP8_FWD_TOL * 1.5
+ for g, r in zip(grads, grads_ref):
+ assert _cos(g, r) > 0.99
+ base = torch.nn.Linear(256, 384, bias = False, dtype = torch.bfloat16).requires_grad_(False)
+ FL.attach_fp8_lora_base(base)
+ x = torch.randn(2, 8, 256).bfloat16().requires_grad_(True)
+ base(x).float().sum().backward()
+ assert x.grad is not None and torch.isfinite(x.grad).all()
+
+
+def test_loaded_adapter_is_converted_when_marked(monkeypatch):
+ model = _tiny_qwen3_peft()
+ calls = []
+ real = f8t.enable_fp8_lora
+ monkeypatch.setattr(
+ f8t, "enable_fp8_lora", lambda m, r = None, **k: calls.append(r) or real(m, r, allow_cpu = True)
+ )
+ f8t.mark_fp8_lora(model, f8t.FP8_LORA_RECIPE)
+ assert calls == [f8t.FP8_LORA_RECIPE]
+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["modules"] == 14
+
+
+@needs_cpu_scaled_mm
+def test_default_no_transposed_cache_matches_cached(monkeypatch):
+ W = _weight()
+ dy = torch.randn(32, 384).bfloat16()
+ lean = FL.Fp8LoRABaseState(W)
+ assert lean.w8t is None
+ monkeypatch.setattr(FL, "_CACHE_T", True)
+ cached = FL.Fp8LoRABaseState(W)
+ assert cached.w8t is not None
+ torch.testing.assert_close(lean.dgrad(dy, W), cached.dgrad(dy, W), rtol = 0, atol = 0)
+
+
+@needs_cpu_scaled_mm
+def test_compiled_graph_has_no_fp8_transpose_outside_the_ops():
+ torch._dynamo.reset()
+ base = torch.nn.Linear(256, 384, bias = False, dtype = torch.bfloat16).requires_grad_(False)
+ FL.attach_fp8_lora_base(base)
+ proj = _Proj(256, 384)
+ FL.attach_fp8_lora_base(proj.base_layer)
+ graphs = []
+
+ def backend(gm, example_inputs):
+ from torch._functorch.aot_autograd import aot_module_simplified
+ def capture(g, _):
+ graphs.append(g)
+ return g
+
+ return aot_module_simplified(gm, example_inputs, fw_compiler = capture, bw_compiler = capture)
+
+ def step(x):
+ y = base(x)
+ W, W_quant, A, B, S = FLK.get_lora_parameters(proj)
+ return FLK.LoRA_W.apply(y[..., :256].contiguous(), W, W_quant, A, B, S).float().sum()
+
+ x = torch.randn(2, 16, 256).bfloat16().requires_grad_(True)
+ torch.compile(step, backend = backend, fullgraph = True)(x).backward()
+ assert len(graphs) >= 2
+ fp8 = (torch.float8_e4m3fn, torch.float8_e5m2)
+ targets = []
+ for g in graphs:
+ for node in g.graph.nodes:
+ val = node.meta.get("val")
+ if node.op == "call_function" and isinstance(val, torch.Tensor) and val.dtype in fp8:
+ targets.append(str(node.target))
+ assert targets == [], targets
+ names = {str(n.target) for g in graphs for n in g.graph.nodes if n.op == "call_function"}
+ assert any("fp8_lora_fwd" in t for t in names) and any(
+ "fp8_lora_dgrad" in t for t in names
+ ), names
+
+
+def offload_on_cpu(monkeypatch):
+ monkeypatch.setattr(FL, "_OFFLOAD_CPU_WEIGHTS", True)
+ monkeypatch.setattr(FL, "_OFFLOAD_MASTER", True)
+ monkeypatch.delenv("WORLD_SIZE", raising = False)
+
+
+def _frozen_linear(
+ K = 256,
+ N = 384,
+ bias = False,
+ seed = 0,
+):
+ torch.manual_seed(seed)
+ lin = torch.nn.Linear(K, N, bias = bias, dtype = torch.bfloat16)
+ return lin.requires_grad_(False)
+
+
+def test_host_master_leaves_only_fp8_bytes_on_the_compute_device(offload_on_cpu):
+ lin, ref = _frozen_linear(), _frozen_linear()
+ W0, param = lin.weight.detach().clone(), lin.weight
+ state = FL.attach_fp8_lora_base(lin)
+ ref_state = FL.attach_fp8_lora_base(ref, offload = False)
+ assert state.offloaded and not ref_state.offloaded
+ assert lin.weight is param
+ assert torch.equal(lin.weight, W0)
+ numel = 256 * 384
+ assert state.resident_bytes(lin) == {
+ "fp8": numel + 8,
+ "master_device": 0,
+ "master_host": 2 * numel,
+ }
+ assert ref_state.resident_bytes(ref) == {
+ "fp8": numel + 8,
+ "master_device": 2 * numel,
+ "master_host": 0,
+ }
+ assert FL.fp8_lora_memory(torch.nn.Sequential(lin, ref))["total"] == {
+ "fp8": 2 * (numel + 8),
+ "master_device": 2 * numel,
+ "master_host": 2 * numel,
+ }
+
+
+@needs_cpu_scaled_mm
+def test_host_master_gives_the_same_fp8_results(offload_on_cpu):
+ lin, ref = _frozen_linear(bias = True), _frozen_linear(bias = True)
+ FL.attach_fp8_lora_base(lin)
+ FL.attach_fp8_lora_base(ref, offload = False)
+ x = torch.randn(2, 16, 256).bfloat16()
+ xa, xb = x.clone().requires_grad_(True), x.clone().requires_grad_(True)
+ ya, yb = lin(xa), ref(xb)
+ dy = torch.randn_like(ya)
+ ya.backward(dy)
+ yb.backward(dy)
+ assert torch.equal(ya, yb) and torch.equal(xa.grad, xb.grad)
+ assert FL.FP8_LORA_STATS["fwd"] == 2 and FL.FP8_LORA_STATS["dgrad"] == 2
+ assert FL.FP8_LORA_STATS["quantize"] == 2
+
+
+def test_host_master_decode_rows_use_the_dequantized_fp8_weight(offload_on_cpu):
+ lin = _frozen_linear()
+ W0 = lin.weight.detach().clone()
+ state = FL.attach_fp8_lora_base(lin)
+ x = torch.randn(1, 1, 256).bfloat16()
+ dense = (state.w8.float() * state.w_inv).bfloat16()
+ torch.testing.assert_close(state.matmul(x, lin.weight), x @ dense.t())
+ torch.testing.assert_close(lin(x), x @ dense.t())
+ dy = torch.randn(1, 1, 384).bfloat16()
+ torch.testing.assert_close(state.matmul(dy, lin.weight.t()), dy @ dense)
+ assert _rel(lin(x), x @ W0.t()) < FP8_FWD_TOL
+ assert FL.FP8_LORA_STATS["fwd"] == 0
+
+
+@needs_cpu_scaled_mm
[email protected]("backend", ["llama", "vision"])
+def test_generation_restores_exact_base_then_training_reenables_fp8(
+ offload_on_cpu, monkeypatch, backend
+):
+ from unsloth.models import llama, vision
+
+ owner = llama if backend == "llama" else vision
+ api = owner.FastLlamaModel if backend == "llama" else owner.FastBaseModel
+ monkeypatch.setenv("UNSLOTH_RETURN_HIDDEN_STATES", "0")
+ monkeypatch.setattr(
+ owner, "enable_fp8_lora", lambda model: f8t.enable_fp8_lora(model, allow_cpu = True)
+ )
+ model = _tiny_qwen3_peft().eval()
+ ids = torch.randint(0, 256, (1, 32))
+ with torch.no_grad():
+ expected = model(ids).logits
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ assert FL.fp8_lora_memory(model)["layers"]
+ api.for_inference(model)
+ with torch.no_grad():
+ actual = model(ids).logits
+ assert torch.equal(actual, expected)
+ assert not FL.fp8_lora_memory(model)["layers"]
+ api.for_training(model, use_gradient_checkpointing = False)
+ assert len(FL.fp8_lora_memory(model)["layers"]) == 14
+ FL.reset_fp8_lora_stats()
+ _, grads = _loss_and_grads(model)
+ assert FL.FP8_LORA_STATS["fwd"] == 14
+ assert any(torch.count_nonzero(grad) for grad in grads.values())
+
+
+def test_enabling_an_offloaded_base_again_keeps_engagement(offload_on_cpu, monkeypatch):
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ expected = f8t.enable_fp8_lora(model, allow_cpu = True)
+ states = [
+ module.__dict__["_unsloth_fp8_lora"]
+ for module in model.modules()
+ if "_unsloth_fp8_lora" in module.__dict__
+ ]
+ for state in states:
+ assert state.offloaded
+ state.device = torch.device("cuda:0")
+ monkeypatch.setattr(torch.cuda, "get_device_capability", lambda device: (8, 9))
+ monkeypatch.setattr(torch.version, "hip", None)
+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")
+ assert f8t.enable_fp8_lora(model) == expected
+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["modules"] == len(expected)
+ assert all(state.offloaded and state.quantizations == 1 for state in states)
+
+
+def test_host_master_dequantize_is_exact(offload_on_cpu):
+ lin = _frozen_linear()
+ W0 = lin.weight.detach().clone()
+ state = FL.attach_fp8_lora_base(lin)
+ from unsloth.kernels.utils import fast_dequantize
+
+ assert torch.equal(fast_dequantize(lin.weight, state), W0)
+ assert torch.equal(fast_dequantize(lin.weight.t(), state), W0.t())
+
+
+def test_repeated_inference_does_not_rescan_restored_layers(offload_on_cpu, monkeypatch):
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ f8t.restore_fp8_lora_for_inference(model)
+ calls = []
+ original = model.modules
+
+ def modules():
+ calls.append(True)
+ return original()
+
+ monkeypatch.setattr(model, "modules", modules)
+ f8t.restore_fp8_lora_for_inference(model)
+ assert calls == []
+
+
+@needs_cpu_scaled_mm
+def test_module_moves_keep_or_restore_the_host_master(offload_on_cpu):
+ lin = _frozen_linear(bias = True)
+ state = FL.attach_fp8_lora_base(lin)
+ w8 = state.w8
+ lin.to(state.device)
+ lin.to(torch.bfloat16)
+ assert state.offloaded and state.w8 is w8
+ W0 = lin.weight.detach().clone()
+ lin.half()
+ assert not state.offloaded and lin.weight.dtype == torch.float16
+ assert torch.equal(lin.weight, W0.half())
+ x = torch.randn(32, 256).half()
+ y = lin(x)
+ assert (
+ y.dtype == torch.float16
+ and _rel(y, x.float() @ W0.float().t() + lin.bias.float()) < FP8_FWD_TOL
+ )
+
+
+def test_no_host_master_for_16bit_dgrad_or_distributed(offload_on_cpu, monkeypatch):
+ monkeypatch.setattr(FL, "_DGRAD_DTYPE", None)
+ assert not FL.offload_supported()
+ monkeypatch.setattr(FL, "_DGRAD_DTYPE", "auto")
+ assert FL.offload_supported()
+ monkeypatch.setenv("WORLD_SIZE", "2")
+ assert not FL.offload_supported()
+ monkeypatch.delenv("WORLD_SIZE")
+ monkeypatch.setattr(FL, "_OFFLOAD_MASTER", False)
+ assert not FL.offload_supported()
+
+
+@needs_cpu_scaled_mm
+def test_save_and_merge_exact_with_host_master(offload_on_cpu):
+ import copy
+
+ model = _tiny_qwen3_peft()
+ ref = copy.deepcopy(model)
+ frozen = {n: p.detach().clone() for n, p in model.named_parameters() if not p.requires_grad}
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ converted = f8t.enable_fp8_lora(model, allow_cpu = True)
+ assert len(converted) == 14
+ assert P.engaged(model)[f8t.FP8_LORA_RECIPE]["detail"]["master_offloaded"] == 14
+ memory = FL.fp8_lora_memory(model)
+ assert len(memory["layers"]) == 14 and memory["total"]["master_device"] == 0
+ opt = torch.optim.SGD([p for p in model.parameters() if p.requires_grad], lr = 0.1)
+ for _ in range(3):
+ _loss_and_grads_keep(model)
+ opt.step()
+ opt.zero_grad(set_to_none = True)
+ sd = model.state_dict()
+ for name, value in frozen.items():
+ assert torch.equal(sd[name], value), name
+ from safetensors.torch import load, save
+
+ base = {n: sd[n].contiguous() for n in frozen if "base_layer" in n}
+ assert len(base) == 14
+ loaded = load(save(base))
+ assert all(torch.equal(loaded[n], frozen[n]) for n in base)
+ ref.load_state_dict({n: p for n, p in sd.items() if "lora_" in n}, strict = False)
+ from unsloth.save import _merge_lora
+
+ mods, ref_mods = dict(model.named_modules()), dict(ref.named_modules())
+ for name in converted:
+ W, _ = _merge_lora(mods[name], name)
+ W_ref, _ = _merge_lora(ref_mods[name], name)
+ assert W.dtype == W_ref.dtype and torch.equal(W, W_ref), name
+ merged = model.merge_and_unload().state_dict()
+ merged_ref = ref.merge_and_unload().state_dict()
+ assert merged.keys() == merged_ref.keys()
+ for name in merged_ref:
+ assert torch.equal(merged[name], merged_ref[name]), name
+ assert all(
+ not m.__dict__["_unsloth_fp8_lora"].offloaded
+ for m in model.modules()
+ if "_unsloth_fp8_lora" in m.__dict__
+ )
+
+
[email protected]("method", ["merge_and_unload", "unload"])
+def test_unloaded_model_matches_its_reloaded_dense_weights(offload_on_cpu, method):
+ import copy
+
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ assert len(f8t.enable_fp8_lora(model, allow_cpu = True)) == 14
+ clone = copy.deepcopy(model)
+ merged = getattr(clone, method)().eval()
+ reloaded = type(merged)(copy.deepcopy(merged.config)).to(torch.bfloat16).eval()
+ reloaded.load_state_dict(merged.state_dict())
+ ids = torch.randint(0, 256, (2, 32))
+ FL.reset_fp8_lora_stats()
+ with torch.no_grad():
+ actual = merged(input_ids = ids).logits
+ expected = reloaded(input_ids = ids).logits
+ assert torch.equal(actual, expected), (actual - expected).abs().max().item()
+ assert FL.FP8_LORA_STATS["fwd"] == 0
+ assert not FL.fp8_lora_memory(merged)["layers"]
+ assert not merged.__dict__.get("_unsloth_fp8_lora_loss_forwards")
+ assert len(FL.fp8_lora_memory(model)["layers"]) == 14
+
+
+def _loss_and_grads_keep(model):
+ torch.manual_seed(1)
+ ids = torch.randint(0, 256, (2, 32))
+ model(input_ids = ids, labels = ids).loss.backward()
+
+
+def test_square_tile_transpose_kernel_byte_exact_under_the_triton_interpreter():
+ triton = pytest.importorskip("triton")
+ import os
+ import subprocess
+ import sys
+ import textwrap
+
+ path = os.path.abspath(FL.__file__)
+ code = textwrap.dedent(
+ f"""
+ import importlib.util, torch
+ spec = importlib.util.spec_from_file_location("fp8_lora_standalone", {path!r})
+ FL = importlib.util.module_from_spec(spec); spec.loader.exec_module(FL)
+ assert FL.triton is not None
+ for R, C in ((64, 64), (128, 192), (100, 37), (1, 300), (65, 129), (256, 64)):
+ x = torch.randint(0, 255, (R, C), dtype = torch.uint8).view(torch.float8_e4m3fn)
+ y = FL.fp8_transpose(x, use_triton = True)
+ assert y.shape == (C, R) and y.is_contiguous() and y.dtype == x.dtype
+ assert torch.equal(y.view(torch.uint8), x.t().contiguous().view(torch.uint8)), (R, C)
+ print("TRANSPOSE_OK")
+ """
+ )
+ env = dict(os.environ, TRITON_INTERPRET = "1", CUDA_VISIBLE_DEVICES = "")
+ p = subprocess.run(
+ [sys.executable, "-c", code], capture_output = True, text = True, env = env, timeout = 600
+ )
+ assert "TRANSPOSE_OK" in p.stdout, p.stdout[-2000:] + p.stderr[-4000:]
+ import inspect
+
+ src = inspect.getsource(FL.fp8_transpose)
+ assert "_TRANSPOSE_BLOCK), triton.cdiv(C, _TRANSPOSE_BLOCK)" in src
+
+
+def test_merge_of_a_deepcopy_merges_the_copy_only(offload_on_cpu):
+ import copy
+
+ model = _tiny_qwen3_peft()
+ plain = copy.deepcopy(model)
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ twin = copy.deepcopy(model)
+ name = "base_model.model.model.layers.0.mlp.down_proj"
+ layer, twin_layer = dict(model.named_modules())[name], dict(twin.named_modules())[name]
+ before = layer.base_layer.weight.detach().clone()
+ twin_layer.merge()
+ assert twin_layer.merged and not layer.merged
+ assert torch.equal(layer.base_layer.weight, before)
+ assert layer.base_layer.__dict__["_unsloth_fp8_lora"].offloaded
+ plain_layer = dict(plain.named_modules())[name]
+ plain_layer.merge()
+ assert torch.equal(twin_layer.base_layer.weight, plain_layer.base_layer.weight)
+
+
+@needs_cpu_scaled_mm
+def test_compiled_fp8_lora_ops_are_counted():
+ torch._dynamo.reset()
+ W = _weight()
+ state = FL.Fp8LoRABaseState(W)
+
+ def run(x, dy):
+ return state.matmul(x, W), state.dgrad(dy, W)
+
+ compiled = torch.compile(run, backend = "aot_eager", fullgraph = True)
+ x, dy = torch.randn(32, 256).bfloat16(), torch.randn(32, 384).bfloat16()
+ compiled(x, dy)
+ with P.count_lowp_gemms() as counts:
+ compiled(x, dy)
+ assert counts["unsloth::fp8_lora_fwd"] == 1 and counts["unsloth::fp8_lora_dgrad"] == 1, dict(
+ counts
+ )
+
+
+@needs_cpu_scaled_mm
+def test_compiled_fused_fp8_lora_ops_are_counted(monkeypatch):
+ torch._dynamo.reset()
+ monkeypatch.setenv("UNSLOTH_STRICT_ENGAGE", "1")
+ W = _weight()
+ state = FL.Fp8LoRABaseState(W)
+ A = torch.randn(16, 256).bfloat16()
+ Bt = torch.randn(16, 384).bfloat16()
+
+ def run(x, dy):
+ return state.lora_first(x, W, x @ A.t(), Bt, 1.0), state.dgrad_accum(
+ dy, W, torch.zeros_like(x)
+ )
+
+ compiled = torch.compile(run, backend = "aot_eager", fullgraph = True)
+ x, dy = torch.randn(32, 256).bfloat16(), torch.randn(32, 384).bfloat16()
+ compiled(x, dy)
+ with P.count_lowp_gemms() as counts:
+ compiled(x, dy)
+ P.require("fp8_lora", sum(counts.values()))
+ assert counts["unsloth::fp8_lora_first"] == counts["unsloth::fp8_lora_accum"] == 1
+ assert counts["_scaled_mm"] == 0
+
+
+@needs_cpu_scaled_mm
[email protected]("compiled_first", [False, True])
+def test_enable_fp8_refreshes_only_an_existing_block_compiler(monkeypatch, compiled_first):
+ from unsloth.models import _block_compile
+
+ model = _tiny_qwen3_peft()
+ model._unsloth_block_compile_installed = compiled_first
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ calls = []
+
+ def refresh(model):
+ assert FL.fp8_lora_memory(model)["layers"]
+ calls.append(model)
+
+ monkeypatch.setattr(_block_compile, "patch_block_compile", refresh)
+ assert f8t.enable_fp8_lora(model, allow_cpu = True)
+ assert calls == ([model] if compiled_first else [])
+
+
+@needs_cpu_scaled_mm
+def test_merge_requantizes_the_fp8_copy(offload_on_cpu):
+ import copy
+
+ model = _tiny_qwen3_peft()
+ ref = copy.deepcopy(model)
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ ids = torch.randint(0, 256, (2, 32))
+ with torch.no_grad():
+ before = model(input_ids = ids).logits.float()
+ model.merge_adapter()
+ ref.merge_adapter()
+ merged = model(input_ids = ids).logits.float()
+ want = ref(input_ids = ids).logits.float()
+ assert _rel(merged, want) < 0.3 and _rel(merged, before) < 0.3
+
+
+def test_dora_layers_keep_the_master_on_the_device(offload_on_cpu):
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+ from peft import LoraConfig, get_peft_model
+
+ torch.manual_seed(0)
+ config = Qwen3Config(
+ vocab_size = 256,
+ hidden_size = 128,
+ intermediate_size = 256,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 32,
+ )
+ model = get_peft_model(
+ Qwen3ForCausalLM(config).to(torch.bfloat16),
+ LoraConfig(r = 8, target_modules = ["q_proj", "v_proj"], use_dora = True),
+ )
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ assert len(f8t.enable_fp8_lora(model, allow_cpu = True)) == 2
+ states = [
+ m.base_layer.__dict__["_unsloth_fp8_lora"] for m in model.modules() if hasattr(m, "lora_A")
+ ]
+ assert states and not any(s.offloaded for s in states)
+
+
+def test_an_added_adapter_lands_on_the_compute_device(offload_on_cpu, monkeypatch):
+ from peft import LoraConfig
+
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ layers = [m for m in model.modules() if hasattr(m, "lora_A")]
+ compute = torch.device("meta") # stands in for cuda; the master stays on the CPU host
+ for m in layers:
+ m.base_layer.__dict__["_unsloth_fp8_lora"].device = compute
+ moved = []
+ real_to = torch.nn.Module.to
+
+ def spy(self, *args, **kwargs):
+ if args and args[0] == compute:
+ moved.append(self)
+ return self
+ return real_to(self, *args, **kwargs)
+
+ monkeypatch.setattr(torch.nn.Module, "to", spy)
+ monkeypatch.setattr(FL.Fp8LoRABaseState, "refresh", lambda self, weight: self)
+ model.add_adapter("second", LoraConfig(r = 4, target_modules = ["q_proj"]))
+ added = [m for m in layers if "second" in m.lora_A]
+ assert added and all(any(x is m.lora_A["second"] for x in moved) for m in added)
+ monkeypatch.setattr(torch.nn.Module, "to", real_to)
+ for m in layers:
+ m.base_layer.__dict__["_unsloth_fp8_lora"].device = torch.device("cpu")
+ model.add_adapter("third", LoraConfig(r = 4, target_modules = ["v_proj"], use_dora = True))
+ dora = [m for m in layers if "third" in m.lora_A]
+ assert dora and not any(m.base_layer.__dict__["_unsloth_fp8_lora"].offloaded for m in dora)
+
+
+def test_low_cpu_mem_adapters_stay_on_meta_for_the_loader(offload_on_cpu):
+ from peft import LoraConfig
+
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ model.add_adapter("second", LoraConfig(r = 4, target_modules = ["q_proj"]), low_cpu_mem_usage = True)
+ added = [m for m in model.modules() if hasattr(m, "lora_A") and "second" in m.lora_A]
+ assert added and all(m.lora_A["second"].weight.device.type == "meta" for m in added)
+
+
+def test_a_dora_adapter_restores_the_master_before_placement(offload_on_cpu, monkeypatch):
+ from peft import LoraConfig
+ from peft.tuners.lora import LoraLayer
+
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ seen = []
+ real = LoraLayer._move_adapter_to_device_of_base_layer
+
+ def spy(self, adapter_name, *args, **kwargs):
+ if adapter_name == "dora":
+ seen.append(self.base_layer.__dict__["_unsloth_fp8_lora"].offloaded)
+ return real(self, adapter_name, *args, **kwargs)
+
+ monkeypatch.setattr(LoraLayer, "_move_adapter_to_device_of_base_layer", spy)
+ model.add_adapter("dora", LoraConfig(r = 4, target_modules = ["v_proj"], use_dora = True))
+ assert seen and not any(seen)
+
+
+@needs_cpu_scaled_mm
+def test_load_state_dict_requantizes_for_compiled_calls():
+ torch._dynamo.reset()
+ lin = _frozen_linear()
+ FL.attach_fp8_lora_base(lin, offload = False)
+ compiled = torch.compile(lin, backend = "aot_eager", fullgraph = True)
+ x = torch.randn(32, 256).bfloat16()
+ assert compiled(x).abs().sum() > 0
+ lin.load_state_dict({"weight": torch.zeros_like(lin.weight)})
+ assert compiled(x).abs().sum() == 0
+
+
+def test_a_dtype_move_requantizes_for_compiled_calls():
+ lin = _frozen_linear()
+ state = FL.attach_fp8_lora_base(lin, offload = False)
+ lin.half()
+ assert state.dtype == torch.float16 and state._key == state._weight_key(lin.weight)
+
+
+def test_a_pissa_adapter_requantizes_the_rewritten_base(offload_on_cpu):
+ from peft import LoraConfig
+
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ model.add_adapter(
+ "pissa",
+ LoraConfig(
+ r = 8, lora_alpha = 16, target_modules = ["q_proj"], init_lora_weights = "pissa_niter_1"
+ ),
+ )
+ layers = [m for m in model.modules() if hasattr(m, "lora_A") and "pissa" in m.lora_A]
+ assert layers
+ for m in layers:
+ state = m.base_layer.__dict__["_unsloth_fp8_lora"]
+ assert state._key == state._weight_key(m.base_layer.weight)
+
+
+def test_mamba_raw_weight_projections_keep_the_master(offload_on_cpu):
+ from peft import LoraConfig, get_peft_model
+
+ class MambaMixer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.in_proj = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)
+ self.out_proj = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)
+
+ class Model(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.mixer = MambaMixer()
+
+ model = get_peft_model(Model(), LoraConfig(r = 4, target_modules = ["in_proj", "out_proj"]))
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ assert len(f8t.enable_fp8_lora(model, allow_cpu = True)) == 2
+ mixer = model.base_model.model.mixer
+ assert not mixer.out_proj.base_layer.__dict__["_unsloth_fp8_lora"].offloaded
+ assert mixer.in_proj.base_layer.__dict__["_unsloth_fp8_lora"].offloaded
+
+
+@needs_cpu_scaled_mm
+def test_legacy_scaled_mm_tuple_return(monkeypatch):
+ real = torch._scaled_mm
+ monkeypatch.setattr(torch, "_scaled_mm", lambda *a, **k: (real(*a, **k), torch.zeros(())))
+ W = _weight()
+ state = FL.Fp8LoRABaseState(W)
+ y = state.matmul(torch.randn(32, 256).bfloat16(), W)
+ dx = state.dgrad(torch.randn(32, 384).bfloat16(), W)
+ assert torch.is_tensor(y) and torch.is_tensor(dx)
+
+
[email protected]("arch", ["falcon_h1", "wavlm"])
+def test_parents_reading_a_child_weight_keep_its_master(offload_on_cpu, arch):
+ if arch == "falcon_h1":
+ mod = pytest.importorskip("transformers.models.falcon_h1.modeling_falcon_h1")
+ parent_cls, leaf = mod.FalconH1Mixer, "out_proj"
+ else:
+ mod = pytest.importorskip("transformers.models.wavlm.modeling_wavlm")
+ parent_cls, leaf = mod.WavLMAttention, "q_proj"
+ parent = parent_cls.__new__(parent_cls)
+ torch.nn.Module.__init__(parent)
+ setattr(parent, leaf, torch.nn.Linear(4, 4))
+ model = torch.nn.Module()
+ model.block = parent
+ assert f8t._mamba_raw_weight_leaf(model, f"block.{leaf}")
+
+
+@needs_cpu_scaled_mm
+def test_refresh_under_inference_mode_builds_trainable_fp8_tensors():
+ lin = _frozen_linear()
+ state = FL.attach_fp8_lora_base(lin, offload = False)
+ with torch.inference_mode():
+ state._key = None
+ state.refresh(lin.weight)
+ assert not state.w8.is_inference()
+ x = torch.randn(32, 256).bfloat16().requires_grad_(True)
+ lin(x).float().sum().backward()
+ assert x.grad is not None
+
+
+def test_grpo_out_of_place_merge_leaves_the_fp8_copy_alone(offload_on_cpu):
+ from unsloth.models.rl_replacements import _unsloth_vllm_peft_merge_out_of_place
+
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ layer = next(m for m in model.modules() if hasattr(m, "lora_A") and hasattr(m, "base_layer"))
+ state = layer.base_layer.__dict__["_unsloth_fp8_lora"]
+ w8, key, offloaded = state.w8, state._key, state.offloaded
+ merged = _unsloth_vllm_peft_merge_out_of_place(layer.base_layer, [layer])
+ assert merged
+ assert state.w8 is w8 and state._key == key and state.offloaded == offloaded
+
+
+def test_a_base_weight_shared_by_two_lora_layers_stays_16bit(offload_on_cpu):
+ from peft import LoraConfig, get_peft_model
+
+ class Tied(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.a = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)
+ self.b = torch.nn.Linear(64, 64, bias = False, dtype = torch.bfloat16)
+ self.b.weight = self.a.weight
+
+ model = get_peft_model(
+ Tied(), LoraConfig(r = 4, target_modules = ["a", "b"], init_lora_weights = False)
+ )
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ assert f8t.enable_fp8_lora(model, allow_cpu = True) == []
+ inner = model.base_model.model
+ model.merge_adapter()
+ x = torch.randn(32, 64).bfloat16()
+ with torch.no_grad():
+ torch.testing.assert_close(inner.a(x), inner.b(x))
+
+
+def test_block_swapped_layers_are_not_converted(offload_on_cpu):
+ import types
+
+ model = _tiny_qwen3_peft()
+ layers = model.base_model.model.model.layers
+ layers._unsloth_block_swap = types.SimpleNamespace(indices = [1])
+ for m in layers[1].modules():
+ if isinstance(m, torch.nn.Linear) and not m.weight.requires_grad:
+ m.weight = torch.nn.Parameter(torch.empty(0, dtype = m.weight.dtype), requires_grad = False)
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ converted = f8t.enable_fp8_lora(model, allow_cpu = True)
+ assert converted and all(".layers.0." in name for name in converted)
+
+
+def test_sharded_or_preloaded_weights_are_not_eligible():
+ linear = torch.nn.Linear(64, 64, bias = False).bfloat16().requires_grad_(False)
+ linear.weight = torch.nn.Parameter(linear.weight.data[:32].clone(), requires_grad = False)
+ assert "shard" in FL.eligible_base(linear)
+ full = torch.nn.Linear(64, 64, bias = False).bfloat16().requires_grad_(False)
+ full.weight.__dict__["_unsloth_fsdp2_preload"] = object()
+ assert "shard" in FL.eligible_base(full)
+
+
+def test_get_peft_model_converts_after_block_swap():
+ import inspect
+ from unsloth.models.llama import FastLlamaModel
+
+ get_src = inspect.getsource(FastLlamaModel.get_peft_model)
+ assert "_defer_fp8_lora = True" in get_src
+ swap = get_src.index("install_block_swap(")
+ assert get_src.index("enable_fp8_lora(model)", swap) > swap
+ patch_src = inspect.getsource(FastLlamaModel.patch_peft_model)
+ assert "if not _defer_fp8_lora:" in patch_src
+
+
+def test_column_major_weights_are_not_eligible():
+ linear = torch.nn.Linear(64, 128, bias = False).bfloat16().requires_grad_(False)
+ linear.weight = torch.nn.Parameter(torch.randn(64, 128).bfloat16().t(), requires_grad = False)
+ assert "contiguous" in FL.eligible_base(linear)
+
+
[email protected]("variant", ["bdlora", "velora"])
+def test_a_variant_adapter_restores_the_master_before_placement(
+ offload_on_cpu, monkeypatch, variant
+):
+ import peft
+ from peft import LoraConfig
+ from peft.tuners.lora import LoraLayer
+
+ if variant == "bdlora":
+ if not hasattr(peft, "BdLoraConfig"):
+ pytest.skip("this PEFT has no BD-LoRA")
+ config = LoraConfig(
+ r = 4,
+ target_modules = ["q_proj"],
+ use_bdlora = peft.BdLoraConfig(nblocks = 2, target_modules_bd_a = ["q_proj"]),
+ )
+ else:
+ if not hasattr(peft, "VeloraConfig"):
+ pytest.skip("this PEFT has no VeLoRA")
+ config = LoraConfig(
+ r = 4,
+ target_modules = ["q_proj"],
+ velora_config = peft.VeloraConfig(num_groups = 2, init_type = "random"),
+ )
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ seen = []
+ real = LoraLayer._move_adapter_to_device_of_base_layer
+
+ def spy(self, adapter_name, *args, **kwargs):
+ if adapter_name == variant:
+ seen.append(self.base_layer.__dict__["_unsloth_fp8_lora"].offloaded)
+ return real(self, adapter_name, *args, **kwargs)
+
+ monkeypatch.setattr(LoraLayer, "_move_adapter_to_device_of_base_layer", spy)
+ try:
+ model.add_adapter(variant, config)
+ except (TypeError, ValueError) as exc:
+ pytest.skip(f"this PEFT cannot build {variant} here: {exc}")
+ assert seen and not any(seen)
+
+
+@needs_cpu_scaled_mm
[email protected]("error_type", [RuntimeError, ValueError])
[email protected]("outputs", ["bf16", "bf16 and fp16"])
+def test_rowwise_fp16_without_float32_out_stays_finite(rowwise, monkeypatch, error_type, outputs):
+ real = torch._scaled_mm
+
+ def refuse_fp32_rowwise(
+ a,
+ b,
+ *args,
+ scale_a = None,
+ out_dtype = None,
+ **kwargs,
+ ):
+ if out_dtype == torch.float32 and scale_a is not None and scale_a.numel() > 1:
+ raise error_type(
+ f"Only {outputs} high precision output types are supported for row-wise scaling."
+ )
+ return real(a, b, *args, scale_a = scale_a, out_dtype = out_dtype, **kwargs)
+
+ monkeypatch.setattr(torch, "_scaled_mm", refuse_fp32_rowwise)
+ monkeypatch.setattr(FL, "_ROWWISE_BF16_ONLY", set())
+ W = torch.full((32, 64), 0.5, dtype = torch.float16)
+ state = FL.Fp8LoRABaseState(W)
+ assert state.w_inv_fwd.dim() == 2
+ x = torch.full((16, 64), 2047.0, dtype = torch.float16)
+ assert torch.isfinite(state.matmul(x, W)).all()
+ assert torch.isfinite(state.matmul(x, W)).all()
+ dy = torch.full((16, 32), 4094.0, dtype = torch.float16)
+ assert torch.isfinite(state.dgrad(dy, W)).all()
+ Wb = W.bfloat16()
+ y = FL.Fp8LoRABaseState(Wb).matmul(torch.randn(16, 64).bfloat16(), Wb)
+ assert y.dtype == torch.bfloat16
+
+
+ "error",
+ [
+ torch.OutOfMemoryError("allocation failed"),
+ RuntimeError("CUDA error: illegal memory access"),
+ ],
+)
+def test_rowwise_failure_does_not_change_the_scaling_recipe(monkeypatch, error):
+ monkeypatch.setattr(FL, "_ROWWISE_BF16_ONLY", set())
+ calls = []
+
+ def fail_once(*args):
+ calls.append(args)
+ if len(calls) == 1:
+ raise error
+ return torch.zeros(16, 16)
+
+ monkeypatch.setattr(FL, "_scaled_mm", fail_once)
+ x = torch.zeros(16, 16)
+ with pytest.raises(type(error), match = str(error)):
+ FL._rowwise_scaled_mm(x, x.t(), torch.ones(16, 1), torch.ones(1, 16), torch.float16, True)
+ assert len(calls) == 1
+ assert FL._ROWWISE_BF16_ONLY == set()
+
+
+def test_vllm_server_merge_of_a_host_master_lands_on_the_adapter_device():
+ from types import SimpleNamespace
+
+ from unsloth.models.rl_replacements import _unsloth_vllm_merged_weight
+
+ lora_A = torch.nn.Linear(64, 4, bias = False, device = "meta")
+ lora_B = torch.nn.Linear(4, 32, bias = False, device = "meta")
+ module = SimpleNamespace(
+ active_adapters = ["default"],
+ lora_A = {"default": lora_A},
+ lora_B = {"default": lora_B},
+ scaling = {"default": 2.0},
+ )
+ weight = torch.nn.Parameter(torch.randn(32, 64).bfloat16(), requires_grad = False)
+ merged = _unsloth_vllm_merged_weight(module, weight)
+ assert merged.device.type == "meta" and merged.dtype == torch.bfloat16
+ lora_A.to_empty(device = "cpu")
+ lora_B.to_empty(device = "cpu")
+ module.lora_A["default"], module.lora_B["default"] = lora_A, lora_B
+ assert _unsloth_vllm_merged_weight(module, weight).device.type == "cpu"
+
+
+@needs_cpu_scaled_mm
+def test_a_merge_under_inference_mode_leaves_a_trainable_master(offload_on_cpu, monkeypatch):
+ real_to = torch.Tensor.to
+
+ def allocating_to(self, *args, **kwargs):
+ if torch.is_inference_mode_enabled():
+ kwargs["copy"] = True
+ return real_to(self, *args, **kwargs)
+
+ model = _tiny_qwen3_peft()
+ f8t.mark_fp8_lora(model.base_model.model, f8t.FP8_LORA_RECIPE)
+ f8t.enable_fp8_lora(model, allow_cpu = True)
+ monkeypatch.setattr(torch.Tensor, "to", allocating_to)
+ with torch.inference_mode():
+ model.merge_adapter()
+ model.unmerge_adapter()
+ monkeypatch.setattr(torch.Tensor, "to", real_to)
+ ids = torch.randint(0, 256, (2, 16))
+ model(input_ids = ids, labels = ids).loss.backward()
+ assert any(p.grad is not None for n, p in model.named_parameters() if "lora_" in n)
+
+
+def test_text_diffusion_refuses_fp8_lora_instead_of_dropping_it():
+ import inspect
+
+ from unsloth.models.loader import FastModel
+
+ src = inspect.getsource(FastModel.from_pretrained)
+ body = src[src.index("def _dispatch_diffusion():") :]
+ body = body[: body.index("FastDiffusionModel.from_pretrained(")]
+ assert "if fp8_lora is not None:" in body and "NotImplementedError" in body
+
+
+def test_vllm_server_sync_of_a_host_master_without_an_active_delta_lands_on_the_device():
+ from types import SimpleNamespace
+
+ from unsloth.models.rl_replacements import _unsloth_vllm_merged_weight
+
+ base = torch.nn.Linear(64, 32, bias = False).bfloat16().requires_grad_(False)
+ base.__dict__["_unsloth_fp8_lora"] = SimpleNamespace(device = torch.device("meta"))
+ module = SimpleNamespace(
+ base_layer = base, active_adapters = ["other"], lora_A = {}, lora_B = {}, scaling = {}
+ )
+ assert _unsloth_vllm_merged_weight(module, base.weight).device.type == "meta"
+ del base.__dict__["_unsloth_fp8_lora"]
+ assert _unsloth_vllm_merged_weight(module, base.weight).device.type == "cpu"
+
+
+def test_fast_language_model_forwards_fp8_lora_to_fastmodel():
+ import inspect
+
+ from unsloth.models.loader import FastLanguageModel
+
+ src = inspect.getsource(FastLanguageModel.from_pretrained)
+ assert '**({"float8_training": fp8_request} if fp8_lora is not None else {})' in src
+
+
+@needs_cpu_scaled_mm
+def test_fp8_lora_accumulates_small_adapter_before_base_rounding(monkeypatch):
+ monkeypatch.setattr(FL, "_SCALING", "tensor")
+ monkeypatch.setattr(FL, "_MIN_ROWS", 16)
+ x = torch.ones(16, 16, dtype = torch.bfloat16)
+ w = torch.zeros(16, 16, dtype = torch.bfloat16)
+ w[:, 0], w[:, 1] = 1, 1 / 512
+ a = torch.zeros(1, 16, dtype = torch.bfloat16)
+ a[0, 0] = 1 / 256
+ b = torch.ones(16, 1, dtype = torch.bfloat16)
+ state = FL.Fp8LoRABaseState(w)
+ delta = torch.empty_like(x).addmm_(x @ a.t(), b.t(), beta = 0)
+ reference = delta.addmm_(x, w.t())
+ actual = FLK.matmul_lora(x, w, state, a, b, 1)
+ old_order = FLK.matmul_lora(x, w, state, a, b, 1, fused = False)
+ assert torch.equal(reference, torch.full_like(x, 1 + 1 / 128))
+ assert torch.equal(actual, reference)
+ assert not torch.equal(old_order, reference)
+
+
[email protected]("projection", ["gate_proj", "up_proj", "down_proj"])
+def test_biased_native_mlp_keeps_its_forward(projection):
+ from peft import LoraConfig, get_peft_model
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ model = get_peft_model(
+ LlamaForCausalLM(
+ LlamaConfig(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ mlp_bias = True,
+ )
+ ),
+ LoraConfig(r = 4, target_modules = ["gate_proj", "up_proj", "down_proj"]),
+ )
+ mlp = model.get_base_model().model.layers[0].mlp
+ for name in ("gate_proj", "up_proj", "down_proj"):
+ base = getattr(mlp, name).base_layer
+ if name != projection:
+ base.bias = None
+ else:
+ base.bias.data.fill_(0.25)
+ base.__dict__["_unsloth_fp8_lora"] = FL.Fp8LoRABaseState(base.weight)
+ original = mlp.forward
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ assert mlp.forward == original
+ x = torch.randn(2, 16, 32)
+ torch.testing.assert_close(mlp(x), original(x), rtol = 0, atol = 0)
+
+
+ "weight_dtype, input_dtype",
+ [
+ (torch.bfloat16, torch.float32),
+ (torch.float16, torch.bfloat16),
+ (torch.bfloat16, torch.float16),
+ ],
+)
+def test_mixed_native_norm_uses_its_original_equation(monkeypatch, weight_dtype, input_dtype):
+ from unsloth.kernels import rms_layernorm
+
+ def unsupported(*args, **kwargs):
+ pytest.fail("mixed-dtype normalization entered the fused kernel")
+
+ monkeypatch.setattr(rms_layernorm, "fast_rms_layernorm", unsupported)
+ model = _tiny_qwen3_peft()
+ norm = model.get_base_model().model.norm.to(weight_dtype)
+ original = norm.forward
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ x = torch.randn(2, 16, norm.weight.numel()).to(input_dtype).requires_grad_(True)
+ expected = original(x)
+ for call in (norm, torch.compile(norm, backend = "aot_eager", fullgraph = True)):
+ actual = call(x)
+ assert actual.dtype == expected.dtype
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
+
+
[email protected]("dtype", [torch.bfloat16, torch.float16])
+def test_fp32_native_norm_keeps_its_promoted_output(dtype):
+ model = _tiny_qwen3_peft()
+ norm = model.get_base_model().model.norm.float()
+ norm.weight.data.fill_(1.001)
+ original = norm.forward
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ assert norm.forward == original
+ x = torch.randn(2, 16, norm.weight.numel()).to(dtype)
+ normalized = x.float() * torch.rsqrt(
+ x.float().square().mean(-1, keepdim = True) + norm.variance_epsilon
+ )
+ expected = norm.weight * normalized.to(dtype)
+ actual = norm(x)
+ assert actual.dtype == expected.dtype == torch.float32
+ torch.testing.assert_close(actual, expected, rtol = 0, atol = 0)
+
+
[email protected]("dtype", [torch.bfloat16, torch.float32])
+def test_norm_upcast_after_binding_keeps_the_native_equation(monkeypatch, dtype):
+ from unsloth.kernels import rms_layernorm
+
+ def unsupported(*args, **kwargs):
+ pytest.fail("FP32 normalization entered the fused kernel")
+
+ monkeypatch.setattr(rms_layernorm, "fast_rms_layernorm", unsupported)
+ model = _tiny_qwen3_peft()
+ norm = model.get_base_model().model.norm
+ original = norm.forward
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ norm.float()
+ x = torch.randn(2, 16, norm.weight.numel()).to(dtype)
+ actual = norm(x)
+ assert actual.dtype == torch.float32
+ torch.testing.assert_close(actual, original(x), rtol = 0, atol = 0)
+
+
+ "mode", ["full", "gelu", "partial", "custom", "custom_class", "private_norm", "generated"]
+)
[email protected]("layout", ["3d", "2d", "strided_3d", "strided_2d"])
+def test_native_operator_binding_preserves_keywords_and_restores_instances(
+ monkeypatch, mode, layout
+):
+ import types
+ from peft import LoraConfig, get_peft_model
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+ from transformers.models.qwen3.modeling_qwen3 import Qwen3MLP, Qwen3RMSNorm
+ from unsloth.kernels import rms_layernorm
+
+ model = get_peft_model(
+ Qwen3ForCausalLM(
+ Qwen3Config(
+ vocab_size = 64,
+ hidden_size = 32,
+ intermediate_size = 64,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ )
+ ).to(torch.bfloat16),
+ LoraConfig(r = 4, target_modules = ["gate_proj", "up_proj", "down_proj"]),
+ )
+ owner = model.get_base_model()
+ mlp = owner.model.layers[0].mlp
+ norm = owner.model.norm
+ original_mlp, original_norm = Qwen3MLP.forward, Qwen3RMSNorm.forward
+ for layer in owner.model.layers:
+ for part in ("gate_proj", "up_proj", "down_proj"):
+ base = getattr(layer.mlp, part).base_layer
+ base.__dict__["_unsloth_fp8_lora"] = FL.Fp8LoRABaseState(base.weight)
+ if mode == "gelu":
+ mlp.act_fn = torch.nn.GELU()
+ elif mode == "partial":
+ mlp.down_proj.base_layer.__dict__.pop("_unsloth_fp8_lora")
+ elif mode in ("custom", "custom_class"):
+
+ def residual(self, x):
+ return original_mlp(self, x) + x
+
+ if mode == "custom":
+ mlp.forward = types.MethodType(residual, mlp)
+ else:
+ residual.__module__ = Qwen3MLP.__module__
+ monkeypatch.setattr(Qwen3MLP, "forward", residual)
+ elif mode == "private_norm":
+ namespace = {k: v for k, v in original_norm.__globals__.items() if k != "__name__"}
+ private = types.FunctionType(original_norm.__code__, namespace)
+ assert private.__module__ is None
+ norm.forward = types.MethodType(private, norm)
+ elif mode == "generated":
+ from unsloth_zoo.temporary_patches.utils import torch_compile_with_fallback
+ for cls, original, argument in (
+ (Qwen3MLP, original_mlp, "x"),
+ (Qwen3RMSNorm, original_norm, "hidden_states"),
+ ):
+ helper = cls.__name__ + "_forward"
+ namespace = {
+ "__name__": "unsloth_compiled_module_test",
+ helper: torch_compile_with_fallback(backend = "aot_eager")(original),
+ }
+ exec(
+ f"def forward(self, {argument}):\n return {helper}(self, {argument}={argument})\n",
+ namespace,
+ )
+ monkeypatch.setattr(cls, "__module__", namespace["__name__"])
+ monkeypatch.setattr(cls, "forward", namespace["forward"])
+ installed_class_forward = Qwen3MLP.forward
+ installed_norm_forward = Qwen3RMSNorm.forward
+ saved_override = mlp.__dict__.get("forward")
+ saved_norm_override = norm.__dict__.get("forward")
+ x = torch.randn(2, 3, 64)[..., ::2] if layout.startswith("strided") else torch.randn(2, 3, 32)
+ if layout.endswith("2d"):
+ x = x.reshape(6, 32)
+ x = x.to(torch.bfloat16).requires_grad_(True)
+ expected_mlp, expected_norm = mlp(x = x), norm(hidden_states = x)
+ calls = {"norm": 0, "mlp": 0}
+
+ def norm_spy(
+ layernorm,
+ X,
+ gemma = False,
+ save_output = False,
+ reuse_grad = False,
+ ):
+ if not torch.compiler.is_compiling():
+ calls["norm"] += 1
+ return original_norm(layernorm, X)
+
+ def mlp_spy(
+ self,
+ X,
+ inplace = True,
+ ):
+ assert X.ndim == 3 and X.is_contiguous()
+ if not torch.compiler.is_compiling():
+ calls["mlp"] += 1
+ return original_mlp(self, X)
+
+ monkeypatch.setattr(rms_layernorm, "fast_rms_layernorm", norm_spy)
+ monkeypatch.setattr(FLK, "apply_lora_mlp_swiglu", mlp_spy)
+ monkeypatch.setenv("UNSLOTH_FP8_LORA_FUSED_OPS", "0")
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ assert "_unsloth_fp8_lora_kernel_forwards" not in owner.__dict__
+ monkeypatch.setenv("UNSLOTH_FP8_LORA_FUSED_OPS", "1")
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ installed = owner.__dict__["_unsloth_fp8_lora_kernel_forwards"]
+ assert installed
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ assert owner.__dict__["_unsloth_fp8_lora_kernel_forwards"] is installed
+ assert torch.equal(mlp(x = x), expected_mlp)
+ assert torch.equal(norm(hidden_states = x), expected_norm)
+ assert calls == {"norm": 1, "mlp": int(mode in ("full", "private_norm", "generated"))}
+ assert (
+ Qwen3MLP.forward is installed_class_forward
+ and Qwen3RMSNorm.forward is installed_norm_forward
+ )
+ if mode == "private_norm" and layout == "strided_2d":
+ reference_grad = torch.autograd.grad(expected_mlp.sum() + expected_norm.sum(), x)[0]
+ compiled_mlp = torch.compile(mlp, backend = "aot_eager", fullgraph = True)
+ compiled_norm = torch.compile(norm, backend = "aot_eager", fullgraph = True)
+ actual_mlp, actual_norm = compiled_mlp(x = x), compiled_norm(hidden_states = x)
+ torch.testing.assert_close(actual_mlp, expected_mlp)
+ torch.testing.assert_close(actual_norm, expected_norm)
+ actual_grad = torch.autograd.grad(actual_mlp.sum() + actual_norm.sum(), x)[0]
+ torch.testing.assert_close(actual_grad, reference_grad)
+ assert calls == {"norm": 1, "mlp": 1}
+ from torch._dynamo.utils import counters
+
+ before = counters["stats"]["unique_graphs"]
+ compiled_mlp(x = x)
+ compiled_norm(hidden_states = x)
+ stable = counters["stats"]["unique_graphs"] - before
+ f8t.restore_fp8_lora_for_inference(model)
+ f8t._prepare_fp8_lora_operators(model, allow_cpu = True)
+ before = counters["stats"]["unique_graphs"]
+ compiled_mlp(x = x)
+ compiled_norm(hidden_states = x)
+ rebound = counters["stats"]["unique_graphs"] - before
+ print("OPERATOR_REBIND_GRAPHS", stable, rebound)
+ assert stable == rebound == 0
+
+ def call(module, value):
+ return module(value)
+
+ shared = torch.compile(call, backend = "aot_eager", fullgraph = True)
+ shared(norm, x)
+ shared(mlp, x)
+ before = counters["stats"]["unique_graphs"]
+ other_norm = owner.model.layers[0].input_layernorm
+ other_mlp = owner.model.layers[1].mlp
+ torch.testing.assert_close(shared(other_norm, x), original_norm(other_norm, x))
+ torch.testing.assert_close(shared(other_mlp, x), original_mlp(other_mlp, x))
+ shared_growth = counters["stats"]["unique_graphs"] - before
+ print("OPERATOR_SHARED_GRAPHS", shared_growth)
+ assert shared_growth == 0
+ f8t.restore_fp8_lora_for_inference(model)
+ assert mlp.__dict__.get("forward") is saved_override
+ assert norm.__dict__.get("forward") is saved_norm_override
+ assert torch.equal(mlp(x = x), expected_mlp)
+ assert torch.equal(norm(hidden_states = x), expected_norm)
+ assert calls == {"norm": 1, "mlp": int(mode in ("full", "private_norm", "generated"))}
diff --git a/tests/test_lora_fused_passes.py b/tests/test_lora_fused_passes.py
index 77bfc7ad51c..c6f14c3c77b 100644
--- a/tests/test_lora_fused_passes.py
+++ b/tests/test_lora_fused_passes.py
@@ -1094,6 +1094,35 @@ def test_decode_keeps_a_subclass_adapter(which, bsz, monkeypatch):
_same(got, want, True)
[email protected]("bsz", [1, 3])
[email protected]("fp8", [False, True])
+def test_decode_packed_base_keeps_its_matmul(bsz, fp8):
+ _, block = _model("cpu", torch.float32)
+ proj = block.o_proj
+ calls = []
+
+ class Packed:
+ _unsloth_packed_weight_state = True
+ _unsloth_fp8_dgrad = fp8
+
+ def matmul(
+ self,
+ X,
+ W,
+ out = None,
+ ):
+ calls.append(X.shape)
+ return torch.matmul(X, W.t(), out = out)
+
+ proj.base_layer.weight.quant_state = Packed()
+ X = torch.randn(bsz, 1, H)
+ with torch.inference_mode():
+ expected = proj(X)
+ actual = kernel_utils.fast_linear_forward(proj, X)
+ torch.testing.assert_close(actual, expected)
+ assert calls == [X.shape]
+
+
@pytest.mark.parametrize("which", ["A", "B"])
def test_the_traced_no_grad_accumulate_keeps_a_subclass_result(which, monkeypatch):
# The opaque op returns nothing, so a subclass operand takes the in-place method and keeps its type.
diff --git a/tests/test_return_logits_survives_for_training.py b/tests/test_return_logits_survives_for_training.py
index 2cc8ba82f7e..36fd4625573 100644
--- a/tests/test_return_logits_survives_for_training.py
+++ b/tests/test_return_logits_survives_for_training.py
@@ -77,7 +77,11 @@ def _fast_base_model():
helpers = {n.name: n for n in utils.body if isinstance(n, ast.FunctionDef)}
# Helpers that import a sibling module (a relative import has no package in this namespace): stubbed with what
# they return for these fakes. A model split by hand (apply_pipeline_parallel) keeps no UNSLOTH_PP_SIZE replica.
- stubbed = {"_pipeline_keeps_replica": lambda model: False}
+ stubbed = {
+ "_pipeline_keeps_replica": lambda model: False,
+ "enable_fp8_lora": lambda model: [],
+ "restore_fp8_lora_for_inference": lambda model: None,
+ }
lifted = {n.name for n in wanted} | set(stubbed)
todo = list(wanted)
while todo:
@@ -174,6 +178,7 @@ def _fast_llama_for_training():
arm_gradient_checkpointing = lambda model, checkpointing = None: False,
set_module_gradient_checkpointing = lambda module, flag: None,
_tensor_parallel_for_training = lambda model: None, # tp_replica.py: no tp model here
+ enable_fp8_lora = lambda model: [],
)
module = ast.fix_missing_locations(ast.Module(body = [fn], type_ignores = []))
exec(compile(module, str(MODELS / "llama.py"), "exec"), namespace)
diff --git a/unsloth/kernels/fast_lora.py b/unsloth/kernels/fast_lora.py
index c3036909935..b61333c1261 100644
--- a/unsloth/kernels/fast_lora.py
+++ b/unsloth/kernels/fast_lora.py
@@ -29,6 +29,7 @@ from .utils import (
torch_amp_custom_bwd,
)
from . import rms_layernorm as _rms_layernorm
+from .fp8_lora import has_fp8_dgrad
_is_compiling = torch.compiler.is_compiling
@@ -669,11 +670,17 @@ class LoRA_MLP(torch.autograd.Function):
gateS,
X if ctx.inplace and not _is_compiling() else None,
)
- upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)
- addmm_(dX, df, upW.t())
+ if has_fp8_dgrad(upW_quant):
+ dX = upW_quant.dgrad_accum(df, upW, dX)
+ else:
+ upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)
+ addmm_(dX, df, upW.t())
del upW
- gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)
- addmm_(dX, de, gateW.t())
+ if has_fp8_dgrad(gateW_quant):
+ dX = gateW_quant.dgrad_accum(de, gateW, dX)
+ else:
+ gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)
+ addmm_(dX, de, gateW.t())
del gateW
d_gateBias = d_upBias = None
d_downBias = _lora_bias_grad(dY, downS) if ctx.biases[2] else None
@@ -732,15 +739,21 @@ class LoRA_MLP(torch.autograd.Function):
d_downBias = _lora_bias_grad(dY, downS) if down_has_bias else None
# dX = matmul_lora(df, upW.t(), ...) + matmul_lora(de, gateW.t(), ...), expanded below.
- upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)
- # Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.
- dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)
- del upW
+ if has_fp8_dgrad(upW_quant):
+ dX = upW_quant.dgrad(df, upW)
+ else:
+ upW = fast_dequantize(upW.t(), upW_quant, dtype = df.dtype)
+ # Eager only: AOT autograd rejects a backward mutating a forward input that requires grad.
+ dX = torch.matmul(df, upW.t(), out = X if ctx.inplace and not _is_compiling() else None)
+ del upW
_add_lora_dX(dX, up_dB, upA, upS, up_mask, up_p)
- gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)
- addmm_(dX, de, gateW.t())
- del gateW
+ if has_fp8_dgrad(gateW_quant):
+ dX.add_(gateW_quant.dgrad(de, gateW))
+ else:
+ gateW = fast_dequantize(gateW.t(), gateW_quant, dtype = de.dtype)
+ addmm_(dX, de, gateW.t())
+ del gateW
_add_lora_dX(dX, gate_dB, gateA, gateS, gate_mask, gate_p)
return (
@@ -1109,12 +1122,15 @@ class LoRA_QKV(torch.autograd.Function):
X if ctx.inplace and not _is_compiling() else None,
)
for W_, W_quant_, dY_ in ((QW, QW_quant, dQ), (KW, KW_quant, dK), (VW, VW_quant, dV)):
- W_ = (
- W_.t()
- if W_quant_ is None
- else fast_dequantize(W_.t(), W_quant_, dtype = dY_.dtype)
- )
- addmm_(dX, dY_, W_.t())
+ if has_fp8_dgrad(W_quant_):
+ dX = W_quant_.dgrad_accum(dY_, W_, dX)
+ else:
+ W_ = (
+ W_.t()
+ if W_quant_ is None
+ else fast_dequantize(W_.t(), W_quant_, dtype = dY_.dtype)
+ )
+ addmm_(dX, dY_, W_.t())
del W_
return (
dX.view(batch, seq_len, hd),
@@ -1175,19 +1191,28 @@ class LoRA_QKV(torch.autograd.Function):
d_VBias = _lora_bias_grad(dV, VS) if V_has_bias else None
# Combine the per-projection derivatives into dX.
- QW = fast_dequantize(QW.t(), QW_quant, dtype = dQ.dtype)
- dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)
- del QW
+ if has_fp8_dgrad(QW_quant):
+ dX = QW_quant.dgrad(dQ, QW)
+ else:
+ QW = fast_dequantize(QW.t(), QW_quant, dtype = dQ.dtype)
+ dX = torch.matmul(dQ, QW.t(), out = X if ctx.inplace and not _is_compiling() else None)
+ del QW
_add_lora_dX(dX, q_dB, QA, QS, Q_mask, Q_p)
- KW = fast_dequantize(KW.t(), KW_quant, dtype = dK.dtype)
- addmm_(dX, dK, KW.t())
- del KW
+ if has_fp8_dgrad(KW_quant):
+ dX.add_(KW_quant.dgrad(dK, KW))
+ else:
+ KW = fast_dequantize(KW.t(), KW_quant, dtype = dK.dtype)
+ addmm_(dX, dK, KW.t())
+ del KW
_add_lora_dX(dX, k_dB, KA, KS, K_mask, K_p)
- VW = fast_dequantize(VW.t(), VW_quant, dtype = dV.dtype)
- addmm_(dX, dV, VW.t())
- del VW
+ if has_fp8_dgrad(VW_quant):
+ dX.add_(VW_quant.dgrad(dV, VW))
+ else:
+ VW = fast_dequantize(VW.t(), VW_quant, dtype = dV.dtype)
+ addmm_(dX, dV, VW.t())
+ del VW
_add_lora_dX(dX, v_dB, VA, VS, V_mask, V_p)
return (
@@ -1365,12 +1390,18 @@ class LoRA_W(torch.autograd.Function):
dX = y_dB.new_empty((y_dB.shape[0], A.shape[0])).addmm_(
y_dB, A.t(), beta = 0, alpha = S
)
- W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)
- addmm_(dX, dY, W.t())
+ if has_fp8_dgrad(W_quant):
+ dX = W_quant.dgrad_accum(dY, W, dX)
+ else:
+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)
+ addmm_(dX, dY, W.t())
del W
else:
- W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)
- dX = dY @ W.t()
+ if has_fp8_dgrad(W_quant):
+ dX = W_quant.dgrad(dY, W)
+ else:
+ W = fast_dequantize(W.t(), W_quant, dtype = dY.dtype)
+ dX = dY @ W.t()
del W
_add_lora_dX(dX, y_dB, A, S, mask, ctx.dropout_p)
diff --git a/unsloth/kernels/fp8_lora.py b/unsloth/kernels/fp8_lora.py
new file mode 100644
index 00000000000..3f565965663
--- /dev/null
+++ b/unsloth/kernels/fp8_lora.py
@@ -0,0 +1,967 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""FP8 frozen-base GEMMs with 16-bit LoRA adapters and an exact master for saving and merging.
+
+cuBLAS needs a K-major dgrad operand; transpose inside the opaque op, or cache it with
+UNSLOTH_FP8_LORA_CACHE_T=1. Offloaded masters remain the same Parameter on the host;
+decode reconstructs the FP8 weight, while save and merge restore the exact master.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import functools
+import os
+from typing import Optional
+
+import torch
+
+__all__ = [
+ "FP8_LORA_STATS",
+ "Fp8LoRABaseState",
+ "fp8_lora_fwd",
+ "fp8_lora_dgrad",
+ "quantize_tensorwise",
+ "quantize_rowwise",
+ "attach_fp8_lora_base",
+ "detach_fp8_lora_base",
+ "fp8_lora_memory",
+ "fp8_transpose",
+ "restore_fp8_lora_master",
+ "has_fp8_dgrad",
+ "reset_fp8_lora_stats",
+]
+
+E4M3 = torch.float8_e4m3fn
+E5M2 = torch.float8_e5m2
+_FP8_MAX = {E4M3: 448.0, E5M2: 57344.0}
+FP8_LORA_STATS = {"fwd": 0, "dgrad": 0, "dgrad_16bit": 0, "fwd_16bit": 0, "quantize": 0}
+
+
+def reset_fp8_lora_stats():
+ for key in FP8_LORA_STATS:
+ FP8_LORA_STATS[key] = 0
+
+
+_SCALING = os.environ.get("UNSLOTH_FP8_LORA_SCALING", "auto").strip().lower()
+_SCALING = {"rowwise": "row", "tensorwise": "tensor"}.get(_SCALING, _SCALING)
+if _SCALING not in ("auto", "row", "tensor"):
+ _SCALING = "auto"
+
+
+def scaling_for_class(gpu_class) -> str:
+ """'row' or 'tensor' for a precision-registry GPU class ('sm89', 'sm120', ...); the env var wins."""
+ if _SCALING != "auto":
+ return _SCALING
+ return "row" if gpu_class == "sm120" else "tensor"
+
+
+def _scaling_for(device):
+ gpu_class = None
+ try:
+ if device.type == "cuda" and torch.version.hip is None:
+ gpu_class = "sm120" if torch.cuda.get_device_capability(device)[0] >= 12 else "other"
+ except Exception:
+ pass
+ return scaling_for_class(gpu_class)
+
+
+def _env_dgrad():
+ value = os.environ.get("UNSLOTH_FP8_LORA_DGRAD", "auto").strip().lower()
+ if value in ("0", "false", "off", "no", "bf16", "16bit"):
+ return None
+ return {"e4m3": E4M3, "e5m2": E5M2}.get(value, "auto")
+
+
+_DGRAD_DTYPE = _env_dgrad()
+_FWD_FAST_ACCUM = os.environ.get("UNSLOTH_FP8_LORA_FWD_FAST_ACCUM", "1") != "0"
+_DGRAD_FAST_ACCUM = os.environ.get("UNSLOTH_FP8_LORA_DGRAD_FAST_ACCUM", "0") == "1"
+_CACHE_T = os.environ.get("UNSLOTH_FP8_LORA_CACHE_T", "0") == "1"
+_OFFLOAD_MASTER = os.environ.get("UNSLOTH_FP8_LORA_OFFLOAD_MASTER", "1") != "0"
+_OFFLOAD_CPU_WEIGHTS = False
+_MIN_ROWS = int(os.environ.get("UNSLOTH_FP8_LORA_MIN_ROWS", "16"))
+
+try:
+ import triton
+ import triton.language as tl
+except Exception:
+ triton = None
+_fast_launch = None
+if triton is not None:
+ try:
+ from .triton_launch import launch as _fast_launch, tag_compile_cache
+ except ImportError:
+ pass
+ else:
+ tag_compile_cache(__file__)
+
+
+def _launch(
+ kernel,
+ grid,
+ args,
+ n_tensors,
+ constexprs,
+ device_index,
+ enable_fp_fusion = True,
+ **options,
+):
+ if _fast_launch is not None and not _compiling():
+ return _fast_launch(
+ kernel,
+ grid,
+ args,
+ n_tensors,
+ constexprs,
+ device_index,
+ enable_fp_fusion = enable_fp_fusion,
+ **options,
+ )
+ kernel[grid](*args, **constexprs, enable_fp_fusion = enable_fp_fusion, **options)
+
+
+# Read once: device_count() inside a compiled region graph-breaks.
+_MULTI_GPU = torch.cuda.is_available() and torch.cuda.device_count() > 1
+
+
+if triton is not None:
+
+ @triton.jit
+ def _cast_fp8_rn(x, IS_E5M2: tl.constexpr):
+ # Triton 3.8 rounds through fp16 on sm89; native f32 conversion avoids double rounding.
+ if IS_E5M2:
+ packed = tl.inline_asm_elementwise(
+ "cvt.rn.satfinite.e5m2x2.f32 $0, $1, $1;",
+ constraints = "=h,f",
+ args = [x],
+ dtype = tl.uint16,
+ is_pure = True,
+ pack = 1,
+ )
+ return packed.to(tl.uint8).to(tl.float8e5, bitcast = True)
+ else:
+ packed = tl.inline_asm_elementwise(
+ "cvt.rn.satfinite.e4m3x2.f32 $0, $1, $1;",
+ constraints = "=h,f",
+ args = [x],
+ dtype = tl.uint16,
+ is_pure = True,
+ pack = 1,
+ )
+ return packed.to(tl.uint8).to(tl.float8e4nv, bitcast = True)
+
+ @triton.jit
+ def _fp8_tensorwise_cast_kernel(
+ x_ptr,
+ y_ptr,
+ amax_ptr,
+ inv_ptr,
+ numel,
+ FP8_MAX: tl.constexpr,
+ IS_E5M2: tl.constexpr,
+ BLOCK: tl.constexpr,
+ ):
+ pid = tl.program_id(0).to(tl.int64)
+ offs = pid * BLOCK + tl.arange(0, BLOCK)
+ mask = offs < numel
+ amax = tl.load(amax_ptr).to(tl.float32)
+ # NaN / inf amax propagates into the output, so a GradScaler still sees the overflow.
+ amax = tl.maximum(amax, 1e-12, propagate_nan = tl.PropagateNan.ALL)
+ # Match PyTorch's reciprocal-then-multiply rounding order (_tensor.py::__rdiv__).
+ scale = tl.div_rn(1.0, amax) * FP8_MAX
+ x = tl.load(x_ptr + offs, mask = mask, other = 0.0).to(tl.float32)
+ y = tl.maximum(x * scale, -FP8_MAX, propagate_nan = tl.PropagateNan.ALL)
+ y = tl.minimum(y, FP8_MAX, propagate_nan = tl.PropagateNan.ALL)
+ tl.store(y_ptr + offs, _cast_fp8_rn(y, IS_E5M2), mask = mask)
+ if pid == 0:
+ tl.store(inv_ptr, amax * (1.0 / FP8_MAX))
+
+
+def quantize_tensorwise(x: torch.Tensor, fp8_dtype = E4M3):
+ """x -> (fp8 x with x's shape, contiguous; fp32 0-d dequant scale amax / fp8_max)."""
+ fp8_max = _FP8_MAX[fp8_dtype]
+ x = x.contiguous()
+ amax = torch.linalg.vector_norm(x, float("inf"), dtype = torch.float32)
+ if triton is not None and x.is_cuda and x.numel() > 0:
+ y = torch.empty(x.shape, dtype = fp8_dtype, device = x.device)
+ inv = torch.empty((), dtype = torch.float32, device = x.device)
+ BLOCK = 2048
+ grid = (triton.cdiv(x.numel(), BLOCK),)
+ with torch.cuda.device(x.device) if _MULTI_GPU else contextlib.nullcontext():
+ _launch(
+ _fp8_tensorwise_cast_kernel,
+ grid,
+ (x, y, amax, inv, x.numel()),
+ 4,
+ {"FP8_MAX": fp8_max, "IS_E5M2": fp8_dtype is E5M2, "BLOCK": BLOCK},
+ x.device.index,
+ num_warps = 4,
+ )
+ return y, inv
+ amax = amax.clamp(min = 1e-12) if torch.isfinite(amax) else amax
+ y = (x.float() * (fp8_max / amax)).clamp(-fp8_max, fp8_max).to(fp8_dtype)
+ return y, (amax / fp8_max).reshape(())
+
+
+if triton is not None:
+
+ @triton.jit
+ def _fp8_rowwise_cast_kernel(
+ x_ptr,
+ y_ptr,
+ inv_ptr,
+ K,
+ stride_xm,
+ FP8_MAX: tl.constexpr,
+ IS_E5M2: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ ONE_BLOCK: tl.constexpr,
+ ):
+ row = tl.program_id(0).to(tl.int64)
+ x_row = x_ptr + row * stride_xm
+ y_row = y_ptr + row * K
+ offs = tl.arange(0, BLOCK_K)
+ if ONE_BLOCK:
+ x = tl.load(x_row + offs, mask = offs < K, other = 0.0).to(tl.float32)
+ amax = tl.max(tl.abs(x), axis = 0)
+ nans = tl.sum((x != x).to(tl.int32), axis = 0)
+ else:
+ acc = tl.zeros((BLOCK_K,), dtype = tl.float32)
+ cnt = tl.zeros((BLOCK_K,), dtype = tl.int32)
+ for k in range(0, K, BLOCK_K):
+ xk = tl.load(x_row + k + offs, mask = k + offs < K, other = 0.0).to(tl.float32)
+ acc = tl.maximum(acc, tl.abs(xk))
+ cnt += (xk != xk).to(tl.int32)
+ amax = tl.max(acc, axis = 0)
+ nans = tl.sum(cnt, axis = 0)
+ # tl.max drops NaN: put it back so a GradScaler still sees the overflow.
+ amax = tl.where(nans > 0, float("nan"), amax)
+ amax = tl.maximum(amax, 1e-12, propagate_nan = tl.PropagateNan.ALL)
+ scale = tl.div_rn(1.0, amax) * FP8_MAX
+ if ONE_BLOCK:
+ y = tl.maximum(x * scale, -FP8_MAX, propagate_nan = tl.PropagateNan.ALL)
+ y = tl.minimum(y, FP8_MAX, propagate_nan = tl.PropagateNan.ALL)
+ tl.store(y_row + offs, _cast_fp8_rn(y, IS_E5M2), mask = offs < K)
+ else:
+ for k in range(0, K, BLOCK_K):
+ mask = k + offs < K
+ xk = tl.load(x_row + k + offs, mask = mask, other = 0.0).to(tl.float32)
+ y = tl.maximum(xk * scale, -FP8_MAX, propagate_nan = tl.PropagateNan.ALL)
+ y = tl.minimum(y, FP8_MAX, propagate_nan = tl.PropagateNan.ALL)
+ tl.store(y_row + k + offs, _cast_fp8_rn(y, IS_E5M2), mask = mask)
+ tl.store(inv_ptr + row, amax * (1.0 / FP8_MAX))
+
+
+def quantize_rowwise(x: torch.Tensor, fp8_dtype = E4M3):
+ """x [M, K] -> (fp8 x [M, K] contiguous; fp32 [M, 1] dequant scales, one per row)."""
+ fp8_max = _FP8_MAX[fp8_dtype]
+ if x.stride(-1) != 1:
+ x = x.contiguous()
+ M, K = x.shape
+ if triton is not None and x.is_cuda and x.numel() > 0:
+ y = torch.empty((M, K), dtype = fp8_dtype, device = x.device)
+ inv = torch.empty((M, 1), dtype = torch.float32, device = x.device)
+ one = K <= 8192
+ BLOCK_K = triton.next_power_of_2(K) if one else 2048
+ with torch.cuda.device(x.device) if _MULTI_GPU else contextlib.nullcontext():
+ _launch(
+ _fp8_rowwise_cast_kernel,
+ (M,),
+ (x, y, inv, K, x.stride(0)),
+ 3,
+ {
+ "FP8_MAX": fp8_max,
+ "IS_E5M2": fp8_dtype is E5M2,
+ "BLOCK_K": BLOCK_K,
+ "ONE_BLOCK": one,
+ },
+ x.device.index,
+ num_warps = 8 if BLOCK_K >= 4096 else 4,
+ )
+ return y, inv
+ xf = x.float()
+ amax = xf.abs().amax(dim = 1, keepdim = True)
+ amax = torch.where(torch.isnan(xf).any(1, keepdim = True), torch.nan, amax).clamp(min = 1e-12)
+ y = (xf * (fp8_max / amax)).clamp(-fp8_max, fp8_max).to(fp8_dtype)
+ return y, amax / fp8_max
+
+
+if triton is not None:
+
+ @triton.jit
+ def _transpose_u8_kernel(x_ptr, y_ptr, R, C, BLOCK: tl.constexpr):
+ # Square tiles on purpose: non-square 2D fp8 transposed stores write corrupt bytes on sm89 (L4, triton 3.x).
+ rows = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+ cols = tl.program_id(1).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+ mask = (rows[:, None] < R) & (cols[None, :] < C)
+ tile = tl.load(x_ptr + rows[:, None] * C + cols[None, :], mask = mask)
+ tl.store(y_ptr + cols[None, :] * R + rows[:, None], tile, mask = mask)
+
+
+_TRANSPOSE_BLOCK = 64
+
+
+def fp8_transpose(w8: torch.Tensor, use_triton: Optional[bool] = None) -> torch.Tensor:
+ """w8 [R, C] (contiguous, 1 byte per element) -> contiguous [C, R], byte for byte."""
+ if use_triton is None:
+ use_triton = triton is not None and w8.is_cuda
+ if not use_triton or w8.dim() != 2 or w8.numel() == 0:
+ return w8.t().contiguous()
+ w8 = w8.contiguous()
+ R, C = w8.shape
+ out = torch.empty((C, R), dtype = w8.dtype, device = w8.device)
+ grid = (triton.cdiv(R, _TRANSPOSE_BLOCK), triton.cdiv(C, _TRANSPOSE_BLOCK))
+ with torch.cuda.device(w8.device) if (_MULTI_GPU and w8.is_cuda) else contextlib.nullcontext():
+ _launch(
+ _transpose_u8_kernel,
+ grid,
+ (w8.view(torch.uint8), out.view(torch.uint8), R, C),
+ 2,
+ {"BLOCK": _TRANSPOSE_BLOCK},
+ w8.device.index,
+ num_warps = 4,
+ )
+ return out
+
+
+def _out_dtype(dtype):
+ return torch.bfloat16 if dtype == torch.bfloat16 else torch.float32
+
+
+_ROWWISE_BF16_ONLY = set()
+
+
+def _rowwise_ok(dtype, device) -> bool:
+ return dtype != torch.float16 or device not in _ROWWISE_BF16_ONLY
+
+
+def _rowwise_scaled_mm(a, b, scale_a, scale_b, dtype, fast_accum):
+ """None for fp16 when this device's rowwise GEMM refuses float32 out: bf16 would round 65504 to an inf."""
+ out_dtype = _out_dtype(dtype)
+ if out_dtype == torch.float32 and a.device not in _ROWWISE_BF16_ONLY:
+ try:
+ return _scaled_mm(a, b, scale_a, scale_b, out_dtype, fast_accum)
+ except (RuntimeError, ValueError) as exc:
+ if str(exc) not in (
+ "Only bf16 high precision output types are supported for row-wise scaling.",
+ "Only bf16 and fp16 high precision output types are supported for row-wise scaling.",
+ ):
+ raise
+ _ROWWISE_BF16_ONLY.add(a.device)
+ if dtype == torch.float16:
+ return None
+ return _scaled_mm(a, b, scale_a, scale_b, torch.bfloat16, fast_accum)
+
+
+def _weight_scale(w_inv):
+ return w_inv.reshape(-1)[0] if w_inv.dim() == 2 else w_inv
+
+
+def _scaled_mm(a, b, scale_a, scale_b, out_dtype, fast_accum):
+ out = torch._scaled_mm(
+ a,
+ b,
+ scale_a = scale_a,
+ scale_b = scale_b,
+ out_dtype = out_dtype,
+ use_fast_accum = fast_accum,
+ )
+ return out[0] if isinstance(out, tuple) else out
+
+
+# Module level: a torch.library.Library deregisters its ops once collected.
+_library = None
+
+
+def _opaque_under_compile(name, fake):
+ def decorator(fn):
+ global _library
+ substitute_in_graph = getattr(torch._dynamo, "substitute_in_graph", None)
+ if substitute_in_graph is None or not hasattr(torch.library, "infer_schema"):
+ return fn
+ try:
+ if _library is None:
+ _library = torch.library.Library("unsloth", "FRAGMENT")
+ _library.define(name + torch.library.infer_schema(fn, mutates_args = ()))
+ _library.impl(name, fn, "CompositeExplicitAutograd")
+ torch.library.register_fake(f"unsloth::{name}", fake, lib = _library)
+ substitute_in_graph(fn)(
+ functools.wraps(fn)(
+ lambda *args, **kwargs: getattr(torch.ops.unsloth, name)(*args, **kwargs)
+ )
+ )
+ except (RuntimeError, TypeError, ValueError): # registered by an earlier import
+ pass
+ return fn
+
+ return decorator
+
+
+def _fwd_fake(x, w8, w_inv):
+ return x.new_empty((x.shape[0], w8.shape[0]))
+
+
+@_opaque_under_compile("fp8_lora_fwd", _fwd_fake)
+def fp8_lora_fwd(x: torch.Tensor, w8: torch.Tensor, w_inv: torch.Tensor) -> torch.Tensor:
+ """x [M, K] (16-bit) @ dequant(w8 [N, K]).T -> [M, N] in x.dtype. w_inv: 0-d (tensorwise activations) or
+ [1, N] (rowwise activations: the weight's tensorwise scale per output column)."""
+ FP8_LORA_STATS["fwd"] += 1
+ out = None
+ if w_inv.dim() == 2 and _rowwise_ok(x.dtype, x.device):
+ x8, x_inv = quantize_rowwise(x, E4M3)
+ out = _rowwise_scaled_mm(x8, w8.t(), x_inv, w_inv, x.dtype, _FWD_FAST_ACCUM)
+ if out is None:
+ x8, x_inv = quantize_tensorwise(x, E4M3)
+ out = _scaled_mm(
+ x8, w8.t(), x_inv, _weight_scale(w_inv), _out_dtype(x.dtype), _FWD_FAST_ACCUM
+ )
+ return out if out.dtype == x.dtype else out.to(x.dtype)
+
+
+def _dgrad_fake(dy, w8, w_inv, w_is_t):
+ return dy.new_empty((dy.shape[0], w8.shape[0] if w_is_t else w8.shape[1]))
+
+
+@_opaque_under_compile("fp8_lora_dgrad", _dgrad_fake)
+def fp8_lora_dgrad(
+ dy: torch.Tensor, w8: torch.Tensor, w_inv: torch.Tensor, w_is_t: bool
+) -> torch.Tensor:
+ """dy [M, N] (16-bit) @ dequant(W [N, K]) -> dx [M, K] in dy.dtype. w8 is the cached K-major copy [K, N]
+ (w_is_t) or the forward copy [N, K], transposed here by the square-tile kernel inside the op, never by an
+ Inductor-generated fp8 transpose (those write corrupt bytes for non-square tile configs on sm89)."""
+ FP8_LORA_STATS["dgrad"] += 1
+ w8t = w8 if w_is_t else fp8_transpose(w8)
+ out = None
+ if w_inv.dim() == 2 and _rowwise_ok(dy.dtype, dy.device):
+ dy8, dy_inv = quantize_rowwise(dy, E4M3)
+ out = _rowwise_scaled_mm(dy8, w8t.t(), dy_inv, w_inv, dy.dtype, _DGRAD_FAST_ACCUM)
+ if out is None:
+ dy8, dy_inv = quantize_tensorwise(
+ dy, E5M2 if _DGRAD_DTYPE in (None, "auto") else _DGRAD_DTYPE
+ )
+ out = _scaled_mm(
+ dy8, w8t.t(), dy_inv, _weight_scale(w_inv), _out_dtype(dy.dtype), _DGRAD_FAST_ACCUM
+ )
+ return out if out.dtype == dy.dtype else out.to(dy.dtype)
+
+
+if triton is not None:
+
+ @triton.jit(do_not_specialize = ["N", "numel", "raw_s0", "raw_s1", "delta_s0", "delta_s1"])
+ def _fp8_accum_epilogue_kernel(
+ raw_ptr,
+ x_inv_ptr,
+ w_inv_ptr,
+ delta_ptr,
+ out_ptr,
+ N,
+ numel,
+ raw_s0,
+ raw_s1,
+ delta_s0,
+ delta_s1,
+ ROW_SCALE: tl.constexpr,
+ BLOCK: tl.constexpr,
+ ):
+ offset = tl.program_id(0).to(tl.int64) * BLOCK + tl.arange(0, BLOCK)
+ row, col = offset // N, offset % N
+ mask = offset < numel
+ raw = tl.load(raw_ptr + row * raw_s0 + col * raw_s1, mask, other = 0)
+ sx = tl.load(x_inv_ptr + tl.where(ROW_SCALE, row, 0), mask, other = 0)
+ sw = tl.load(w_inv_ptr + tl.where(ROW_SCALE, col, 0), mask, other = 0)
+ delta = tl.load(delta_ptr + row * delta_s0 + col * delta_s1, mask, other = 0).to(tl.float32)
+ out = tl.inline_asm_elementwise(
+ "{ .reg .f32 v; mul.rn.f32 v, $1, $2; mul.rn.f32 v, v, $3; add.rn.f32 $0, v, $4; }",
+ constraints = "=f,f,f,f,f",
+ args = (raw, sx, sw, delta),
+ dtype = tl.float32,
+ is_pure = True,
+ pack = 1,
+ )
+ tl.store(out_ptr + offset, out, mask)
+
+
+def _fp8_accum_epilogue(raw, x_inv, w_inv, delta):
+ if triton is None or not raw.is_cuda or torch.version.hip is not None:
+ return (raw * x_inv * w_inv + delta.float()).to(delta.dtype)
+ out = torch.empty(raw.shape, device = raw.device, dtype = delta.dtype)
+ with torch.cuda.device(raw.device) if _MULTI_GPU else contextlib.nullcontext():
+ _launch(
+ _fp8_accum_epilogue_kernel,
+ ((raw.numel() + 1023) // 1024,),
+ (
+ raw,
+ x_inv,
+ w_inv,
+ delta,
+ out,
+ raw.shape[1],
+ raw.numel(),
+ *raw.stride(),
+ *delta.stride(),
+ ),
+ 5,
+ {"ROW_SCALE": w_inv.dim() == 2, "BLOCK": 1024},
+ raw.device.index,
+ num_warps = 4,
+ )
+ return out
+
+
+def _accum_fake(x, w8, w_inv, delta, unit_scale, backward, transpose_weight):
+ return delta.new_empty(delta.shape)
+
+
+@_opaque_under_compile("fp8_lora_accum", _accum_fake)
+def fp8_lora_accum(
+ x: torch.Tensor,
+ w8: torch.Tensor,
+ w_inv: torch.Tensor,
+ delta: torch.Tensor,
+ unit_scale: torch.Tensor,
+ backward: bool,
+ transpose_weight: bool,
+) -> torch.Tensor:
+ FP8_LORA_STATS["dgrad" if backward else "fwd"] += 1
+ if transpose_weight:
+ w8 = fp8_transpose(w8)
+ if w_inv.dim() == 2:
+ x8, x_inv = quantize_rowwise(x, E4M3)
+ else:
+ dtype = (
+ E5M2
+ if backward and _DGRAD_DTYPE in (None, "auto")
+ else (_DGRAD_DTYPE if backward else E4M3)
+ )
+ x8, x_inv = quantize_tensorwise(x, dtype)
+ raw = _scaled_mm(
+ x8,
+ w8.t(),
+ unit_scale,
+ unit_scale,
+ torch.float32,
+ _DGRAD_FAST_ACCUM if backward else _FWD_FAST_ACCUM,
+ )
+ return _fp8_accum_epilogue(raw, x_inv, w_inv, delta)
+
+
+def _first_fake(x, w8, w_inv, xa, bt, alpha, unit_scale, backward, transpose_weight):
+ return x.new_empty((x.shape[0], bt.shape[1]))
+
+
+@_opaque_under_compile("fp8_lora_first", _first_fake)
+def fp8_lora_first(
+ x: torch.Tensor,
+ w8: torch.Tensor,
+ w_inv: torch.Tensor,
+ xa: torch.Tensor,
+ bt: torch.Tensor,
+ alpha: float,
+ unit_scale: torch.Tensor,
+ backward: bool,
+ transpose_weight: bool,
+) -> torch.Tensor:
+ delta = x.new_empty((x.shape[0], bt.shape[1]))
+ delta.addmm_(xa, bt, alpha = alpha, beta = 0)
+ return fp8_lora_accum(x, w8, w_inv, delta, unit_scale, backward, transpose_weight)
+
+
+def _is_transposed(W):
+ return W.dim() == 2 and W.stride(0) == 1 and W.stride(1) != 1
+
+
+def _compiling():
+ try:
+ return torch.compiler.is_compiling()
+ except Exception:
+ return False
+
+
+class Fp8LoRABaseState:
+ """Quant state of a frozen LoRA base in Unsloth's LoRA kernels (a packed weight state): ``W`` is the
+ 16-bit weight [out, in] (forward) or its ``.t()`` view (input gradient)."""
+
+ _unsloth_packed_weight_state = True
+ _unsloth_fp8_dgrad = True
+
+ def __init__(self, weight: torch.Tensor):
+ self.dtype = weight.dtype
+ self.device = weight.device # where the fp8 copy and the GEMMs live
+ self.w8 = self.w8t = self.w_inv = self.w_inv_fwd = self.w_inv_dgrad = self.unit_scale = None
+ self.scaling = "tensor"
+ self.offloaded = False # True: the module's 16-bit weight is the host copy
+ self._key = None
+ self.quantizations = 0
+ self.refresh(weight)
+
+ @staticmethod
+ def _weight_key(weight):
+ return (
+ weight.data_ptr(),
+ weight._version,
+ weight.device,
+ weight.dtype,
+ tuple(weight.shape),
+ )
+
+ def refresh(self, weight: torch.Tensor):
+ """Quantize `weight` unless the cache already holds it (same storage, version, device)."""
+ if _compiling():
+ return self
+ key = self._weight_key(weight)
+ if key == self._key:
+ return self
+ with torch.inference_mode(False), torch.no_grad():
+ w = weight.detach()
+ if self.offloaded:
+ w = w.to(self.device)
+ else:
+ self.device = w.device
+ self.w8, self.w_inv = quantize_tensorwise(w, E4M3)
+ self.unit_scale = torch.ones((), device = w.device, dtype = torch.float32)
+ self.w8t = fp8_transpose(self.w8) if _CACHE_T else None
+ self.scaling = _scaling_for(w.device)
+ if self.scaling == "row":
+ self.w_inv_fwd = self.w_inv.reshape(1, 1).expand(1, w.shape[0]).contiguous()
+ self.w_inv_dgrad = self.w_inv.reshape(1, 1).expand(1, w.shape[1]).contiguous()
+ else:
+ self.w_inv_fwd = self.w_inv_dgrad = self.w_inv
+ self.dtype = weight.dtype
+ self._key = key
+ self.quantizations += 1
+ FP8_LORA_STATS["quantize"] += 1
+ return self
+
+ def _dgrad_weight(self):
+ return (self.w8t, True) if self.w8t is not None else (self.w8, False)
+
+ def offload(self, module):
+ """Move module.weight (the 16-bit master) to host RAM, keeping the Parameter object."""
+ weight = module.weight
+ if self.offloaded or (weight.device.type == "cpu" and not _OFFLOAD_CPU_WEIGHTS):
+ return False
+ self.refresh(weight)
+ host = None
+ if weight.is_cuda:
+ try:
+ host = torch.empty(weight.shape, dtype = weight.dtype, device = "cpu", pin_memory = True)
+ except (
+ RuntimeError
+ ): # pinned host memory exhausted: pageable works, restores are slower
+ host = None
+ if host is None:
+ host = torch.empty(weight.shape, dtype = weight.dtype, device = "cpu")
+ host.copy_(weight.detach())
+ weight.data = host
+ self.offloaded = True
+ self._key = self._weight_key(weight)
+ return True
+
+ def restore(self, module):
+ if not self.offloaded:
+ return False
+ weight = module.weight
+ with torch.inference_mode(False), torch.no_grad():
+ weight.data = weight.data.to(self.device, copy = True)
+ self.offloaded = False
+ self._key = self._weight_key(weight)
+ return True
+
+ def _dense(self, W):
+ """A 16-bit operand on the compute device for the non-fp8 fallbacks: W itself, or (master on the host)
+ the dequantized fp8 weight in W's orientation, which is what the fp8 forward uses anyway."""
+ if W is not None and not self.offloaded and W.device == self.device:
+ return W
+ dense = self.w8.to(torch.float32) * self.w_inv
+ dense = dense.to(self.dtype if W is None else W.dtype)
+ return dense.t() if W is not None and _is_transposed(W) else dense
+
+ def resident_bytes(self, module = None):
+ fp8 = sum(
+ t.numel() * t.element_size()
+ for t in {
+ id(t): t
+ for t in (
+ self.w8,
+ self.w8t,
+ self.w_inv,
+ self.w_inv_fwd,
+ self.w_inv_dgrad,
+ self.unit_scale,
+ )
+ if t is not None
+ }.values()
+ )
+ out = {"fp8": fp8, "master_device": 0, "master_host": 0}
+ if module is not None:
+ w = module.weight
+ nbytes = w.numel() * w.element_size()
+ if self.offloaded:
+ out["master_host"] = nbytes
+ elif w.device == self.device:
+ out["master_device"] = nbytes
+ return out
+
+ def dequantize(
+ self,
+ W = None,
+ dtype = None,
+ ):
+ if W is not None and W.device != self.device:
+ return W.to(self.device)
+ return W
+
+ def _use_fp8(self, X):
+ return X.dtype in (torch.bfloat16, torch.float16, torch.float32) and (
+ X.numel() // max(X.shape[-1], 1) >= _MIN_ROWS
+ )
+
+ def matmul(
+ self,
+ X,
+ W,
+ out = None,
+ ):
+ """``X @ W.t()``: the forward for the stored W, the input gradient for its transposed view."""
+ shape = X.shape
+ X2 = X.reshape(-1, shape[-1])
+ if not self._use_fp8(X2):
+ FP8_LORA_STATS["fwd_16bit"] += 1
+ result = torch.matmul(X2, self._dense(W).t().to(X2.dtype))
+ elif _is_transposed(W):
+ result = self.dgrad(X2, W.t())
+ else:
+ result = fp8_lora_fwd(X2, self.w8, self.w_inv_fwd)
+ result = result.view(*shape[:-1], result.shape[-1])
+ return result if out is None else out.copy_(result)
+
+ def lora_first(
+ self,
+ X,
+ W,
+ XA,
+ Bt,
+ alpha,
+ out = None,
+ ):
+ backward = _is_transposed(W)
+ if not self._use_fp8(X) or (backward and _DGRAD_DTYPE is None):
+ delta = X.new_empty((X.shape[0], Bt.shape[1]))
+ delta.addmm_(XA, Bt, alpha = alpha, beta = 0)
+ result = delta.addmm_(X, self._dense(W).t().to(X.dtype))
+ else:
+ w8 = self._dgrad_weight()[0] if backward else self.w8
+ inv = self.w_inv_dgrad if backward else self.w_inv_fwd
+ result = fp8_lora_first(
+ X, w8, inv, XA, Bt, alpha, self.unit_scale, backward, backward and self.w8t is None
+ )
+ return result if out is None else out.copy_(result)
+
+ def dgrad_accum(self, dY, W, delta):
+ if _DGRAD_DTYPE is None or not self._use_fp8(dY):
+ return delta.addmm_(dY, self._dense(W).to(dY.dtype))
+ w8, transposed = self._dgrad_weight()
+ return fp8_lora_accum(
+ dY, w8, self.w_inv_dgrad, delta, self.unit_scale, True, not transposed
+ )
+
+ def dgrad(
+ self,
+ dY,
+ W = None,
+ ):
+ """dY [..., out] @ W -> [..., in]: the base layer's input gradient."""
+ shape = dY.shape
+ dY2 = dY.reshape(-1, shape[-1])
+ if _DGRAD_DTYPE is None or not self._use_fp8(dY2):
+ FP8_LORA_STATS["dgrad_16bit"] += 1
+ if W is None and not self.offloaded:
+ raise RuntimeError("Unsloth: 16-bit dgrad needs the base weight")
+ result = torch.matmul(dY2, self._dense(W).to(dY2.dtype))
+ else:
+ w8, w_is_t = self._dgrad_weight()
+ result = fp8_lora_dgrad(dY2, w8, self.w_inv_dgrad, w_is_t)
+ return result.view(*shape[:-1], result.shape[-1])
+
+
+def has_fp8_dgrad(W_quant) -> bool:
+ return getattr(type(W_quant), "_unsloth_fp8_dgrad", False)
+
+
+class _Fp8FrozenLinear(torch.autograd.Function):
+ @staticmethod
+ def forward(ctx, x, weight, w8, w8t, w_inv_fwd, w_inv_dgrad):
+ ctx.w_is_t = w8t is not None
+ ctx.save_for_backward(
+ weight,
+ w8t if w8t is not None else w8,
+ w_inv_fwd if w_inv_dgrad is None else w_inv_dgrad,
+ )
+ shape = x.shape
+ x2 = x.reshape(-1, shape[-1])
+ y = fp8_lora_fwd(x2, w8, w_inv_fwd)
+ return y.view(*shape[:-1], y.shape[-1])
+
+ @staticmethod
+ def backward(ctx, dy):
+ weight, w8t, w_inv = ctx.saved_tensors
+ shape = dy.shape
+ dy2 = dy.reshape(-1, shape[-1])
+ if _DGRAD_DTYPE is None:
+ FP8_LORA_STATS["dgrad_16bit"] += 1
+ dx = torch.matmul(dy2, weight.to(dy2.dtype))
+ else:
+ dx = fp8_lora_dgrad(dy2, w8t, w_inv, ctx.w_is_t)
+ return dx.view(*shape[:-1], dx.shape[-1]), None, None, None, None, None
+
+
+def _fp8_base_forward(self, x):
+ state = self.__dict__.get("_unsloth_fp8_lora")
+ weight = self.weight
+ if state is None or weight.requires_grad or not state._use_fp8(x.reshape(-1, x.shape[-1])):
+ FP8_LORA_STATS["fwd_16bit"] += 1
+ if state is not None and state.offloaded:
+ if weight.requires_grad: # unfrozen: train the exact weight
+ state.restore(self)
+ else:
+ weight = state._dense(weight)
+ return torch.nn.functional.linear(x, weight.to(x.dtype), self.bias)
+ state.refresh(weight)
+ # Tensorwise: one scale serves both GEMMs (Dynamo rejects the same tensor passed twice to apply).
+ dgrad_scale = None if state.w_inv_dgrad is state.w_inv_fwd else state.w_inv_dgrad
+ y = _Fp8FrozenLinear.apply(x, weight, state.w8, state.w8t, state.w_inv_fwd, dgrad_scale)
+ if self.bias is not None:
+ y = y + self.bias.to(y.dtype)
+ return y
+
+
+def eligible_base(module) -> Optional[str]:
+ if type(module) is not torch.nn.Linear:
+ return f"{type(module).__name__} is not a plain nn.Linear"
+ weight = module.weight
+ if type(weight) not in (torch.Tensor, torch.nn.Parameter):
+ return f"weight is a {type(weight).__name__}"
+ if getattr(weight, "_unsloth_fsdp2_preload", None) is not None or tuple(weight.shape) != (
+ module.out_features,
+ module.in_features,
+ ):
+ return f"weight is a {tuple(weight.shape)} shard of {module.out_features} x {module.in_features}"
+ if not weight.is_contiguous():
+ return "weight is not contiguous"
+ if weight.dtype not in (torch.bfloat16, torch.float16):
+ return f"weight dtype {weight.dtype}"
+ if weight.requires_grad:
+ return "weight is trainable"
+ if module.in_features % 16 or module.out_features % 16:
+ return f"{module.in_features} x {module.out_features} is not a multiple of 16"
+ if getattr(module, "weight_fake_quantizer", None) is not None:
+ return "QAT fake-quantized"
+ return None
+
+
+def offload_supported() -> bool:
+ """The host master needs a quantized-weight dgrad and one process owning the weights."""
+ if not _OFFLOAD_MASTER or _DGRAD_DTYPE is None:
+ return False
+ try:
+ if int(os.environ.get("WORLD_SIZE", "1")) > 1:
+ return False
+ import torch.distributed as dist
+ if dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1:
+ return False
+ except Exception:
+ return False
+ return True
+
+
+def _fp8_base_apply(
+ self,
+ fn,
+ recurse = True,
+):
+ state = self.__dict__.get("_unsloth_fp8_lora")
+ if state is not None and state.offloaded:
+ probe = fn(torch.empty(0, dtype = state.dtype, device = state.device))
+ if probe.device == state.device and probe.dtype == state.dtype:
+ weight = self._parameters.pop("weight")
+ try:
+ return torch.nn.Module._apply(self, fn, recurse)
+ finally:
+ self._parameters["weight"] = weight
+ state.restore(self)
+ out = torch.nn.Module._apply(self, fn, recurse)
+ if state is not None:
+ state._key = None
+ state.refresh(self.weight)
+ return out
+
+
+def attach_fp8_lora_base(module, offload: Optional[bool] = None) -> Fp8LoRABaseState:
+ """Quantize the frozen base `module` (nn.Linear) once and route its GEMMs through fp8. offload: move the
+ 16-bit master to host RAM (None: offload_supported())."""
+ import types
+
+ state = module.__dict__.get("_unsloth_fp8_lora")
+ if state is None:
+ state = Fp8LoRABaseState(module.weight)
+ module.__dict__["_unsloth_fp8_lora"] = state
+ module.forward = types.MethodType(_fp8_base_forward, module)
+ module._apply = types.MethodType(_fp8_base_apply, module)
+ module.__dict__["_unsloth_fp8_lora_load_hook"] = module.register_load_state_dict_post_hook(
+ _fp8_requantize_after_load
+ )
+ if offload is None:
+ offload = offload_supported()
+ if offload:
+ state.offload(module)
+ return state
+
+
+def _fp8_requantize_after_load(module, incompatible_keys):
+ state = module.__dict__.get("_unsloth_fp8_lora")
+ if state is not None:
+ state._key = None
+ state.refresh(module.weight)
+
+
+def restore_fp8_lora_master(module) -> bool:
+ """Bring a base layer's exact 16-bit weight back to its compute device (PEFT merge, export)."""
+ state = module.__dict__.get("_unsloth_fp8_lora")
+ return state.restore(module) if state is not None else False
+
+
+def detach_fp8_lora_base(module):
+ restore_fp8_lora_master(module)
+ module.__dict__.pop("_unsloth_fp8_lora", None)
+ hook = module.__dict__.pop("_unsloth_fp8_lora_load_hook", None)
+ if hook is not None:
+ hook.remove()
+ module.__dict__.pop("forward", None)
+ module.__dict__.pop("_apply", None)
+
+
+def fp8_lora_memory(model) -> dict:
+ """fp8 / device-master / host-master bytes per converted base layer and in total."""
+ layers, total = {}, {"fp8": 0, "master_device": 0, "master_host": 0}
+ for name, module in model.named_modules():
+ state = module.__dict__.get("_unsloth_fp8_lora")
+ if state is None:
+ continue
+ layers[name] = state.resident_bytes(module)
+ for key in total:
+ total[key] += layers[name][key]
+ return {"layers": layers, "total": total}
diff --git a/unsloth/kernels/utils.py b/unsloth/kernels/utils.py
index 69030e330da..e9e66f1c249 100644
--- a/unsloth/kernels/utils.py
+++ b/unsloth/kernels/utils.py
@@ -484,6 +484,14 @@ def _has_packed_qweight(proj):
)
+def _fp8_lora_state(base_layer, W):
+ """The fp8 frozen-base state (kernels/fp8_lora.py, float8_training with LoRA), refreshed if W changed."""
+ state = base_layer.__dict__.get("_unsloth_fp8_lora")
+ if state is None or W.requires_grad:
+ return None
+ return state.refresh(W)
+
+
def _is_packed_state(quant_state):
return getattr(type(quant_state), "_unsloth_packed_weight_state", False)
@@ -527,6 +535,8 @@ def get_lora_parameters(proj):
if packed is not None:
W_quant = packed[1]
+ elif W_quant is None:
+ W_quant = _fp8_lora_state(base_layer, W)
if getattr(proj, "disable_adapters", True) or proj.merged:
return W, W_quant, None, None, None
@@ -590,6 +600,8 @@ def get_lora_parameters_bias(proj):
if packed is not None:
W_quant = packed[1]
+ elif W_quant is None:
+ W_quant = _fp8_lora_state(base_layer, W)
if getattr(proj, "disable_adapters", True) or proj.merged:
return W, W_quant, None, None, None, base_layer.bias
@@ -2024,6 +2036,8 @@ def _lora_fused_ok(W, W_quant, dtype):
return False
if not _is_compiling() and W.layout != torch.strided:
return False
+ if getattr(type(W_quant), "_unsloth_fp8_dgrad", False):
+ return W_quant.dtype == dtype
if type(W_quant) is Int4QuantState or _is_packed_state(W_quant):
return False
if W_quant is None:
@@ -2081,22 +2095,7 @@ def matmul_lora(
if XA is not None and out is not None and _shares_storage(out, XA):
XA = None
- if isinstance(W, Float8Tensor):
- assert W.ndim == 2
- if W.block_size[0] == W.shape[0] and W.block_size[1] == 1:
- # Rowwise scaling becomes colwise after transpose, so this detects the backward pass.
- W = W.dequantize()
- else:
- W = W.contiguous()
- out = torch_matmul(X, W.t(), out = out)
- elif W.dtype == torch.float8_e4m3fn:
- out = fp8_linear(X, W, W_quant)
- elif type(W_quant) is Int4QuantState and W.stride(-1) == 1:
- # Forward orientation: few rows use the fused kernel, the rest decode once + cuBLAS.
- out = _int4_matmul(X, W, W_quant, out = out)
- elif _is_packed_state(W_quant):
- out = W_quant.matmul(X, W, out = out)
- elif (
+ lora_first = (
A is not None
and fused
# Compiled inside torch.func transforms or native forward-mode AD: the op has no forward-mode or
@@ -2119,11 +2118,31 @@ def matmul_lora(
and not _shares_storage(out, W, X, A, B, XA)
)
)
+ )
+ if isinstance(W, Float8Tensor):
+ assert W.ndim == 2
+ if W.block_size[0] == W.shape[0] and W.block_size[1] == 1:
+ # Rowwise scaling becomes colwise after transpose, so this detects the backward pass.
+ W = W.dequantize()
+ else:
+ W = W.contiguous()
+ out = torch_matmul(X, W.t(), out = out)
+ elif W.dtype == torch.float8_e4m3fn:
+ out = fp8_linear(X, W, W_quant)
+ elif type(W_quant) is Int4QuantState and W.stride(-1) == 1:
+ # Forward orientation: few rows use the fused kernel, the rest decode once + cuBLAS.
+ out = _int4_matmul(X, W, W_quant, out = out)
+ elif _is_packed_state(W_quant) and not (
+ lora_first and getattr(type(W_quant), "_unsloth_fp8_dgrad", False)
):
+ out = W_quant.matmul(X, W, out = out)
+ elif lora_first:
if XA is None:
XA = torch_matmul(X, A.t() if A.dtype is dtype else A.t().to(dtype))
B = B.t() if B.dtype is dtype else B.t().to(dtype)
- if out is not None:
+ if getattr(type(W_quant), "_unsloth_fp8_dgrad", False):
+ out = W_quant.lora_first(X, W, XA, B, s, out = out)
+ elif out is not None:
# Eager only: a traced caller buffer cannot be checked for aliasing (_shares_storage). Dequantize
# first: the caller's buffer may share storage with the quant state (absmax, code, ...).
if W_quant is not None:
diff --git a/unsloth/models/_block_compile.py b/unsloth/models/_block_compile.py
index b46911975e2..ff6b617964a 100644
--- a/unsloth/models/_block_compile.py
+++ b/unsloth/models/_block_compile.py
@@ -1406,8 +1406,10 @@ def _wrap(
cls,
compile_options,
kinds = 1,
+ *,
+ install = True,
):
- original = cls.forward
+ original = getattr(cls.forward, "_unsloth_block_original", cls.forward)
# Host-read packed lengths / PrefixGrouper segments stay stock.
unsloth_forward = getattr(original, "__module__", "").startswith("unsloth.models.")
try:
@@ -1662,7 +1664,9 @@ def _wrap(
forward._unsloth_block_original = original
forward._unsloth_block_state = state
setattr(forward, _MARK, True)
- cls.forward = forward
+ if install:
+ cls.forward = forward
+ return forward
_TOWER_SEGMENTS = (
@@ -1971,10 +1975,28 @@ def patch_block_compile(model):
continue
_wrap(cls, _COMPILE_OPTIONS, len(kinds.get(cls, ())) or 1)
wrapped.append(cls.__name__)
+ for layer in compiled_layers:
+ if not any(m.__dict__.get("_unsloth_fp8_lora") is not None for m in layer.modules()):
+ continue
+ default = type(layer).forward
+ precise = getattr(default, "_unsloth_fp8_block_forward", None)
+ if precise is None:
+ # FP8 dgrad amplifies elided BF16 rounding; other instances of this class keep their options.
+ precise = _wrap(
+ type(layer),
+ {**_COMPILE_OPTIONS, "emulate_precision_casts": True},
+ len(kinds.get(type(layer), ())) or 1,
+ install = False,
+ )
+ default._unsloth_fp8_block_forward = precise
+ if getattr(layer.forward, "__func__", None) is not precise:
+ layer.forward = types.MethodType(precise, layer)
+ layer.__dict__.pop("_unsloth_block_checked", None)
if _COMPILE_OPTIONS.get("triton.cudagraphs") or (
_CUDAGRAPH_MODE == "auto" and _auto_platform_reason() is None
):
_watch_hidden_states(model)
+ model.__dict__["_unsloth_block_compile_installed"] = True
if _CUDAGRAPH_MODE == "auto":
# A new model decides again: a launch-bound model loaded first must not turn graphs on for
# a GPU-bound one. Graphs that went off for shape churn stay off.
diff --git a/unsloth/models/float8_training.py b/unsloth/models/float8_training.py
index 1cf9916d977..4b6742ece7f 100644
--- a/unsloth/models/float8_training.py
+++ b/unsloth/models/float8_training.py
@@ -6,6 +6,8 @@ torchao.float8.Float8LinearConfig); with native FSDP2 convert before sharding.""
from __future__ import annotations
+from functools import lru_cache
+
import torch
@@ -49,6 +51,10 @@ __all__ = [
"resolve_float8_training",
"float8_training_filter",
"convert_model_to_float8_training",
+ "resolve_fp8_lora",
+ "mark_fp8_lora",
+ "enable_fp8_lora",
+ "restore_fp8_lora_for_inference",
]
# Names and gates come from the precision registry (unsloth/precision/recipes.py).
@@ -408,3 +414,609 @@ def convert_model_to_float8_training(
except Exception:
pass
return converted
+
+
+FP8_LORA_RECIPE = "fp8_lora_tensorwise"
+_FP8_LORA_BY_NAME = ("tensorwise",)
+_FP8_LORA_EXPLICIT = ("fp8_lora", FP8_LORA_RECIPE)
+
+
+def resolve_fp8_lora(
+ float8_training,
+ *,
+ dtype = None,
+ load_in_4bit = False,
+ load_in_8bit = False,
+ load_in_fp8 = False,
+ qat_scheme = None,
+ gpu = "current",
+):
+ """`float8_training` with LoRA / QLoRA -> FP8_LORA_RECIPE or None; refuses (PrecisionRefused, a ValueError)
+ before anything loads. `gpu` is a registry GPU class ('sm89', ...); 'current' reads the CUDA device."""
+ if float8_training is None or float8_training is False:
+ return None
+ refused = _precision.PrecisionRefused
+ if float8_training is True:
+ explicit = False
+ elif isinstance(float8_training, str) and float8_training.strip().lower() in (
+ _FP8_LORA_BY_NAME + _FP8_LORA_EXPLICIT
+ ):
+ explicit = float8_training.strip().lower() in _FP8_LORA_EXPLICIT
+ else:
+ raise ValueError(
+ f"Unsloth: float8_training = {float8_training!r} is a full finetuning recipe "
+ "(`full_finetuning = True`). With LoRA, float8_training = 'fp8_lora' (or True where it is "
+ "measured faster) runs the frozen base's forward and input gradient in fp8."
+ )
+ if load_in_4bit or load_in_8bit or (load_in_fp8 not in (False, None)):
+ raise refused(
+ "Unsloth: float8_training with LoRA quantizes a 16-bit base to fp8 itself; pass "
+ "load_in_4bit = False (and no load_in_8bit / load_in_fp8). QLoRA bases are already 4 / 8 bit."
+ )
+ if qat_scheme is not None:
+ raise refused(
+ "Unsloth: float8_training with LoRA and qat_scheme cannot be combined: QAT fake-quantizes the "
+ "base weight that fp8 LoRA would quantize."
+ )
+ if gpu == "current":
+ gpu = _precision.current_gpu_class()
+ if gpu is None:
+ raise refused(
+ "Unsloth: float8_training with LoRA needs an NVIDIA GPU with compute capability 8.9+ "
+ "(L4, RTX 40 / 50, RTX PRO 6000, H100, B200); no CUDA device is visible."
+ )
+ recipe = fp8_lora_recipe_for(gpu)
+ gate = _check_precision_recipe(recipe, gpu = gpu, input_dtype = dtype, mode = "lora")
+ if not explicit and gate.status != "named":
+ raise refused(
+ f"Unsloth: float8_training = {float8_training!r} with LoRA has not been measured faster than "
+ f"bfloat16 LoRA on {gpu} ({gate.status}), so it is not enabled by name there. Pass "
+ "float8_training = 'fp8_lora' to run the fp8 frozen base anyway."
+ )
+ if gate.status == "unprobed":
+ print(f"Unsloth: fp8 LoRA has not been tested on {gpu}; running it as requested.")
+ return recipe
+
+
+def fp8_lora_recipe_for(gpu):
+ """The frozen-base recipe the kernels run on this GPU class: per-token scales on sm120, tensorwise
+ elsewhere (UNSLOTH_FP8_LORA_SCALING overrides; kernels/fp8_lora.scaling_for_class is the one rule)."""
+ scaling = _load_fp8_lora_kernels().scaling_for_class(gpu)
+ return "fp8_lora_rowwise" if scaling == "row" else FP8_LORA_RECIPE
+
+
+def mark_fp8_lora(model, recipe):
+ """Remember the request on the loaded model; get_peft_model converts the LoRA bases. A model that is
+ already a PeftModel (an adapter repo was loaded, so get_peft_model is skipped) is converted now."""
+ if recipe is not None and model is not None:
+ try:
+ model._unsloth_fp8_lora_recipe = recipe
+ except Exception:
+ pass
+ if getattr(model, "peft_config", None):
+ enable_fp8_lora(model, recipe)
+ return model
+
+
+def _fp8_lora_request(model):
+ stack, seen = [model], set()
+ while stack and len(seen) < 8:
+ m = stack.pop()
+ if m is None or id(m) in seen:
+ continue
+ seen.add(id(m))
+ recipe = getattr(m, "_unsloth_fp8_lora_recipe", None)
+ if isinstance(recipe, str):
+ return recipe
+ stack.extend(getattr(m, attr, None) for attr in ("base_model", "model"))
+ return None
+
+
+def _load_fp8_lora_kernels():
+ try:
+ from ..kernels import fp8_lora
+ return fp8_lora
+ except ImportError:
+ import importlib.util
+ import os
+ import sys
+
+ name = "_unsloth_fp8_lora_standalone"
+ if name in sys.modules:
+ return sys.modules[name]
+ path = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "kernels", "fp8_lora.py"
+ )
+ spec = importlib.util.spec_from_file_location(name, path)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return module
+
+
+def _merge_with_master(
+ self,
+ *args,
+ _unsloth_method = "merge",
+ **kwargs,
+):
+ base = getattr(self, "base_layer", None)
+ if base is not None:
+ _load_fp8_lora_kernels().restore_fp8_lora_master(base)
+ out = getattr(type(self), _unsloth_method)(self, *args, **kwargs)
+ state = base.__dict__.get("_unsloth_fp8_lora") if base is not None else None
+ if state is not None:
+ state._key = None
+ state.refresh(base.weight)
+ return out
+
+
+def _unload_fp8_lora(self, *args, **kwargs):
+ restore_fp8_lora_for_inference(self.model)
+ return type(self)._unload_and_optionally_merge(self, *args, **kwargs)
+
+
+def _has_variant(lora_module, adapter_name = None) -> bool:
+ """Variants may read the master directly or initialize tensors on its device."""
+ dora = getattr(lora_module, "use_dora", {}) or {}
+ variants = getattr(lora_module, "lora_variant", {}) or {}
+ if adapter_name is not None:
+ return bool(dora.get(adapter_name, False)) or adapter_name in variants
+ return any(dora.values()) or bool(variants)
+
+
+def _move_adapter_with_master(self, adapter_name, *args, **kwargs):
+ """Place adapters on the compute device; variants also need their master there."""
+ base = getattr(self, "base_layer", None)
+ state = base.__dict__.get("_unsloth_fp8_lora") if base is not None else None
+ if state is not None and state.offloaded and _has_variant(self, adapter_name):
+ _load_fp8_lora_kernels().restore_fp8_lora_master(base)
+ out = type(self)._move_adapter_to_device_of_base_layer(self, adapter_name, *args, **kwargs)
+ if state is None:
+ return out
+ state._key = None
+ state.refresh(base.weight)
+ if not state.offloaded:
+ return out
+ for name in tuple(getattr(self, "adapter_layer_names", ())) + tuple(
+ getattr(self, "other_param_names", ())
+ ):
+ layer = getattr(self, name, None)
+ if (
+ not isinstance(layer, (torch.nn.ModuleDict, torch.nn.ParameterDict))
+ or adapter_name not in layer
+ ):
+ continue
+ entry = layer[adapter_name]
+ tensors = list(entry.parameters()) if isinstance(entry, torch.nn.Module) else [entry]
+ if any(t.device.type == "meta" for t in tensors):
+ # low_cpu_mem_usage: PEFT leaves meta parameters for the checkpoint loader to materialize.
+ continue
+ if isinstance(entry, torch.nn.Module):
+ layer[adapter_name] = entry.to(state.device)
+ else:
+ layer[adapter_name] = torch.nn.Parameter(
+ entry.data.to(state.device), requires_grad = entry.requires_grad
+ )
+ return out
+
+
+def _restore_master_before_merge(lora_module):
+ import functools
+ import types
+
+ for method in ("merge", "unmerge"):
+ if method in lora_module.__dict__ or not callable(getattr(type(lora_module), method, None)):
+ continue
+ fn = functools.partial(_merge_with_master, _unsloth_method = method)
+ object.__setattr__(lora_module, method, types.MethodType(fn, lora_module))
+ move = "_move_adapter_to_device_of_base_layer"
+ if move not in lora_module.__dict__ and callable(getattr(type(lora_module), move, None)):
+ object.__setattr__(
+ lora_module, move, types.MethodType(_move_adapter_with_master, lora_module)
+ )
+
+
+_RAW_READS = {}
+
+
+def _reads_raw_weight(parent, leaf) -> bool:
+ """Direct weight readers bypass the patched forward, so their masters must stay resident."""
+ import inspect
+ import re
+
+ key = (type(parent), leaf)
+ if key not in _RAW_READS:
+ try:
+ source = inspect.getsource(type(parent))
+ except (OSError, TypeError):
+ source = ""
+ _RAW_READS[key] = re.search(rf"\b{re.escape(leaf)}\.weight\b", source) is not None
+ return _RAW_READS[key]
+
+
+def _mamba_raw_weight_leaf(model, name) -> bool:
+ parent_name, _, leaf = name.rpartition(".")
+ try:
+ parent = model.get_submodule(parent_name) if parent_name else model
+ except AttributeError:
+ return False
+ if _reads_raw_weight(parent, leaf):
+ return True
+ try:
+ from ._mamba_lora import _RAW_WEIGHT_LEAVES, _is_mamba_mixer
+ except Exception:
+ return False
+ # Unsloth's fused Mamba LoRA path builds W + delta from these leaves of any Mamba-style mixer.
+ return leaf in _RAW_WEIGHT_LEAVES and _is_mamba_mixer(parent)
+
+
+def _block_swapped_modules(model) -> set:
+ out = set()
+ for module in model.modules():
+ swapper = module.__dict__.get("_unsloth_block_swap")
+ if swapper is None or not isinstance(module, torch.nn.ModuleList):
+ continue
+ for index in getattr(swapper, "indices", ()) or ():
+ if 0 <= index < len(module):
+ out.update(id(m) for m in module[index].modules())
+ return out
+
+
+def _fp8_lora_owner(model):
+ getter = getattr(model, "get_base_model", None)
+ return getter() if getattr(model, "peft_config", None) and callable(getter) else model
+
+
+@lru_cache(maxsize = 128)
+def _fp8_lora_operator_code(code, names):
+ return code.replace(co_varnames = names + code.co_varnames[2:])
+
+
+@lru_cache(maxsize = 128)
+def _fp8_lora_proxy_code(name, receiver, value):
+ source = f"def forward({receiver}, {value}):\n return {name}({receiver}, {value}={value})\n"
+ return compile(source, "<Unsloth FP8 forwarding contract>", "exec").co_consts[0]
+
+
+def _prepare_fp8_lora_operators(model, allow_cpu = False):
+ import inspect
+ import os
+ import types
+ from transformers.activations import ACT2FN
+ from ..kernels.fast_lora import apply_lora_mlp_swiglu
+ from ..kernels.rms_layernorm import fast_rms_layernorm
+ from unsloth_zoo.temporary_patches.common import unwrap_already_compiled
+
+ owner = _fp8_lora_owner(model)
+ if os.environ.get("UNSLOTH_FP8_LORA_FUSED_OPS", "1") == "0" or owner.__dict__.get(
+ "_unsloth_fp8_lora_kernel_forwards"
+ ):
+ return
+
+ def fingerprint(function):
+ if isinstance(function, types.CodeType):
+ code = function
+ elif isinstance(function, types.FunctionType):
+ code = function.__code__
+ else:
+ return None
+ return (
+ code.co_code,
+ code.co_consts,
+ code.co_names,
+ code.co_argcount,
+ code.co_kwonlyargcount,
+ code.co_posonlyargcount,
+ code.co_flags & (inspect.CO_VARARGS | inspect.CO_VARKEYWORDS),
+ )
+
+ def fused_mlp(self, X):
+ X = X.contiguous()
+ if X.ndim == 2:
+ return apply_lora_mlp_swiglu(self, X.unsqueeze(0), inplace = False).squeeze(0)
+ return apply_lora_mlp_swiglu(self, X, inplace = False)
+
+ def rms_equation(self, x):
+ dtype = x.dtype
+ x = x.to(torch.float32)
+ variance = x.pow(2).mean(-1, keepdim = True)
+ x = x * torch.rsqrt(variance + self.variance_epsilon)
+ return self.weight * x.to(dtype)
+
+ def fused_rms(self, X):
+ if X.dtype not in (torch.float16, torch.bfloat16) or X.dtype != self.weight.dtype:
+ return rms_equation(self, X)
+ return fast_rms_layernorm(self, X)
+
+ def mlp_equation(self, x):
+ output = self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+ return output
+
+ def mlp_expression(self, x):
+ return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))
+
+ templates = (
+ (rms_equation, fused_rms),
+ (mlp_equation, fused_mlp),
+ (mlp_expression, fused_mlp),
+ )
+ cache = owner.__dict__.setdefault("_unsloth_fp8_lora_kernel_cache", {})
+ forwards = []
+ for module in model.modules():
+ if not type(module).__module__.startswith(
+ ("transformers.models.", "unsloth_compiled_module_")
+ ):
+ continue
+ for name in ("forward", "_old_forward"):
+ method = getattr(module, name, None)
+ function = getattr(method, "__func__", None)
+ if function is None or function.__code__.co_argcount != 2:
+ continue
+ equation = function
+ if type(module).__module__.startswith("unsloth_compiled_module_"):
+ name_forward = type(module).__name__ + "_forward"
+ if function.__code__.co_names != (name_forward,):
+ continue
+ proxy = _fp8_lora_proxy_code(name_forward, *function.__code__.co_varnames[:2])
+ if fingerprint(function) != fingerprint(proxy):
+ continue
+ equation = unwrap_already_compiled(function.__globals__.get(name_forward))
+ for template, kernel in templates:
+ # Match the installed native equation, including constants, before replacing it.
+ if fingerprint(equation) != fingerprint(template):
+ continue
+ if function.__code__.co_argcount != 2:
+ continue
+ if kernel is fused_mlp:
+ activation = getattr(module, "act_fn", None)
+ if type(activation) not in (type(ACT2FN["silu"]), torch.nn.SiLU):
+ continue
+ if (
+ getattr(activation, "act", torch.nn.functional.silu)
+ is not torch.nn.functional.silu
+ ):
+ continue
+ projs = [
+ getattr(module, part, None)
+ for part in ("gate_proj", "up_proj", "down_proj")
+ ]
+ if any(
+ getattr(getattr(proj, "base_layer", None), "bias", None) is not None
+ for proj in projs
+ ):
+ continue
+ states = [
+ getattr(getattr(proj, "base_layer", None), "__dict__", {}).get(
+ "_unsloth_fp8_lora"
+ )
+ for proj in projs
+ ]
+ if not all(
+ state is not None and (allow_cpu or state.device.type == "cuda")
+ for state in states
+ ):
+ continue
+ else:
+ weight = getattr(module, "weight", None)
+ if equation.__globals__.get("torch") is not torch or weight is None:
+ continue
+ if weight.dtype not in (torch.float16, torch.bfloat16):
+ continue
+ if not allow_cpu and weight.device.type != "cuda":
+ continue
+ operator = apply_lora_mlp_swiglu if kernel is fused_mlp else fast_rms_layernorm
+ key = (id(module), name, function, operator, kernel.__code__)
+ replacement = cache.get(key)
+ if replacement is None:
+ # Preserve native keyword names without a Python wrapper in each block.
+ code = _fp8_lora_operator_code(
+ kernel.__code__, function.__code__.co_varnames[:2]
+ )
+ replacement = types.FunctionType(
+ code,
+ kernel.__globals__,
+ function.__name__,
+ kernel.__defaults__,
+ kernel.__closure__,
+ )
+ replacement = types.MethodType(replacement, method.__self__)
+ # Reuse the binding across GRPO inference/training cycles to retain compiled graphs.
+ cache[key] = replacement
+ forwards.append((module, name, module.__dict__.get(name)))
+ setattr(module, name, replacement)
+ break
+ owner.__dict__["_unsloth_fp8_lora_kernel_forwards"] = tuple(forwards)
+
+
+def _prepare_fp8_lora_loss(model):
+ import functools
+ import types
+
+ owner = _fp8_lora_owner(model)
+ if owner.__dict__.get("_unsloth_fp8_lora_loss_forwards"):
+ return
+ getter = getattr(owner, "get_output_embeddings", None)
+ if not callable(getter) or not isinstance(getter(), torch.nn.Module):
+ return
+ from unsloth_zoo.fused_losses import forward_adapter
+
+ original = forward_adapter.unsloth_fused_lm_head_loss
+
+ @functools.wraps(original)
+ def routed(hidden_states, lm_head, labels, *args, **kwargs):
+ # HF's fixed_cross_entropy ignores these flags; they must not select another loss.
+ kwargs.pop("output_attentions", None)
+ kwargs.pop("output_hidden_states", None)
+ return original(hidden_states, lm_head, labels, *args, **kwargs)
+
+ memo = {}
+
+ def isolate(function):
+ if not isinstance(function, types.FunctionType):
+ return function
+ if function in memo:
+ return memo[function]
+ memo[function] = function
+ inner = getattr(function, "__wrapped__", None)
+ changed_inner = isolate(inner)
+ closure = function.__closure__
+ # Decorators call their closure; changing __wrapped__ alone does not reroute them.
+ if changed_inner is not inner and closure:
+ closure = tuple(
+ types.CellType(changed_inner) if cell.cell_contents is inner else cell
+ for cell in closure
+ )
+ direct = (
+ "unsloth_fused_lm_head_loss" in function.__code__.co_names
+ and function.__globals__.get("unsloth_fused_lm_head_loss") is original
+ )
+ if not direct and closure == function.__closure__:
+ return function
+ namespace = function.__globals__
+ if direct:
+ namespace = {**namespace, "unsloth_fused_lm_head_loss": routed}
+ # Older Dynamo resolves named scopes through the original module instead of this copy.
+ namespace.pop("__name__", None)
+ replacement = types.FunctionType(
+ function.__code__,
+ namespace,
+ function.__name__,
+ function.__defaults__,
+ closure,
+ )
+ replacement.__dict__.update(function.__dict__)
+ for attribute in (
+ "__kwdefaults__",
+ "__annotations__",
+ "__qualname__",
+ "__module__",
+ "__doc__",
+ ):
+ setattr(replacement, attribute, getattr(function, attribute))
+ if changed_inner is not inner:
+ replacement.__wrapped__ = changed_inner
+ memo[function] = replacement
+ return replacement
+
+ forwards = []
+ for module in model.modules():
+ for name in ("forward", "_old_forward"):
+ method = getattr(module, name, None)
+ function = getattr(method, "__func__", method)
+ replacement = isolate(function)
+ if replacement is function:
+ continue
+ forwards.append((module, name, module.__dict__.get(name)))
+ if isinstance(method, types.MethodType):
+ replacement = types.MethodType(replacement, method.__self__)
+ setattr(module, name, replacement)
+ owner.__dict__["_unsloth_fp8_lora_loss_forwards"] = tuple(forwards)
+
+
+def restore_fp8_lora_for_inference(model):
+ owner = _fp8_lora_owner(model)
+ for module, name, method in owner.__dict__.pop("_unsloth_fp8_lora_kernel_forwards", ()):
+ if method is None:
+ module.__dict__.pop(name, None)
+ else:
+ setattr(module, name, method)
+ for module, name, method in owner.__dict__.pop("_unsloth_fp8_lora_loss_forwards", ()):
+ if method is None:
+ module.__dict__.pop(name, None)
+ else:
+ setattr(module, name, method)
+ bases = owner.__dict__.get("_unsloth_fp8_lora_bases", ())
+ if not bases:
+ return
+ kernels = _load_fp8_lora_kernels()
+ for module in bases:
+ kernels.detach_fp8_lora_base(module)
+ owner.__dict__["_unsloth_fp8_lora_bases"] = ()
+
+
+def enable_fp8_lora(
+ model,
+ recipe = None,
+ allow_cpu = False,
+):
+ """Attach the fp8 frozen-base state to every LoRA base nn.Linear (frozen, 16-bit, dims % 16, not a
+ high-precision layer). Returns the converted names. A no-op unless the model was loaded with it."""
+ recipe = recipe or _fp8_lora_request(model)
+ if recipe is None:
+ return []
+ kernels = _load_fp8_lora_kernels()
+ policy = _precision.get_recipe(recipe).hp_layers
+ try:
+ is_hp = _precision.hp_filter(model, policy)
+ except Exception:
+ is_hp = lambda module, fqn: False
+ converted, skipped = [], {}
+ uses = {}
+ for _, p in model.named_parameters(remove_duplicate = False):
+ uses[id(p)] = uses.get(id(p), 0) + 1
+ swapped = _block_swapped_modules(model)
+ offload = kernels.offload_supported()
+ offloaded = 0
+ for name, module in model.named_modules():
+ base = getattr(module, "base_layer", None)
+ if base is None or not hasattr(module, "lora_A") or not isinstance(base, torch.nn.Module):
+ continue
+ if is_hp(base, name + ".base_layer") or is_hp(module, name):
+ skipped[name] = "high precision layer"
+ continue
+ why = kernels.eligible_base(base)
+ if why is None and not allow_cpu:
+ state = base.__dict__.get("_unsloth_fp8_lora")
+ device = state.device if state is not None and state.offloaded else base.weight.device
+ if device.type != "cuda" or torch.version.hip is not None:
+ why = f"weight on {device}"
+ elif torch.cuda.get_device_capability(device) < (8, 9):
+ why = "compute capability below 8.9"
+ if why is None and id(base) in swapped:
+ why = "block-swapped layer"
+ if why is None and uses.get(id(base.weight), 0) > 1:
+ why = "weight shared with another module"
+ if why is not None:
+ skipped[name] = why
+ continue
+ reads_raw = _has_variant(module) or _mamba_raw_weight_leaf(model, name)
+ state = kernels.attach_fp8_lora_base(base, offload = offload and not reads_raw)
+ offloaded += int(state.offloaded)
+ _restore_master_before_merge(module)
+ converted.append(name)
+ if offloaded and torch.cuda.is_available():
+ torch.cuda.empty_cache()
+ _fp8_lora_owner(model).__dict__["_unsloth_fp8_lora_bases"] = tuple(
+ module for module in model.modules() if "_unsloth_fp8_lora" in module.__dict__
+ )
+ detail = {"skipped": len(skipped)} if skipped else {}
+ if offloaded:
+ detail["master_offloaded"] = offloaded
+ _record_engagement(model, recipe, len(converted), detail = detail or None)
+ if not _require_engagement(recipe, len(converted), "no frozen 16-bit LoRA base Linear layers"):
+ print(
+ "Unsloth: float8_training with LoRA found no eligible base layers"
+ + (f" ({next(iter(skipped.values()))})" if skipped else "")
+ + "; training stays in 16 bit."
+ )
+ else:
+ print(f"Unsloth: fp8 LoRA base on {len(converted)} Linear layers.")
+ if converted:
+ _prepare_fp8_lora_loss(model)
+ _prepare_fp8_lora_operators(model)
+ import types
+ from peft.tuners.tuners_utils import BaseTuner
+
+ tuner = model if isinstance(model, BaseTuner) else getattr(model, "base_model", None)
+ unload = "_unload_and_optionally_merge"
+ if (
+ isinstance(tuner, BaseTuner)
+ and unload not in tuner.__dict__
+ and callable(getattr(type(tuner), unload, None))
+ ):
+ object.__setattr__(tuner, unload, types.MethodType(_unload_fp8_lora, tuner))
+ if converted and getattr(model, "_unsloth_block_compile_installed", False):
+ from ._block_compile import patch_block_compile
+ patch_block_compile(model)
+ return converted
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index cc25b3c52eb..57405afb428 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -88,6 +88,7 @@ from .loader_utils import (
warn_if_bitsandbytes_quantized_nothing,
)
from .fsdp2_qlora import prepare_fsdp2_qlora, set_fsdp2_quant_storage
+from .float8_training import enable_fp8_lora, restore_fp8_lora_for_inference
from ..utils.packing import (
get_packed_info_from_kwargs,
mask_packed_boundary_labels,
@@ -5228,7 +5229,9 @@ class FastLlamaModel:
model._saved_temp_tokenizer = _saved_temp_tokenizer
- model = FastLlamaModel.patch_peft_model(model, use_gradient_checkpointing)
+ model = FastLlamaModel.patch_peft_model(
+ model, use_gradient_checkpointing, _defer_fp8_lora = True
+ )
offload_embedding_if_tight(model)
install_block_swap(
model,
@@ -5236,6 +5239,7 @@ class FastLlamaModel:
prefetch_depth = prefetch_depth,
use_gradient_checkpointing = use_gradient_checkpointing,
)
+ enable_fp8_lora(model)
skip_checkpointing(model, checkpoint_skip_layers)
if ensure_weight_tying:
@@ -5336,7 +5340,11 @@ class FastLlamaModel:
return model
@staticmethod
- def patch_peft_model(model, use_gradient_checkpointing = "unsloth"):
+ def patch_peft_model(
+ model,
+ use_gradient_checkpointing = "unsloth",
+ _defer_fp8_lora = False,
+ ):
# Persist the GC mode: for_inference() clears it every GRPO step and TrainingArguments defaults
# it to False, which would silently disable it at train time.
model._unsloth_gradient_checkpointing = use_gradient_checkpointing
@@ -5565,6 +5573,8 @@ class FastLlamaModel:
m.for_training = functools.partial(FastBaseModel.for_training, m)
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
+ if not _defer_fp8_lora:
+ enable_fp8_lora(model)
_unsloth_install_pretrain_detector(model)
_legacy_block_compile(model)
return model
@@ -5576,6 +5586,7 @@ class FastLlamaModel:
"Unsloth: I think you're passing a tokenizer, not the model to for_inference!"
)
_tensor_parallel_for_inference(model)
+ restore_fp8_lora_for_inference(model)
def _for_inference(m):
if hasattr(m, "gradient_checkpointing"):
@@ -5648,6 +5659,7 @@ class FastLlamaModel:
m = m.model
_for_training(m)
apply_training_mode()
+ enable_fp8_lora(model)
# Since transformers 4.53, this must be turned on explicitly.
for module in checkpointing:
set_module_gradient_checkpointing(module, use_gradient_checkpointing)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 0e6a48accea..dbf53df547f 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -36,7 +36,12 @@ from peft import PeftConfig, PeftModel
from .grouped_linear_lora import register_grouped_linear_lora_for_adapter
from .fp8_to_nf4 import track_explicit_4bit_request
from .fsdp2_load import sharded_load_env as fsdp2_sharded_load_env
-from .float8_training import convert_model_to_float8_training, resolve_float8_training
+from .float8_training import (
+ convert_model_to_float8_training,
+ mark_fp8_lora,
+ resolve_float8_training,
+ resolve_fp8_lora,
+)
from .mistral_format import (
MistralFormatRedirect,
is_mistral_format_view,
@@ -1341,10 +1346,16 @@ class FastLanguageModel(FastLlamaModel):
device_map = distributed_device_map
# @_offline_aware_load already forced offline when needed; delegations inherit it.
+ fp8_lora = None
+ fp8_request = kwargs.get("float8_training", None)
if kwargs.get("float8_training", None) not in (None, False) and not full_finetuning:
- raise ValueError(
- "Unsloth: `float8_training` is for `full_finetuning = True` only; LoRA / QLoRA keep "
- "their base weights frozen, so there is no fp8 training step to take."
+ fp8_lora = resolve_fp8_lora(
+ kwargs.pop("float8_training"),
+ dtype = dtype,
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = load_in_8bit,
+ load_in_fp8 = load_in_fp8,
+ qat_scheme = qat_scheme,
)
if kwargs.get("float8_training", None) not in (None, False) and qat_scheme is not None:
raise ValueError(
@@ -1926,6 +1937,7 @@ class FastLanguageModel(FastLlamaModel):
unsloth_tiled_mlp = unsloth_tiled_mlp,
text_only = text_only,
*args,
+ **({"float8_training": fp8_request} if fp8_lora is not None else {}),
**kwargs,
)
return _mark_requested_float32(delegated, user_float32), tokenizer
@@ -2168,6 +2180,7 @@ class FastLanguageModel(FastLlamaModel):
model = _mark_forced_float32(model, False)
# Full finetuning delegates to FastModel above, so this path is always LoRA.
model = _mark_full_finetuning(model, False)
+ model = mark_fp8_lora(model, fp8_lora)
return _mark_requested_float32(model, user_float32), tokenizer
@@ -2281,12 +2294,17 @@ class FastModel(FastBaseModel):
):
rope_scaling = _fold_rope_parameters(rope_scaling, kwargs)
user_config = kwargs.pop("config", None)
- float8_training = resolve_float8_training(kwargs.pop("float8_training", None), dtype = dtype)
- if float8_training is not None and not full_finetuning:
- raise ValueError(
- "Unsloth: `float8_training` is for `full_finetuning = True` only; LoRA / QLoRA keep "
- "their base weights frozen, so there is no fp8 training step to take."
+ fp8_lora = None
+ if not full_finetuning and kwargs.get("float8_training", None) not in (None, False):
+ fp8_lora = resolve_fp8_lora(
+ kwargs.pop("float8_training"),
+ dtype = dtype,
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = load_in_8bit,
+ load_in_fp8 = load_in_fp8,
+ qat_scheme = qat_scheme,
)
+ float8_training = resolve_float8_training(kwargs.pop("float8_training", None), dtype = dtype)
if float8_training is not None and qat_scheme is not None:
raise ValueError(
"Unsloth: `float8_training` and `qat_scheme` cannot be combined. QAT swaps the Linear "
@@ -2556,6 +2574,12 @@ class FastModel(FastBaseModel):
# Text-diffusion slow-path dispatch, factored so the normal route and the legacy-config fallback share one call site.
def _dispatch_diffusion():
+ if fp8_lora is not None:
+ raise NotImplementedError(
+ f"Unsloth: float8_training = {fp8_lora!r} (fp8 LoRA) is not supported for text-diffusion "
+ "models yet. Instead: drop float8_training for LoRA, or float8_training with "
+ "full_finetuning = True."
+ )
model, tokenizer = FastDiffusionModel.from_pretrained(
model_name = model_name,
max_seq_length = max_seq_length,
@@ -3672,6 +3696,7 @@ class FastModel(FastBaseModel):
print(f"Unsloth: float8 training on {len(converted)} Linear layers.")
model = _mark_forced_float32(model, do_forced_float32)
model = _mark_full_finetuning(model, full_finetuning)
+ model = mark_fp8_lora(model, fp8_lora)
return _mark_requested_float32(model, user_float32), tokenizer
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index a62541c5885..82db7d57c25 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -3319,11 +3319,16 @@ def _unsloth_vllm_merged_weight(module, weight):
delta = module.get_delta_weight(adapter).float()
else:
continue
- delta = delta.to(weight.device).reshape(weight.shape)
- merged = delta if merged is None else merged + delta
+ device = weight.device if weight.device.type != "cpu" else delta.device
+ delta = delta.to(device).reshape(weight.shape)
+ merged = delta if merged is None else merged + delta.to(merged.device)
if merged is None:
+ base = getattr(module, "base_layer", None)
+ state = base.__dict__.get("_unsloth_fp8_lora") if base is not None else None
+ if state is not None and weight.device != state.device:
+ return weight.data.to(state.device)
return weight.data
- return (weight.float() + merged).to(weight.dtype)
+ return (weight.to(merged.device).float() + merged).to(weight.dtype)
def _unsloth_vllm_tuner_groups(model):
@@ -3347,6 +3352,8 @@ def _unsloth_vllm_peft_merge_out_of_place(base, layers):
the base tensors and return {id(param): merged tensor}; the live parameters keep their exact bytes."""
params = list(base.parameters(recurse = False))
originals = [param.data for param in params]
+ fp8 = base.__dict__.get("_unsloth_fp8_lora")
+ fp8_saved = dict(fp8.__dict__) if fp8 is not None else None
states = [
(layer, list(getattr(layer, "merged_adapters", [])), dict(getattr(layer, "_caches", {})))
for layer in layers
@@ -3369,6 +3376,9 @@ def _unsloth_vllm_peft_merge_out_of_place(base, layers):
module.training = training
for param, data in zip(params, originals):
param.data = data
+ if fp8 is not None:
+ fp8.__dict__.clear()
+ fp8.__dict__.update(fp8_saved)
for layer, merged_adapters, caches in states:
if hasattr(layer, "merged_adapters"):
layer.merged_adapters[:] = merged_adapters
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index fe0b3a7d094..70539820213 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -283,6 +283,7 @@ from ._utils import (
)
from ._utils import *
from ._utils import _ignore_block_swap_params_for_ddp
+from .float8_training import enable_fp8_lora, restore_fp8_lora_for_inference
from ._utils import _tensor_parallel_for_inference, _tensor_parallel_for_training
from ._uma_safetensors import is_integrated_unified_memory_gpu
from ._utils import estimate_training_reserve_bytes as _zoo_reserve_estimate
@@ -5818,6 +5819,7 @@ class FastBaseModel:
m.for_training = functools.partial(FastBaseModel.for_training, m)
m.for_inference = functools.partial(FastBaseModel.for_inference, m)
m = m.model
+ enable_fp8_lora(model)
# Detect a stray pre-train forward so train() can drop the torch.compile graph cache it would otherwise poison (see prepare_for_training_mode).
_unsloth_install_pretrain_detector(model)
model = _exclude_rope_inv_freq_from_ddp(model)
@@ -6013,6 +6015,7 @@ class FastBaseModel:
"Unsloth: I think you're passing a tokenizer, not the model to for_inference!"
)
_tensor_parallel_for_inference(model)
+ restore_fp8_lora_for_inference(model)
def _for_inference(m):
if hasattr(m, "gradient_checkpointing"):
@@ -6091,6 +6094,7 @@ class FastBaseModel:
m = m.model
_for_training(m)
apply_training_mode()
+ enable_fp8_lora(model)
# Since transformers 4.53, this must be turned on explicitly.
for module in checkpointing:
set_module_gradient_checkpointing(module, use_gradient_checkpointing)
diff --git a/unsloth/precision/engage.py b/unsloth/precision/engage.py
index ea891206763..a5567c89733 100644
--- a/unsloth/precision/engage.py
+++ b/unsloth/precision/engage.py
@@ -85,6 +85,10 @@ LOWP_GEMM_CUSTOM_OPS = (
"unsloth::fp8_block_matmul_triton",
"unsloth::fp8_block_matmul_torchao",
"unsloth::fp8_block_small_m_linear",
+ "unsloth::fp8_lora_fwd",
+ "unsloth::fp8_lora_dgrad",
+ "unsloth::fp8_lora_first",
+ "unsloth::fp8_lora_accum",
"unsloth_zoo::mxfp4_ogs_moe",
"fbgemm::f8f8bf16",
"fbgemm::f8f8bf16_tensorwise",
diff --git a/unsloth/precision/recipes.py b/unsloth/precision/recipes.py
index 07d99d80fe6..b56c6628e8b 100644
--- a/unsloth/precision/recipes.py
+++ b/unsloth/precision/recipes.py
@@ -512,6 +512,83 @@ _add(
)
)
+_FP8_LORA_RUNS = (
+ "Qwen3 LoRA r=16, seq 2048, bs 2, 200 steps on alpaca-cleaned, eval every 50 steps on 32 held-out "
+ "rows, 2 seeds, vs bf16 LoRA in the same Colab VM"
+)
+_add(
+ Recipe(
+ name = "fp8_lora_tensorwise",
+ kind = "frozen_base",
+ modes = ("lora",),
+ fwd = GemmPolicy(_op(E4M3, "tensor"), _op(E4M3, "tensor")),
+ dgrad = GemmPolicy(_op(E5M2, "tensor"), _op(E4M3, "tensor")),
+ hp_layers = _HP_FP8,
+ input_dtypes = ("bf16", "fp16"),
+ grad_scaler = "per_dtype",
+ compile = "opaque",
+ backend = "unsloth.kernels.fp8_lora (float8_training with LoRA on a 16-bit base)",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ "sm89": Gate(
+ "explicit",
+ evidence = "L4, 16-bit master in host RAM (default): step 1.15x bf16 at Qwen3-0.6B (A/A 1.0%), "
+ "1.30x at 1.7B (A/A 0.05%); peak memory -11% / -33% vs bf16 LoRA. Eval loss (fp8 forward) "
+ "+0.53-0.64% at 0.6B, +0.39-0.55% at 1.7B (bf16 seed band 0.03%); the trained adapter on the "
+ "exact 16-bit base +0.07-0.17%. Explicit, not named: the fp8-forward eval gap passes "
+ "jobs/compare.py (rtol 5%) but not the 0.5% bar at 0.6B. Saved and merged weights bitwise equal "
+ "to the checkpoint. " + _FP8_LORA_RUNS,
+ ),
+ "sm90": Gate("unprobed", "H100 is not available to test on"),
+ "sm100": Gate("unprobed"),
+ "sm120": Gate(
+ "explicit",
+ evidence = "RTX PRO 6000: 1.08-1.10x at Qwen3-0.6B, 1.17x at 1.7B; rowwise "
+ "(fp8_lora_rowwise) is faster there. " + _FP8_LORA_RUNS,
+ ),
+ "rocm": Gate("refused", "ROCm fp8 (fnuz) GEMMs are not wired for the frozen base"),
+ },
+ notes = "base GEMMs fp8, LoRA branch 16-bit (Megatron-Bridge TEFusedLoRALinear split); weight "
+ "quantized once; the fp8 GEMM writes bf16 for fp16 activations",
+ )
+)
+_add(
+ Recipe(
+ name = "fp8_lora_rowwise",
+ kind = "frozen_base",
+ modes = ("lora",),
+ fwd = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "tensor")),
+ dgrad = GemmPolicy(_op(E4M3, "row"), _op(E4M3, "tensor")),
+ hp_layers = _HP_FP8,
+ input_dtypes = ("bf16", "fp16"),
+ grad_scaler = "per_dtype",
+ compile = "opaque",
+ backend = "unsloth.kernels.fp8_lora (UNSLOTH_FP8_LORA_SCALING=row; the default on sm120)",
+ gates = {
+ "sm75": _NO_FP8_HW,
+ "sm80": _NO_FP8_HW,
+ "sm89": Gate(
+ "explicit",
+ evidence = "L4: rowwise _scaled_mm 0.55-0.90x bf16 at op level, no faster than bf16 LoRA "
+ "end to end; use fp8_lora_tensorwise there. " + _FP8_LORA_RUNS,
+ ),
+ "sm90": Gate("unprobed", "H100 is not available to test on"),
+ "sm100": Gate("unprobed"),
+ "sm120": Gate(
+ "explicit",
+ evidence = "RTX PRO 6000, 16-bit master in host RAM (default): step 1.16x bf16 at Qwen3-0.6B "
+ "(A/A 0.02%), 1.22x at 1.7B (A/A 0.17%); peak memory -12% / -33% vs bf16 LoRA. Eval loss "
+ "(fp8 forward) +0.48-0.56% at 0.6B, +0.32-0.54% at 1.7B (bf16 seed band 0.02%); the trained "
+ "adapter on the exact 16-bit base +0.06-0.12%. Explicit, not named: the fp8-forward eval gap "
+ "passes jobs/compare.py (rtol 5%) but not the 0.5% bar at 0.6B. " + _FP8_LORA_RUNS,
+ ),
+ "rocm": Gate("refused", "ROCm fp8 (fnuz) GEMMs are not wired for the frozen base"),
+ },
+ notes = "per-token activation / gradient scales, the weight's tensorwise scale broadcast per column",
+ )
+)
+
def get_recipe(name: str) -> Recipe:
try: