Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Keep Linear layers an FP8 checkpoint stores in bf16 unconverted (#12124)
Daniel Hanb779aca61fea
diff --git a/tests/test_fp8_unscaled_checkpoint_linears.py b/tests/test_fp8_unscaled_checkpoint_linears.pynew file mode 100644index 00000000000..6868a7c6c7b--- /dev/null+++ b/tests/test_fp8_unscaled_checkpoint_linears.py@@ -0,0 +1,202 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A Linear an FP8 checkpoint stores in bf16 (Step-3.7-Flash-FP8's projector) must stay a plain Linear."""++import pytest++torch = pytest.importorskip("torch")+pytest.importorskip("transformers")+safetensors_torch = pytest.importorskip("safetensors.torch")++import transformers # noqa: E402++try:+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer+ from transformers.utils.quantization_config import FineGrainedFP8Config+except Exception: # pragma: no cover+ pytest.skip("no FineGrainedFP8 quantizer in this transformers", allow_module_level = True)++if not hasattr(torch, "float8_e4m3fn"): # pragma: no cover+ pytest.skip("torch without float8", allow_module_level = True)++try:+ from transformers.quantizers.quantizers_utils import should_convert_module # noqa: F401+ def exact(name):+ import re+ return re.escape(name) + "$"+except Exception: # transformers 4.x matches skip entries as substrings++ def exact(name):+ return name+++from unsloth.import_fixes import ( # noqa: E402+ _fp8_checkpoint_tensor_dtypes,+ _fp8_unscaled_linear_patterns,+ fix_transformers_fp8_unscaled_checkpoint_linears,+)++F8 = torch.float8_e4m3fn+++def _write(path, tensors):+ safetensors_torch.save_file({k: v.contiguous() for k, v in tensors.items()}, str(path))+ return str(path)+++def _fp8(*shape):+ return torch.zeros(*shape).to(F8)+++def _quantizer(skip):+ config = FineGrainedFP8Config(weight_block_size = [128, 128], modules_to_not_convert = skip)+ return FineGrainedFP8HfQuantizer(config, pre_quantized = True), config+++def _llama():+ from transformers import LlamaConfig, LlamaForCausalLM+ config = LlamaConfig(+ hidden_size = 256,+ intermediate_size = 512,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ vocab_size = 512,+ )+ with torch.device("meta"):+ return LlamaForCausalLM(config)+++def _llama_checkpoint(tmp_path, bf16_module):+ tensors = {}+ for name, module in _llama().named_modules():+ if isinstance(module, torch.nn.Linear) and name != "lm_head":+ if name == bf16_module:+ tensors[name + ".weight"] = torch.zeros(2, 2, dtype = torch.bfloat16)+ else:+ tensors[name + ".weight"] = _fp8(2, 2)+ tensors[name + ".weight_scale_inv"] = torch.ones(1, 1)+ tensors["lm_head.weight"] = torch.zeros(2, 2, dtype = torch.bfloat16)+ return _write(tmp_path / "model.safetensors", tensors)+++def _preprocess(model, skip, files):+ quantizer, config = _quantizer(list(skip))+ quantizer.preprocess_model(model = model, dtype = torch.bfloat16, checkpoint_files = files)+ return {n: type(m).__name__ for n, m in model.named_modules()}, config+++BF16 = "model.layers.1.self_attn.o_proj"+++def test_header_dtypes_mark_the_bf16_linear(tmp_path):+ files = [_llama_checkpoint(tmp_path, BF16)]+ dtypes = _fp8_checkpoint_tensor_dtypes(files, None)+ assert dtypes[BF16 + ".weight"] == "BF16"+ assert set(_fp8_unscaled_linear_patterns(_llama(), dtypes)) == {exact("lm_head"), exact(BF16)}+++def test_bf16_linear_stays_linear_and_skip_list_is_restored(tmp_path):+ files = [_llama_checkpoint(tmp_path, BF16)]+ fix_transformers_fp8_unscaled_checkpoint_linears()+ types, config = _preprocess(_llama(), ["lm_head"], files)+ assert types[BF16] == "Linear"+ assert types["model.layers.0.self_attn.o_proj"] == "FP8Linear"+ assert types["model.layers.1.self_attn.q_proj"] == "FP8Linear"+ assert config.modules_to_not_convert == ["lm_head"]+++def test_skip_entry_the_checkpoint_already_lists_survives_the_restore(tmp_path):+ files = [_llama_checkpoint(tmp_path, BF16)]+ fix_transformers_fp8_unscaled_checkpoint_linears()+ types, config = _preprocess(_llama(), ["lm_head", exact(BF16)], files)+ assert types[BF16] == "Linear"+ assert config.modules_to_not_convert == ["lm_head", exact(BF16)]+++def test_not_an_fp8_checkpoint_changes_nothing(tmp_path):+ files = [+ _write(+ tmp_path / "model.safetensors", {"a.weight": torch.zeros(2, 2, dtype = torch.bfloat16)}+ )+ ]+ assert _fp8_unscaled_linear_patterns(_llama(), _fp8_checkpoint_tensor_dtypes(files, None)) == []+++def test_index_only_falls_back_to_missing_scale(tmp_path):+ import json++ (tmp_path / "model.safetensors.index.json").write_text(+ json.dumps(+ {+ "weight_map": {+ "model.layers.0.self_attn.q_proj.weight": "x.safetensors",+ "model.layers.0.self_attn.q_proj.weight_scale_inv": "x.safetensors",+ "model.layers.0.self_attn.k_proj.weight": "x.safetensors",+ }+ }+ )+ )+ dtypes = _fp8_checkpoint_tensor_dtypes([str(tmp_path / "x.safetensors")], None)+ assert dtypes and all(v is None for v in dtypes.values())+ assert _fp8_unscaled_linear_patterns(_llama(), dtypes) == [+ exact("model.layers.0.self_attn.k_proj")+ ]+++Step3p7 = getattr(transformers, "Step3p7ForConditionalGeneration", None)+++def _step():+ config = transformers.Step3p7Config()+ config.text_config.num_hidden_layers = 1+ with torch.device("meta"):+ return Step3p7(config)+++def _step_checkpoint(tmp_path):+ return [+ _write(+ tmp_path / "model.safetensors",+ {+ "vit_large_projector.weight": torch.zeros(2, 2, dtype = torch.bfloat16),+ "model.layers.0.self_attn.q_proj.weight": _fp8(2, 2),+ "model.layers.0.self_attn.q_proj.weight_scale_inv": torch.ones(1, 1),+ },+ )+ ]++[email protected](Step3p7 is None, reason = "no native step3p7 in this transformers")+def test_step37_config_skip_list_alone_converts_the_projector(tmp_path):+ fix_transformers_fp8_unscaled_checkpoint_linears()+ patched = FineGrainedFP8HfQuantizer._process_model_before_weight_loading+ quantizer, _ = _quantizer(["lm_head", "vit_large_projector"])+ model = _step()+ patched.__wrapped__(quantizer, model, checkpoint_files = _step_checkpoint(tmp_path))+ assert type(model.model.multi_modal_projector).__name__ == "FP8Linear"++[email protected](Step3p7 is None, reason = "no native step3p7 in this transformers")+def test_step37_projector_under_its_checkpoint_name(tmp_path):+ files = _step_checkpoint(tmp_path)+ fix_transformers_fp8_unscaled_checkpoint_linears()+ types, config = _preprocess(_step(), ["lm_head", "vit_large_projector"], files)+ assert types["model.multi_modal_projector"] == "Linear"+ assert types["model.language_model.layers.0.self_attn.q_proj"] == "FP8Linear"+ assert "vit_large_projector" in config.modules_to_not_convert+++def test_fp8_forward_falls_back_for_a_bf16_weight():+ try:+ import unsloth.kernels.fp8 as fp8+ except ImportError as e: # environment (e.g. vLLM refusing transformers 4.x), not this fix+ pytest.skip(f"unsloth.kernels.fp8 not importable: {e}")+ forward = fp8.module_forward_patch(+ lambda *a: (_ for _ in ()).throw(AssertionError("fp8 kernel")), "weight_scale_inv"+ )+ layer = torch.nn.Linear(4, 3, bias = True)+ layer.weight_scale_inv = torch.nn.Parameter(torch.ones(1, 1))+ x = torch.randn(2, 4)+ torch.testing.assert_close(forward(layer, x), layer(x))diff --git a/tests/test_fp8linear_high_precision_weight.py b/tests/test_fp8linear_high_precision_weight.pynew file mode 100644index 00000000000..5d331fc221a--- /dev/null+++ b/tests/test_fp8linear_high_precision_weight.py@@ -0,0 +1,35 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team.+"""An FP8Linear holding a bf16 weight (GLM-5.3-Flash forget gate) must run a plain linear."""++import pytest+import torch+import torch.nn.functional as F++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")+++ "bias", [None, torch.bfloat16, torch.float32], ids = ["no_bias", "bf16_bias", "fp32_bias"]+)+def test_bf16_weight_in_fp8linear_runs_plain_linear(bias):+ from unsloth.kernels import fp8 # installs the patched forward++ FP8Linear = fp8.FP8Linear+ if FP8Linear is None:+ pytest.skip("this transformers has no FP8Linear")+ layer = FP8Linear(256, 128, block_size = (128, 128)).cuda()+ torch.manual_seed(0)+ layer.weight = torch.nn.Parameter(+ torch.randn(128, 256, device = "cuda", dtype = torch.bfloat16) * 0.05, requires_grad = False+ )+ layer.bias = (+ torch.nn.Parameter(torch.randn(128, device = "cuda", dtype = bias))+ if bias is not None+ else None+ )+ x = torch.randn(4, 7, 256, device = "cuda", dtype = torch.bfloat16)+ out = layer(x)+ assert out.dtype == torch.bfloat16+ ref = F.linear(x, layer.weight, None if layer.bias is None else layer.bias.to(x.dtype))+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.pyindex fa9569dc7f3..febadfaf369 100644--- a/tests/test_modelopt_fp8_loading.py+++ b/tests/test_modelopt_fp8_loading.py@@ -901,18 +901,22 @@ print("CHECK", q.dtype, w_rel, float((got - want).norm() / want.norm()))def test_fp8_linear_forward_patch_adds_the_bias():from unsloth.kernels.fp8 import module_forward_patch- forward = module_forward_patch(lambda X, weight, scale: X @ weight.t(), "weight_scale_inv")+ forward = module_forward_patch(+ lambda X, weight, scale: X @ weight.to(X.dtype).t(), "weight_scale_inv"+ )biased, plain = nn.Linear(4, 3), nn.Linear(4, 3, bias = False)for module in (biased, plain):module.weight_scale_inv = torch.ones(())# fbgemm keeps its bias in fp32; the output must stay in the activation dtype.biased.bias.data = biased.bias.data.float()X = torch.randn(2, 4, dtype = torch.bfloat16)- biased.weight.data, plain.weight.data = (m.weight.data.bfloat16() for m in (biased, plain))+ biased.weight.data, plain.weight.data = (+ m.weight.data.to(torch.float8_e4m3fn) for m in (biased, plain)+ )out = forward(biased, X)assert out.dtype == torch.bfloat16- torch.testing.assert_close(out, X @ biased.weight.t() + biased.bias.bfloat16())- assert torch.equal(forward(plain, X), X @ plain.weight.t())+ torch.testing.assert_close(out, X @ biased.weight.bfloat16().t() + biased.bias.bfloat16())+ assert torch.equal(forward(plain, X), X @ plain.weight.bfloat16().t())def test_save_keeps_transformers_fp8_scale_names():diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.pyindex 32a159c5955..cb58166ae4b 100644--- a/unsloth/_gpu_init.py+++ b/unsloth/_gpu_init.py@@ -258,6 +258,7 @@ from .import_fixes import (fix_transformers_longcat_lsa_config,fix_transformers_rope_scaling_drops_theta,fix_transformers_fp8_modulelist_experts,+ fix_transformers_fp8_unscaled_checkpoint_linears,fix_transformers_validate_rope_ignore_keys,fix_transformers5_remote_code_legacy_defaults,fix_transformers_config_only_remote_code,@@ -326,6 +327,7 @@ del check_transformers_prequantized_vlm_quant_state# retry in models/llama.py sees a config that kept its base (#2405).fix_transformers_rope_scaling_drops_theta()fix_transformers_fp8_modulelist_experts()+fix_transformers_fp8_unscaled_checkpoint_linears()fix_transformers_validate_rope_ignore_keys()fix_transformers5_remote_code_legacy_defaults()fix_transformers_config_only_remote_code()@@ -393,6 +395,7 @@ del fix_transformers5_bare_annotation_configsdel fix_transformers5_legacy_config_typesdel fix_transformers_rope_scaling_drops_thetadel fix_transformers_fp8_modulelist_experts+del fix_transformers_fp8_unscaled_checkpoint_linearsdel fix_transformers_validate_rope_ignore_keysdel fix_transformers_longcat_lsa_configdel fix_transformers_remote_rope_scaling_nonediff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.pyindex 59284a6c430..439d75e2e3e 100644--- a/unsloth/import_fixes.py+++ b/unsloth/import_fixes.py@@ -2799,6 +2799,170 @@ def fix_transformers_fp8_modulelist_experts():quantizer_cls._process_model_before_weight_loading = _process_model_before_weight_loading+_FP8_SCALE_SUFFIXES = (".weight_scale_inv", ".weight_scale")+++def _safetensors_header_dtypes(path):+ import json+ with open(path, "rb") as f:+ n = int.from_bytes(f.read(8), "little")+ header = json.loads(f.read(n))+ return {+ k: v.get("dtype") for k, v in header.items() if k != "__metadata__" and isinstance(v, dict)+ }+++def _fp8_checkpoint_files(checkpoint_files, config):+ files = [str(f) for f in (checkpoint_files or []) if f]+ if files:+ return files+ # transformers 4.x does not pass checkpoint_files to the quantizer.+ import glob++ name = getattr(config, "_name_or_path", None) or getattr(config, "name_or_path", None)+ if not name:+ return []+ directory = str(name) if os.path.isdir(str(name)) else None+ if directory is None:+ try:+ from huggingface_hub import try_to_load_from_cache+ for filename in ("model.safetensors.index.json", "model.safetensors"):+ hit = try_to_load_from_cache(+ name, filename, revision = getattr(config, "_commit_hash", None)+ )+ if isinstance(hit, str):+ directory = os.path.dirname(hit)+ break+ except Exception:+ return []+ if directory is None:+ return []+ return sorted(glob.glob(os.path.join(directory, "*.safetensors")))+++def _fp8_checkpoint_tensor_dtypes(checkpoint_files, config):+ # Shard headers, not the index: Step-3.7-Flash-FP8's index omits weight_scale_inv.+ import json++ files = _fp8_checkpoint_files(checkpoint_files, config)+ dtypes = {}+ for path in files:+ if path.endswith(".safetensors") and os.path.isfile(path):+ try:+ dtypes.update(_safetensors_header_dtypes(path))+ except Exception:+ pass+ if dtypes:+ return dtypes+ for directory in {os.path.dirname(f) for f in files}:+ index = os.path.join(directory, "model.safetensors.index.json")+ if os.path.isfile(index):+ try:+ with open(index, "r", encoding = "utf-8") as f:+ dtypes.update(dict.fromkeys(json.load(f).get("weight_map", {})))+ except Exception:+ pass+ return dtypes+++def _fp8_unscaled_linear_patterns(model, tensor_dtypes):+ import torch.nn as nn++ if not tensor_dtypes:+ return []+ known = {k: v for k, v in tensor_dtypes.items() if v is not None}+ if known:+ if not any(str(v).upper().startswith("F8") for v in known.values()):+ return []+ unscaled = [+ k[: -len(".weight")]+ for k, v in known.items()+ if k.endswith(".weight") and not str(v).upper().startswith("F8")+ ]+ else:+ names = set(tensor_dtypes)+ if not any(n.endswith(_FP8_SCALE_SUFFIXES) for n in names):+ return []+ unscaled = [+ n[: -len(".weight")]+ for n in names+ if n.endswith(".weight")+ and not any(n[: -len(".weight")] + s in names for s in _FP8_SCALE_SUFFIXES)+ ]+ if not unscaled:+ return []+ linears = {n for n, m in model.named_modules() if isinstance(m, nn.Linear)}+ try:+ from transformers.conversion_mapping import get_model_conversion_mapping+ renamings = get_model_conversion_mapping(model) or []+ except Exception:+ renamings = []+ try:+ # transformers 5 matches skip entries as regexes; 4.x as substrings.+ from transformers.quantizers.quantizers_utils import should_convert_module # noqa: F401+ exact = lambda n: re.escape(n) + "$"+ except Exception:+ exact = lambda n: n+ patterns = []+ for name in unscaled:+ # Rename the key, not the module name: renames need the trailing dot (`^vit_large_projector\.`).+ renamed = name + ".weight"+ for rename in renamings:+ try:+ renamed, _ = rename.rename_source_key(renamed)+ except Exception:+ pass+ renamed = renamed[: -len(".weight")] if renamed.endswith(".weight") else renamed+ for candidate in (renamed, name):+ if candidate in linears:+ pattern = exact(candidate)+ if pattern not in patterns:+ patterns.append(pattern)+ break+ return patterns+++def fix_transformers_fp8_unscaled_checkpoint_linears():+ """Keep Linear layers an FP8 checkpoint stores unscaled in bf16 unconverted; modules_to_not_convert misses renamed ones."""+ try:+ from transformers.quantizers import quantizer_finegrained_fp8+ except Exception:+ return+ quantizer_cls = getattr(quantizer_finegrained_fp8, "FineGrainedFP8HfQuantizer", None)+ method = getattr(quantizer_cls, "_process_model_before_weight_loading", None)+ if method is None or getattr(method, "_unsloth_unscaled_linears", False):+ return++ @functools.wraps(method)+ def _process_model_before_weight_loading(self, model, *args, **kwargs):+ extra = []+ qconfig = getattr(self, "quantization_config", None)+ if getattr(self, "pre_quantized", False) and qconfig is not None:+ try:+ dtypes = _fp8_checkpoint_tensor_dtypes(+ kwargs.get("checkpoint_files"), getattr(model, "config", None)+ )+ extra = _fp8_unscaled_linear_patterns(model, dtypes)+ except Exception:+ extra = []+ saved = getattr(qconfig, "modules_to_not_convert", None) if extra else None+ original = list(saved or [])+ added = [p for p in extra if p not in original]+ if not added:+ return method(self, model, *args, **kwargs)+ qconfig.modules_to_not_convert = original + added+ try:+ return method(self, model, *args, **kwargs)+ finally:+ # Strip only what was added: the checkpoint may list the same pattern itself.+ current = list(getattr(qconfig, "modules_to_not_convert", None) or [])+ restored = [p for p in current if p not in added]+ qconfig.modules_to_not_convert = restored if restored or saved is not None else None++ _process_model_before_weight_loading._unsloth_unscaled_linears = True+ quantizer_cls._process_model_before_weight_loading = _process_model_before_weight_loading++def _cast_fp8_dequantize_to_model_dtype(op_cls):# transformers 5.4 dequantizes to the scale's fp32; cast to the replaced parameter's dtype.convert = getattr(op_cls, "convert", None)diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.pyindex 75de151a339..0a5834cbe1d 100644--- a/unsloth/kernels/fp8.py+++ b/unsloth/kernels/fp8.py@@ -1007,9 +1007,23 @@ def fp8_linear(return out+_FP8_WEIGHT_DTYPES = tuple(+ getattr(torch, n)+ for n in ("float8_e4m3fn", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz")+ if hasattr(torch, n)+)++def module_forward_patch(forward_function, scale_attr = "weight_scale"):def patched_forward(self, X):- out = forward_function(X, self.weight, getattr(self, scale_attr))+ weight = self.weight+ if weight.dtype not in _FP8_WEIGHT_DTYPES:+ # bf16 layer the skip list missed: its scale was never loaded.+ bias = self._parameters.get("bias")+ return torch.nn.functional.linear(+ X, weight.to(X.dtype), None if bias is None else bias.to(X.dtype)+ )+ out = forward_function(X, weight, getattr(self, scale_attr))# The kernels take no bias, so a biased Linear (Qwen2-style q/k/v) adds it here; fbgemm keeps it fp32.bias = self._parameters.get("bias")return out if bias is None else out + bias.to(out.dtype)