Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Keep Linear layers an FP8 checkpoint stores in bf16 unconverted (#12124)

Daniel Hanb779aca61fea
diff --git a/tests/test_fp8_unscaled_checkpoint_linears.py b/tests/test_fp8_unscaled_checkpoint_linears.py
new file mode 100644
index 00000000000..6868a7c6c7b
--- /dev/null
+++ b/tests/test_fp8_unscaled_checkpoint_linears.py
@@ -0,0 +1,202 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""A Linear an FP8 checkpoint stores in bf16 (Step-3.7-Flash-FP8's projector) must stay a plain Linear."""
+
+import pytest
+
+torch = pytest.importorskip("torch")
+pytest.importorskip("transformers")
+safetensors_torch = pytest.importorskip("safetensors.torch")
+
+import transformers # noqa: E402
+
+try:
+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer
+ from transformers.utils.quantization_config import FineGrainedFP8Config
+except Exception: # pragma: no cover
+ pytest.skip("no FineGrainedFP8 quantizer in this transformers", allow_module_level = True)
+
+if not hasattr(torch, "float8_e4m3fn"): # pragma: no cover
+ pytest.skip("torch without float8", allow_module_level = True)
+
+try:
+ from transformers.quantizers.quantizers_utils import should_convert_module # noqa: F401
+ def exact(name):
+ import re
+ return re.escape(name) + "$"
+except Exception: # transformers 4.x matches skip entries as substrings
+
+ def exact(name):
+ return name
+
+
+from unsloth.import_fixes import ( # noqa: E402
+ _fp8_checkpoint_tensor_dtypes,
+ _fp8_unscaled_linear_patterns,
+ fix_transformers_fp8_unscaled_checkpoint_linears,
+)
+
+F8 = torch.float8_e4m3fn
+
+
+def _write(path, tensors):
+ safetensors_torch.save_file({k: v.contiguous() for k, v in tensors.items()}, str(path))
+ return str(path)
+
+
+def _fp8(*shape):
+ return torch.zeros(*shape).to(F8)
+
+
+def _quantizer(skip):
+ config = FineGrainedFP8Config(weight_block_size = [128, 128], modules_to_not_convert = skip)
+ return FineGrainedFP8HfQuantizer(config, pre_quantized = True), config
+
+
+def _llama():
+ from transformers import LlamaConfig, LlamaForCausalLM
+ config = LlamaConfig(
+ hidden_size = 256,
+ intermediate_size = 512,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 512,
+ )
+ with torch.device("meta"):
+ return LlamaForCausalLM(config)
+
+
+def _llama_checkpoint(tmp_path, bf16_module):
+ tensors = {}
+ for name, module in _llama().named_modules():
+ if isinstance(module, torch.nn.Linear) and name != "lm_head":
+ if name == bf16_module:
+ tensors[name + ".weight"] = torch.zeros(2, 2, dtype = torch.bfloat16)
+ else:
+ tensors[name + ".weight"] = _fp8(2, 2)
+ tensors[name + ".weight_scale_inv"] = torch.ones(1, 1)
+ tensors["lm_head.weight"] = torch.zeros(2, 2, dtype = torch.bfloat16)
+ return _write(tmp_path / "model.safetensors", tensors)
+
+
+def _preprocess(model, skip, files):
+ quantizer, config = _quantizer(list(skip))
+ quantizer.preprocess_model(model = model, dtype = torch.bfloat16, checkpoint_files = files)
+ return {n: type(m).__name__ for n, m in model.named_modules()}, config
+
+
+BF16 = "model.layers.1.self_attn.o_proj"
+
+
+def test_header_dtypes_mark_the_bf16_linear(tmp_path):
+ files = [_llama_checkpoint(tmp_path, BF16)]
+ dtypes = _fp8_checkpoint_tensor_dtypes(files, None)
+ assert dtypes[BF16 + ".weight"] == "BF16"
+ assert set(_fp8_unscaled_linear_patterns(_llama(), dtypes)) == {exact("lm_head"), exact(BF16)}
+
+
+def test_bf16_linear_stays_linear_and_skip_list_is_restored(tmp_path):
+ files = [_llama_checkpoint(tmp_path, BF16)]
+ fix_transformers_fp8_unscaled_checkpoint_linears()
+ types, config = _preprocess(_llama(), ["lm_head"], files)
+ assert types[BF16] == "Linear"
+ assert types["model.layers.0.self_attn.o_proj"] == "FP8Linear"
+ assert types["model.layers.1.self_attn.q_proj"] == "FP8Linear"
+ assert config.modules_to_not_convert == ["lm_head"]
+
+
+def test_skip_entry_the_checkpoint_already_lists_survives_the_restore(tmp_path):
+ files = [_llama_checkpoint(tmp_path, BF16)]
+ fix_transformers_fp8_unscaled_checkpoint_linears()
+ types, config = _preprocess(_llama(), ["lm_head", exact(BF16)], files)
+ assert types[BF16] == "Linear"
+ assert config.modules_to_not_convert == ["lm_head", exact(BF16)]
+
+
+def test_not_an_fp8_checkpoint_changes_nothing(tmp_path):
+ files = [
+ _write(
+ tmp_path / "model.safetensors", {"a.weight": torch.zeros(2, 2, dtype = torch.bfloat16)}
+ )
+ ]
+ assert _fp8_unscaled_linear_patterns(_llama(), _fp8_checkpoint_tensor_dtypes(files, None)) == []
+
+
+def test_index_only_falls_back_to_missing_scale(tmp_path):
+ import json
+
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps(
+ {
+ "weight_map": {
+ "model.layers.0.self_attn.q_proj.weight": "x.safetensors",
+ "model.layers.0.self_attn.q_proj.weight_scale_inv": "x.safetensors",
+ "model.layers.0.self_attn.k_proj.weight": "x.safetensors",
+ }
+ }
+ )
+ )
+ dtypes = _fp8_checkpoint_tensor_dtypes([str(tmp_path / "x.safetensors")], None)
+ assert dtypes and all(v is None for v in dtypes.values())
+ assert _fp8_unscaled_linear_patterns(_llama(), dtypes) == [
+ exact("model.layers.0.self_attn.k_proj")
+ ]
+
+
+Step3p7 = getattr(transformers, "Step3p7ForConditionalGeneration", None)
+
+
+def _step():
+ config = transformers.Step3p7Config()
+ config.text_config.num_hidden_layers = 1
+ with torch.device("meta"):
+ return Step3p7(config)
+
+
+def _step_checkpoint(tmp_path):
+ return [
+ _write(
+ tmp_path / "model.safetensors",
+ {
+ "vit_large_projector.weight": torch.zeros(2, 2, dtype = torch.bfloat16),
+ "model.layers.0.self_attn.q_proj.weight": _fp8(2, 2),
+ "model.layers.0.self_attn.q_proj.weight_scale_inv": torch.ones(1, 1),
+ },
+ )
+ ]
+
+
[email protected](Step3p7 is None, reason = "no native step3p7 in this transformers")
+def test_step37_config_skip_list_alone_converts_the_projector(tmp_path):
+ fix_transformers_fp8_unscaled_checkpoint_linears()
+ patched = FineGrainedFP8HfQuantizer._process_model_before_weight_loading
+ quantizer, _ = _quantizer(["lm_head", "vit_large_projector"])
+ model = _step()
+ patched.__wrapped__(quantizer, model, checkpoint_files = _step_checkpoint(tmp_path))
+ assert type(model.model.multi_modal_projector).__name__ == "FP8Linear"
+
+
[email protected](Step3p7 is None, reason = "no native step3p7 in this transformers")
+def test_step37_projector_under_its_checkpoint_name(tmp_path):
+ files = _step_checkpoint(tmp_path)
+ fix_transformers_fp8_unscaled_checkpoint_linears()
+ types, config = _preprocess(_step(), ["lm_head", "vit_large_projector"], files)
+ assert types["model.multi_modal_projector"] == "Linear"
+ assert types["model.language_model.layers.0.self_attn.q_proj"] == "FP8Linear"
+ assert "vit_large_projector" in config.modules_to_not_convert
+
+
+def test_fp8_forward_falls_back_for_a_bf16_weight():
+ try:
+ import unsloth.kernels.fp8 as fp8
+ except ImportError as e: # environment (e.g. vLLM refusing transformers 4.x), not this fix
+ pytest.skip(f"unsloth.kernels.fp8 not importable: {e}")
+ forward = fp8.module_forward_patch(
+ lambda *a: (_ for _ in ()).throw(AssertionError("fp8 kernel")), "weight_scale_inv"
+ )
+ layer = torch.nn.Linear(4, 3, bias = True)
+ layer.weight_scale_inv = torch.nn.Parameter(torch.ones(1, 1))
+ x = torch.randn(2, 4)
+ torch.testing.assert_close(forward(layer, x), layer(x))
diff --git a/tests/test_fp8linear_high_precision_weight.py b/tests/test_fp8linear_high_precision_weight.py
new file mode 100644
index 00000000000..5d331fc221a
--- /dev/null
+++ b/tests/test_fp8linear_high_precision_weight.py
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team.
+"""An FP8Linear holding a bf16 weight (GLM-5.3-Flash forget gate) must run a plain linear."""
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason = "needs CUDA")
+
+
+ "bias", [None, torch.bfloat16, torch.float32], ids = ["no_bias", "bf16_bias", "fp32_bias"]
+)
+def test_bf16_weight_in_fp8linear_runs_plain_linear(bias):
+ from unsloth.kernels import fp8 # installs the patched forward
+
+ FP8Linear = fp8.FP8Linear
+ if FP8Linear is None:
+ pytest.skip("this transformers has no FP8Linear")
+ layer = FP8Linear(256, 128, block_size = (128, 128)).cuda()
+ torch.manual_seed(0)
+ layer.weight = torch.nn.Parameter(
+ torch.randn(128, 256, device = "cuda", dtype = torch.bfloat16) * 0.05, requires_grad = False
+ )
+ layer.bias = (
+ torch.nn.Parameter(torch.randn(128, device = "cuda", dtype = bias))
+ if bias is not None
+ else None
+ )
+ x = torch.randn(4, 7, 256, device = "cuda", dtype = torch.bfloat16)
+ out = layer(x)
+ assert out.dtype == torch.bfloat16
+ ref = F.linear(x, layer.weight, None if layer.bias is None else layer.bias.to(x.dtype))
+ torch.testing.assert_close(out, ref, rtol = 0, atol = 0)
diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.py
index fa9569dc7f3..febadfaf369 100644
--- a/tests/test_modelopt_fp8_loading.py
+++ b/tests/test_modelopt_fp8_loading.py
@@ -901,18 +901,22 @@ print("CHECK", q.dtype, w_rel, float((got - want).norm() / want.norm()))
def test_fp8_linear_forward_patch_adds_the_bias():
from unsloth.kernels.fp8 import module_forward_patch
- forward = module_forward_patch(lambda X, weight, scale: X @ weight.t(), "weight_scale_inv")
+ forward = module_forward_patch(
+ lambda X, weight, scale: X @ weight.to(X.dtype).t(), "weight_scale_inv"
+ )
biased, plain = nn.Linear(4, 3), nn.Linear(4, 3, bias = False)
for module in (biased, plain):
module.weight_scale_inv = torch.ones(())
# fbgemm keeps its bias in fp32; the output must stay in the activation dtype.
biased.bias.data = biased.bias.data.float()
X = torch.randn(2, 4, dtype = torch.bfloat16)
- biased.weight.data, plain.weight.data = (m.weight.data.bfloat16() for m in (biased, plain))
+ biased.weight.data, plain.weight.data = (
+ m.weight.data.to(torch.float8_e4m3fn) for m in (biased, plain)
+ )
out = forward(biased, X)
assert out.dtype == torch.bfloat16
- torch.testing.assert_close(out, X @ biased.weight.t() + biased.bias.bfloat16())
- assert torch.equal(forward(plain, X), X @ plain.weight.t())
+ torch.testing.assert_close(out, X @ biased.weight.bfloat16().t() + biased.bias.bfloat16())
+ assert torch.equal(forward(plain, X), X @ plain.weight.bfloat16().t())
def test_save_keeps_transformers_fp8_scale_names():
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 32a159c5955..cb58166ae4b 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -258,6 +258,7 @@ from .import_fixes import (
fix_transformers_longcat_lsa_config,
fix_transformers_rope_scaling_drops_theta,
fix_transformers_fp8_modulelist_experts,
+ fix_transformers_fp8_unscaled_checkpoint_linears,
fix_transformers_validate_rope_ignore_keys,
fix_transformers5_remote_code_legacy_defaults,
fix_transformers_config_only_remote_code,
@@ -326,6 +327,7 @@ del check_transformers_prequantized_vlm_quant_state
# retry in models/llama.py sees a config that kept its base (#2405).
fix_transformers_rope_scaling_drops_theta()
fix_transformers_fp8_modulelist_experts()
+fix_transformers_fp8_unscaled_checkpoint_linears()
fix_transformers_validate_rope_ignore_keys()
fix_transformers5_remote_code_legacy_defaults()
fix_transformers_config_only_remote_code()
@@ -393,6 +395,7 @@ del fix_transformers5_bare_annotation_configs
del fix_transformers5_legacy_config_types
del fix_transformers_rope_scaling_drops_theta
del fix_transformers_fp8_modulelist_experts
+del fix_transformers_fp8_unscaled_checkpoint_linears
del fix_transformers_validate_rope_ignore_keys
del fix_transformers_longcat_lsa_config
del fix_transformers_remote_rope_scaling_none
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index 59284a6c430..439d75e2e3e 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2799,6 +2799,170 @@ def fix_transformers_fp8_modulelist_experts():
quantizer_cls._process_model_before_weight_loading = _process_model_before_weight_loading
+_FP8_SCALE_SUFFIXES = (".weight_scale_inv", ".weight_scale")
+
+
+def _safetensors_header_dtypes(path):
+ import json
+ with open(path, "rb") as f:
+ n = int.from_bytes(f.read(8), "little")
+ header = json.loads(f.read(n))
+ return {
+ k: v.get("dtype") for k, v in header.items() if k != "__metadata__" and isinstance(v, dict)
+ }
+
+
+def _fp8_checkpoint_files(checkpoint_files, config):
+ files = [str(f) for f in (checkpoint_files or []) if f]
+ if files:
+ return files
+ # transformers 4.x does not pass checkpoint_files to the quantizer.
+ import glob
+
+ name = getattr(config, "_name_or_path", None) or getattr(config, "name_or_path", None)
+ if not name:
+ return []
+ directory = str(name) if os.path.isdir(str(name)) else None
+ if directory is None:
+ try:
+ from huggingface_hub import try_to_load_from_cache
+ for filename in ("model.safetensors.index.json", "model.safetensors"):
+ hit = try_to_load_from_cache(
+ name, filename, revision = getattr(config, "_commit_hash", None)
+ )
+ if isinstance(hit, str):
+ directory = os.path.dirname(hit)
+ break
+ except Exception:
+ return []
+ if directory is None:
+ return []
+ return sorted(glob.glob(os.path.join(directory, "*.safetensors")))
+
+
+def _fp8_checkpoint_tensor_dtypes(checkpoint_files, config):
+ # Shard headers, not the index: Step-3.7-Flash-FP8's index omits weight_scale_inv.
+ import json
+
+ files = _fp8_checkpoint_files(checkpoint_files, config)
+ dtypes = {}
+ for path in files:
+ if path.endswith(".safetensors") and os.path.isfile(path):
+ try:
+ dtypes.update(_safetensors_header_dtypes(path))
+ except Exception:
+ pass
+ if dtypes:
+ return dtypes
+ for directory in {os.path.dirname(f) for f in files}:
+ index = os.path.join(directory, "model.safetensors.index.json")
+ if os.path.isfile(index):
+ try:
+ with open(index, "r", encoding = "utf-8") as f:
+ dtypes.update(dict.fromkeys(json.load(f).get("weight_map", {})))
+ except Exception:
+ pass
+ return dtypes
+
+
+def _fp8_unscaled_linear_patterns(model, tensor_dtypes):
+ import torch.nn as nn
+
+ if not tensor_dtypes:
+ return []
+ known = {k: v for k, v in tensor_dtypes.items() if v is not None}
+ if known:
+ if not any(str(v).upper().startswith("F8") for v in known.values()):
+ return []
+ unscaled = [
+ k[: -len(".weight")]
+ for k, v in known.items()
+ if k.endswith(".weight") and not str(v).upper().startswith("F8")
+ ]
+ else:
+ names = set(tensor_dtypes)
+ if not any(n.endswith(_FP8_SCALE_SUFFIXES) for n in names):
+ return []
+ unscaled = [
+ n[: -len(".weight")]
+ for n in names
+ if n.endswith(".weight")
+ and not any(n[: -len(".weight")] + s in names for s in _FP8_SCALE_SUFFIXES)
+ ]
+ if not unscaled:
+ return []
+ linears = {n for n, m in model.named_modules() if isinstance(m, nn.Linear)}
+ try:
+ from transformers.conversion_mapping import get_model_conversion_mapping
+ renamings = get_model_conversion_mapping(model) or []
+ except Exception:
+ renamings = []
+ try:
+ # transformers 5 matches skip entries as regexes; 4.x as substrings.
+ from transformers.quantizers.quantizers_utils import should_convert_module # noqa: F401
+ exact = lambda n: re.escape(n) + "$"
+ except Exception:
+ exact = lambda n: n
+ patterns = []
+ for name in unscaled:
+ # Rename the key, not the module name: renames need the trailing dot (`^vit_large_projector\.`).
+ renamed = name + ".weight"
+ for rename in renamings:
+ try:
+ renamed, _ = rename.rename_source_key(renamed)
+ except Exception:
+ pass
+ renamed = renamed[: -len(".weight")] if renamed.endswith(".weight") else renamed
+ for candidate in (renamed, name):
+ if candidate in linears:
+ pattern = exact(candidate)
+ if pattern not in patterns:
+ patterns.append(pattern)
+ break
+ return patterns
+
+
+def fix_transformers_fp8_unscaled_checkpoint_linears():
+ """Keep Linear layers an FP8 checkpoint stores unscaled in bf16 unconverted; modules_to_not_convert misses renamed ones."""
+ try:
+ from transformers.quantizers import quantizer_finegrained_fp8
+ except Exception:
+ return
+ quantizer_cls = getattr(quantizer_finegrained_fp8, "FineGrainedFP8HfQuantizer", None)
+ method = getattr(quantizer_cls, "_process_model_before_weight_loading", None)
+ if method is None or getattr(method, "_unsloth_unscaled_linears", False):
+ return
+
+ @functools.wraps(method)
+ def _process_model_before_weight_loading(self, model, *args, **kwargs):
+ extra = []
+ qconfig = getattr(self, "quantization_config", None)
+ if getattr(self, "pre_quantized", False) and qconfig is not None:
+ try:
+ dtypes = _fp8_checkpoint_tensor_dtypes(
+ kwargs.get("checkpoint_files"), getattr(model, "config", None)
+ )
+ extra = _fp8_unscaled_linear_patterns(model, dtypes)
+ except Exception:
+ extra = []
+ saved = getattr(qconfig, "modules_to_not_convert", None) if extra else None
+ original = list(saved or [])
+ added = [p for p in extra if p not in original]
+ if not added:
+ return method(self, model, *args, **kwargs)
+ qconfig.modules_to_not_convert = original + added
+ try:
+ return method(self, model, *args, **kwargs)
+ finally:
+ # Strip only what was added: the checkpoint may list the same pattern itself.
+ current = list(getattr(qconfig, "modules_to_not_convert", None) or [])
+ restored = [p for p in current if p not in added]
+ qconfig.modules_to_not_convert = restored if restored or saved is not None else None
+
+ _process_model_before_weight_loading._unsloth_unscaled_linears = True
+ quantizer_cls._process_model_before_weight_loading = _process_model_before_weight_loading
+
+
def _cast_fp8_dequantize_to_model_dtype(op_cls):
# transformers 5.4 dequantizes to the scale's fp32; cast to the replaced parameter's dtype.
convert = getattr(op_cls, "convert", None)
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 75de151a339..0a5834cbe1d 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -1007,9 +1007,23 @@ def fp8_linear(
return out
+_FP8_WEIGHT_DTYPES = tuple(
+ getattr(torch, n)
+ for n in ("float8_e4m3fn", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz")
+ if hasattr(torch, n)
+)
+
+
def module_forward_patch(forward_function, scale_attr = "weight_scale"):
def patched_forward(self, X):
- out = forward_function(X, self.weight, getattr(self, scale_attr))
+ weight = self.weight
+ if weight.dtype not in _FP8_WEIGHT_DTYPES:
+ # bf16 layer the skip list missed: its scale was never loaded.
+ bias = self._parameters.get("bias")
+ return torch.nn.functional.linear(
+ X, weight.to(X.dtype), None if bias is None else bias.to(X.dtype)
+ )
+ out = forward_function(X, weight, getattr(self, scale_attr))
# The kernels take no bias, so a biased Linear (Qwen2-style q/k/v) adds it here; fbgemm keeps it fp32.
bias = self._parameters.get("bias")
return out if bias is None else out + bias.to(out.dtype)