Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Dequantize fine-grained fp8 checkpoints on a 16bit load in FastLanguageModel, and save them as 16bit (#14516)

Daniel Han795f8d08812c
diff --git a/tests/test_fp8_16bit_text_loader_dequant.py b/tests/test_fp8_16bit_text_loader_dequant.py
new file mode 100644
index 00000000000..e110e8fff9e
--- /dev/null
+++ b/tests/test_fp8_16bit_text_loader_dequant.py
@@ -0,0 +1,196 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""An explicit 16bit load of a block-fp8 checkpoint dequantizes it on the FastLanguageModel path too."""
+
+import inspect
+import json
+import os
+import subprocess
+import sys
+from types import SimpleNamespace
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda
+
+import unsloth # noqa: F401
+from unsloth.models.loader_utils import _fp8_dequantize_quantization_config
+
+try:
+ from transformers import FineGrainedFP8Config
+ _HAS_DEQUANTIZE = "dequantize" in inspect.signature(FineGrainedFP8Config).parameters
+except Exception:
+ _HAS_DEQUANTIZE = False
+
+_BLOCK_FP8 = {
+ "quant_method": "fp8",
+ "activation_scheme": "dynamic",
+ "weight_block_size": [128, 128],
+}
+
+
[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")
[email protected]("as_object", [False, True])
+def test_block_fp8_config_gets_dequantize(as_object):
+ qc = FineGrainedFP8Config.from_dict(dict(_BLOCK_FP8)) if as_object else dict(_BLOCK_FP8)
+ out = _fp8_dequantize_quantization_config(SimpleNamespace(quantization_config = qc))
+ assert out is not None and out.dequantize is True
+ assert list(out.weight_block_size) == [128, 128]
+
+
[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")
+def test_enum_quant_method_is_read_by_value():
+ from transformers.utils.quantization_config import QuantizationMethod
+
+ qc = dict(_BLOCK_FP8, quant_method = QuantizationMethod.FP8)
+ out = _fp8_dequantize_quantization_config(SimpleNamespace(quantization_config = qc))
+ assert out is not None and out.dequantize is True
+
+
+ "qc",
+ [None, {"quant_method": "bitsandbytes", "load_in_4bit": True}, {"quant_method": "modelopt"}],
+)
+def test_other_configs_are_left_alone(qc):
+ assert _fp8_dequantize_quantization_config(SimpleNamespace(quantization_config = qc)) is None
+ assert _fp8_dequantize_quantization_config(None) is None
+
+
+def _write_tiny_block_fp8_llama(path):
+ from safetensors.torch import save_file
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ torch.manual_seed(0)
+ config = LlamaConfig(
+ hidden_size = 256,
+ intermediate_size = 512,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 512,
+ )
+ reference = LlamaForCausalLM(config).to(torch.bfloat16)
+ tensors = {}
+ for key, value in reference.state_dict().items():
+ if key.endswith("proj.weight"):
+ n, k = value.shape
+ blocks = value.float().view(n // 128, 128, k // 128, 128)
+ scale = blocks.abs().amax(dim = (1, 3)) / 448.0
+ packed = (blocks / scale[:, None, :, None]).view(n, k).to(torch.float8_e4m3fn)
+ tensors[key] = packed
+ tensors[key.replace(".weight", ".weight_scale_inv")] = scale.float()
+ else:
+ tensors[key] = value.contiguous()
+ save_file(tensors, os.path.join(path, "model.safetensors"))
+ raw = config.to_dict()
+ raw["architectures"] = ["LlamaForCausalLM"]
+ raw["quantization_config"] = dict(_BLOCK_FP8)
+ with open(os.path.join(path, "config.json"), "w") as f:
+ json.dump(raw, f)
+
+
[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")
[email protected](not has_real_cuda(), reason = "fp8 checkpoints load on CUDA")
[email protected]("load_in_16bit", [True, False])
+def test_fast_language_model_16bit_load_dequantizes(tmp_path, load_in_16bit):
+ """Subprocess: FastLanguageModel patches the Llama classes process-wide."""
+ _write_tiny_block_fp8_llama(str(tmp_path))
+ code = f"""
+import unsloth, torch, transformers
+from unsloth import FastLanguageModel
+real = transformers.AutoModelForCausalLM.from_pretrained.__func__
+def load(cls, *args, **kwargs):
+ model = real(cls, *args, **kwargs)
+ fp8 = sum(p.dtype == torch.float8_e4m3fn for p in model.parameters())
+ print("FP8", int(fp8), getattr(kwargs.get("quantization_config"), "dequantize", None), flush = True)
+ return model
+transformers.AutoModelForCausalLM.from_pretrained = classmethod(load)
+try:
+ FastLanguageModel.from_pretrained(
+ {str(tmp_path)!r}, max_seq_length = 64, dtype = torch.bfloat16,
+ load_in_4bit = False, load_in_16bit = {load_in_16bit},
+ )
+except Exception:
+ pass # the tiny checkpoint ships no tokenizer; the weights are loaded before that
+"""
+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 600)
+ seen = [line.split() for line in out.stdout.splitlines() if line.startswith("FP8")]
+ assert seen, (out.stdout[-2000:], out.stderr[-2000:])
+ fp8, dequantize = int(seen[-1][1]), seen[-1][2]
+ if load_in_16bit:
+ assert fp8 == 0 and dequantize == "True", seen[-1]
+ else:
+ # Without the explicit flag nothing changes: transformers decides, as before.
+ assert dequantize == "None", seen[-1]
+
+
+def _write_tiny_block_fp8_qwen3_moe(path):
+ from safetensors.torch import load_file, save_file
+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM
+
+ torch.manual_seed(0)
+ bf16 = os.path.join(path, "bf16")
+ config = Qwen3MoeConfig(
+ hidden_size = 256,
+ intermediate_size = 512,
+ moe_intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 64,
+ num_experts = 4,
+ num_experts_per_tok = 2,
+ vocab_size = 512,
+ tie_word_embeddings = False,
+ )
+ Qwen3MoeForCausalLM(config).to(torch.bfloat16).save_pretrained(bf16)
+ tensors = {}
+ for f in os.listdir(bf16):
+ if f.endswith(".safetensors"):
+ for key, value in load_file(os.path.join(bf16, f)).items():
+ if key.endswith("proj.weight") and value.dim() == 2:
+ n, k = value.shape
+ blocks = value.float().view(n // 128, 128, k // 128, 128)
+ scale = blocks.abs().amax(dim = (1, 3)) / 448.0
+ tensors[key] = (
+ (blocks / scale[:, None, :, None]).view(n, k).to(torch.float8_e4m3fn)
+ )
+ tensors[key.replace(".weight", ".weight_scale_inv")] = scale.float()
+ else:
+ tensors[key] = value
+ save_file(tensors, os.path.join(path, "model.safetensors"))
+ with open(os.path.join(bf16, "config.json")) as f:
+ raw = json.load(f)
+ raw["quantization_config"] = dict(_BLOCK_FP8)
+ with open(os.path.join(path, "config.json"), "w") as f:
+ json.dump(raw, f)
+
+
[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")
[email protected](not has_real_cuda(), reason = "fp8 checkpoints load on CUDA")
[email protected]("arch", ["llama", "qwen3_moe"])
+def test_dequantized_load_saves_and_reloads_16bit_weights(tmp_path, arch):
+ from safetensors import safe_open
+ from transformers import AutoModelForCausalLM
+ from unsloth.models.loader_utils import _drop_fp8_dequantize_conversions
+
+ src, dst = tmp_path / "src", tmp_path / "saved"
+ src.mkdir()
+ (_write_tiny_block_fp8_llama if arch == "llama" else _write_tiny_block_fp8_qwen3_moe)(str(src))
+ config = FineGrainedFP8Config.from_dict(dict(_BLOCK_FP8), dequantize = True)
+ model = AutoModelForCausalLM.from_pretrained(
+ str(src), dtype = torch.bfloat16, device_map = "cuda", quantization_config = config
+ )
+ assert _drop_fp8_dequantize_conversions(model) > 0
+ before = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}
+ model.save_pretrained(str(dst))
+ for name in os.listdir(dst):
+ if name.endswith(".safetensors"):
+ with safe_open(str(dst / name), "pt") as f:
+ assert all(f.get_slice(k).get_dtype() == "BF16" for k in f.keys())
+ with open(dst / "config.json") as f:
+ assert "quantization_config" not in json.load(f)
+ after = AutoModelForCausalLM.from_pretrained(str(dst), dtype = torch.bfloat16).state_dict()
+ assert before.keys() == after.keys()
+ assert all(torch.equal(before[k], after[k].cpu()) for k in before)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 6389535a8df..d428c181307 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -51,6 +51,8 @@ from .loader_utils import (
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
_fp8_load_as_is,
+ _fp8_dequantize_quantization_config,
+ _drop_fp8_dequantize_conversions,
_fp8_needs_offline_quantize,
_offline_quantize_to_fp8,
_tag_model_with_fp8_torchao_config,
@@ -1797,6 +1799,18 @@ class FastLanguageModel(FastLlamaModel):
if quantization_config is not None and not fast_inference:
load_in_4bit_kwargs = False
load_in_8bit_kwargs = False
+ # Mirror FastModel: an explicit 16bit load dequantizes a fine-grained fp8 checkpoint.
+ _fp8_dequantize_config = None
+ if (
+ load_in_16bit
+ and not load_in_4bit
+ and not load_in_8bit
+ and not fast_inference
+ and kwargs.get("quantization_config", None) is None
+ ):
+ _fp8_dequantize_config = _fp8_dequantize_quantization_config(model_config)
+ if _fp8_dequantize_config is not None:
+ kwargs["quantization_config"] = _fp8_dequantize_config
# Mirror FastModel: bitsandbytes < 0.46.0 needs dynamo disabled. Best effort, never crash the load (an old zoo without the #710 fix raises NameError here on Python 3.13).
try:
@@ -1838,6 +1852,8 @@ class FastLanguageModel(FastLlamaModel):
**kwargs,
)
_record_modelscope_repo_id(model, modelscope_repo_id, modelscope_dir)
+ if _fp8_dequantize_config is not None:
+ _drop_fp8_dequantize_conversions(model)
if resize_model_vocab is not None:
_resize_vocab(model, resize_model_vocab)
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 42a1f5beab5..bce51330d94 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -2032,6 +2032,89 @@ def _fp8_needs_offline_quantize(resolved_name, requested_name, fast_inference):
return Version(vllm.__version__) < Version("0.12.0")
+def _drop_fp8_dequantize_conversions(model):
+ """Forget the Fp8Dequantize load conversions of a dequantizing fp8 load. save_pretrained reverses
+ them into Fp8Quantize, writing fp8 weights under a config with no quantization_config, so a reload
+ reads the raw fp8 values as 16bit weights. Returns how many conversions changed."""
+ conversions = getattr(model, "_weight_conversions", None)
+ if not conversions:
+ return 0
+ try:
+ from transformers.integrations.finegrained_fp8 import Fp8Dequantize
+ except Exception:
+ return 0
+
+ def _targets(conversion):
+ patterns = getattr(conversion, "_original_target_patterns", None) or []
+ return tuple(p[:-1] if p.endswith("$") else p for p in patterns)
+
+ plain = None
+ kept, changed = [], 0
+ for conversion in conversions:
+ operations = getattr(conversion, "operations", None) or []
+ if not any(isinstance(op, Fp8Dequantize) for op in operations):
+ kept.append(conversion)
+ continue
+ if all(isinstance(op, Fp8Dequantize) for op in operations):
+ # weight + weight_scale_inv -> weight: the 16bit weight saves as it is.
+ changed += 1
+ continue
+ # A model converter (expert merge, ...) with the dequant prepended: keep its quantizer-free form.
+ if plain is None:
+ try:
+ from transformers.conversion_mapping import get_model_conversion_mapping
+ plain = get_model_conversion_mapping(model, None, None)
+ except Exception:
+ plain = []
+ match = next(
+ (
+ c
+ for c in plain
+ if type(c) is type(conversion) and _targets(c) == _targets(conversion)
+ ),
+ None,
+ )
+ if match is None:
+ print(
+ "Unsloth: an fp8 load conversion has no 16bit counterpart; save_pretrained may write fp8 "
+ f"weights for {list(_targets(conversion))}."
+ )
+ kept.append(conversion)
+ continue
+ kept.append(match)
+ changed += 1
+ if changed:
+ model._weight_conversions = kept
+ return changed
+
+
+def _fp8_dequantize_quantization_config(model_config):
+ """The checkpoint's fine-grained fp8 config with `dequantize = True` for an explicit 16bit load, else None.
+ On FP8-capable GPUs transformers otherwise keeps FP8Linear layers; FastModel passes the same flag."""
+ quantization_config = getattr(model_config, "quantization_config", None)
+ if quantization_config is None:
+ return None
+ as_dict = (
+ quantization_config
+ if isinstance(quantization_config, dict)
+ else getattr(quantization_config, "to_dict", lambda: {})()
+ )
+ quant_method = as_dict.get("quant_method", "")
+ # A config object's to_dict() can keep the QuantizationMethod enum, whose str() is not its value.
+ if str(getattr(quant_method, "value", quant_method)).lower() != "fp8":
+ return None
+ try:
+ from transformers import FineGrainedFP8Config
+ except Exception:
+ return None
+ if "dequantize" not in inspect.signature(FineGrainedFP8Config).parameters:
+ return None
+ try:
+ return FineGrainedFP8Config.from_dict(dict(as_dict), dequantize = True)
+ except Exception:
+ return None
+
+
def _fp8_load_as_is(
model_name,
revision = None,
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 2771170fcf7..7d22eba694f 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -155,6 +155,7 @@ from .loader_utils import (
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
_dequantize_leftover_fp8_params,
+ _drop_fp8_dequantize_conversions,
_restore_dropped_fp8_scales,
_dequantize_multihead_attention_out_proj,
_prepare_compressed_tensors_model,
@@ -4016,6 +4017,7 @@ class FastBaseModel:
elif fsdp2_requested():
prepare_fsdp2_qlora(model, bnb_compute_dtype)
if load_in_16bit and not load_in_4bit and not load_in_8bit:
+ _drop_fp8_dequantize_conversions(model)
_dequantize_leftover_fp8_params(
model,
model_name,