Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Dequantize fine-grained fp8 checkpoints on a 16bit load in FastLanguageModel, and save them as 16bit (#14516)
Daniel Han795f8d08812c
diff --git a/tests/test_fp8_16bit_text_loader_dequant.py b/tests/test_fp8_16bit_text_loader_dequant.pynew file mode 100644index 00000000000..e110e8fff9e--- /dev/null+++ b/tests/test_fp8_16bit_text_loader_dequant.py@@ -0,0 +1,196 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""An explicit 16bit load of a block-fp8 checkpoint dequantizes it on the FastLanguageModel path too."""++import inspect+import json+import os+import subprocess+import sys+from types import SimpleNamespace++import pytest+import torch+from real_accelerator import has_real_cuda++import unsloth # noqa: F401+from unsloth.models.loader_utils import _fp8_dequantize_quantization_config++try:+ from transformers import FineGrainedFP8Config+ _HAS_DEQUANTIZE = "dequantize" in inspect.signature(FineGrainedFP8Config).parameters+except Exception:+ _HAS_DEQUANTIZE = False++_BLOCK_FP8 = {+ "quant_method": "fp8",+ "activation_scheme": "dynamic",+ "weight_block_size": [128, 128],+}++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")[email protected]("as_object", [False, True])+def test_block_fp8_config_gets_dequantize(as_object):+ qc = FineGrainedFP8Config.from_dict(dict(_BLOCK_FP8)) if as_object else dict(_BLOCK_FP8)+ out = _fp8_dequantize_quantization_config(SimpleNamespace(quantization_config = qc))+ assert out is not None and out.dequantize is True+ assert list(out.weight_block_size) == [128, 128]++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")+def test_enum_quant_method_is_read_by_value():+ from transformers.utils.quantization_config import QuantizationMethod++ qc = dict(_BLOCK_FP8, quant_method = QuantizationMethod.FP8)+ out = _fp8_dequantize_quantization_config(SimpleNamespace(quantization_config = qc))+ assert out is not None and out.dequantize is True+++ "qc",+ [None, {"quant_method": "bitsandbytes", "load_in_4bit": True}, {"quant_method": "modelopt"}],+)+def test_other_configs_are_left_alone(qc):+ assert _fp8_dequantize_quantization_config(SimpleNamespace(quantization_config = qc)) is None+ assert _fp8_dequantize_quantization_config(None) is None+++def _write_tiny_block_fp8_llama(path):+ from safetensors.torch import save_file+ from transformers import LlamaConfig, LlamaForCausalLM++ torch.manual_seed(0)+ config = LlamaConfig(+ hidden_size = 256,+ intermediate_size = 512,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ vocab_size = 512,+ )+ reference = LlamaForCausalLM(config).to(torch.bfloat16)+ tensors = {}+ for key, value in reference.state_dict().items():+ if key.endswith("proj.weight"):+ n, k = value.shape+ blocks = value.float().view(n // 128, 128, k // 128, 128)+ scale = blocks.abs().amax(dim = (1, 3)) / 448.0+ packed = (blocks / scale[:, None, :, None]).view(n, k).to(torch.float8_e4m3fn)+ tensors[key] = packed+ tensors[key.replace(".weight", ".weight_scale_inv")] = scale.float()+ else:+ tensors[key] = value.contiguous()+ save_file(tensors, os.path.join(path, "model.safetensors"))+ raw = config.to_dict()+ raw["architectures"] = ["LlamaForCausalLM"]+ raw["quantization_config"] = dict(_BLOCK_FP8)+ with open(os.path.join(path, "config.json"), "w") as f:+ json.dump(raw, f)++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")[email protected](not has_real_cuda(), reason = "fp8 checkpoints load on CUDA")[email protected]("load_in_16bit", [True, False])+def test_fast_language_model_16bit_load_dequantizes(tmp_path, load_in_16bit):+ """Subprocess: FastLanguageModel patches the Llama classes process-wide."""+ _write_tiny_block_fp8_llama(str(tmp_path))+ code = f"""+import unsloth, torch, transformers+from unsloth import FastLanguageModel+real = transformers.AutoModelForCausalLM.from_pretrained.__func__+def load(cls, *args, **kwargs):+ model = real(cls, *args, **kwargs)+ fp8 = sum(p.dtype == torch.float8_e4m3fn for p in model.parameters())+ print("FP8", int(fp8), getattr(kwargs.get("quantization_config"), "dequantize", None), flush = True)+ return model+transformers.AutoModelForCausalLM.from_pretrained = classmethod(load)+try:+ FastLanguageModel.from_pretrained(+ {str(tmp_path)!r}, max_seq_length = 64, dtype = torch.bfloat16,+ load_in_4bit = False, load_in_16bit = {load_in_16bit},+ )+except Exception:+ pass # the tiny checkpoint ships no tokenizer; the weights are loaded before that+"""+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 600)+ seen = [line.split() for line in out.stdout.splitlines() if line.startswith("FP8")]+ assert seen, (out.stdout[-2000:], out.stderr[-2000:])+ fp8, dequantize = int(seen[-1][1]), seen[-1][2]+ if load_in_16bit:+ assert fp8 == 0 and dequantize == "True", seen[-1]+ else:+ # Without the explicit flag nothing changes: transformers decides, as before.+ assert dequantize == "None", seen[-1]+++def _write_tiny_block_fp8_qwen3_moe(path):+ from safetensors.torch import load_file, save_file+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM++ torch.manual_seed(0)+ bf16 = os.path.join(path, "bf16")+ config = Qwen3MoeConfig(+ hidden_size = 256,+ intermediate_size = 512,+ moe_intermediate_size = 128,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 64,+ num_experts = 4,+ num_experts_per_tok = 2,+ vocab_size = 512,+ tie_word_embeddings = False,+ )+ Qwen3MoeForCausalLM(config).to(torch.bfloat16).save_pretrained(bf16)+ tensors = {}+ for f in os.listdir(bf16):+ if f.endswith(".safetensors"):+ for key, value in load_file(os.path.join(bf16, f)).items():+ if key.endswith("proj.weight") and value.dim() == 2:+ n, k = value.shape+ blocks = value.float().view(n // 128, 128, k // 128, 128)+ scale = blocks.abs().amax(dim = (1, 3)) / 448.0+ tensors[key] = (+ (blocks / scale[:, None, :, None]).view(n, k).to(torch.float8_e4m3fn)+ )+ tensors[key.replace(".weight", ".weight_scale_inv")] = scale.float()+ else:+ tensors[key] = value+ save_file(tensors, os.path.join(path, "model.safetensors"))+ with open(os.path.join(bf16, "config.json")) as f:+ raw = json.load(f)+ raw["quantization_config"] = dict(_BLOCK_FP8)+ with open(os.path.join(path, "config.json"), "w") as f:+ json.dump(raw, f)++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")[email protected](not has_real_cuda(), reason = "fp8 checkpoints load on CUDA")[email protected]("arch", ["llama", "qwen3_moe"])+def test_dequantized_load_saves_and_reloads_16bit_weights(tmp_path, arch):+ from safetensors import safe_open+ from transformers import AutoModelForCausalLM+ from unsloth.models.loader_utils import _drop_fp8_dequantize_conversions++ src, dst = tmp_path / "src", tmp_path / "saved"+ src.mkdir()+ (_write_tiny_block_fp8_llama if arch == "llama" else _write_tiny_block_fp8_qwen3_moe)(str(src))+ config = FineGrainedFP8Config.from_dict(dict(_BLOCK_FP8), dequantize = True)+ model = AutoModelForCausalLM.from_pretrained(+ str(src), dtype = torch.bfloat16, device_map = "cuda", quantization_config = config+ )+ assert _drop_fp8_dequantize_conversions(model) > 0+ before = {k: v.detach().cpu().clone() for k, v in model.state_dict().items()}+ model.save_pretrained(str(dst))+ for name in os.listdir(dst):+ if name.endswith(".safetensors"):+ with safe_open(str(dst / name), "pt") as f:+ assert all(f.get_slice(k).get_dtype() == "BF16" for k in f.keys())+ with open(dst / "config.json") as f:+ assert "quantization_config" not in json.load(f)+ after = AutoModelForCausalLM.from_pretrained(str(dst), dtype = torch.bfloat16).state_dict()+ assert before.keys() == after.keys()+ assert all(torch.equal(before[k], after[k].cpu()) for k in before)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 6389535a8df..d428c181307 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -51,6 +51,8 @@ from .loader_utils import (_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,_fp8_load_as_is,+ _fp8_dequantize_quantization_config,+ _drop_fp8_dequantize_conversions,_fp8_needs_offline_quantize,_offline_quantize_to_fp8,_tag_model_with_fp8_torchao_config,@@ -1797,6 +1799,18 @@ class FastLanguageModel(FastLlamaModel):if quantization_config is not None and not fast_inference:load_in_4bit_kwargs = Falseload_in_8bit_kwargs = False+ # Mirror FastModel: an explicit 16bit load dequantizes a fine-grained fp8 checkpoint.+ _fp8_dequantize_config = None+ if (+ load_in_16bit+ and not load_in_4bit+ and not load_in_8bit+ and not fast_inference+ and kwargs.get("quantization_config", None) is None+ ):+ _fp8_dequantize_config = _fp8_dequantize_quantization_config(model_config)+ if _fp8_dequantize_config is not None:+ kwargs["quantization_config"] = _fp8_dequantize_config# Mirror FastModel: bitsandbytes < 0.46.0 needs dynamo disabled. Best effort, never crash the load (an old zoo without the #710 fix raises NameError here on Python 3.13).try:@@ -1838,6 +1852,8 @@ class FastLanguageModel(FastLlamaModel):**kwargs,)_record_modelscope_repo_id(model, modelscope_repo_id, modelscope_dir)+ if _fp8_dequantize_config is not None:+ _drop_fp8_dequantize_conversions(model)if resize_model_vocab is not None:_resize_vocab(model, resize_model_vocab)diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 42a1f5beab5..bce51330d94 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -2032,6 +2032,89 @@ def _fp8_needs_offline_quantize(resolved_name, requested_name, fast_inference):return Version(vllm.__version__) < Version("0.12.0")+def _drop_fp8_dequantize_conversions(model):+ """Forget the Fp8Dequantize load conversions of a dequantizing fp8 load. save_pretrained reverses+ them into Fp8Quantize, writing fp8 weights under a config with no quantization_config, so a reload+ reads the raw fp8 values as 16bit weights. Returns how many conversions changed."""+ conversions = getattr(model, "_weight_conversions", None)+ if not conversions:+ return 0+ try:+ from transformers.integrations.finegrained_fp8 import Fp8Dequantize+ except Exception:+ return 0++ def _targets(conversion):+ patterns = getattr(conversion, "_original_target_patterns", None) or []+ return tuple(p[:-1] if p.endswith("$") else p for p in patterns)++ plain = None+ kept, changed = [], 0+ for conversion in conversions:+ operations = getattr(conversion, "operations", None) or []+ if not any(isinstance(op, Fp8Dequantize) for op in operations):+ kept.append(conversion)+ continue+ if all(isinstance(op, Fp8Dequantize) for op in operations):+ # weight + weight_scale_inv -> weight: the 16bit weight saves as it is.+ changed += 1+ continue+ # A model converter (expert merge, ...) with the dequant prepended: keep its quantizer-free form.+ if plain is None:+ try:+ from transformers.conversion_mapping import get_model_conversion_mapping+ plain = get_model_conversion_mapping(model, None, None)+ except Exception:+ plain = []+ match = next(+ (+ c+ for c in plain+ if type(c) is type(conversion) and _targets(c) == _targets(conversion)+ ),+ None,+ )+ if match is None:+ print(+ "Unsloth: an fp8 load conversion has no 16bit counterpart; save_pretrained may write fp8 "+ f"weights for {list(_targets(conversion))}."+ )+ kept.append(conversion)+ continue+ kept.append(match)+ changed += 1+ if changed:+ model._weight_conversions = kept+ return changed+++def _fp8_dequantize_quantization_config(model_config):+ """The checkpoint's fine-grained fp8 config with `dequantize = True` for an explicit 16bit load, else None.+ On FP8-capable GPUs transformers otherwise keeps FP8Linear layers; FastModel passes the same flag."""+ quantization_config = getattr(model_config, "quantization_config", None)+ if quantization_config is None:+ return None+ as_dict = (+ quantization_config+ if isinstance(quantization_config, dict)+ else getattr(quantization_config, "to_dict", lambda: {})()+ )+ quant_method = as_dict.get("quant_method", "")+ # A config object's to_dict() can keep the QuantizationMethod enum, whose str() is not its value.+ if str(getattr(quant_method, "value", quant_method)).lower() != "fp8":+ return None+ try:+ from transformers import FineGrainedFP8Config+ except Exception:+ return None+ if "dequantize" not in inspect.signature(FineGrainedFP8Config).parameters:+ return None+ try:+ return FineGrainedFP8Config.from_dict(dict(as_dict), dequantize = True)+ except Exception:+ return None++def _fp8_load_as_is(model_name,revision = None,diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 2771170fcf7..7d22eba694f 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -155,6 +155,7 @@ from .loader_utils import (_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,_dequantize_leftover_fp8_params,+ _drop_fp8_dequantize_conversions,_restore_dropped_fp8_scales,_dequantize_multihead_attention_out_proj,_prepare_compressed_tensors_model,@@ -4016,6 +4017,7 @@ class FastBaseModel:elif fsdp2_requested():prepare_fsdp2_qlora(model, bnb_compute_dtype)if load_in_16bit and not load_in_4bit and not load_in_8bit:+ _drop_fp8_dequantize_conversions(model)_dequantize_leftover_fp8_params(model,model_name,