Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
FP8: keep every scoped converter and clean up every dequantizing load before save (#14678)
Daniel Han1b76d5f9bc3a
diff --git a/tests/test_fp8_dequant_save_policy.py b/tests/test_fp8_dequant_save_policy.pynew file mode 100644index 00000000000..edc8c3fc6f4--- /dev/null+++ b/tests/test_fp8_dequant_save_policy.py@@ -0,0 +1,244 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""A dequantized fp8 load saves 16bit weights in the plain layout: every submodel of a composite+model, and every dequantizing route (Unsloth's 16bit flag, a caller's config, transformers' own)."""++import ast+import inspect+import json+from pathlib import Path++import pytest+import torch++try:+ from transformers import FineGrainedFP8Config+ _HAS_DEQUANTIZE = "dequantize" in inspect.signature(FineGrainedFP8Config).parameters+except Exception:+ _HAS_DEQUANTIZE = False++ROOT = Path(__file__).resolve().parents[1]+BLOCK = 8+++def _helper():+ # Exact source, without importing unsloth (which needs an accelerator).+ path = ROOT / "unsloth" / "models" / "loader_utils.py"+ tree = ast.parse(path.read_text())+ fn = [+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef) and n.name == "_drop_fp8_dequantize_conversions"+ ]+ ns = {}+ exec(compile(ast.Module(fn, []), str(path), "exec"), ns)+ return ns["_drop_fp8_dequantize_conversions"]+++def _pair_classes():+ from transformers import PreTrainedModel, Qwen3MoeConfig, Qwen3MoeForCausalLM++ try:+ from transformers import PreTrainedConfig+ except ImportError:+ from transformers import PretrainedConfig as PreTrainedConfig++ class PairConfig(PreTrainedConfig):+ model_type = "unsloth_test_fp8_pair"++ class Pair(PreTrainedModel):+ config_class = PairConfig++ def __init__(self, config):+ super().__init__(config)+ sub = Qwen3MoeConfig(+ hidden_size = 16,+ intermediate_size = 32,+ moe_intermediate_size = 8,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 8,+ num_experts = 2,+ num_experts_per_tok = 1,+ vocab_size = 32,+ )+ self.left = Qwen3MoeForCausalLM(sub)+ self.right = Qwen3MoeForCausalLM(sub)+ self.post_init()++ return PairConfig, Pair+++def _quantize_checkpoint(src):+ from safetensors.torch import load_file, save_file++ weights = load_file(src / "model.safetensors")+ out = {}+ for key, v in weights.items():+ if key.endswith("proj.weight") and v.ndim == 2:+ n, k = v.shape+ b = v.float().reshape(n // BLOCK, BLOCK, k // BLOCK, BLOCK)+ s = b.abs().amax(dim = (1, 3)).clamp_min(1e-12) / 448+ out[key] = (b / s[:, None, :, None]).reshape(n, k).to(torch.float8_e4m3fn)+ out[key.replace(".weight", ".weight_scale_inv")] = s+ else:+ out[key] = v+ save_file(out, src / "model.safetensors")+ cfg = json.loads((src / "config.json").read_text())+ cfg["quantization_config"] = {+ "quant_method": "fp8",+ "activation_scheme": "dynamic",+ "weight_block_size": [BLOCK, BLOCK],+ }+ (src / "config.json").write_text(json.dumps(cfg))++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")+def test_composite_model_keeps_every_scoped_counterpart(tmp_path):+ from safetensors.torch import load_file+ from transformers.conversion_mapping import get_model_conversion_mapping++ PairConfig, Pair = _pair_classes()+ torch.manual_seed(0)+ src = tmp_path / "src"+ Pair(PairConfig()).to(torch.bfloat16).save_pretrained(src)+ _quantize_checkpoint(src)+ qc = FineGrainedFP8Config(dequantize = True, weight_block_size = [BLOCK, BLOCK])+ model = Pair.from_pretrained(+ src, device_map = "cpu", dtype = torch.bfloat16, quantization_config = qc+ )+ assert not any(p.dtype == torch.float8_e4m3fn for p in model.parameters())+ assert _helper()(model) > 0++ model.save_pretrained(tmp_path / "saved")+ model._weight_conversions = get_model_conversion_mapping(model, None, None)+ model.save_pretrained(tmp_path / "canonical")+ saved = load_file(tmp_path / "saved" / "model.safetensors")+ canonical = load_file(tmp_path / "canonical" / "model.safetensors")+ # Same per-expert layout as a plain 16bit save of both submodels, so older transformers+ # (ModuleList experts) reload it too.+ assert sorted(saved) == sorted(canonical)+ assert "right.model.layers.0.mlp.experts.0.gate_proj.weight" in saved+ assert all(torch.equal(saved[k], canonical[k]) for k in saved)+++def _calls_and_guards(path, cls_name):+ tree = ast.parse(path.read_text())+ cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == cls_name)+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")+ found = []++ def visit(node, guards):+ for child in ast.iter_child_nodes(node):+ if (+ isinstance(child, ast.Call)+ and isinstance(child.func, ast.Name)+ and child.func.id == "_drop_fp8_dequantize_conversions"+ ):+ found.append(guards)+ g = guards + [ast.unparse(child.test)] if isinstance(child, ast.If) else guards+ visit(child, g)++ visit(fn, [])+ return found+++ "rel, cls_name",+ [+ ("unsloth/models/loader.py", "FastLanguageModel"),+ ("unsloth/models/vision.py", "FastBaseModel"),+ ],+)+def test_cleanup_runs_for_every_dequantizing_route(rel, cls_name):+ # A caller's FineGrainedFP8Config(dequantize = True) and transformers' automatic dequantize+ # (pre-sm89, CPU) leave the same Fp8Dequantize conversions as Unsloth's 16bit flag.+ found = _calls_and_guards(ROOT / rel, cls_name)+ assert found, "cleanup call missing"+ for guards in found:+ text = " ".join(guards)+ assert "_fp8_dequantize_config" not in text and "load_in_16bit" not in text, guards++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")[email protected]("explicit", [True, False])+def test_explicit_and_automatic_dequantize_reload(tmp_path, explicit):+ from transformers import LlamaConfig, LlamaForCausalLM++ torch.manual_seed(0)+ src = tmp_path / "src"+ config = LlamaConfig(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ vocab_size = 64,+ )+ LlamaForCausalLM(config).to(torch.bfloat16).save_pretrained(src)+ _quantize_checkpoint(src)+ kwargs = {}+ if explicit:+ kwargs["quantization_config"] = FineGrainedFP8Config(+ dequantize = True, weight_block_size = [BLOCK, BLOCK]+ )+ # On CPU transformers dequantizes by itself, as on pre-sm89 GPUs.+ model = LlamaForCausalLM.from_pretrained(src, device_map = "cpu", dtype = torch.bfloat16, **kwargs)+ assert not any(p.dtype == torch.float8_e4m3fn for p in model.parameters())+ assert _helper()(model) > 0+ before = {k: v.detach().clone() for k, v in model.state_dict().items()}+ model.save_pretrained(tmp_path / "saved")+ after = LlamaForCausalLM.from_pretrained(tmp_path / "saved", dtype = torch.bfloat16).state_dict()+ assert before.keys() == after.keys()+ assert all(torch.equal(before[k], after[k]) for k in before)++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")+def test_heterogeneous_composite_keeps_one_copy_of_an_unwrapped_converter(monkeypatch):+ # Two submodels whose converters share a type and target but differ in sources: only the+ # `.weight`-sourced one gets the fp8 dequant prepended. The other one is kept as loaded and must+ # not gain a second copy, or save_pretrained would reverse it twice.+ import transformers.conversion_mapping as cm+ from types import SimpleNamespace+ from transformers.core_model_loading import (+ Concatenate,+ MergeModulelist,+ Transpose,+ WeightConverter,+ )+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer++ target = "mlp.experts.gate_up_proj"++ def plain():+ merge = WeightConverter(+ source_patterns = ["mlp.experts.*.gate_proj.weight", "mlp.experts.*.up_proj.weight"],+ target_patterns = target,+ operations = [MergeModulelist(dim = 0), Concatenate(dim = 1)],+ )+ merge.scope_prefix = "left"+ transpose = WeightConverter(+ source_patterns = target, target_patterns = target, operations = [Transpose(1, 2)]+ )+ transpose.scope_prefix = "right"+ return [merge, transpose]++ quantizer = FineGrainedFP8HfQuantizer(+ FineGrainedFP8Config(dequantize = True, weight_block_size = [BLOCK, BLOCK])+ )+ quantizer.pre_quantized = True+ loaded = quantizer.update_weight_conversions(plain())+ model = SimpleNamespace(_weight_conversions = loaded)+ monkeypatch.setattr(cm, "get_model_conversion_mapping", lambda *a, **k: plain())+ assert _helper()(model) > 0++ convs = [c for c in model._weight_conversions if isinstance(c, WeightConverter)]+ transposes = [c for c in convs if any(isinstance(op, Transpose) for op in c.operations)]+ merges = [c for c in convs if any(isinstance(op, MergeModulelist) for op in c.operations)]+ assert len(transposes) == 1, [(c.scope_prefix, c._original_source_patterns) for c in transposes]+ assert len(merges) == 1 and merges[0].scope_prefix == "left"+ from transformers.integrations.finegrained_fp8 import Fp8Dequantize++ assert not any(isinstance(op, Fp8Dequantize) for c in convs for op in c.operations)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 1c3b0279f63..5dcddb5cf14 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -1853,8 +1853,9 @@ class FastLanguageModel(FastLlamaModel):**kwargs,)_record_modelscope_repo_id(model, modelscope_repo_id, modelscope_dir)- if _fp8_dequantize_config is not None:- _drop_fp8_dequantize_conversions(model)+ # Also a caller's FineGrainedFP8Config(dequantize = True) and transformers' own pre-sm89 / CPU+ # dequantize; a no-op unless the load kept Fp8Dequantize conversions.+ _drop_fp8_dequantize_conversions(model)if resize_model_vocab is not None:_resize_vocab(model, resize_model_vocab)diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex bce51330d94..be375da3a0e 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -2048,11 +2048,42 @@ def _drop_fp8_dequantize_conversions(model):patterns = getattr(conversion, "_original_target_patterns", None) or []return tuple(p[:-1] if p.endswith("$") else p for p in patterns)+ def _sources(patterns):+ # The dequantizing copy anchors `.weight` sources with `$` and adds `.weight_scale_inv$` ones.+ patterns = [p[:-1] if p.endswith("$") else p for p in patterns or []]+ return tuple(sorted(p for p in patterns if not p.endswith(".weight_scale_inv")))++ def _source_forms(conversion):+ return {+ _sources(getattr(conversion, "_original_source_patterns", None)),+ _sources(getattr(conversion, "source_patterns", None)),+ }++ def _ops(conversion):+ operations = getattr(conversion, "operations", None) or []+ return tuple(type(op).__name__ for op in operations if not isinstance(op, Fp8Dequantize))++ def _signature(conversion):+ return (+ type(conversion),+ getattr(conversion, "scope_prefix", None),+ min(_source_forms(conversion)),+ _targets(conversion),+ _ops(conversion),+ )++ def _has_dequant(conversion):+ return any(+ isinstance(op, Fp8Dequantize) for op in getattr(conversion, "operations", None) or []+ )+plain = Nonekept, changed = [], 0+ # Converters the load left untouched stay as they are; a counterpart equal to one is not re-added.+ retained = {_signature(c) for c in conversions if not _has_dequant(c)}for conversion in conversions:operations = getattr(conversion, "operations", None) or []- if not any(isinstance(op, Fp8Dequantize) for op in operations):+ if not _has_dequant(conversion):kept.append(conversion)continueif all(isinstance(op, Fp8Dequantize) for op in operations):@@ -2066,22 +2097,32 @@ def _drop_fp8_dequantize_conversions(model):plain = get_model_conversion_mapping(model, None, None)except Exception:plain = []- match = next(- (- c- for c in plain- if type(c) is type(conversion) and _targets(c) == _targets(conversion)- ),- None,- )- if match is None:+ # A composite model scopes one converter per submodel; the dequantizing copy can lose its+ # scope_prefix, so an unscoped conversion is replaced by every scoped counterpart with the same+ # sources, targets and operations (another submodel's converter can share type and targets).+ scope = getattr(conversion, "scope_prefix", None)+ sources = _source_forms(conversion)+ matches = [+ c+ for c in plain+ if type(c) is type(conversion)+ and _targets(c) == _targets(conversion)+ and _source_forms(c) & sources+ and _ops(c) == _ops(conversion)+ and (scope is None or getattr(c, "scope_prefix", None) == scope)+ ]+ if not matches:print("Unsloth: an fp8 load conversion has no 16bit counterpart; save_pretrained may write fp8 "f"weights for {list(_targets(conversion))}.")kept.append(conversion)continue- kept.append(match)+ for match in matches:+ signature = _signature(match)+ if signature not in retained:+ retained.add(signature)+ kept.append(match)changed += 1if changed:model._weight_conversions = keptdiff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 64b325fa279..3cf06446f7b 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -4020,8 +4020,9 @@ class FastBaseModel:_dequantize_bitsandbytes_for_full_finetuning(model, torch_dtype, model_name)elif fsdp2_requested():prepare_fsdp2_qlora(model, bnb_compute_dtype)+ # Any dequantizing fp8 load (explicit config, pre-sm89 / CPU fallback); else a no-op.+ _drop_fp8_dequantize_conversions(model)if load_in_16bit and not load_in_4bit and not load_in_8bit:- _drop_fp8_dequantize_conversions(model)_dequantize_leftover_fp8_params(model,model_name,