Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

loader: do not restore an fp8 quantization_config over dequantized weights (#15364)

Daniel Hanc3a290777e86
diff --git a/tests/test_fp8_16bit_text_loader_dequant.py b/tests/test_fp8_16bit_text_loader_dequant.py
index 015d0d7cbae..762e85c0ee3 100644
--- a/tests/test_fp8_16bit_text_loader_dequant.py
+++ b/tests/test_fp8_16bit_text_loader_dequant.py
@@ -306,3 +306,69 @@ def test_only_a_load_without_fp8_config_is_marked_dequantized():
dequantized = model(SimpleNamespace())
assert _drop_fp8_dequantize_conversions(dequantized) == 1
assert dequantized._weight_conversions == [] and dequantized._unsloth_fp8_dequantized
+
+
+def _write_tiny_tokenizer(path):
+ from tokenizers import Tokenizer
+ from tokenizers.models import WordLevel
+ from tokenizers.pre_tokenizers import Whitespace
+ from transformers import PreTrainedTokenizerFast
+
+ vocab = {"[UNK]": 0, "[BOS]": 1, "[EOS]": 2, "hello": 3, "world": 4}
+ tok = Tokenizer(WordLevel(vocab, unk_token = "[UNK]"))
+ tok.pre_tokenizer = Whitespace()
+ PreTrainedTokenizerFast(
+ tokenizer_object = tok,
+ unk_token = "[UNK]",
+ bos_token = "[BOS]",
+ eos_token = "[EOS]",
+ pad_token = "[EOS]",
+ ).save_pretrained(path)
+
+
[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")
[email protected](not has_real_cuda(), reason = "fp8 checkpoints load on CUDA")
[email protected]("cls_name", ["FastLanguageModel", "FastModel"])
+def test_explicit_dequantize_config_saves_without_fp8_config(tmp_path, cls_name):
+ src, dst = tmp_path / "src", tmp_path / "saved"
+ src.mkdir()
+ _write_tiny_block_fp8_llama(str(src))
+ _write_tiny_tokenizer(str(src))
+ code = f"""
+import json, os, torch
+import unsloth
+from unsloth import {cls_name} as cls
+from transformers import AutoModelForCausalLM, FineGrainedFP8Config
+from safetensors import safe_open
+config = FineGrainedFP8Config(dequantize = True, weight_block_size = [128, 128])
+model, _ = cls.from_pretrained(
+ {str(src)!r}, max_seq_length = 64, dtype = torch.bfloat16, quantization_config = config,
+ local_files_only = True,
+)
+print("MARKED", int(getattr(model, "_unsloth_fp8_dequantized", False)))
+before = {{k: v.detach().cpu().clone() for k, v in model.state_dict().items()}}
+model.save_pretrained({str(dst)!r})
+with open(os.path.join({str(dst)!r}, "config.json")) as f:
+ print("QUANT", json.dumps(json.load(f).get("quantization_config")))
+dtypes = set()
+for name in os.listdir({str(dst)!r}):
+ if name.endswith(".safetensors"):
+ with safe_open(os.path.join({str(dst)!r}, name), "pt") as f:
+ dtypes |= {{f.get_slice(k).get_dtype() for k in f.keys()}}
+print("DTYPES", ",".join(sorted(dtypes)))
+# A GPU without bf16 (T4) loads float16: reload in whatever dtype was saved.
+after = AutoModelForCausalLM.from_pretrained({str(dst)!r}, dtype = next(iter(before.values())).dtype).state_dict()
+same = all(torch.equal(before[k], after[k].cpu()) for k in before if k in after)
+print("RELOAD", int(same and set(before) <= set(after)))
+"""
+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 600)
+ lines = dict(
+ l.split(" ", 1)
+ for l in out.stdout.splitlines()
+ if l.startswith(("MARKED", "QUANT", "DTYPES", "RELOAD"))
+ )
+ assert "RELOAD" in lines, (out.stdout[-2000:], out.stderr[-2000:])
+ assert lines["MARKED"] == "1", lines
+ assert lines["QUANT"] == "null", lines
+ assert lines["DTYPES"] in ("BF16", "F16"), lines
+ assert lines["RELOAD"] == "1", lines
diff --git a/tests/test_fp8_dequant_save_policy.py b/tests/test_fp8_dequant_save_policy.py
index e733826dd3d..89a849bbdc0 100644
--- a/tests/test_fp8_dequant_save_policy.py
+++ b/tests/test_fp8_dequant_save_policy.py
@@ -265,3 +265,45 @@ def test_heterogeneous_composite_keeps_one_copy_of_an_unwrapped_converter(monkey
from transformers.integrations.finegrained_fp8 import Fp8Dequantize
assert not any(isinstance(op, Fp8Dequantize) for c in convs for op in c.operations)
+
+
+def _quantization_config_restores(path, cls_name):
+ tree = ast.parse(path.read_text(encoding = "utf-8"))
+ cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == cls_name)
+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")
+ found = []
+
+ def restores(call):
+ if not (isinstance(call.func, ast.Attribute) and call.func.attr == "update"):
+ return False
+ if ast.unparse(call.func.value) != "model.config" or not call.args:
+ return False
+ arg = call.args[0]
+ if not isinstance(arg, ast.Dict):
+ return False
+ return any(
+ isinstance(k, ast.Constant)
+ and k.value == "quantization_config"
+ and ast.unparse(v).startswith("quantization_config")
+ for k, v in zip(arg.keys, arg.values)
+ )
+
+ def visit(node, guards):
+ for child in ast.iter_child_nodes(node):
+ if isinstance(child, ast.Call) and restores(child):
+ found.append(guards)
+ g = guards + [ast.unparse(child.test)] if isinstance(child, ast.If) else guards
+ visit(child, g)
+
+ visit(fn, [])
+ return found
+
+
[email protected]("cls_name", ["FastLanguageModel", "FastModel"])
+def test_caller_fp8_config_is_not_restored_over_dequantized_weights(cls_name):
+ # Restoring an fp8 config over dequantized weights saves bf16 as fp8 with no scales (unloadable).
+ found = _quantization_config_restores(ROOT / "unsloth/models/loader.py", cls_name)
+ assert found, "quantization_config restore missing"
+ for guards in found:
+ text = " ".join(guards)
+ assert "_unsloth_fp8_dequantized" in text or "_stamp_bnb" in text, guards
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 7ec5c574a16..17194d619a8 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -2022,7 +2022,8 @@ class FastLanguageModel(FastLlamaModel):
"quant_method": "bitsandbytes",
}
model.config.update({"quantization_config": quantization_config})
- else:
+ elif not getattr(model, "_unsloth_fp8_dequantized", False):
+ # Weights are 16bit after fp8 dequantize: restoring the config would save them as fp8.
if hasattr(quantization_config, "to_dict"):
model.config.update({"quantization_config": quantization_config.to_dict()})
elif isinstance(quantization_config, dict):
@@ -3447,7 +3448,8 @@ class FastModel(FastBaseModel):
"quant_method": "bitsandbytes",
}
model.config.update({"quantization_config": quantization_config})
- else:
+ elif not getattr(model, "_unsloth_fp8_dequantized", False):
+ # Weights are 16bit after fp8 dequantize: restoring the config would save them as fp8.
if hasattr(quantization_config, "to_dict"):
model.config.update({"quantization_config": quantization_config.to_dict()})
elif isinstance(quantization_config, dict):