Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
loader: do not restore an fp8 quantization_config over dequantized weights (#15364)
Daniel Hanc3a290777e86
diff --git a/tests/test_fp8_16bit_text_loader_dequant.py b/tests/test_fp8_16bit_text_loader_dequant.pyindex 015d0d7cbae..762e85c0ee3 100644--- a/tests/test_fp8_16bit_text_loader_dequant.py+++ b/tests/test_fp8_16bit_text_loader_dequant.py@@ -306,3 +306,69 @@ def test_only_a_load_without_fp8_config_is_marked_dequantized():dequantized = model(SimpleNamespace())assert _drop_fp8_dequantize_conversions(dequantized) == 1assert dequantized._weight_conversions == [] and dequantized._unsloth_fp8_dequantized+++def _write_tiny_tokenizer(path):+ from tokenizers import Tokenizer+ from tokenizers.models import WordLevel+ from tokenizers.pre_tokenizers import Whitespace+ from transformers import PreTrainedTokenizerFast++ vocab = {"[UNK]": 0, "[BOS]": 1, "[EOS]": 2, "hello": 3, "world": 4}+ tok = Tokenizer(WordLevel(vocab, unk_token = "[UNK]"))+ tok.pre_tokenizer = Whitespace()+ PreTrainedTokenizerFast(+ tokenizer_object = tok,+ unk_token = "[UNK]",+ bos_token = "[BOS]",+ eos_token = "[EOS]",+ pad_token = "[EOS]",+ ).save_pretrained(path)++[email protected](not _HAS_DEQUANTIZE, reason = "transformers has no fp8 dequantize flag")[email protected](not has_real_cuda(), reason = "fp8 checkpoints load on CUDA")[email protected]("cls_name", ["FastLanguageModel", "FastModel"])+def test_explicit_dequantize_config_saves_without_fp8_config(tmp_path, cls_name):+ src, dst = tmp_path / "src", tmp_path / "saved"+ src.mkdir()+ _write_tiny_block_fp8_llama(str(src))+ _write_tiny_tokenizer(str(src))+ code = f"""+import json, os, torch+import unsloth+from unsloth import {cls_name} as cls+from transformers import AutoModelForCausalLM, FineGrainedFP8Config+from safetensors import safe_open+config = FineGrainedFP8Config(dequantize = True, weight_block_size = [128, 128])+model, _ = cls.from_pretrained(+ {str(src)!r}, max_seq_length = 64, dtype = torch.bfloat16, quantization_config = config,+ local_files_only = True,+)+print("MARKED", int(getattr(model, "_unsloth_fp8_dequantized", False)))+before = {{k: v.detach().cpu().clone() for k, v in model.state_dict().items()}}+model.save_pretrained({str(dst)!r})+with open(os.path.join({str(dst)!r}, "config.json")) as f:+ print("QUANT", json.dumps(json.load(f).get("quantization_config")))+dtypes = set()+for name in os.listdir({str(dst)!r}):+ if name.endswith(".safetensors"):+ with safe_open(os.path.join({str(dst)!r}, name), "pt") as f:+ dtypes |= {{f.get_slice(k).get_dtype() for k in f.keys()}}+print("DTYPES", ",".join(sorted(dtypes)))+# A GPU without bf16 (T4) loads float16: reload in whatever dtype was saved.+after = AutoModelForCausalLM.from_pretrained({str(dst)!r}, dtype = next(iter(before.values())).dtype).state_dict()+same = all(torch.equal(before[k], after[k].cpu()) for k in before if k in after)+print("RELOAD", int(same and set(before) <= set(after)))+"""+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 600)+ lines = dict(+ l.split(" ", 1)+ for l in out.stdout.splitlines()+ if l.startswith(("MARKED", "QUANT", "DTYPES", "RELOAD"))+ )+ assert "RELOAD" in lines, (out.stdout[-2000:], out.stderr[-2000:])+ assert lines["MARKED"] == "1", lines+ assert lines["QUANT"] == "null", lines+ assert lines["DTYPES"] in ("BF16", "F16"), lines+ assert lines["RELOAD"] == "1", linesdiff --git a/tests/test_fp8_dequant_save_policy.py b/tests/test_fp8_dequant_save_policy.pyindex e733826dd3d..89a849bbdc0 100644--- a/tests/test_fp8_dequant_save_policy.py+++ b/tests/test_fp8_dequant_save_policy.py@@ -265,3 +265,45 @@ def test_heterogeneous_composite_keeps_one_copy_of_an_unwrapped_converter(monkeyfrom transformers.integrations.finegrained_fp8 import Fp8Dequantizeassert not any(isinstance(op, Fp8Dequantize) for c in convs for op in c.operations)+++def _quantization_config_restores(path, cls_name):+ tree = ast.parse(path.read_text(encoding = "utf-8"))+ cls = next(n for n in tree.body if isinstance(n, ast.ClassDef) and n.name == cls_name)+ fn = next(n for n in cls.body if isinstance(n, ast.FunctionDef) and n.name == "from_pretrained")+ found = []++ def restores(call):+ if not (isinstance(call.func, ast.Attribute) and call.func.attr == "update"):+ return False+ if ast.unparse(call.func.value) != "model.config" or not call.args:+ return False+ arg = call.args[0]+ if not isinstance(arg, ast.Dict):+ return False+ return any(+ isinstance(k, ast.Constant)+ and k.value == "quantization_config"+ and ast.unparse(v).startswith("quantization_config")+ for k, v in zip(arg.keys, arg.values)+ )++ def visit(node, guards):+ for child in ast.iter_child_nodes(node):+ if isinstance(child, ast.Call) and restores(child):+ found.append(guards)+ g = guards + [ast.unparse(child.test)] if isinstance(child, ast.If) else guards+ visit(child, g)++ visit(fn, [])+ return found++[email protected]("cls_name", ["FastLanguageModel", "FastModel"])+def test_caller_fp8_config_is_not_restored_over_dequantized_weights(cls_name):+ # Restoring an fp8 config over dequantized weights saves bf16 as fp8 with no scales (unloadable).+ found = _quantization_config_restores(ROOT / "unsloth/models/loader.py", cls_name)+ assert found, "quantization_config restore missing"+ for guards in found:+ text = " ".join(guards)+ assert "_unsloth_fp8_dequantized" in text or "_stamp_bnb" in text, guardsdiff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 7ec5c574a16..17194d619a8 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -2022,7 +2022,8 @@ class FastLanguageModel(FastLlamaModel):"quant_method": "bitsandbytes",}model.config.update({"quantization_config": quantization_config})- else:+ elif not getattr(model, "_unsloth_fp8_dequantized", False):+ # Weights are 16bit after fp8 dequantize: restoring the config would save them as fp8.if hasattr(quantization_config, "to_dict"):model.config.update({"quantization_config": quantization_config.to_dict()})elif isinstance(quantization_config, dict):@@ -3447,7 +3448,8 @@ class FastModel(FastBaseModel):"quant_method": "bitsandbytes",}model.config.update({"quantization_config": quantization_config})- else:+ elif not getattr(model, "_unsloth_fp8_dequantized", False):+ # Weights are 16bit after fp8 dequantize: restoring the config would save them as fp8.if hasattr(quantization_config, "to_dict"):model.config.update({"quantization_config": quantization_config.to_dict()})elif isinstance(quantization_config, dict):