Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Load block-FP8 checkpoints in 4-bit (NF4) when load_in_4bit=True is passed (#12146)
Daniel Han91bcb185f9ac
diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.pyindex 74d9a819921..f8ea88facf9 100644--- a/studio/backend/core/export/export.py+++ b/studio/backend/core/export/export.py@@ -54,6 +54,11 @@ if not _IS_MLX:logger = get_logger(__name__)+def _load_in_4bit_kwargs(load_in_4bit: bool) -> dict:+ # True is the loaders' default; passing it reads as an explicit request to requantize fp8 checkpoints to NF4.+ return {} if load_in_4bit else {"load_in_4bit": False}++def _export_runtime_available() -> bool:"""True if export can run: MLX active, or Unsloth imported (only succeeds on a GPU host)."""return bool(_IS_MLX) or (FastLanguageModel is not None)@@ -750,7 +755,7 @@ class ExportBackend:model_name = checkpoint_path,max_seq_length = max_seq_length,dtype = None,- load_in_4bit = load_in_4bit,+ **_load_in_4bit_kwargs(load_in_4bit),trust_remote_code = trust_remote_code,token = token,local_files_only = local_files_only,@@ -790,7 +795,7 @@ class ExportBackend:model_name = checkpoint_path,max_seq_length = max_seq_length,dtype = None,- load_in_4bit = load_in_4bit,+ **_load_in_4bit_kwargs(load_in_4bit),trust_remote_code = trust_remote_code,token = token,local_files_only = local_files_only,@@ -804,7 +809,7 @@ class ExportBackend:model_name = checkpoint_path,max_seq_length = max_seq_length,dtype = None,- load_in_4bit = load_in_4bit,+ **_load_in_4bit_kwargs(load_in_4bit),trust_remote_code = trust_remote_code,token = token,local_files_only = local_files_only,diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex 2efc064d4ea..5487b5671b6 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -72,6 +72,11 @@ from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_mlogger = get_logger(__name__)+def _load_in_4bit_kwargs(load_in_4bit: bool) -> dict:+ # True is the loaders' default; passing it reads as an explicit request to requantize fp8 checkpoints to NF4.+ return {} if load_in_4bit else {"load_in_4bit": False}++def _hf_token_for_loader(hf_token: Optional[str] | bool) -> Optional[str] | bool:if hf_token is False:return False@@ -880,7 +885,7 @@ class InferenceBackend:model_name = load_path,max_seq_length = max_seq_length,dtype = dtype,- load_in_4bit = load_in_4bit,+ **_load_in_4bit_kwargs(load_in_4bit),device_map = device_map,token = _hf_token_for_loader(hf_token),trust_remote_code = trust_remote_code,@@ -929,7 +934,7 @@ class InferenceBackend:model_name = load_path,max_seq_length = max_seq_length,dtype = dtype,- load_in_4bit = load_in_4bit,+ **_load_in_4bit_kwargs(load_in_4bit),device_map = device_map,token = _hf_token_for_loader(hf_token),trust_remote_code = trust_remote_code,diff --git a/studio/backend/tests/test_fp8_chat_load_is_not_a_4bit_request.py b/studio/backend/tests/test_fp8_chat_load_is_not_a_4bit_request.pynew file mode 100644index 00000000000..604978e030d--- /dev/null+++ b/studio/backend/tests/test_fp8_chat_load_is_not_a_4bit_request.py@@ -0,0 +1,48 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""Chat and export loads must not pass `load_in_4bit = True`: Unsloth reads a passed flag as an+explicit request to requantize fp8 checkpoints to NF4, and the chat UI always sends True."""++from __future__ import annotations++import ast+from pathlib import Path++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+_FILES = ("core/inference/inference.py", "core/export/export.py")+++def _helper(path):+ tree = ast.parse((_BACKEND / path).read_text(encoding = "utf-8"))+ node = next(+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_load_in_4bit_kwargs"+ )+ scope = {}+ exec(compile(ast.Module([node], []), path, "exec"), scope)+ return scope["_load_in_4bit_kwargs"]++[email protected]("path", _FILES)+def test_true_is_left_to_the_loader_default(path):+ helper = _helper(path)+ assert helper(True) == {}+ assert helper(False) == {"load_in_4bit": False}++[email protected]("path", _FILES)+def test_no_unsloth_loader_call_forwards_the_flag_verbatim(path):+ tree = ast.parse((_BACKEND / path).read_text(encoding = "utf-8"))+ forwarded = [+ node.lineno+ for node in ast.walk(tree)+ if isinstance(node, ast.Call)+ and isinstance(node.func, ast.Attribute)+ and node.func.attr == "from_pretrained"+ and isinstance(node.func.value, ast.Name)+ and node.func.value.id.startswith("Fast")+ and any(kw.arg == "load_in_4bit" and isinstance(kw.value, ast.Name) for kw in node.keywords)+ ]+ assert not forwarded, f"{path}: from_pretrained forwards load_in_4bit at lines {forwarded}"diff --git a/tests/test_fp8_to_nf4_load.py b/tests/test_fp8_to_nf4_load.pynew file mode 100644index 00000000000..34437b0c565--- /dev/null+++ b/tests/test_fp8_to_nf4_load.py@@ -0,0 +1,627 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""`load_in_4bit = True` on a block-FP8 checkpoint must give NF4 bytes identical to loading the+same checkpoint dequantized to bf16 by an independent formula."""++import json+import os+import shutil+from types import SimpleNamespace++import pytest+import torch++import unsloth # noqa: F401 (patches before transformers)+from unsloth import FastLanguageModel, FastModel+from unsloth.models.loader_utils import check_and_disable_bitsandbytes_loading++_BLOCK = 128+_TOKENIZER_REPO = "trl-internal-testing/tiny-Qwen3ForCausalLM"+_KEEP_16BIT = "model.layers.1.self_attn.o_proj"++needs_cuda = pytest.mark.skipif(+ not torch.cuda.is_available(), reason = "bitsandbytes 4bit needs a GPU"+)+++def _feature_unavailable():+ try:+ from unsloth.models.fp8_to_nf4 import fp8_to_nf4_unavailable_reason+ except ImportError:+ return None # main: no module, the tests must fail there+ return fp8_to_nf4_unavailable_reason()+++needs_feature = pytest.mark.skipif(+ _feature_unavailable() is not None,+ reason = f"fp8 -> 4bit loading unavailable: {_feature_unavailable()}",+)+needs_fp8_gpu = pytest.mark.skipif(+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),+ reason = "fp8 checkpoints load natively only on sm89+",+)+++def _quantize_block_fp8(weight):+ rows, cols = weight.shape+ grid_r, grid_c = -(-rows // _BLOCK), -(-cols // _BLOCK)+ padded = torch.zeros(grid_r * _BLOCK, grid_c * _BLOCK, dtype = torch.float32)+ padded[:rows, :cols] = weight.float()+ blocks = padded.view(grid_r, _BLOCK, grid_c, _BLOCK)+ scale = blocks.abs().amax(dim = (1, 3)).clamp(min = 1e-12) / 448.0+ quant = (blocks / scale[:, None, :, None]).clamp(-448, 448).to(torch.float8_e4m3fn)+ quant = quant.view(grid_r * _BLOCK, grid_c * _BLOCK)[:rows, :cols].contiguous()+ return quant, scale.contiguous()+++def _dequantize_reference(quant, scale):+ rows, cols = quant.shape+ expanded = (+ scale.float().repeat_interleave(_BLOCK, 0)[:rows].repeat_interleave(_BLOCK, 1)[:, :cols]+ )+ return (quant.float() * expanded).to(torch.bfloat16)+++def _is_fp8_candidate(key, tensor):+ if not key.endswith(".weight") or tensor.dim() != 2:+ return False+ if ".layers." not in key or "norm" in key or ".mlp.gate.weight" in key:+ return False+ return not key.startswith(_KEEP_16BIT + ".")+++def _write_pair(bf16_dir, root):+ """bf16 save -> (fp8 dir, dequantized-bf16 dir)."""+ from safetensors.torch import load_file, save_file++ fp8_dir, deq_dir = os.path.join(root, "fp8"), os.path.join(root, "deq")+ for d in (fp8_dir, deq_dir):+ os.makedirs(d, exist_ok = True)+ for name in os.listdir(bf16_dir):+ if not name.endswith(".safetensors") and not name.endswith(".index.json"):+ shutil.copy2(os.path.join(bf16_dir, name), os.path.join(d, name))+ state = {}+ for name in sorted(os.listdir(bf16_dir)):+ if name.endswith(".safetensors"):+ state.update(load_file(os.path.join(bf16_dir, name)))+ fp8_state, deq_state = {}, {}+ for key, tensor in state.items():+ if _is_fp8_candidate(key, tensor):+ quant, scale = _quantize_block_fp8(tensor)+ fp8_state[key] = quant+ fp8_state[key + "_scale_inv"] = scale+ deq_state[key] = _dequantize_reference(quant, scale)+ else:+ fp8_state[key] = tensor+ deq_state[key] = tensor+ save_file(fp8_state, os.path.join(fp8_dir, "model.safetensors"), metadata = {"format": "pt"})+ save_file(deq_state, os.path.join(deq_dir, "model.safetensors"), metadata = {"format": "pt"})+ with open(os.path.join(fp8_dir, "config.json")) as f:+ config = json.load(f)+ config["quantization_config"] = {+ "quant_method": "fp8",+ "fmt": "e4m3",+ "activation_scheme": "dynamic",+ "weight_block_size": [_BLOCK, _BLOCK],+ "modules_to_not_convert": ["lm_head", "model.embed_tokens", _KEEP_16BIT],+ }+ with open(os.path.join(fp8_dir, "config.json"), "w") as f:+ json.dump(config, f, indent = 2)+ return fp8_dir, deq_dir+++def _build(root, moe):+ from transformers import AutoTokenizer++ tokenizer = AutoTokenizer.from_pretrained(_TOKENIZER_REPO)+ common = dict(+ vocab_size = len(tokenizer),+ hidden_size = 256,+ num_hidden_layers = 2,+ num_attention_heads = 4,+ num_key_value_heads = 2,+ head_dim = 64,+ max_position_embeddings = 512,+ tie_word_embeddings = False,+ torch_dtype = "bfloat16",+ )+ torch.manual_seed(0)+ if moe:+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM+ config = Qwen3MoeConfig(+ intermediate_size = 320,+ moe_intermediate_size = 192, # ragged: 1.5 blocks+ num_experts = 4,+ num_experts_per_tok = 2,+ decoder_sparse_step = 1,+ **common,+ )+ model = Qwen3MoeForCausalLM(config)+ else:+ from transformers import Qwen3Config, Qwen3ForCausalLM+ config = Qwen3Config(intermediate_size = 320, **common) # ragged: 2.5 blocks+ model = Qwen3ForCausalLM(config)+ model = model.to(torch.bfloat16)+ bf16_dir = os.path.join(root, "bf16")+ model.save_pretrained(bf16_dir, safe_serialization = True)+ tokenizer.save_pretrained(bf16_dir)+ return _write_pair(bf16_dir, root)++[email protected](scope = "module")+def dense_pair(tmp_path_factory):+ return _build(str(tmp_path_factory.mktemp("fp8_dense")), moe = False)++[email protected](scope = "module")+def moe_pair(tmp_path_factory):+ return _build(str(tmp_path_factory.mktemp("fp8_moe")), moe = True)+++def _fingerprint(model):+ """name -> tensors that define the loaded weight: packed NF4 bytes and the full quant state."""+ out = {}+ for name, param in model.named_parameters():+ state = getattr(param, "quant_state", None)+ entry = {"data": param.data.detach().clone().cpu()}+ if state is not None:+ entry["absmax"] = state.absmax.detach().cpu()+ entry["shape"] = tuple(state.shape)+ entry["dtype"] = state.dtype+ entry["blocksize"] = state.blocksize+ entry["quant_type"] = state.quant_type+ if getattr(state, "nested", False):+ entry["offset"] = state.offset.detach().cpu()+ entry["absmax2"] = state.state2.absmax.detach().cpu()+ entry["code2"] = state.state2.code.detach().cpu()+ out[name] = entry+ return out+++def _assert_same(a, b):+ assert a.keys() == b.keys()+ for name in a:+ ea, eb = a[name], b[name]+ assert ea.keys() == eb.keys(), name+ for key in ea:+ va, vb = ea[key], eb[key]+ if isinstance(va, torch.Tensor):+ assert va.dtype == vb.dtype and va.shape == vb.shape, (name, key)+ assert torch.equal(+ va.reshape(-1).view(torch.uint8), vb.reshape(-1).view(torch.uint8)+ ), (name, key)+ else:+ assert va == vb, (name, key)+++def _count_4bit(model):+ return sum(1 for _, p in model.named_parameters() if type(p).__name__ == "Params4bit")+++def _count_fp8(model):+ return sum(1 for _, p in model.named_parameters() if p.dtype == torch.float8_e4m3fn)+++def _load(loader, path, **kwargs):+ model, _ = loader.from_pretrained(path, max_seq_length = 64, dtype = torch.bfloat16, **kwargs)+ return model+++def _reference_config():+ from transformers import BitsAndBytesConfig+ from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES+ return BitsAndBytesConfig(+ load_in_4bit = True,+ bnb_4bit_use_double_quant = True,+ bnb_4bit_quant_type = "nf4",+ bnb_4bit_compute_dtype = torch.bfloat16,+ llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES + [_KEEP_16BIT],+ )+++def _free(*models):+ for model in models:+ del model+ import gc++ gc.collect()+ torch.cuda.empty_cache()+++@needs_cuda+@needs_feature+def test_dense_fp8_loads_nf4_bit_identical_to_dequantized_bf16(dense_pair, capsys):+ fp8_dir, deq_dir = dense_pair+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)+ printed = capsys.readouterr().out+ assert _count_4bit(model) == 13 # 2 x (q, k, v, o, gate, up, down) minus the 16-bit o_proj+ assert _count_fp8(model) == 0+ assert "dequantizing each fp8 tensor" in printed+ assert not hasattr(model.config, "_unsloth_fp8_to_nf4")+ ours = _fingerprint(model)+ _free(model)+ reference = _load(FastLanguageModel, deq_dir, quantization_config = _reference_config())+ theirs = _fingerprint(reference)+ _free(reference)+ _assert_same(ours, theirs)+++@needs_cuda+@needs_feature+def test_moe_fp8_loads_nf4_experts_bit_identical(moe_pair):+ fp8_dir, deq_dir = moe_pair+ model = _load(FastModel, fp8_dir, load_in_4bit = True)+ experts = [+ name+ for name, p in model.named_parameters()+ if ".experts." in name and type(p).__name__ == "Params4bit"+ ]+ # gate_up_proj and down_proj stacks on both layers go through Unsloth's bnb expert path.+ assert len(experts) == 4, experts+ assert _count_fp8(model) == 0+ ours = _fingerprint(model)+ _free(model)+ reference = _load(FastModel, deq_dir, quantization_config = _reference_config())+ theirs = _fingerprint(reference)+ _free(reference)+ _assert_same(ours, theirs)+++@needs_cuda+@needs_feature+def test_checkpoint_16bit_modules_stay_16bit(dense_pair):+ from safetensors.torch import load_file++ fp8_dir, _ = dense_pair+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)+ module = model.get_submodule(_KEEP_16BIT)+ assert type(module) is torch.nn.Linear+ stored = load_file(os.path.join(fp8_dir, "model.safetensors"))[_KEEP_16BIT + ".weight"]+ assert stored.dtype == torch.bfloat16+ assert torch.equal(module.weight.detach().cpu(), stored)+ quantization_config = model.config.quantization_config+ if not isinstance(quantization_config, dict):+ quantization_config = quantization_config.to_dict()+ # Saved with the model, so a reload of the 4bit save keeps it in 16bit too.+ assert any(+ p.replace("\\", "").rstrip("$") == _KEEP_16BIT+ for p in quantization_config["llm_int8_skip_modules"]+ )+ _free(model)+++@needs_fp8_gpu+@needs_feature+def test_default_load_in_4bit_and_kill_switch_keep_fp8(dense_pair, monkeypatch):+ fp8_dir, _ = dense_pair+ # load_in_4bit left at its default is not a request: unchanged fp8 load.+ model = _load(FastLanguageModel, fp8_dir)+ assert _count_4bit(model) == 0 and _count_fp8(model) > 0+ _free(model)+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "0")+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)+ assert _count_4bit(model) == 0 and _count_fp8(model) > 0+ _free(model)+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")+ model = _load(FastLanguageModel, fp8_dir)+ assert _count_4bit(model) == 13 and _count_fp8(model) == 0+ _free(model)+++@needs_cuda+@needs_feature+def test_fp8_keys_the_model_does_not_have_are_ignored(dense_pair, tmp_path):+ """MTP layers ship in fp8 but transformers drops them as unexpected (DeepSeek-V3, GLM-4 MoE,+ Qwen3-Next): they must not count as unapplied scales."""+ from safetensors.torch import load_file, save_file++ fp8_dir, _ = dense_pair+ extra_dir = str(tmp_path / "fp8_mtp")+ shutil.copytree(fp8_dir, extra_dir)+ state = load_file(os.path.join(extra_dir, "model.safetensors"))+ quant, scale = _quantize_block_fp8(torch.randn(320, 256) * 0.02)+ state["mtp.layers.0.mlp.gate_proj.weight"] = quant+ state["mtp.layers.0.mlp.gate_proj.weight_scale_inv"] = scale+ save_file(state, os.path.join(extra_dir, "model.safetensors"), metadata = {"format": "pt"})+ model = _load(FastLanguageModel, extra_dir, load_in_4bit = True)+ assert _count_4bit(model) == 13+ ours = _fingerprint(model)+ _free(model)+ plain = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)+ theirs = _fingerprint(plain)+ _free(plain)+ _assert_same(ours, theirs)+++@needs_cuda+@needs_feature+def test_quantize_16bit_switch_matches_plain_4bit_skip_list(dense_pair, monkeypatch):+ fp8_dir, deq_dir = dense_pair+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT", "1")+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)+ assert _count_4bit(model) == 14+ ours = _fingerprint(model)+ _free(model)+ reference = _load(FastLanguageModel, deq_dir, load_in_4bit = True)+ theirs = _fingerprint(reference)+ _free(reference)+ _assert_same(ours, theirs)+++def _zoo_reads_fp8_block_size_from_disk():+ try:+ from unsloth_zoo import saving_utils+ except Exception:+ return False+ return hasattr(saving_utils, "_fp8_block_size_on_disk")+++@needs_cuda+@needs_feature+ not _zoo_reads_fp8_block_size_from_disk(),+ reason = "unsloth_zoo takes the fp8 block size of a 16bit merge from the in-memory config only",+)+def test_moe_merged_16bit_save_equals_dequantized_checkpoint(moe_pair, tmp_path):+ from safetensors.torch import load_file++ fp8_dir, deq_dir = moe_pair+ model, tokenizer = FastModel.from_pretrained(+ fp8_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True+ )+ assert _count_4bit(model) > 0+ model = FastModel.get_peft_model(+ model, r = 8, target_modules = ["q_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]+ )+ # lora_B starts at zero, so the merge must give back the dequantized fp8 weights exactly,+ # read from the fp8 shards on disk rather than the NF4 copy in memory.+ model.save_pretrained_merged(str(tmp_path / "merged"), tokenizer, save_method = "merged_16bit")+ _free(model)+ merged = {}+ for name in os.listdir(tmp_path / "merged"):+ if name.endswith(".safetensors"):+ merged.update(load_file(str(tmp_path / "merged" / name)))+ reference = load_file(os.path.join(deq_dir, "model.safetensors"))+ assert merged.keys() == reference.keys()+ for key, tensor in reference.items():+ assert torch.equal(merged[key], tensor), key+++@needs_fp8_gpu[email protected](_feature_unavailable() is None, reason = "only where fp8 -> 4bit cannot run")+def test_unavailable_keeps_todays_fp8_load(dense_pair, capsys):+ fp8_dir, _ = dense_pair+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)+ assert _count_4bit(model) == 0 and _count_fp8(model) > 0+ assert "cannot quantize to 4bit here" in capsys.readouterr().out+ _free(model)+++def _fp8_config():+ return SimpleNamespace(+ model_type = "qwen3",+ quantization_config = {+ "quant_method": "fp8",+ "fmt": "e4m3",+ "weight_block_size": [128, 128],+ "modules_to_not_convert": ["lm_head"],+ },+ )+++@needs_feature+def test_arming_needs_an_explicit_request(monkeypatch):+ from unsloth.models import fp8_to_nf4++ monkeypatch.delenv("UNSLOTH_FP8_TO_NF4", raising = False)+ config = _fp8_config()+ assert check_and_disable_bitsandbytes_loading(config, load_in_4bit = True, verbose = False)[:2] == (+ False,+ False,+ )+ assert config.quantization_config["quant_method"] == "fp8"++ inside = {}++ @fp8_to_nf4.track_explicit_4bit_request+ def from_pretrained(+ model_name = None,+ load_in_4bit = True,+ **kwargs,+ ):+ flags = check_and_disable_bitsandbytes_loading(+ config, load_in_4bit = load_in_4bit, verbose = False+ )+ inside["armed"] = fp8_to_nf4.fp8_to_nf4_armed(config)+ inside["stripped"] = not hasattr(config, "quantization_config")+ return flags++ assert from_pretrained("x")[:2] == (False, False)+ config = _fp8_config()+ assert from_pretrained("x", load_in_4bit = True)[:2] == (True, False)+ assert inside == {"armed": True, "stripped": True}+ # The outermost from_pretrained hands the caller's config back as it was.+ assert config.quantization_config["quant_method"] == "fp8"+ assert not fp8_to_nf4.fp8_to_nf4_armed(config)++ config = _fp8_config()+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "0")+ assert from_pretrained("x", load_in_4bit = True)[:2] == (False, False)+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")+ assert from_pretrained("x")[:2] == (True, False)+++def test_non_block_fp8_and_8bit_are_not_armed(monkeypatch):+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")+ config = _fp8_config()+ config.quantization_config.pop("weight_block_size")+ assert check_and_disable_bitsandbytes_loading(config, load_in_4bit = True, verbose = False)[:2] == (+ False,+ False,+ )+ config = _fp8_config()+ assert check_and_disable_bitsandbytes_loading(+ config, load_in_4bit = False, load_in_8bit = True, verbose = False+ )[:2] == (False, False)+ assert config.quantization_config["quant_method"] == "fp8"+++@needs_feature+def test_scaled_weights_stored_as_integers_are_refused():+ from unsloth.models import fp8_to_nf4+ fp8_to_nf4._refuse_packed_scaled_weights(+ {"a.weight": "F8_E4M3", "a.weight_scale_inv": "F32", "b.weight": "BF16", "n.scale": "F32"}+ )+ with pytest.raises(RuntimeError, match = "cannot dequantize"):+ fp8_to_nf4._refuse_packed_scaled_weights({"e.weight": "I8", "e.scale": "F8_E8M0"})+++@needs_feature+def test_packed_fp4_expert_checkpoints_are_not_armed(monkeypatch):+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")+ config = _fp8_config()+ config.quantization_config["expert_dtype"] = "fp4"+ assert check_and_disable_bitsandbytes_loading(config, load_in_4bit = True, verbose = False)[:2] == (+ False,+ False,+ )+ assert config.quantization_config["quant_method"] == "fp8"++[email protected]("shape", [(256, 384), (320, 256), (192, 200), (3, 320, 192)])+def test_block_dequant_matches_reference_formula(shape):+ torch.manual_seed(1)+ weight = torch.randn(*shape) * 0.02+ if len(shape) == 2:+ quant, scale = _quantize_block_fp8(weight)+ expected = _dequantize_reference(quant, scale)+ else:+ pairs = [_quantize_block_fp8(w) for w in weight]+ quant = torch.stack([q for q, _ in pairs])+ scale = torch.stack([s for _, s in pairs])+ expected = torch.stack([_dequantize_reference(q, s) for q, s in pairs])+ from unsloth.models import fp8_to_nf4++ got = fp8_to_nf4._dequantize_block_fp8(quant, scale, [128, 128], torch.bfloat16)+ assert torch.equal(got.view(torch.int16), expected.view(torch.int16))+++@needs_feature+def test_stacked_16bit_weight_with_a_scale_is_copied_into_the_stack():+ import unsloth.models.fp8_to_nf4 as fp8_to_nf4++ dequantize_op, _ = fp8_to_nf4._build_classes()+ torch.manual_seed(2)+ quant, scale = _quantize_block_fp8(torch.randn(128, 128) * 0.02)+ stored_16bit = (torch.randn(128, 128) * 0.02).to(torch.bfloat16)+ fp8_to_nf4._STATE = SimpleNamespace(+ block = [128, 128], dtype = torch.bfloat16, kept_fp8 = 0, dequantized = 0+ )+ try:+ result = dequantize_op(stack = True).convert(+ {+ "mlp.experts.*.down_proj.weight": [quant, stored_16bit],+ "mlp.experts.*.down_proj.weight_scale_inv": [scale, torch.ones(1, 1)],+ },+ target_patterns = ["mlp.experts.down_proj"],+ )+ finally:+ fp8_to_nf4._STATE = None+ stack = result["mlp.experts.*.down_proj.weight"]+ assert torch.equal(stack[0], _dequantize_reference(quant, scale))+ assert torch.equal(stack[1], stored_16bit)+++@needs_feature+def test_a_load_that_raises_before_loading_hands_the_config_back(monkeypatch):+ # Prefetch or device-map planning can raise after arming, before the load restores the config.+ from unsloth.models import fp8_to_nf4++ monkeypatch.delenv("UNSLOTH_FP8_TO_NF4", raising = False)+ config = _fp8_config()+ original = dict(config.quantization_config)++ @fp8_to_nf4.track_explicit_4bit_request+ def from_pretrained(+ model_name = None,+ load_in_4bit = True,+ **kwargs,+ ):+ assert check_and_disable_bitsandbytes_loading(+ config, load_in_4bit = load_in_4bit, verbose = False+ )[0]+ assert fp8_to_nf4.fp8_to_nf4_armed(config)+ raise RuntimeError("prefetch stalled")++ with pytest.raises(RuntimeError, match = "prefetch stalled"):+ from_pretrained("x", load_in_4bit = True)+ assert config.quantization_config == original+ assert not fp8_to_nf4.fp8_to_nf4_armed(config)+++def test_planner_sizes_checkpoint_16bit_linears_as_4bit_in_quantize_all_mode(monkeypatch):+ from unsloth.models import fp8_to_nf4++ config = SimpleNamespace(+ **{+ fp8_to_nf4.UNSLOTH_FP8_TO_NF4_ATTR: {+ "modules_to_not_convert": ["vision_tower", "lm_head"]+ }+ }+ )+ monkeypatch.delenv("UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT", raising = False)+ kept = fp8_to_nf4.fp8_to_nf4_planner_quantization_config(config, ["lm_head"])+ assert kept["llm_int8_skip_modules"] == ["lm_head", "vision_tower"]+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT", "1")+ quantized = fp8_to_nf4.fp8_to_nf4_planner_quantization_config(config, ["lm_head"])+ assert quantized["llm_int8_skip_modules"] == ["lm_head"]+++@needs_feature+def test_planner_sizes_an_armed_load_expert_merge_in_the_load_dtype():+ # The fused expert stack is built in bf16 before bitsandbytes packs it: plan that transient.+ planner = pytest.importorskip("unsloth_zoo.device_map_planner")+ from transformers import AutoConfig++ from unsloth.models import fp8_to_nf4++ assert fp8_to_nf4.install_fp8_to_nf4_quantizer()+ repo = "trl-internal-testing/tiny-Qwen3MoeForCausalLM"+ config = AutoConfig.from_pretrained(repo)+ rewritten = {"quant_method": "bitsandbytes", "load_in_4bit": True, "bnb_4bit_quant_type": "nf4"}+ measured = {}+ for armed in (False, True):+ token = fp8_to_nf4._ARMED_CONFIGS.set([config] if armed else None)+ try:+ model, quantizer, _ = planner.build_meta_model(+ repo,+ config = config,+ dtype = torch.bfloat16,+ rewritten_quantization_config = dict(rewritten),+ )+ units = [(name, 0) for name, _ in model.named_modules() if name.endswith(".mlp")]+ transient = planner._load_transient_by_unit(model, units, quantizer)+ measured[armed] = (+ quantizer.pre_quantized,+ planner._compute_module_sizes(model, quantizer)[""],+ max(transient.values()),+ )+ finally:+ fp8_to_nf4._ARMED_CONFIGS.reset(token)+ assert measured[False][0] and not measured[True][0]+ assert measured[True][1] == measured[False][1] # weights still sized as 4bit+ assert measured[True][2] == 4 * measured[False][2] # bf16 stack, not packed nibblesdiff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.pyindex febadfaf369..9f375738730 100644--- a/tests/test_modelopt_fp8_loading.py+++ b/tests/test_modelopt_fp8_loading.py@@ -531,7 +531,7 @@ def test_config_branch_moves_rope_extension_onto_the_config():from unsloth.models import llamasource = inspect.getsource(llama.FastLlamaModel.from_pretrained)- branch = source.split("if user_config is not None or _modelopt_rewritten:", 1)[1]+ branch = source.split("if user_config is not None or _modelopt_rewritten", 1)[1]branch = branch.split("AutoModelForCausalLM.from_pretrained(", 1)[0]assert 'kwargs.pop("rope_scaling", None)' in branchdiff --git a/unsloth/models/fp8_to_nf4.py b/unsloth/models/fp8_to_nf4.pynew file mode 100644index 00000000000..4122a6a1dfa--- /dev/null+++ b/unsloth/models/fp8_to_nf4.py@@ -0,0 +1,1169 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published by+# the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.+"""Load a block-FP8 checkpoint straight into bitsandbytes NF4 (``load_in_4bit = True``).++The fp8 block is parked off the config so transformers runs a bnb 4bit load; each FP8 tensor is+dequantized inside its weight converter right before bnb quantizes it, so only one converter's+tensors are ever 16-bit and NF4 is bit-identical to loading the dequantized checkpoint. The+checkpoint's ``modules_to_not_convert`` stay out of bitsandbytes.+"""++import contextvars+import copy+import functools+import inspect+import os+import re+from typing import Optional++import torch++__all__ = [+ "UNSLOTH_FP8_TO_NF4_ATTR",+ "track_explicit_4bit_request",+ "requests_bnb_4bit",+ "explicit_4bit_requested",+ "fp8_block_quantization_config",+ "fp8_to_nf4_unavailable_reason",+ "maybe_arm_fp8_to_nf4",+ "fp8_to_nf4_armed",+ "fp8_to_nf4_disabled",+ "disarm_fp8_to_nf4",+ "fp8_to_nf4_planner_quantization_config",+]++UNSLOTH_FP8_TO_NF4_ATTR = "_unsloth_fp8_to_nf4"+_ENV = "UNSLOTH_FP8_TO_NF4"+_ENV_QUANTIZE_ALL = "UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT"++# None outside a load; True / False once the outermost from_pretrained has looked at its call.+_EXPLICIT_4BIT = contextvars.ContextVar("unsloth_explicit_load_in_4bit", default = None)+# Configs armed inside the outermost from_pretrained; disarmed when it returns or raises, so an+# exception before the load's own restore (prefetch, device-map planning) cannot leak a stripped config.+_ARMED_CONFIGS = contextvars.ContextVar("unsloth_fp8_to_nf4_armed_configs", default = None)++_FP8_DTYPES = tuple(+ getattr(torch, name)+ for name in ("float8_e4m3fn", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz")+ if hasattr(torch, name)+)+_FP8_SAFETENSORS = ("F8_E4M3", "F8_E5M2", "F8_E4M3FNUZ", "F8_E5M2FNUZ")+_SCALE_SUFFIXES = (".weight_scale_inv", ".activation_scale")+# Checkpoint dtypes a scaled weight may be stored in; anything else (int8-packed FP4) is refused.+_FLOAT_SAFETENSORS = _FP8_SAFETENSORS + ("BF16", "F16", "F32")+_PACKED_EXPERT_DTYPES = ("fp4", "mxfp4", "nvfp4", "int4")+++def requests_bnb_4bit(quantization_config) -> bool:+ if quantization_config is None:+ return False+ if isinstance(quantization_config, dict):+ get = quantization_config.get+ else:+ get = lambda key, default = None: getattr(quantization_config, key, default)+ method = get("quant_method", "") or ""+ method = str(getattr(method, "value", method)).lower()+ if method and "bitsandbytes" not in method:+ return False+ return bool(get("load_in_4bit", False))+++def track_explicit_4bit_request(fn):+ """Record whether the caller itself asked for 4bit: ``load_in_4bit`` defaults to True, so only a+ passed argument is a request (a user ``quantization_config`` clears the loader's 4bit flag, so it+ never arms). The outermost from_pretrained decides; the FastModel delegations inside it pass the+ flag on explicitly."""+ try:+ signature = inspect.signature(fn)+ except (TypeError, ValueError):+ signature = None++ @functools.wraps(fn)+ def _wrapper(*args, **kwargs):+ if _EXPLICIT_4BIT.get() is not None:+ return fn(*args, **kwargs)+ explicit = False+ try:+ if signature is not None:+ arguments = signature.bind_partial(*args, **kwargs).arguments+ explicit = bool(arguments.get("load_in_4bit", False))+ except TypeError:+ explicit = bool(kwargs.get("load_in_4bit", False))+ explicit = explicit or requests_bnb_4bit(kwargs.get("quantization_config", None))+ token = _EXPLICIT_4BIT.set(explicit)+ armed_token = _ARMED_CONFIGS.set([])+ try:+ return fn(*args, **kwargs)+ finally:+ for armed in _ARMED_CONFIGS.get() or []:+ disarm_fp8_to_nf4(armed)+ _ARMED_CONFIGS.reset(armed_token)+ _EXPLICIT_4BIT.reset(token)++ return _wrapper+++def explicit_4bit_requested() -> bool:+ return bool(_EXPLICIT_4BIT.get())+++def _quant_dict(quant) -> Optional[dict]:+ if quant is None:+ return None+ if isinstance(quant, dict):+ return quant+ to_dict = getattr(quant, "to_dict", None)+ if callable(to_dict):+ try:+ return to_dict()+ except Exception:+ return None+ return None+++def _configs_with_quantization(config) -> list:+ """The config objects holding a quantization_config: the root, else its text config+ (transformers' get_hf_quantizer reads both)."""+ found = []+ if config is None:+ return found+ if getattr(config, "quantization_config", None) is not None:+ found.append(config)+ get_text_config = getattr(config, "get_text_config", None)+ if callable(get_text_config):+ try:+ text_config = get_text_config(decoder = True)+ except TypeError:+ text_config = get_text_config()+ except Exception:+ text_config = None+ if (+ text_config is not None+ and text_config is not config+ and getattr(text_config, "quantization_config", None) is not None+ ):+ found.append(text_config)+ return found+++def _block_size(quant: dict):+ block = quant.get("weight_block_size")+ if isinstance(block, (int, float)):+ block = [block, block]+ if not isinstance(block, (list, tuple)) or len(block) != 2:+ return None+ try:+ block = [int(block[0]), int(block[1])]+ except (TypeError, ValueError):+ return None+ if block[0] <= 0 or block[1] <= 0:+ return None+ return block+++def fp8_block_quantization_config(config) -> Optional[dict]:+ """The checkpoint's fine-grained (block) fp8 quantization_config as a dict, else None."""+ for holder in _configs_with_quantization(config):+ quant = _quant_dict(getattr(holder, "quantization_config", None))+ if not quant or str(quant.get("quant_method", "")).lower() != "fp8":+ continue+ if _block_size(quant) is None:+ return None+ return quant+ return None+++def _transformers_supports_fp8_to_nf4() -> Optional[str]:+ try:+ from transformers import core_model_loading as cml+ from transformers.conversion_mapping import get_model_conversion_mapping # noqa: F401+ from transformers.quantizers import auto as quantizers_auto+ from transformers.quantizers.quantizer_bnb_4bit import Bnb4BitHfQuantizer+ except Exception:+ return "this transformers has no weight-conversion loader (transformers >= 5.16 is needed)"+ for name in (+ "WeightConverter",+ "WeightRenaming",+ "ConversionOps",+ "rename_source_key",+ "spawn_materialize",+ ):+ if not hasattr(cml, name):+ return f"transformers.core_model_loading has no `{name}`"+ if not hasattr(cml.WeightTransform, "add_tensor"):+ return "transformers' WeightTransform has no add_tensor"+ if not isinstance(getattr(quantizers_auto, "AUTO_QUANTIZER_MAPPING", None), dict):+ return "transformers has no AUTO_QUANTIZER_MAPPING"+ # Converters are rebuilt with these (5.5 to 5.16 added them one by one).+ if not hasattr(cml, "_IdentityOp"):+ return "transformers.core_model_loading has no `_IdentityOp`"+ if not callable(getattr(Bnb4BitHfQuantizer, "update_weight_conversions", None)):+ return "transformers' quantizers have no update_weight_conversions hook"+ # Slots, not the signature: Unsloth wraps WeightConverter.__init__ as (*args, **kwargs).+ converter_fields = set(getattr(cml.WeightConverter, "__slots__", ())) | set(+ getattr(cml.WeightConverter, "__dataclass_fields__", {})+ )+ if "force_cpu" not in converter_fields:+ return "transformers' WeightConverter predates force_cpu (transformers >= 5.16 is needed)"+ return None+++def fp8_to_nf4_unavailable_reason() -> Optional[str]:+ reason = _transformers_supports_fp8_to_nf4()+ if reason is not None:+ return reason+ try:+ import bitsandbytes # noqa: F401+ except Exception:+ return "bitsandbytes is not installed"+ return None+++def fp8_to_nf4_disabled() -> bool:+ return os.environ.get(_ENV, "").strip().lower() in ("0", "false", "off", "no")+++def fp8_to_nf4_armed(config) -> bool:+ return config is not None and getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) is not None+++def maybe_arm_fp8_to_nf4(+ config,+ load_in_4bit,+ load_in_8bit,+ verbose = True,+) -> bool:+ """Arm an fp8 -> NF4 load. True means: keep load_in_4bit, the fp8 block was parked on `config`+ and removed from it, and the load must pass this `config` to from_pretrained."""+ if not load_in_4bit or load_in_8bit or config is None:+ return False+ quant = fp8_block_quantization_config(config)+ if quant is None:+ return False+ if fp8_to_nf4_disabled():+ return False+ if str(quant.get("expert_dtype") or "").lower() in _PACKED_EXPERT_DTYPES:+ # DeepSeek-V4-Flash: fp8 attention, packed FP4 experts. Not handled here.+ if verbose and explicit_4bit_requested():+ print(+ "Unsloth: this fp8 checkpoint packs its experts in FP4, which the fp8 -> 4bit load does "+ "not handle. Loading it as it is."+ )+ return False+ mode = os.environ.get(_ENV, "").strip().lower()+ if not (explicit_4bit_requested() or mode in ("1", "true", "on", "yes", "force")):+ return False+ reason = fp8_to_nf4_unavailable_reason()+ if reason is not None:+ if verbose:+ print(+ f"Unsloth: `load_in_4bit = True` on an fp8 checkpoint cannot quantize to 4bit here: "+ f"{reason}. Loading the fp8 weights instead."+ )+ return False+ if quant.get("modules_to_convert"):+ try:+ from transformers.integrations.finegrained_fp8 import ( # noqa: F401+ FP8Embedding,+ replace_with_fp8_embedding,+ )+ except Exception:+ if verbose:+ print(+ "Unsloth: this fp8 checkpoint keeps fp8 embedding tables (modules_to_convert), which "+ "need transformers >= 5.17 to load in 4bit. Loading the fp8 weights instead."+ )+ return False+ if not install_fp8_to_nf4_quantizer():+ if verbose:+ print(+ "Unsloth: could not register the fp8 -> 4bit loader with transformers. Loading the "+ "fp8 weights instead."+ )+ return False+ plan = {+ "original_quantization_config": dict(quant),+ "weight_block_size": _block_size(quant),+ "fmt": quant.get("fmt", "e4m3"),+ "activation_scheme": quant.get("activation_scheme"),+ "modules_to_not_convert": list(quant.get("modules_to_not_convert") or []),+ "modules_to_convert": list(quant.get("modules_to_convert") or []),+ }+ holders = _configs_with_quantization(config)+ try:+ stripped = []+ for holder in holders:+ if (+ str((_quant_dict(holder.quantization_config) or {}).get("quant_method", "")).lower()+ == "fp8"+ ):+ stripped.append(holder is not config)+ try:+ delattr(holder, "quantization_config")+ except AttributeError:+ holder.__dict__.pop("quantization_config", None)+ plan["stripped_text_config"] = any(stripped)+ plan["stripped_root_config"] = not all(stripped) if stripped else False+ setattr(config, UNSLOTH_FP8_TO_NF4_ATTR, plan)+ armed_configs = _ARMED_CONFIGS.get()+ if armed_configs is not None:+ armed_configs.append(config)+ except Exception:+ for holder in holders:+ if getattr(holder, "quantization_config", None) is None:+ holder.quantization_config = quant+ return False+ if verbose:+ print(+ "Unsloth: fp8 checkpoint with `load_in_4bit = True`: dequantizing each fp8 tensor to 16bit "+ "while loading and quantizing it to 4bit (NF4) right away, so no 16bit copy of the model is "+ f"ever held. Set {_ENV}=0 to load the fp8 weights instead."+ )+ return True+++def disarm_fp8_to_nf4(config) -> bool:+ """Undo maybe_arm_fp8_to_nf4 when the load ends up not quantizing (e.g. full finetuning)."""+ global _STATE+ plan = getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) if config is not None else None+ if plan is None:+ return False+ # A load that raised never reached _process_model_after_weight_loading.+ _STATE = None+ original = plan["original_quantization_config"]+ if plan.get("stripped_root_config"):+ config.quantization_config = dict(original)+ if plan.get("stripped_text_config"):+ get_text_config = getattr(config, "get_text_config", None)+ text_config = None+ if callable(get_text_config):+ try:+ text_config = get_text_config(decoder = True)+ except TypeError:+ text_config = get_text_config()+ if text_config is not None:+ text_config.quantization_config = dict(original)+ try:+ delattr(config, UNSLOTH_FP8_TO_NF4_ATTR)+ except AttributeError:+ config.__dict__.pop(UNSLOTH_FP8_TO_NF4_ATTR, None)+ return True+++def _quantize_16bit_requested() -> bool:+ return os.environ.get(_ENV_QUANTIZE_ALL, "0").strip().lower() in ("1", "true", "on", "yes")+++def fp8_to_nf4_planner_quantization_config(config, llm_int8_skip_modules = None) -> Optional[dict]:+ """What the device-map planner must size an armed load as: a bitsandbytes 4bit load."""+ plan = getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) if config is not None else None+ if plan is None:+ return None+ skip = list(llm_int8_skip_modules or [])+ # Quantize-all mode quantizes the checkpoint's 16-bit Linears too: size them as 4bit.+ for name in [] if _quantize_16bit_requested() else plan.get("modules_to_not_convert") or []:+ if name not in skip:+ skip.append(name)+ return {+ "quant_method": "bitsandbytes",+ "load_in_4bit": True,+ "bnb_4bit_quant_type": "nf4",+ "bnb_4bit_use_double_quant": True,+ "llm_int8_skip_modules": skip,+ }+++class _LoadState:+ """Book-keeping for the one fp8 -> NF4 load in flight. Module level, never on an op or a+ converter: the loader deep-copies those once per target key."""++ def __init__(self, plan, dtype, headers):+ self.block = plan["weight_block_size"]+ self.dtype = dtype+ self.fp8_keys = {k for k, v in headers.items() if v in _FP8_SAFETENSORS}+ # `<module>.scale` is DeepSeek-V4's name for weight_scale_inv (renamed by the fp8 quantizer too).+ self.dot_scale_keys = {+ k+ for k in headers+ if k.endswith(".scale") and (k[: -len(".scale")] + ".weight") in self.fp8_keys+ }+ # Activation scales (static fp8) are not needed for a weight dequant, so they are not tracked.+ self.scale_keys = {+ k for k in headers if k.endswith(".weight_scale_inv")+ } | self.dot_scale_keys+ self.native_keys = self.fp8_keys | self.scale_keys+ # Refined once renaming is known: dropped MTP layers never reach a converter.+ self.expected_fp8 = set(self.fp8_keys)+ self.expected_scales = set(self.scale_keys)+ self.headers = headers+ self.routed_native = set()+ self.dequantized = 0+ self.kept_fp8 = 0+++_STATE: Optional[_LoadState] = None+++def _is_scale_pattern(pattern: str) -> bool:+ base = pattern[:-1] if pattern.endswith("$") else pattern+ return base.endswith(("weight_scale_inv", "activation_scale"))+++def _scale_pattern_for(pattern: str) -> str:+ anchored = pattern.endswith("$")+ base = pattern[:-1] if anchored else pattern+ if base.endswith(".weight"):+ scale = base[: -len(".weight")] + ".weight_scale_inv"+ elif base == "weight":+ scale = "weight_scale_inv"+ else:+ scale = base + "_scale_inv"+ return scale + "$" if anchored else scale+++def _scale_as_fp32(scale: torch.Tensor) -> torch.Tensor:+ if scale.dtype == torch.uint8:+ return (scale.to(torch.float32) - 127.0).exp2()+ return scale.to(torch.float32)+++def _dequantize_block_fp8(+ weight,+ scale,+ block,+ dtype,+ out = None,+):+ """``(weight.float() * scale_per_block).to(dtype)``, element by element the same product as a+ full expand of the scale grid, so the result is bit-identical to any dequant using that formula.+ Ragged last blocks (dims not a multiple of the block) are supported. Works in row chunks to+ bound the fp32 transient."""+ if weight.dtype not in _FP8_DTYPES:+ raise TypeError(f"expected an fp8 weight, got {weight.dtype}")+ if weight.dim() > 2:+ weight2 = weight.reshape(-1, *weight.shape[-2:])+ if scale.numel() == 1:+ scale2 = scale.reshape(1, 1, 1).expand(weight2.shape[0], 1, 1)+ else:+ scale2 = scale.reshape(-1, *scale.shape[-2:])+ if out is None:+ out = torch.empty(weight.shape, dtype = dtype, device = weight.device)+ out2 = out.view(-1, *weight.shape[-2:])+ for i in range(weight2.shape[0]):+ _dequantize_block_fp8(weight2[i], scale2[i], block, dtype, out = out2[i])+ return out+ rows, cols = weight.shape+ if scale.dim() == 0 or scale.numel() == 1:+ scale = scale.reshape(1, 1)+ scale = scale.reshape(scale.shape[-2], scale.shape[-1]) if scale.dim() != 2 else scale+ scale_rows, scale_cols = scale.shape+ if scale_rows == 1 and scale_cols == 1:+ block_m, block_n = rows, cols+ elif -(-rows // block[0]) == scale_rows and -(-cols // block[1]) == scale_cols:+ block_m, block_n = block+ elif rows % scale_rows == 0 and cols % scale_cols == 0:+ block_m, block_n = rows // scale_rows, cols // scale_cols+ else:+ raise ValueError(+ f"fp8 weight {tuple(weight.shape)} does not fit its scale grid {tuple(scale.shape)} "+ f"with block {block}"+ )+ scale = _scale_as_fp32(scale).to(weight.device)+ if out is None:+ out = torch.empty((rows, cols), dtype = dtype, device = weight.device)+ # About 64M fp32 elements per chunk, a whole number of row blocks.+ chunk_blocks = max(1, (1 << 26) // max(1, block_m * cols))+ for block_row in range(0, scale_rows, chunk_blocks):+ row_start = block_row * block_m+ row_end = min(rows, (block_row + chunk_blocks) * block_m)+ piece = weight[row_start:row_end].to(torch.float32)+ piece_scale = scale[block_row : block_row + chunk_blocks]+ expanded = piece_scale.repeat_interleave(block_m, dim = 0)[: row_end - row_start]+ expanded = expanded.repeat_interleave(block_n, dim = 1)[:, :cols]+ piece.mul_(expanded)+ out[row_start:row_end].copy_(piece)+ del piece, expanded+ return out+++def _output_dtype(model, full_layer_name, default):+ """The load dtype, except fp32 for a target the model keeps in fp32 (_keep_in_fp32_modules)."""+ if model is None or not full_layer_name:+ return default+ module_name, _, attr = full_layer_name.rpartition(".")+ try:+ module = model.get_submodule(module_name) if module_name else model+ except AttributeError:+ return default+ param = getattr(module, attr, None)+ if (+ isinstance(param, torch.Tensor)+ and param.dtype == torch.float32+ and type(param).__name__ != "Params4bit"+ and not type(module).__module__.startswith("bitsandbytes")+ ):+ return torch.float32+ return default+++def _target_keeps_fp8(model, full_layer_name) -> bool:+ """An fp8 container module (FP8Embedding) that holds its own scale: load its bytes as stored."""+ if model is None or not full_layer_name:+ return False+ module_name, _, attr = full_layer_name.rpartition(".")+ try:+ module = model.get_submodule(module_name) if module_name else model+ except AttributeError:+ return False+ param = getattr(module, attr, None)+ if not isinstance(param, torch.Tensor) or param.dtype not in _FP8_DTYPES:+ return False+ return any(+ isinstance(getattr(module, name, None), torch.Tensor)+ for name in (attr + "_scale", attr + "_scale_inv", "weight_scale", "weight_scale_inv")+ )+++def _merge_accepts_stacked_tensor() -> bool:+ try:+ from transformers.core_model_loading import MergeModulelist++ stacked = torch.zeros(2, 1, 1)+ merged = MergeModulelist(dim = 0).convert(+ {"a": stacked}, source_patterns = ["a"], target_patterns = ["b"]+ )+ return next(iter(merged.values())) is stacked+ except Exception:+ return False+++def _build_classes():+ from transformers.core_model_loading import ConversionOps, WeightConverter, _IdentityOp++ class Fp8BlockDequantizeToHalf(ConversionOps):+ """Dequantize fp8 sources with their scales to the load dtype; drop scale entries. `stack`+ pre-allocates the stacked output a following MergeModulelist(dim=0) accepts as is."""++ def __init__(+ self,+ generic = False,+ stack = False,+ fuse = None,+ ):+ self.generic = generic+ self.stack = stack+ # (patterns in merge order, concat dim): replaces MergeModulelist + Concatenate by writing+ # each expert straight into the final stack, the only 16-bit tensor ever held.+ self.fuse = fuse++ def _fused(+ self,+ input_dict,+ target_patterns,+ state,+ model = None,+ full_layer_name = None,+ ):+ patterns, cat_dim = self.fuse+ groups = []+ for pattern in patterns:+ weights = input_dict.get(pattern)+ if weights is None:+ continue+ weights = weights if isinstance(weights, list) else [weights]+ scales = input_dict.get(_scale_pattern_for(pattern))+ scales = (+ (scales if isinstance(scales, list) else [scales])+ if scales is not None+ else None+ )+ if scales is not None and len(scales) != len(weights):+ raise ValueError(+ f"Unsloth: {len(weights)} fp8 weights but {len(scales)} scales"+ )+ groups.append((weights, scales))+ if not groups:+ return {}+ shapes = [(len(w), *w[0].shape) for w, _ in groups]+ for weights, _ in groups:+ if any(tuple(w.shape) != tuple(weights[0].shape) for w in weights):+ raise ValueError("Unsloth: experts of one projection have different shapes")+ for shape in shapes[1:]:+ if any(a != b for i, (a, b) in enumerate(zip(shape, shapes[0])) if i != cat_dim):+ raise ValueError(+ f"Unsloth: expert stacks {shapes} cannot be concatenated on dim {cat_dim}"+ )+ final = list(shapes[0])+ final[cat_dim] = sum(shape[cat_dim] for shape in shapes)+ device = groups[0][0][0].device+ dtype = _output_dtype(model, full_layer_name, state.dtype)+ out = torch.empty(final, dtype = dtype, device = device)+ offset = 0+ for (weights, scales), shape in zip(groups, shapes):+ part = out.narrow(cat_dim, offset, shape[cat_dim])+ offset += shape[cat_dim]+ for i in range(len(weights)):+ weight = weights[i]+ weights[i] = None+ if weight.dtype in _FP8_DTYPES:+ if scales is None:+ raise RuntimeError(+ "Unsloth: an fp8 expert weight has no weight_scale_inv"+ )+ _dequantize_block_fp8(weight, scales[i], state.block, dtype, out = part[i])+ state.dequantized += 1+ elif weight.is_floating_point() and weight.element_size() >= 2:+ part[i].copy_(weight)+ else:+ raise RuntimeError(+ f"Unsloth: an expert weight is stored as {weight.dtype}, which the fp8 -> 4bit "+ f"load cannot dequantize. Set {_ENV}=0 to load the checkpoint as it is."+ )+ del weight+ return {target_patterns[0]: out}++ @torch.no_grad()+ def convert(+ self,+ input_dict,+ source_patterns = None,+ target_patterns = None,+ full_layer_name = None,+ model = None,+ **kwargs,+ ):+ state = _STATE+ if state is None:+ raise RuntimeError(+ "Unsloth: fp8 -> 4bit dequantize ran outside an fp8 -> 4bit load"+ )+ block, dtype = state.block, _output_dtype(model, full_layer_name, state.dtype)+ if _target_keeps_fp8(model, full_layer_name):+ # modules_to_convert (Qwen3.8's n-gram table): stays fp8 with its own per-tensor scale.+ state.kept_fp8 += sum(+ len(v) if isinstance(v, list) else 1+ for k, v in input_dict.items()+ if not _is_scale_pattern(k)+ )+ return input_dict+ if self.fuse is not None:+ return self._fused(input_dict, target_patterns, state, model, full_layer_name)+ result = {}+ for key in list(input_dict.keys()):+ if _is_scale_pattern(key):+ continue+ value = input_dict[key]+ scale_key = _scale_pattern_for(key)+ if scale_key not in input_dict:+ alt = scale_key[:-1] if scale_key.endswith("$") else scale_key + "$"+ scale_key = alt if alt in input_dict else None+ weights = value if isinstance(value, list) else [value]+ if scale_key is None:+ if any(isinstance(w, torch.Tensor) and w.dtype in _FP8_DTYPES for w in weights):+ raise RuntimeError(+ f"Unsloth: fp8 tensor for `{full_layer_name}` has no weight_scale_inv in the checkpoint"+ )+ if self.generic:+ target = full_layer_name if full_layer_name is not None else "weight"+ result[target] = value+ else:+ result[key] = value+ continue+ scales = input_dict[scale_key]+ scales = scales if isinstance(scales, list) else [scales]+ if len(scales) != len(weights):+ raise ValueError(+ f"Unsloth: {len(weights)} fp8 weights but {len(scales)} scales for `{full_layer_name}`"+ )+ out = None+ if self.stack and isinstance(value, list) and len(weights) > 0:+ first = weights[0]+ out = torch.empty(+ (len(weights), *first.shape), dtype = dtype, device = first.device+ )+ outputs = out+ else:+ outputs = []+ for i in range(len(weights)):+ weight, scale = weights[i], scales[i]+ weights[i] = None+ if weight.dtype in _FP8_DTYPES:+ converted = _dequantize_block_fp8(+ weight, scale, block, dtype, out = out[i] if out is not None else None+ )+ state.dequantized += 1+ elif weight.is_floating_point() and weight.element_size() >= 2:+ # A 16-bit tensor that shipped a scale anyway: take it as stored.+ converted = weight.to(dtype)+ if out is not None:+ out[i].copy_(converted)+ else:+ raise RuntimeError(+ f"Unsloth: `{full_layer_name}` is stored as {weight.dtype} with a scale, which the "+ f"fp8 -> 4bit load cannot dequantize. Set {_ENV}=0 to load the checkpoint as it is."+ )+ if out is None:+ outputs.append(converted)+ del weight+ if self.generic:+ target = full_layer_name if full_layer_name is not None else "weight"+ result[target] = outputs if out is not None else outputs[0]+ else:+ result[key] = (+ outputs if (out is not None or isinstance(value, list)) else outputs[0]+ )+ return result++ @property+ def reverse_op(self):+ return _IdentityOp()++ class Fp8SourceConverter(WeightConverter):+ """A WeightConverter that loads fp8 weights and their scales at their stored dtype. An+ on-the-fly bitsandbytes load casts every source to the model dtype, which would round the+ fp32 scales to bf16."""++ __slots__ = ()++ def add_tensor(self, target_key, source_key, source_pattern, future):+ state = _STATE+ if state is not None and source_key in state.native_keys:+ native = getattr(future, "_unsloth_native_materialize", None)+ if native is None:+ # Cast to the model dtype, a fp32 scale would round to bf16: refuse, do not guess.+ raise RuntimeError(+ f"Unsloth: cannot read `{source_key}` at its stored dtype during an fp8 -> 4bit load. "+ f"Set {_ENV}=0 to load the fp8 weights as they are."+ )+ future = native+ state.routed_native.add(source_key)+ return super().add_tensor(target_key, source_key, source_pattern, future)++ return Fp8BlockDequantizeToHalf, Fp8SourceConverter+++def _install_spawn_materialize_tag() -> bool:+ """Tag each lazily materialized source with a twin that keeps the stored dtype."""+ from transformers import core_model_loading as cml++ original = cml.spawn_materialize+ if getattr(original, "_unsloth_fp8_to_nf4", False):+ return True+ try:+ signature = inspect.signature(original)+ except (TypeError, ValueError):+ return False+ if not {"thread_pool", "tensor", "device", "dtype"} <= set(signature.parameters):+ return False+ materialize_copy = getattr(cml, "_materialize_copy", None)++ @functools.wraps(original)+ def spawn_materialize(*args, **kwargs):+ job = original(*args, **kwargs)+ if _STATE is None or not callable(job) or materialize_copy is None:+ return job+ try:+ bound = signature.bind(*args, **kwargs)+ except TypeError:+ return job+ arguments = bound.arguments+ if (+ arguments.get("sharding_op", None) is not None+ or arguments.get("thread_pool", None) is not None+ ):+ return job+ tensor = arguments.get("tensor")+ device = arguments.get("device", None)+ try:+ job._unsloth_native_materialize = functools.partial(+ materialize_copy, tensor, device, None+ )+ except Exception:+ pass+ return job++ spawn_materialize._unsloth_fp8_to_nf4 = True+ cml.spawn_materialize = spawn_materialize+ return True+++def _refuse_packed_scaled_weights(headers):+ """A scaled weight stored as int8 / uint8 is packed FP4 (or another format): it would be cast+ to bf16 as integers. Fail before loading anything."""+ for key, dtype in headers.items():+ if not key.endswith((".weight_scale_inv", ".scale")):+ continue+ weight = headers.get(_weight_key_of_scale(key))+ if weight is not None and weight not in _FLOAT_SAFETENSORS:+ raise RuntimeError(+ f"Unsloth: `{_weight_key_of_scale(key)}` is stored as {weight} with a block scale, which the "+ f"fp8 -> 4bit load cannot dequantize. Set {_ENV}=0 to load the checkpoint as it is."+ )+++def _keep_fp8_embeddings(model, patterns) -> int:+ """The checkpoint's modules_to_convert are fp8 embedding tables too large to expand (Qwen3.8's+ 51B-weight n-gram table): keep them fp8 with their per-tensor scale, as an fp8 load does."""+ try:+ from transformers.integrations.finegrained_fp8 import (+ FP8Embedding,+ replace_with_fp8_embedding,+ )+ except Exception:+ return 0+ replace_with_fp8_embedding(model, list(patterns))+ return sum(1 for module in model.modules() if isinstance(module, FP8Embedding))+++def _read_headers(checkpoint_files) -> dict:+ from safetensors import safe_open++ headers = {}+ for path in checkpoint_files or ():+ path = str(path)+ if not path.endswith(".safetensors"):+ raise RuntimeError(+ f"Unsloth: fp8 -> 4bit loading needs safetensors shards, got `{os.path.basename(path)}`"+ )+ with safe_open(path, framework = "pt") as handle:+ for key in handle.keys():+ headers[key] = handle.get_slice(key).get_dtype()+ return headers+++def _generalize(name: str) -> str:+ escaped = re.escape(name)+ return re.sub(r"(?<=\\\.)\d+(?=\\\.|$)", r"\\d+", escaped)+++def _compress_patterns(names, others) -> list:+ """Regexes matching every name in `names` and none in `others`: layer indices generalized+ where that stays exact, else the exact names. Anchored, since transformers re.match()es them."""+ families = {}+ for name in sorted(names):+ families.setdefault(_generalize(name), []).append(name)+ patterns = []+ for pattern, members in families.items():+ regex = re.compile(pattern + "$")+ if any(regex.match(other) for other in others):+ patterns.extend(re.escape(member) + "$" for member in members)+ else:+ patterns.append(pattern + "$")+ return patterns+++def _high_precision_linears(model, headers, weight_conversions) -> tuple:+ """nn.Linear modules whose checkpoint weight is stored in 16/32 bits (the fp8 checkpoint's+ modules_to_not_convert), found by renaming every checkpoint key exactly as the loader will."""+ from copy import deepcopy+ from transformers import core_model_loading as cml++ # Copies: renaming marks a transform used, and the used ones are what saving reverses.+ conversions = deepcopy(list(weight_conversions))+ renamings = [c for c in conversions if isinstance(c, cml.WeightRenaming)]+ converters = [c for c in conversions if isinstance(c, cml.WeightConverter)]+ meta_state_dict = model.state_dict()+ prefix = getattr(model, "base_model_prefix", None)+ sources_fp8 = {}+ reached = set()+ for key, dtype in headers.items():+ if key.endswith(_SCALE_SUFFIXES) or key.endswith(".scale"):+ continue+ renamed, _ = cml.rename_source_key(key, renamings, converters, prefix, meta_state_dict)+ if renamed not in meta_state_dict and key in meta_state_dict:+ renamed = key+ if renamed in meta_state_dict:+ reached.add(key)+ sources_fp8[renamed] = sources_fp8.get(renamed, False) or dtype in _FP8_SAFETENSORS+ keep, quantize = [], []+ for name, module in model.named_modules():+ if not isinstance(module, torch.nn.Linear):+ continue+ target = f"{name}.weight"+ if target not in sources_fp8:+ continue+ (quantize if sources_fp8[target] else keep).append(name)+ return keep, quantize, reached+++def _weight_key_of_scale(key):+ for suffix in (".weight_scale_inv", ".scale"):+ if key.endswith(suffix):+ return key[: -len(suffix)] + ".weight"+ return key+++def _restore_plain_linears(model, names, dtype) -> int:+ """Swap bitsandbytes Linear4bit back to nn.Linear (still on meta, before any weight loads)."""+ try:+ import bitsandbytes as bnb+ except Exception:+ return 0+ restored = 0+ for name in names:+ try:+ module = model.get_submodule(name)+ except AttributeError:+ continue+ if not isinstance(module, bnb.nn.Linear4bit):+ continue+ parent_name, _, child = name.rpartition(".")+ parent = model.get_submodule(parent_name) if parent_name else model+ device = module.weight.device+ plain = torch.nn.Linear(+ module.in_features,+ module.out_features,+ bias = module.bias is not None,+ device = device,+ dtype = dtype,+ )+ setattr(parent, child, plain)+ restored += 1+ return restored+++_installed_class = None+++def install_fp8_to_nf4_quantizer() -> bool:+ """Register a Bnb4BitHfQuantizer subclass for `bitsandbytes_4bit` that only differs from its+ parent on a load whose config carries the parked fp8 block. Composes with another subclass."""+ global _installed_class+ if _transformers_supports_fp8_to_nf4() is not None:+ return False+ from transformers.quantizers import auto as quantizers_auto+ from transformers.quantizers.quantizer_bnb_4bit import Bnb4BitHfQuantizer+ from transformers.core_model_loading import WeightConverter, WeightRenaming++ mapping = quantizers_auto.AUTO_QUANTIZER_MAPPING+ current = mapping.get("bitsandbytes_4bit")+ if current is None or not issubclass(current, Bnb4BitHfQuantizer):+ return False+ if getattr(current, "_unsloth_fp8_to_nf4", False):+ return _install_spawn_materialize_tag()+ if not _install_spawn_materialize_tag():+ return False+ DequantOp, SourceConverter = _build_classes()+ stack_ok = _merge_accepts_stacked_tensor()++ def _rebuild(conv):+ from transformers.core_model_loading import Concatenate, MergeModulelist++ weight_sources = [p for p in conv.source_patterns if p.endswith(".weight")]+ if not weight_sources:+ return None+ other = [p for p in conv.source_patterns if not p.endswith(".weight")]+ sources = (+ [p + "$" for p in weight_sources]+ + [p[: -len(".weight")] + ".weight_scale_inv$" for p in weight_sources]+ + other+ )+ ops = list(conv.operations)+ first = ops[0] if ops else None+ merges = isinstance(first, MergeModulelist) and getattr(first, "dim", None) == 0+ if (+ merges+ and len(ops) == 2+ and type(ops[1]) is Concatenate+ and not other+ and len(conv.target_patterns) == 1+ ):+ operations = [DequantOp(fuse = ([p + "$" for p in weight_sources], ops[1].dim % 3))]+ else:+ operations = [DequantOp(stack = bool(stack_ok and merges))] + ops+ rebuilt = SourceConverter(+ source_patterns = sources,+ target_patterns = list(conv._original_target_patterns),+ operations = operations,+ force_cpu = getattr(conv, "force_cpu", False),+ )+ rebuilt.scope_prefix = conv.scope_prefix+ rebuilt.base_model_prefix = conv.base_model_prefix+ return rebuilt++ class UnslothFp8ToNf4Bnb4BitHfQuantizer(current):+ _unsloth_fp8_to_nf4 = True+ _unsloth_fp8_plan = None+ _unsloth_fp8_rebuilt = None+ _unsloth_fp8_model = None+ _unsloth_fp8_kept = 0++ def __init__(self, quantization_config, **kwargs):+ # The device-map planner builds this from the rewritten config without `pre_quantized`+ # (transformers' own load always passes it). An armed load quantizes at runtime, so size+ # its expert merges in the load dtype, not as packed 4bit (a 4x smaller transient).+ if "pre_quantized" not in kwargs and _ARMED_CONFIGS.get():+ kwargs["pre_quantized"] = False+ super().__init__(quantization_config, **kwargs)++ def _process_model_before_weight_loading(self, model, **kwargs):+ global _STATE+ config = getattr(model, "config", None)+ plan = getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) if config is not None else None+ if plan is not None:+ self._unsloth_fp8_plan = plan+ # preprocess_model does not forward dtype; the meta model was built in the load dtype.+ dtype = kwargs.get("dtype", None)+ if not isinstance(dtype, torch.dtype):+ dtype = getattr(model, "dtype", None)+ if dtype not in (torch.bfloat16, torch.float16, torch.float32):+ dtype = torch.bfloat16+ self._unsloth_fp8_model = model+ headers = _read_headers(kwargs.get("checkpoint_files"))+ _refuse_packed_scaled_weights(headers)+ _STATE = _LoadState(plan, dtype, headers)+ if plan.get("modules_to_convert"):+ _keep_fp8_embeddings(model, plan["modules_to_convert"])+ return super()._process_model_before_weight_loading(model, **kwargs)++ def update_weight_conversions(self, weight_conversions):+ if self._unsloth_fp8_plan is None or _STATE is None:+ return super().update_weight_conversions(weight_conversions)+ model = self._unsloth_fp8_model+ self._unsloth_fp8_model = None+ # Here, not before loading: only now are the caller's key_mapping renamings known.+ keep, quantize = [], []+ quantize_all = _quantize_16bit_requested()+ if model is not None:+ keep, quantize, reached = _high_precision_linears(+ model, _STATE.headers, weight_conversions+ )+ _STATE.expected_fp8 = _STATE.fp8_keys & reached+ _STATE.expected_scales = {+ key+ for key in _STATE.scale_keys+ if _weight_key_of_scale(key) in _STATE.expected_fp8+ }+ if quantize_all:+ # UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT=1 quantizes them too, like a load of the bf16 repo.+ keep = []+ self._unsloth_fp8_kept = _restore_plain_linears(model, keep, _STATE.dtype)+ if keep:+ patterns = _compress_patterns(keep, quantize)+ # A copy: the caller may reuse their BitsAndBytesConfig for another load.+ quantization_config = copy.copy(self.quantization_config)+ skip = list(quantization_config.llm_int8_skip_modules or [])+ quantization_config.llm_int8_skip_modules = skip + patterns+ self.quantization_config = quantization_config+ if isinstance(getattr(self, "modules_to_not_convert", None), list):+ self.modules_to_not_convert.extend(patterns)+ updated = []+ rebuilt_from = {}+ if _STATE.dot_scale_keys:+ updated.append(+ WeightRenaming(+ source_patterns = r"^(.+)\.scale$", target_patterns = r"\1.weight_scale_inv"+ )+ )+ for conv in weight_conversions:+ if isinstance(conv, WeightConverter):+ rebuilt = _rebuild(conv)+ if rebuilt is not None:+ rebuilt_from[id(rebuilt)] = conv+ updated.append(rebuilt)+ continue+ updated.append(conv)+ updated.append(+ SourceConverter(+ source_patterns = ["weight$", "weight_scale_inv", "activation_scale"],+ target_patterns = "weight",+ operations = [DequantOp(generic = True)],+ )+ )+ self._unsloth_fp8_rebuilt = rebuilt_from+ return super().update_weight_conversions(updated)++ def _process_model_after_weight_loading(self, model, **kwargs):+ global _STATE+ if self._unsloth_fp8_plan is not None:+ state = _STATE+ _STATE = None+ self._unsloth_fp8_plan = None+ self._unsloth_fp8_model = None+ rebuilt_from = self._unsloth_fp8_rebuilt or {}+ self._unsloth_fp8_rebuilt = None+ _finish_fp8_to_nf4_load(+ model, state, rebuilt_from, getattr(self, "_unsloth_fp8_kept", 0)+ )+ return super()._process_model_after_weight_loading(model, **kwargs)++ mapping["bitsandbytes_4bit"] = UnslothFp8ToNf4Bnb4BitHfQuantizer+ _installed_class = UnslothFp8ToNf4Bnb4BitHfQuantizer+ return True+++def _finish_fp8_to_nf4_load(model, state, rebuilt_from, kept):+ config = getattr(model, "config", None)+ if config is not None:+ try:+ delattr(config, UNSLOTH_FP8_TO_NF4_ATTR)+ except AttributeError:+ config.__dict__.pop(UNSLOTH_FP8_TO_NF4_ATTR, None)+ # Saving reverses model._weight_conversions: put the model's own converters back.+ conversions = getattr(model, "_weight_conversions", None)+ if isinstance(conversions, list):+ restored = []+ for conv in conversions:+ original = rebuilt_from.get(id(conv))+ if original is not None:+ try:+ original._was_used = True+ except Exception:+ pass+ restored.append(original)+ elif type(conv).__name__ == "Fp8SourceConverter":+ continue+ elif getattr(conv, "_original_source_patterns", None) == [r"^(.+)\.scale$"]:+ continue+ else:+ restored.append(conv)+ model._weight_conversions = restored+ if state is None:+ raise RuntimeError("Unsloth: the fp8 -> 4bit load lost its state")+ problems = []+ missed_scales = sorted(state.expected_scales - state.routed_native)+ if missed_scales:+ problems.append(+ f"{len(missed_scales)} fp8 scales were never applied (e.g. `{missed_scales[0]}`), so their "+ "weights would load unscaled"+ )+ leftover = [+ name+ for name, param in model.named_parameters()+ if param.dtype in _FP8_DTYPES and not _target_keeps_fp8(model, name)+ ]+ if leftover:+ problems.append(f"{len(leftover)} parameters are still fp8 (e.g. `{leftover[0]}`)")+ if state.dequantized + state.kept_fp8 < len(state.expected_fp8):+ problems.append(+ f"only {state.dequantized} of {len(state.expected_fp8) - state.kept_fp8} fp8 tensors were dequantized"+ )+ if problems:+ raise RuntimeError(+ "Unsloth: loading this fp8 checkpoint in 4bit failed: " + "; ".join(problems) + ". "+ f"Set {_ENV}=0 to load the fp8 weights as they are."+ )+ kept_fp8 = (+ f" {state.kept_fp8} fp8 embedding shards (modules_to_convert) stay fp8."+ if state.kept_fp8+ else ""+ )+ kept_note = (+ f" {kept} Linear layers the checkpoint stores in 16bit stay in 16bit "+ f"({_ENV_QUANTIZE_ALL}=1 quantizes them too)."+ if kept+ else ""+ )+ print(+ f"Unsloth: dequantized {state.dequantized} fp8 tensors to 4bit while loading.{kept_note}{kept_fp8}"+ )diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex ff83c94620a..9b373ec9430 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2625,6 +2625,8 @@ class FastLlamaModel:token = token,model_name = model_name,revision = revision,+ # vLLM reads the checkpoint itself, so it cannot take the in-process fp8 -> 4bit route.+ allow_fp8_to_nf4 = not _vllm_will_load_weights(fast_inference, num_labels),hub_kwargs = {"cache_dir": kwargs.get("cache_dir"),"subfolder": kwargs.get("subfolder"),@@ -2640,6 +2642,14 @@ class FastLlamaModel:pop_modelopt_key_mapping,)+ from .fp8_to_nf4 import (+ disarm_fp8_to_nf4,+ fp8_to_nf4_armed,+ fp8_to_nf4_planner_quantization_config,+ )++ # The fp8 config was parked on model_config, so the load must be handed this config.+ _fp8_to_nf4 = fp8_to_nf4_armed(model_config)_modelopt_rewritten = modelopt_rewritten(model_config)if _modelopt_rewritten:verify_fp8_support_if_applicable(model_config)@@ -2714,7 +2724,10 @@ class FastLlamaModel:quantization_config = kwargs.get("quantization_config", None),rewritten_quantization_config = modelopt_planner_quantization_config(model_config)if _modelopt_rewritten- else None,+ else fp8_to_nf4_planner_quantization_config(+ model_config,+ SKIP_QUANTIZATION_MODULES + (["out_proj"] if IS_FALCON_H1 else []),+ ),# The same extra the bnb config below adds.extra_skip_modules = ["out_proj"] if IS_FALCON_H1 else None,),@@ -2781,16 +2794,19 @@ class FastLlamaModel:set_task_config_attr(model_config, _cfg_key, _cfg_val)else:setattr(model_config, _cfg_key, _cfg_val)- model = AutoModelForSequenceClassification.from_pretrained(- model_name,- config = model_config,- device_map = device_map,- token = token,- trust_remote_code = trust_remote_code,- attn_implementation = preferred_attn_impl,- revision = revision,- **kwargs,- )+ try:+ model = AutoModelForSequenceClassification.from_pretrained(+ model_name,+ config = model_config,+ device_map = device_map,+ token = token,+ trust_remote_code = trust_remote_code,+ attn_implementation = preferred_attn_impl,+ revision = revision,+ **kwargs,+ )+ finally:+ disarm_fp8_to_nf4(model_config)# Defensive: ensure the task head is in a floating dtype, guarding against any path leaving it# as integer storage (#5027).for _head_name in ("score", "classifier", "qa_outputs"):@@ -2836,7 +2852,7 @@ class FastLlamaModel:)_prepare_compressed_tensors_model(model)elif not fast_inference:- if user_config is not None or _modelopt_rewritten:+ if user_config is not None or _modelopt_rewritten or _fp8_to_nf4:# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override# on the config and pass the single config object through.if max_position_embeddings is not None:@@ -2844,18 +2860,22 @@ class FastLlamaModel:_rope_scaling = kwargs.pop("rope_scaling", None)if _rope_scaling is not None:model_config.rope_scaling = _rope_scaling- if _modelopt_rewritten and user_config is None:+ if (_modelopt_rewritten or _fp8_to_nf4) and user_config is None:move_config_overrides_onto_config(model_config, kwargs)- model = AutoModelForCausalLM.from_pretrained(- model_name,- config = model_config,- device_map = device_map,- token = token,- trust_remote_code = trust_remote_code,- attn_implementation = preferred_attn_impl,- revision = revision,- **kwargs,- )+ try:+ model = AutoModelForCausalLM.from_pretrained(+ model_name,+ config = model_config,+ device_map = device_map,+ token = token,+ trust_remote_code = trust_remote_code,+ attn_implementation = preferred_attn_impl,+ revision = revision,+ **kwargs,+ )+ finally:+ # The load deep-copied the config; give the caller's object its fp8 block back.+ disarm_fp8_to_nf4(model_config)else:model = AutoModelForCausalLM.from_pretrained(model_name,diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 44b14338867..670a469c0c0 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -33,6 +33,7 @@ from transformers import AutoConfigfrom transformers import __version__ as transformers_versionfrom peft import PeftConfig, PeftModelfrom .grouped_linear_lora import register_grouped_linear_lora_for_adapter+from .fp8_to_nf4 import track_explicit_4bit_requestfrom .loader_utils import (DEFAULT_DEVICE_MAP,OFFLOAD_EMBEDDING_AUTO,@@ -700,6 +701,7 @@ def _fix_rope_inv_freq(model):class FastLanguageModel(FastLlamaModel):@staticmethod@_offline_aware_load+ @track_explicit_4bit_requestdef from_pretrained(model_name = "unsloth/Llama-3.2-1B-Instruct",max_seq_length = 2048,@@ -1503,6 +1505,7 @@ class FastModel(FastBaseModel):@staticmethod@_offline_aware_load+ @track_explicit_4bit_requestdef from_pretrained(model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",max_seq_length = 2048,diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex d3333080a53..2478ab68f9f 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -2185,6 +2185,7 @@ def check_and_disable_bitsandbytes_loading(model_name = None,revision = None,hub_kwargs = None,+ allow_fp8_to_nf4 = True,):"""Disable bitsandbytes loading (load_in_4bit/load_in_8bit) when the model already carries a non-bitsandbytes quantization config. Returns ``(load_in_4bit, load_in_8bit, quant_method)``, with both flags False if they were disabled and quant_method the detected method or None. ``rewrite_modelopt`` converts ModelOpt FP8 to fp8; pass False when vLLM loads it natively."""quant_method = get_quant_type(model_config)@@ -2212,6 +2213,12 @@ def check_and_disable_bitsandbytes_loading(if arm_modelopt_fp8_loading(model_config, verbose = verbose) is not None:quant_method = "fp8"+ # An explicit 4bit request on a block-fp8 checkpoint: dequantize each fp8 tensor and quantize it to NF4 while loading.+ if allow_fp8_to_nf4 and str(quant_method).lower() == "fp8":+ from .fp8_to_nf4 import maybe_arm_fp8_to_nf4+ if maybe_arm_fp8_to_nf4(model_config, load_in_4bit, load_in_8bit, verbose = verbose):+ return load_in_4bit, load_in_8bit, None+# A non-bitsandbytes quantization config (compressed-tensors, gptq, awq) means BOTH bitsandbytes loading flags must be disabled to avoid config conflicts.if load_in_4bit or load_in_8bit:if verbose:@@ -2219,6 +2226,21 @@ def check_and_disable_bitsandbytes_loading(f"Unsloth: Model already quantized with {quant_method}. "f"Disabling `load_in_4bit` and `load_in_8bit` to avoid quantization config conflict.")+ if load_in_4bit and not load_in_8bit and str(quant_method).lower() == "fp8":+ from .fp8_to_nf4 import (+ explicit_4bit_requested,+ fp8_block_quantization_config,+ fp8_to_nf4_disabled,+ )+ if (+ not explicit_4bit_requested()+ and not fp8_to_nf4_disabled()+ and fp8_block_quantization_config(model_config) is not None+ ):+ print(+ "Unsloth: Pass `load_in_4bit = True` explicitly (or set UNSLOTH_FP8_TO_NF4=1) to "+ "quantize this fp8 checkpoint to 4bit while it loads."+ )load_in_4bit = Falseload_in_8bit = Falsediff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex c2dc47efa59..e4e2b61ba23 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -2202,6 +2202,7 @@ class FastBaseModel:check_and_disable_bitsandbytes_loading,sync_unsloth_model_name_bnb_flags,)+ from .fp8_to_nf4 import fp8_to_nf4_planner_quantization_configload_in_4bit, load_in_8bit, _ = check_and_disable_bitsandbytes_loading(auto_config,@@ -2211,6 +2212,8 @@ class FastBaseModel:token = token,model_name = model_name,revision = _revision,+ # vLLM reads the checkpoint itself, so it cannot take the in-process fp8 -> 4bit route.+ allow_fp8_to_nf4 = not (fast_inference and is_vLLM_available()),hub_kwargs = {"cache_dir": kwargs.get("cache_dir"),"subfolder": kwargs.get("subfolder"),@@ -2239,6 +2242,13 @@ class FastBaseModel:load_in_4bit = Falseload_in_8bit = Falseload_in_16bit = False+ from .fp8_to_nf4 import disarm_fp8_to_nf4, requests_bnb_4bit++ if not load_in_4bit or (+ user_quantization_config is not None and not requests_bnb_4bit(user_quantization_config)+ ):+ # An fp8 -> 4bit load parked the checkpoint's fp8 config; give it back.+ disarm_fp8_to_nf4(auto_config)# text_only builds the bare decoder; from model_name the planner would plan the whole VLM._planner_skip_reason = None@@ -2292,7 +2302,10 @@ class FastBaseModel:auto_config, dequantize = load_in_16bit)if _modelopt_rewritten- else None,+ else fp8_to_nf4_planner_quantization_config(+ auto_config,+ SKIP_QUANTIZATION_MODULES + _architecture_skip_modules(model_types),+ ),extra_skip_modules = _architecture_skip_modules(model_types) or None,),)@@ -2499,14 +2512,18 @@ class FastBaseModel:or kwargs.get("quantization_config") is not None)):- model = auto_model.from_pretrained(- model_name,- config = model_config,- device_map = device_map,- token = token,- trust_remote_code = trust_remote_code,- **kwargs,- )+ try:+ model = auto_model.from_pretrained(+ model_name,+ config = model_config,+ device_map = device_map,+ token = token,+ trust_remote_code = trust_remote_code,+ **kwargs,+ )+ finally:+ # The load deep-copied the config; give the caller's object its fp8 block back.+ disarm_fp8_to_nf4(model_config)model = _text_trainable_core(model, text_intent = bool(text_only) if text_intent is None else bool(text_intent))