Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load block-FP8 checkpoints in 4-bit (NF4) when load_in_4bit=True is passed (#12146)

Daniel Han91bcb185f9ac
diff --git a/studio/backend/core/export/export.py b/studio/backend/core/export/export.py
index 74d9a819921..f8ea88facf9 100644
--- a/studio/backend/core/export/export.py
+++ b/studio/backend/core/export/export.py
@@ -54,6 +54,11 @@ if not _IS_MLX:
logger = get_logger(__name__)
+def _load_in_4bit_kwargs(load_in_4bit: bool) -> dict:
+ # True is the loaders' default; passing it reads as an explicit request to requantize fp8 checkpoints to NF4.
+ return {} if load_in_4bit else {"load_in_4bit": False}
+
+
def _export_runtime_available() -> bool:
"""True if export can run: MLX active, or Unsloth imported (only succeeds on a GPU host)."""
return bool(_IS_MLX) or (FastLanguageModel is not None)
@@ -750,7 +755,7 @@ class ExportBackend:
model_name = checkpoint_path,
max_seq_length = max_seq_length,
dtype = None,
- load_in_4bit = load_in_4bit,
+ **_load_in_4bit_kwargs(load_in_4bit),
trust_remote_code = trust_remote_code,
token = token,
local_files_only = local_files_only,
@@ -790,7 +795,7 @@ class ExportBackend:
model_name = checkpoint_path,
max_seq_length = max_seq_length,
dtype = None,
- load_in_4bit = load_in_4bit,
+ **_load_in_4bit_kwargs(load_in_4bit),
trust_remote_code = trust_remote_code,
token = token,
local_files_only = local_files_only,
@@ -804,7 +809,7 @@ class ExportBackend:
model_name = checkpoint_path,
max_seq_length = max_seq_length,
dtype = None,
- load_in_4bit = load_in_4bit,
+ **_load_in_4bit_kwargs(load_in_4bit),
trust_remote_code = trust_remote_code,
token = token,
local_files_only = local_files_only,
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index 2efc064d4ea..5487b5671b6 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -72,6 +72,11 @@ from utils.gpu_memory_events import invalidates_gpu_memory as _invalidates_gpu_m
logger = get_logger(__name__)
+def _load_in_4bit_kwargs(load_in_4bit: bool) -> dict:
+ # True is the loaders' default; passing it reads as an explicit request to requantize fp8 checkpoints to NF4.
+ return {} if load_in_4bit else {"load_in_4bit": False}
+
+
def _hf_token_for_loader(hf_token: Optional[str] | bool) -> Optional[str] | bool:
if hf_token is False:
return False
@@ -880,7 +885,7 @@ class InferenceBackend:
model_name = load_path,
max_seq_length = max_seq_length,
dtype = dtype,
- load_in_4bit = load_in_4bit,
+ **_load_in_4bit_kwargs(load_in_4bit),
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
@@ -929,7 +934,7 @@ class InferenceBackend:
model_name = load_path,
max_seq_length = max_seq_length,
dtype = dtype,
- load_in_4bit = load_in_4bit,
+ **_load_in_4bit_kwargs(load_in_4bit),
device_map = device_map,
token = _hf_token_for_loader(hf_token),
trust_remote_code = trust_remote_code,
diff --git a/studio/backend/tests/test_fp8_chat_load_is_not_a_4bit_request.py b/studio/backend/tests/test_fp8_chat_load_is_not_a_4bit_request.py
new file mode 100644
index 00000000000..604978e030d
--- /dev/null
+++ b/studio/backend/tests/test_fp8_chat_load_is_not_a_4bit_request.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""Chat and export loads must not pass `load_in_4bit = True`: Unsloth reads a passed flag as an
+explicit request to requantize fp8 checkpoints to NF4, and the chat UI always sends True."""
+
+from __future__ import annotations
+
+import ast
+from pathlib import Path
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+_FILES = ("core/inference/inference.py", "core/export/export.py")
+
+
+def _helper(path):
+ tree = ast.parse((_BACKEND / path).read_text(encoding = "utf-8"))
+ node = next(
+ n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "_load_in_4bit_kwargs"
+ )
+ scope = {}
+ exec(compile(ast.Module([node], []), path, "exec"), scope)
+ return scope["_load_in_4bit_kwargs"]
+
+
[email protected]("path", _FILES)
+def test_true_is_left_to_the_loader_default(path):
+ helper = _helper(path)
+ assert helper(True) == {}
+ assert helper(False) == {"load_in_4bit": False}
+
+
[email protected]("path", _FILES)
+def test_no_unsloth_loader_call_forwards_the_flag_verbatim(path):
+ tree = ast.parse((_BACKEND / path).read_text(encoding = "utf-8"))
+ forwarded = [
+ node.lineno
+ for node in ast.walk(tree)
+ if isinstance(node, ast.Call)
+ and isinstance(node.func, ast.Attribute)
+ and node.func.attr == "from_pretrained"
+ and isinstance(node.func.value, ast.Name)
+ and node.func.value.id.startswith("Fast")
+ and any(kw.arg == "load_in_4bit" and isinstance(kw.value, ast.Name) for kw in node.keywords)
+ ]
+ assert not forwarded, f"{path}: from_pretrained forwards load_in_4bit at lines {forwarded}"
diff --git a/tests/test_fp8_to_nf4_load.py b/tests/test_fp8_to_nf4_load.py
new file mode 100644
index 00000000000..34437b0c565
--- /dev/null
+++ b/tests/test_fp8_to_nf4_load.py
@@ -0,0 +1,627 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""`load_in_4bit = True` on a block-FP8 checkpoint must give NF4 bytes identical to loading the
+same checkpoint dequantized to bf16 by an independent formula."""
+
+import json
+import os
+import shutil
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+import unsloth # noqa: F401 (patches before transformers)
+from unsloth import FastLanguageModel, FastModel
+from unsloth.models.loader_utils import check_and_disable_bitsandbytes_loading
+
+_BLOCK = 128
+_TOKENIZER_REPO = "trl-internal-testing/tiny-Qwen3ForCausalLM"
+_KEEP_16BIT = "model.layers.1.self_attn.o_proj"
+
+needs_cuda = pytest.mark.skipif(
+ not torch.cuda.is_available(), reason = "bitsandbytes 4bit needs a GPU"
+)
+
+
+def _feature_unavailable():
+ try:
+ from unsloth.models.fp8_to_nf4 import fp8_to_nf4_unavailable_reason
+ except ImportError:
+ return None # main: no module, the tests must fail there
+ return fp8_to_nf4_unavailable_reason()
+
+
+needs_feature = pytest.mark.skipif(
+ _feature_unavailable() is not None,
+ reason = f"fp8 -> 4bit loading unavailable: {_feature_unavailable()}",
+)
+needs_fp8_gpu = pytest.mark.skipif(
+ not torch.cuda.is_available() or torch.cuda.get_device_capability() < (8, 9),
+ reason = "fp8 checkpoints load natively only on sm89+",
+)
+
+
+def _quantize_block_fp8(weight):
+ rows, cols = weight.shape
+ grid_r, grid_c = -(-rows // _BLOCK), -(-cols // _BLOCK)
+ padded = torch.zeros(grid_r * _BLOCK, grid_c * _BLOCK, dtype = torch.float32)
+ padded[:rows, :cols] = weight.float()
+ blocks = padded.view(grid_r, _BLOCK, grid_c, _BLOCK)
+ scale = blocks.abs().amax(dim = (1, 3)).clamp(min = 1e-12) / 448.0
+ quant = (blocks / scale[:, None, :, None]).clamp(-448, 448).to(torch.float8_e4m3fn)
+ quant = quant.view(grid_r * _BLOCK, grid_c * _BLOCK)[:rows, :cols].contiguous()
+ return quant, scale.contiguous()
+
+
+def _dequantize_reference(quant, scale):
+ rows, cols = quant.shape
+ expanded = (
+ scale.float().repeat_interleave(_BLOCK, 0)[:rows].repeat_interleave(_BLOCK, 1)[:, :cols]
+ )
+ return (quant.float() * expanded).to(torch.bfloat16)
+
+
+def _is_fp8_candidate(key, tensor):
+ if not key.endswith(".weight") or tensor.dim() != 2:
+ return False
+ if ".layers." not in key or "norm" in key or ".mlp.gate.weight" in key:
+ return False
+ return not key.startswith(_KEEP_16BIT + ".")
+
+
+def _write_pair(bf16_dir, root):
+ """bf16 save -> (fp8 dir, dequantized-bf16 dir)."""
+ from safetensors.torch import load_file, save_file
+
+ fp8_dir, deq_dir = os.path.join(root, "fp8"), os.path.join(root, "deq")
+ for d in (fp8_dir, deq_dir):
+ os.makedirs(d, exist_ok = True)
+ for name in os.listdir(bf16_dir):
+ if not name.endswith(".safetensors") and not name.endswith(".index.json"):
+ shutil.copy2(os.path.join(bf16_dir, name), os.path.join(d, name))
+ state = {}
+ for name in sorted(os.listdir(bf16_dir)):
+ if name.endswith(".safetensors"):
+ state.update(load_file(os.path.join(bf16_dir, name)))
+ fp8_state, deq_state = {}, {}
+ for key, tensor in state.items():
+ if _is_fp8_candidate(key, tensor):
+ quant, scale = _quantize_block_fp8(tensor)
+ fp8_state[key] = quant
+ fp8_state[key + "_scale_inv"] = scale
+ deq_state[key] = _dequantize_reference(quant, scale)
+ else:
+ fp8_state[key] = tensor
+ deq_state[key] = tensor
+ save_file(fp8_state, os.path.join(fp8_dir, "model.safetensors"), metadata = {"format": "pt"})
+ save_file(deq_state, os.path.join(deq_dir, "model.safetensors"), metadata = {"format": "pt"})
+ with open(os.path.join(fp8_dir, "config.json")) as f:
+ config = json.load(f)
+ config["quantization_config"] = {
+ "quant_method": "fp8",
+ "fmt": "e4m3",
+ "activation_scheme": "dynamic",
+ "weight_block_size": [_BLOCK, _BLOCK],
+ "modules_to_not_convert": ["lm_head", "model.embed_tokens", _KEEP_16BIT],
+ }
+ with open(os.path.join(fp8_dir, "config.json"), "w") as f:
+ json.dump(config, f, indent = 2)
+ return fp8_dir, deq_dir
+
+
+def _build(root, moe):
+ from transformers import AutoTokenizer
+
+ tokenizer = AutoTokenizer.from_pretrained(_TOKENIZER_REPO)
+ common = dict(
+ vocab_size = len(tokenizer),
+ hidden_size = 256,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ head_dim = 64,
+ max_position_embeddings = 512,
+ tie_word_embeddings = False,
+ torch_dtype = "bfloat16",
+ )
+ torch.manual_seed(0)
+ if moe:
+ from transformers import Qwen3MoeConfig, Qwen3MoeForCausalLM
+ config = Qwen3MoeConfig(
+ intermediate_size = 320,
+ moe_intermediate_size = 192, # ragged: 1.5 blocks
+ num_experts = 4,
+ num_experts_per_tok = 2,
+ decoder_sparse_step = 1,
+ **common,
+ )
+ model = Qwen3MoeForCausalLM(config)
+ else:
+ from transformers import Qwen3Config, Qwen3ForCausalLM
+ config = Qwen3Config(intermediate_size = 320, **common) # ragged: 2.5 blocks
+ model = Qwen3ForCausalLM(config)
+ model = model.to(torch.bfloat16)
+ bf16_dir = os.path.join(root, "bf16")
+ model.save_pretrained(bf16_dir, safe_serialization = True)
+ tokenizer.save_pretrained(bf16_dir)
+ return _write_pair(bf16_dir, root)
+
+
[email protected](scope = "module")
+def dense_pair(tmp_path_factory):
+ return _build(str(tmp_path_factory.mktemp("fp8_dense")), moe = False)
+
+
[email protected](scope = "module")
+def moe_pair(tmp_path_factory):
+ return _build(str(tmp_path_factory.mktemp("fp8_moe")), moe = True)
+
+
+def _fingerprint(model):
+ """name -> tensors that define the loaded weight: packed NF4 bytes and the full quant state."""
+ out = {}
+ for name, param in model.named_parameters():
+ state = getattr(param, "quant_state", None)
+ entry = {"data": param.data.detach().clone().cpu()}
+ if state is not None:
+ entry["absmax"] = state.absmax.detach().cpu()
+ entry["shape"] = tuple(state.shape)
+ entry["dtype"] = state.dtype
+ entry["blocksize"] = state.blocksize
+ entry["quant_type"] = state.quant_type
+ if getattr(state, "nested", False):
+ entry["offset"] = state.offset.detach().cpu()
+ entry["absmax2"] = state.state2.absmax.detach().cpu()
+ entry["code2"] = state.state2.code.detach().cpu()
+ out[name] = entry
+ return out
+
+
+def _assert_same(a, b):
+ assert a.keys() == b.keys()
+ for name in a:
+ ea, eb = a[name], b[name]
+ assert ea.keys() == eb.keys(), name
+ for key in ea:
+ va, vb = ea[key], eb[key]
+ if isinstance(va, torch.Tensor):
+ assert va.dtype == vb.dtype and va.shape == vb.shape, (name, key)
+ assert torch.equal(
+ va.reshape(-1).view(torch.uint8), vb.reshape(-1).view(torch.uint8)
+ ), (name, key)
+ else:
+ assert va == vb, (name, key)
+
+
+def _count_4bit(model):
+ return sum(1 for _, p in model.named_parameters() if type(p).__name__ == "Params4bit")
+
+
+def _count_fp8(model):
+ return sum(1 for _, p in model.named_parameters() if p.dtype == torch.float8_e4m3fn)
+
+
+def _load(loader, path, **kwargs):
+ model, _ = loader.from_pretrained(path, max_seq_length = 64, dtype = torch.bfloat16, **kwargs)
+ return model
+
+
+def _reference_config():
+ from transformers import BitsAndBytesConfig
+ from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
+ return BitsAndBytesConfig(
+ load_in_4bit = True,
+ bnb_4bit_use_double_quant = True,
+ bnb_4bit_quant_type = "nf4",
+ bnb_4bit_compute_dtype = torch.bfloat16,
+ llm_int8_skip_modules = SKIP_QUANTIZATION_MODULES + [_KEEP_16BIT],
+ )
+
+
+def _free(*models):
+ for model in models:
+ del model
+ import gc
+
+ gc.collect()
+ torch.cuda.empty_cache()
+
+
+@needs_cuda
+@needs_feature
+def test_dense_fp8_loads_nf4_bit_identical_to_dequantized_bf16(dense_pair, capsys):
+ fp8_dir, deq_dir = dense_pair
+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)
+ printed = capsys.readouterr().out
+ assert _count_4bit(model) == 13 # 2 x (q, k, v, o, gate, up, down) minus the 16-bit o_proj
+ assert _count_fp8(model) == 0
+ assert "dequantizing each fp8 tensor" in printed
+ assert not hasattr(model.config, "_unsloth_fp8_to_nf4")
+ ours = _fingerprint(model)
+ _free(model)
+ reference = _load(FastLanguageModel, deq_dir, quantization_config = _reference_config())
+ theirs = _fingerprint(reference)
+ _free(reference)
+ _assert_same(ours, theirs)
+
+
+@needs_cuda
+@needs_feature
+def test_moe_fp8_loads_nf4_experts_bit_identical(moe_pair):
+ fp8_dir, deq_dir = moe_pair
+ model = _load(FastModel, fp8_dir, load_in_4bit = True)
+ experts = [
+ name
+ for name, p in model.named_parameters()
+ if ".experts." in name and type(p).__name__ == "Params4bit"
+ ]
+ # gate_up_proj and down_proj stacks on both layers go through Unsloth's bnb expert path.
+ assert len(experts) == 4, experts
+ assert _count_fp8(model) == 0
+ ours = _fingerprint(model)
+ _free(model)
+ reference = _load(FastModel, deq_dir, quantization_config = _reference_config())
+ theirs = _fingerprint(reference)
+ _free(reference)
+ _assert_same(ours, theirs)
+
+
+@needs_cuda
+@needs_feature
+def test_checkpoint_16bit_modules_stay_16bit(dense_pair):
+ from safetensors.torch import load_file
+
+ fp8_dir, _ = dense_pair
+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)
+ module = model.get_submodule(_KEEP_16BIT)
+ assert type(module) is torch.nn.Linear
+ stored = load_file(os.path.join(fp8_dir, "model.safetensors"))[_KEEP_16BIT + ".weight"]
+ assert stored.dtype == torch.bfloat16
+ assert torch.equal(module.weight.detach().cpu(), stored)
+ quantization_config = model.config.quantization_config
+ if not isinstance(quantization_config, dict):
+ quantization_config = quantization_config.to_dict()
+ # Saved with the model, so a reload of the 4bit save keeps it in 16bit too.
+ assert any(
+ p.replace("\\", "").rstrip("$") == _KEEP_16BIT
+ for p in quantization_config["llm_int8_skip_modules"]
+ )
+ _free(model)
+
+
+@needs_fp8_gpu
+@needs_feature
+def test_default_load_in_4bit_and_kill_switch_keep_fp8(dense_pair, monkeypatch):
+ fp8_dir, _ = dense_pair
+ # load_in_4bit left at its default is not a request: unchanged fp8 load.
+ model = _load(FastLanguageModel, fp8_dir)
+ assert _count_4bit(model) == 0 and _count_fp8(model) > 0
+ _free(model)
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "0")
+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)
+ assert _count_4bit(model) == 0 and _count_fp8(model) > 0
+ _free(model)
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")
+ model = _load(FastLanguageModel, fp8_dir)
+ assert _count_4bit(model) == 13 and _count_fp8(model) == 0
+ _free(model)
+
+
+@needs_cuda
+@needs_feature
+def test_fp8_keys_the_model_does_not_have_are_ignored(dense_pair, tmp_path):
+ """MTP layers ship in fp8 but transformers drops them as unexpected (DeepSeek-V3, GLM-4 MoE,
+ Qwen3-Next): they must not count as unapplied scales."""
+ from safetensors.torch import load_file, save_file
+
+ fp8_dir, _ = dense_pair
+ extra_dir = str(tmp_path / "fp8_mtp")
+ shutil.copytree(fp8_dir, extra_dir)
+ state = load_file(os.path.join(extra_dir, "model.safetensors"))
+ quant, scale = _quantize_block_fp8(torch.randn(320, 256) * 0.02)
+ state["mtp.layers.0.mlp.gate_proj.weight"] = quant
+ state["mtp.layers.0.mlp.gate_proj.weight_scale_inv"] = scale
+ save_file(state, os.path.join(extra_dir, "model.safetensors"), metadata = {"format": "pt"})
+ model = _load(FastLanguageModel, extra_dir, load_in_4bit = True)
+ assert _count_4bit(model) == 13
+ ours = _fingerprint(model)
+ _free(model)
+ plain = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)
+ theirs = _fingerprint(plain)
+ _free(plain)
+ _assert_same(ours, theirs)
+
+
+@needs_cuda
+@needs_feature
+def test_quantize_16bit_switch_matches_plain_4bit_skip_list(dense_pair, monkeypatch):
+ fp8_dir, deq_dir = dense_pair
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT", "1")
+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)
+ assert _count_4bit(model) == 14
+ ours = _fingerprint(model)
+ _free(model)
+ reference = _load(FastLanguageModel, deq_dir, load_in_4bit = True)
+ theirs = _fingerprint(reference)
+ _free(reference)
+ _assert_same(ours, theirs)
+
+
+def _zoo_reads_fp8_block_size_from_disk():
+ try:
+ from unsloth_zoo import saving_utils
+ except Exception:
+ return False
+ return hasattr(saving_utils, "_fp8_block_size_on_disk")
+
+
+@needs_cuda
+@needs_feature
+ not _zoo_reads_fp8_block_size_from_disk(),
+ reason = "unsloth_zoo takes the fp8 block size of a 16bit merge from the in-memory config only",
+)
+def test_moe_merged_16bit_save_equals_dequantized_checkpoint(moe_pair, tmp_path):
+ from safetensors.torch import load_file
+
+ fp8_dir, deq_dir = moe_pair
+ model, tokenizer = FastModel.from_pretrained(
+ fp8_dir, max_seq_length = 64, dtype = torch.bfloat16, load_in_4bit = True
+ )
+ assert _count_4bit(model) > 0
+ model = FastModel.get_peft_model(
+ model, r = 8, target_modules = ["q_proj", "o_proj", "gate_proj", "up_proj", "down_proj"]
+ )
+ # lora_B starts at zero, so the merge must give back the dequantized fp8 weights exactly,
+ # read from the fp8 shards on disk rather than the NF4 copy in memory.
+ model.save_pretrained_merged(str(tmp_path / "merged"), tokenizer, save_method = "merged_16bit")
+ _free(model)
+ merged = {}
+ for name in os.listdir(tmp_path / "merged"):
+ if name.endswith(".safetensors"):
+ merged.update(load_file(str(tmp_path / "merged" / name)))
+ reference = load_file(os.path.join(deq_dir, "model.safetensors"))
+ assert merged.keys() == reference.keys()
+ for key, tensor in reference.items():
+ assert torch.equal(merged[key], tensor), key
+
+
+@needs_fp8_gpu
[email protected](_feature_unavailable() is None, reason = "only where fp8 -> 4bit cannot run")
+def test_unavailable_keeps_todays_fp8_load(dense_pair, capsys):
+ fp8_dir, _ = dense_pair
+ model = _load(FastLanguageModel, fp8_dir, load_in_4bit = True)
+ assert _count_4bit(model) == 0 and _count_fp8(model) > 0
+ assert "cannot quantize to 4bit here" in capsys.readouterr().out
+ _free(model)
+
+
+def _fp8_config():
+ return SimpleNamespace(
+ model_type = "qwen3",
+ quantization_config = {
+ "quant_method": "fp8",
+ "fmt": "e4m3",
+ "weight_block_size": [128, 128],
+ "modules_to_not_convert": ["lm_head"],
+ },
+ )
+
+
+@needs_feature
+def test_arming_needs_an_explicit_request(monkeypatch):
+ from unsloth.models import fp8_to_nf4
+
+ monkeypatch.delenv("UNSLOTH_FP8_TO_NF4", raising = False)
+ config = _fp8_config()
+ assert check_and_disable_bitsandbytes_loading(config, load_in_4bit = True, verbose = False)[:2] == (
+ False,
+ False,
+ )
+ assert config.quantization_config["quant_method"] == "fp8"
+
+ inside = {}
+
+ @fp8_to_nf4.track_explicit_4bit_request
+ def from_pretrained(
+ model_name = None,
+ load_in_4bit = True,
+ **kwargs,
+ ):
+ flags = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = load_in_4bit, verbose = False
+ )
+ inside["armed"] = fp8_to_nf4.fp8_to_nf4_armed(config)
+ inside["stripped"] = not hasattr(config, "quantization_config")
+ return flags
+
+ assert from_pretrained("x")[:2] == (False, False)
+ config = _fp8_config()
+ assert from_pretrained("x", load_in_4bit = True)[:2] == (True, False)
+ assert inside == {"armed": True, "stripped": True}
+ # The outermost from_pretrained hands the caller's config back as it was.
+ assert config.quantization_config["quant_method"] == "fp8"
+ assert not fp8_to_nf4.fp8_to_nf4_armed(config)
+
+ config = _fp8_config()
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "0")
+ assert from_pretrained("x", load_in_4bit = True)[:2] == (False, False)
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")
+ assert from_pretrained("x")[:2] == (True, False)
+
+
+def test_non_block_fp8_and_8bit_are_not_armed(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")
+ config = _fp8_config()
+ config.quantization_config.pop("weight_block_size")
+ assert check_and_disable_bitsandbytes_loading(config, load_in_4bit = True, verbose = False)[:2] == (
+ False,
+ False,
+ )
+ config = _fp8_config()
+ assert check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = False, load_in_8bit = True, verbose = False
+ )[:2] == (False, False)
+ assert config.quantization_config["quant_method"] == "fp8"
+
+
+@needs_feature
+def test_scaled_weights_stored_as_integers_are_refused():
+ from unsloth.models import fp8_to_nf4
+ fp8_to_nf4._refuse_packed_scaled_weights(
+ {"a.weight": "F8_E4M3", "a.weight_scale_inv": "F32", "b.weight": "BF16", "n.scale": "F32"}
+ )
+ with pytest.raises(RuntimeError, match = "cannot dequantize"):
+ fp8_to_nf4._refuse_packed_scaled_weights({"e.weight": "I8", "e.scale": "F8_E8M0"})
+
+
+@needs_feature
+def test_packed_fp4_expert_checkpoints_are_not_armed(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4", "1")
+ config = _fp8_config()
+ config.quantization_config["expert_dtype"] = "fp4"
+ assert check_and_disable_bitsandbytes_loading(config, load_in_4bit = True, verbose = False)[:2] == (
+ False,
+ False,
+ )
+ assert config.quantization_config["quant_method"] == "fp8"
+
+
[email protected]("shape", [(256, 384), (320, 256), (192, 200), (3, 320, 192)])
+def test_block_dequant_matches_reference_formula(shape):
+ torch.manual_seed(1)
+ weight = torch.randn(*shape) * 0.02
+ if len(shape) == 2:
+ quant, scale = _quantize_block_fp8(weight)
+ expected = _dequantize_reference(quant, scale)
+ else:
+ pairs = [_quantize_block_fp8(w) for w in weight]
+ quant = torch.stack([q for q, _ in pairs])
+ scale = torch.stack([s for _, s in pairs])
+ expected = torch.stack([_dequantize_reference(q, s) for q, s in pairs])
+ from unsloth.models import fp8_to_nf4
+
+ got = fp8_to_nf4._dequantize_block_fp8(quant, scale, [128, 128], torch.bfloat16)
+ assert torch.equal(got.view(torch.int16), expected.view(torch.int16))
+
+
+@needs_feature
+def test_stacked_16bit_weight_with_a_scale_is_copied_into_the_stack():
+ import unsloth.models.fp8_to_nf4 as fp8_to_nf4
+
+ dequantize_op, _ = fp8_to_nf4._build_classes()
+ torch.manual_seed(2)
+ quant, scale = _quantize_block_fp8(torch.randn(128, 128) * 0.02)
+ stored_16bit = (torch.randn(128, 128) * 0.02).to(torch.bfloat16)
+ fp8_to_nf4._STATE = SimpleNamespace(
+ block = [128, 128], dtype = torch.bfloat16, kept_fp8 = 0, dequantized = 0
+ )
+ try:
+ result = dequantize_op(stack = True).convert(
+ {
+ "mlp.experts.*.down_proj.weight": [quant, stored_16bit],
+ "mlp.experts.*.down_proj.weight_scale_inv": [scale, torch.ones(1, 1)],
+ },
+ target_patterns = ["mlp.experts.down_proj"],
+ )
+ finally:
+ fp8_to_nf4._STATE = None
+ stack = result["mlp.experts.*.down_proj.weight"]
+ assert torch.equal(stack[0], _dequantize_reference(quant, scale))
+ assert torch.equal(stack[1], stored_16bit)
+
+
+@needs_feature
+def test_a_load_that_raises_before_loading_hands_the_config_back(monkeypatch):
+ # Prefetch or device-map planning can raise after arming, before the load restores the config.
+ from unsloth.models import fp8_to_nf4
+
+ monkeypatch.delenv("UNSLOTH_FP8_TO_NF4", raising = False)
+ config = _fp8_config()
+ original = dict(config.quantization_config)
+
+ @fp8_to_nf4.track_explicit_4bit_request
+ def from_pretrained(
+ model_name = None,
+ load_in_4bit = True,
+ **kwargs,
+ ):
+ assert check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = load_in_4bit, verbose = False
+ )[0]
+ assert fp8_to_nf4.fp8_to_nf4_armed(config)
+ raise RuntimeError("prefetch stalled")
+
+ with pytest.raises(RuntimeError, match = "prefetch stalled"):
+ from_pretrained("x", load_in_4bit = True)
+ assert config.quantization_config == original
+ assert not fp8_to_nf4.fp8_to_nf4_armed(config)
+
+
+def test_planner_sizes_checkpoint_16bit_linears_as_4bit_in_quantize_all_mode(monkeypatch):
+ from unsloth.models import fp8_to_nf4
+
+ config = SimpleNamespace(
+ **{
+ fp8_to_nf4.UNSLOTH_FP8_TO_NF4_ATTR: {
+ "modules_to_not_convert": ["vision_tower", "lm_head"]
+ }
+ }
+ )
+ monkeypatch.delenv("UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT", raising = False)
+ kept = fp8_to_nf4.fp8_to_nf4_planner_quantization_config(config, ["lm_head"])
+ assert kept["llm_int8_skip_modules"] == ["lm_head", "vision_tower"]
+ monkeypatch.setenv("UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT", "1")
+ quantized = fp8_to_nf4.fp8_to_nf4_planner_quantization_config(config, ["lm_head"])
+ assert quantized["llm_int8_skip_modules"] == ["lm_head"]
+
+
+@needs_feature
+def test_planner_sizes_an_armed_load_expert_merge_in_the_load_dtype():
+ # The fused expert stack is built in bf16 before bitsandbytes packs it: plan that transient.
+ planner = pytest.importorskip("unsloth_zoo.device_map_planner")
+ from transformers import AutoConfig
+
+ from unsloth.models import fp8_to_nf4
+
+ assert fp8_to_nf4.install_fp8_to_nf4_quantizer()
+ repo = "trl-internal-testing/tiny-Qwen3MoeForCausalLM"
+ config = AutoConfig.from_pretrained(repo)
+ rewritten = {"quant_method": "bitsandbytes", "load_in_4bit": True, "bnb_4bit_quant_type": "nf4"}
+ measured = {}
+ for armed in (False, True):
+ token = fp8_to_nf4._ARMED_CONFIGS.set([config] if armed else None)
+ try:
+ model, quantizer, _ = planner.build_meta_model(
+ repo,
+ config = config,
+ dtype = torch.bfloat16,
+ rewritten_quantization_config = dict(rewritten),
+ )
+ units = [(name, 0) for name, _ in model.named_modules() if name.endswith(".mlp")]
+ transient = planner._load_transient_by_unit(model, units, quantizer)
+ measured[armed] = (
+ quantizer.pre_quantized,
+ planner._compute_module_sizes(model, quantizer)[""],
+ max(transient.values()),
+ )
+ finally:
+ fp8_to_nf4._ARMED_CONFIGS.reset(token)
+ assert measured[False][0] and not measured[True][0]
+ assert measured[True][1] == measured[False][1] # weights still sized as 4bit
+ assert measured[True][2] == 4 * measured[False][2] # bf16 stack, not packed nibbles
diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.py
index febadfaf369..9f375738730 100644
--- a/tests/test_modelopt_fp8_loading.py
+++ b/tests/test_modelopt_fp8_loading.py
@@ -531,7 +531,7 @@ def test_config_branch_moves_rope_extension_onto_the_config():
from unsloth.models import llama
source = inspect.getsource(llama.FastLlamaModel.from_pretrained)
- branch = source.split("if user_config is not None or _modelopt_rewritten:", 1)[1]
+ branch = source.split("if user_config is not None or _modelopt_rewritten", 1)[1]
branch = branch.split("AutoModelForCausalLM.from_pretrained(", 1)[0]
assert 'kwargs.pop("rope_scaling", None)' in branch
diff --git a/unsloth/models/fp8_to_nf4.py b/unsloth/models/fp8_to_nf4.py
new file mode 100644
index 00000000000..4122a6a1dfa
--- /dev/null
+++ b/unsloth/models/fp8_to_nf4.py
@@ -0,0 +1,1169 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""Load a block-FP8 checkpoint straight into bitsandbytes NF4 (``load_in_4bit = True``).
+
+The fp8 block is parked off the config so transformers runs a bnb 4bit load; each FP8 tensor is
+dequantized inside its weight converter right before bnb quantizes it, so only one converter's
+tensors are ever 16-bit and NF4 is bit-identical to loading the dequantized checkpoint. The
+checkpoint's ``modules_to_not_convert`` stay out of bitsandbytes.
+"""
+
+import contextvars
+import copy
+import functools
+import inspect
+import os
+import re
+from typing import Optional
+
+import torch
+
+__all__ = [
+ "UNSLOTH_FP8_TO_NF4_ATTR",
+ "track_explicit_4bit_request",
+ "requests_bnb_4bit",
+ "explicit_4bit_requested",
+ "fp8_block_quantization_config",
+ "fp8_to_nf4_unavailable_reason",
+ "maybe_arm_fp8_to_nf4",
+ "fp8_to_nf4_armed",
+ "fp8_to_nf4_disabled",
+ "disarm_fp8_to_nf4",
+ "fp8_to_nf4_planner_quantization_config",
+]
+
+UNSLOTH_FP8_TO_NF4_ATTR = "_unsloth_fp8_to_nf4"
+_ENV = "UNSLOTH_FP8_TO_NF4"
+_ENV_QUANTIZE_ALL = "UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT"
+
+# None outside a load; True / False once the outermost from_pretrained has looked at its call.
+_EXPLICIT_4BIT = contextvars.ContextVar("unsloth_explicit_load_in_4bit", default = None)
+# Configs armed inside the outermost from_pretrained; disarmed when it returns or raises, so an
+# exception before the load's own restore (prefetch, device-map planning) cannot leak a stripped config.
+_ARMED_CONFIGS = contextvars.ContextVar("unsloth_fp8_to_nf4_armed_configs", default = None)
+
+_FP8_DTYPES = tuple(
+ getattr(torch, name)
+ for name in ("float8_e4m3fn", "float8_e5m2", "float8_e4m3fnuz", "float8_e5m2fnuz")
+ if hasattr(torch, name)
+)
+_FP8_SAFETENSORS = ("F8_E4M3", "F8_E5M2", "F8_E4M3FNUZ", "F8_E5M2FNUZ")
+_SCALE_SUFFIXES = (".weight_scale_inv", ".activation_scale")
+# Checkpoint dtypes a scaled weight may be stored in; anything else (int8-packed FP4) is refused.
+_FLOAT_SAFETENSORS = _FP8_SAFETENSORS + ("BF16", "F16", "F32")
+_PACKED_EXPERT_DTYPES = ("fp4", "mxfp4", "nvfp4", "int4")
+
+
+def requests_bnb_4bit(quantization_config) -> bool:
+ if quantization_config is None:
+ return False
+ if isinstance(quantization_config, dict):
+ get = quantization_config.get
+ else:
+ get = lambda key, default = None: getattr(quantization_config, key, default)
+ method = get("quant_method", "") or ""
+ method = str(getattr(method, "value", method)).lower()
+ if method and "bitsandbytes" not in method:
+ return False
+ return bool(get("load_in_4bit", False))
+
+
+def track_explicit_4bit_request(fn):
+ """Record whether the caller itself asked for 4bit: ``load_in_4bit`` defaults to True, so only a
+ passed argument is a request (a user ``quantization_config`` clears the loader's 4bit flag, so it
+ never arms). The outermost from_pretrained decides; the FastModel delegations inside it pass the
+ flag on explicitly."""
+ try:
+ signature = inspect.signature(fn)
+ except (TypeError, ValueError):
+ signature = None
+
+ @functools.wraps(fn)
+ def _wrapper(*args, **kwargs):
+ if _EXPLICIT_4BIT.get() is not None:
+ return fn(*args, **kwargs)
+ explicit = False
+ try:
+ if signature is not None:
+ arguments = signature.bind_partial(*args, **kwargs).arguments
+ explicit = bool(arguments.get("load_in_4bit", False))
+ except TypeError:
+ explicit = bool(kwargs.get("load_in_4bit", False))
+ explicit = explicit or requests_bnb_4bit(kwargs.get("quantization_config", None))
+ token = _EXPLICIT_4BIT.set(explicit)
+ armed_token = _ARMED_CONFIGS.set([])
+ try:
+ return fn(*args, **kwargs)
+ finally:
+ for armed in _ARMED_CONFIGS.get() or []:
+ disarm_fp8_to_nf4(armed)
+ _ARMED_CONFIGS.reset(armed_token)
+ _EXPLICIT_4BIT.reset(token)
+
+ return _wrapper
+
+
+def explicit_4bit_requested() -> bool:
+ return bool(_EXPLICIT_4BIT.get())
+
+
+def _quant_dict(quant) -> Optional[dict]:
+ if quant is None:
+ return None
+ if isinstance(quant, dict):
+ return quant
+ to_dict = getattr(quant, "to_dict", None)
+ if callable(to_dict):
+ try:
+ return to_dict()
+ except Exception:
+ return None
+ return None
+
+
+def _configs_with_quantization(config) -> list:
+ """The config objects holding a quantization_config: the root, else its text config
+ (transformers' get_hf_quantizer reads both)."""
+ found = []
+ if config is None:
+ return found
+ if getattr(config, "quantization_config", None) is not None:
+ found.append(config)
+ get_text_config = getattr(config, "get_text_config", None)
+ if callable(get_text_config):
+ try:
+ text_config = get_text_config(decoder = True)
+ except TypeError:
+ text_config = get_text_config()
+ except Exception:
+ text_config = None
+ if (
+ text_config is not None
+ and text_config is not config
+ and getattr(text_config, "quantization_config", None) is not None
+ ):
+ found.append(text_config)
+ return found
+
+
+def _block_size(quant: dict):
+ block = quant.get("weight_block_size")
+ if isinstance(block, (int, float)):
+ block = [block, block]
+ if not isinstance(block, (list, tuple)) or len(block) != 2:
+ return None
+ try:
+ block = [int(block[0]), int(block[1])]
+ except (TypeError, ValueError):
+ return None
+ if block[0] <= 0 or block[1] <= 0:
+ return None
+ return block
+
+
+def fp8_block_quantization_config(config) -> Optional[dict]:
+ """The checkpoint's fine-grained (block) fp8 quantization_config as a dict, else None."""
+ for holder in _configs_with_quantization(config):
+ quant = _quant_dict(getattr(holder, "quantization_config", None))
+ if not quant or str(quant.get("quant_method", "")).lower() != "fp8":
+ continue
+ if _block_size(quant) is None:
+ return None
+ return quant
+ return None
+
+
+def _transformers_supports_fp8_to_nf4() -> Optional[str]:
+ try:
+ from transformers import core_model_loading as cml
+ from transformers.conversion_mapping import get_model_conversion_mapping # noqa: F401
+ from transformers.quantizers import auto as quantizers_auto
+ from transformers.quantizers.quantizer_bnb_4bit import Bnb4BitHfQuantizer
+ except Exception:
+ return "this transformers has no weight-conversion loader (transformers >= 5.16 is needed)"
+ for name in (
+ "WeightConverter",
+ "WeightRenaming",
+ "ConversionOps",
+ "rename_source_key",
+ "spawn_materialize",
+ ):
+ if not hasattr(cml, name):
+ return f"transformers.core_model_loading has no `{name}`"
+ if not hasattr(cml.WeightTransform, "add_tensor"):
+ return "transformers' WeightTransform has no add_tensor"
+ if not isinstance(getattr(quantizers_auto, "AUTO_QUANTIZER_MAPPING", None), dict):
+ return "transformers has no AUTO_QUANTIZER_MAPPING"
+ # Converters are rebuilt with these (5.5 to 5.16 added them one by one).
+ if not hasattr(cml, "_IdentityOp"):
+ return "transformers.core_model_loading has no `_IdentityOp`"
+ if not callable(getattr(Bnb4BitHfQuantizer, "update_weight_conversions", None)):
+ return "transformers' quantizers have no update_weight_conversions hook"
+ # Slots, not the signature: Unsloth wraps WeightConverter.__init__ as (*args, **kwargs).
+ converter_fields = set(getattr(cml.WeightConverter, "__slots__", ())) | set(
+ getattr(cml.WeightConverter, "__dataclass_fields__", {})
+ )
+ if "force_cpu" not in converter_fields:
+ return "transformers' WeightConverter predates force_cpu (transformers >= 5.16 is needed)"
+ return None
+
+
+def fp8_to_nf4_unavailable_reason() -> Optional[str]:
+ reason = _transformers_supports_fp8_to_nf4()
+ if reason is not None:
+ return reason
+ try:
+ import bitsandbytes # noqa: F401
+ except Exception:
+ return "bitsandbytes is not installed"
+ return None
+
+
+def fp8_to_nf4_disabled() -> bool:
+ return os.environ.get(_ENV, "").strip().lower() in ("0", "false", "off", "no")
+
+
+def fp8_to_nf4_armed(config) -> bool:
+ return config is not None and getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) is not None
+
+
+def maybe_arm_fp8_to_nf4(
+ config,
+ load_in_4bit,
+ load_in_8bit,
+ verbose = True,
+) -> bool:
+ """Arm an fp8 -> NF4 load. True means: keep load_in_4bit, the fp8 block was parked on `config`
+ and removed from it, and the load must pass this `config` to from_pretrained."""
+ if not load_in_4bit or load_in_8bit or config is None:
+ return False
+ quant = fp8_block_quantization_config(config)
+ if quant is None:
+ return False
+ if fp8_to_nf4_disabled():
+ return False
+ if str(quant.get("expert_dtype") or "").lower() in _PACKED_EXPERT_DTYPES:
+ # DeepSeek-V4-Flash: fp8 attention, packed FP4 experts. Not handled here.
+ if verbose and explicit_4bit_requested():
+ print(
+ "Unsloth: this fp8 checkpoint packs its experts in FP4, which the fp8 -> 4bit load does "
+ "not handle. Loading it as it is."
+ )
+ return False
+ mode = os.environ.get(_ENV, "").strip().lower()
+ if not (explicit_4bit_requested() or mode in ("1", "true", "on", "yes", "force")):
+ return False
+ reason = fp8_to_nf4_unavailable_reason()
+ if reason is not None:
+ if verbose:
+ print(
+ f"Unsloth: `load_in_4bit = True` on an fp8 checkpoint cannot quantize to 4bit here: "
+ f"{reason}. Loading the fp8 weights instead."
+ )
+ return False
+ if quant.get("modules_to_convert"):
+ try:
+ from transformers.integrations.finegrained_fp8 import ( # noqa: F401
+ FP8Embedding,
+ replace_with_fp8_embedding,
+ )
+ except Exception:
+ if verbose:
+ print(
+ "Unsloth: this fp8 checkpoint keeps fp8 embedding tables (modules_to_convert), which "
+ "need transformers >= 5.17 to load in 4bit. Loading the fp8 weights instead."
+ )
+ return False
+ if not install_fp8_to_nf4_quantizer():
+ if verbose:
+ print(
+ "Unsloth: could not register the fp8 -> 4bit loader with transformers. Loading the "
+ "fp8 weights instead."
+ )
+ return False
+ plan = {
+ "original_quantization_config": dict(quant),
+ "weight_block_size": _block_size(quant),
+ "fmt": quant.get("fmt", "e4m3"),
+ "activation_scheme": quant.get("activation_scheme"),
+ "modules_to_not_convert": list(quant.get("modules_to_not_convert") or []),
+ "modules_to_convert": list(quant.get("modules_to_convert") or []),
+ }
+ holders = _configs_with_quantization(config)
+ try:
+ stripped = []
+ for holder in holders:
+ if (
+ str((_quant_dict(holder.quantization_config) or {}).get("quant_method", "")).lower()
+ == "fp8"
+ ):
+ stripped.append(holder is not config)
+ try:
+ delattr(holder, "quantization_config")
+ except AttributeError:
+ holder.__dict__.pop("quantization_config", None)
+ plan["stripped_text_config"] = any(stripped)
+ plan["stripped_root_config"] = not all(stripped) if stripped else False
+ setattr(config, UNSLOTH_FP8_TO_NF4_ATTR, plan)
+ armed_configs = _ARMED_CONFIGS.get()
+ if armed_configs is not None:
+ armed_configs.append(config)
+ except Exception:
+ for holder in holders:
+ if getattr(holder, "quantization_config", None) is None:
+ holder.quantization_config = quant
+ return False
+ if verbose:
+ print(
+ "Unsloth: fp8 checkpoint with `load_in_4bit = True`: dequantizing each fp8 tensor to 16bit "
+ "while loading and quantizing it to 4bit (NF4) right away, so no 16bit copy of the model is "
+ f"ever held. Set {_ENV}=0 to load the fp8 weights instead."
+ )
+ return True
+
+
+def disarm_fp8_to_nf4(config) -> bool:
+ """Undo maybe_arm_fp8_to_nf4 when the load ends up not quantizing (e.g. full finetuning)."""
+ global _STATE
+ plan = getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) if config is not None else None
+ if plan is None:
+ return False
+ # A load that raised never reached _process_model_after_weight_loading.
+ _STATE = None
+ original = plan["original_quantization_config"]
+ if plan.get("stripped_root_config"):
+ config.quantization_config = dict(original)
+ if plan.get("stripped_text_config"):
+ get_text_config = getattr(config, "get_text_config", None)
+ text_config = None
+ if callable(get_text_config):
+ try:
+ text_config = get_text_config(decoder = True)
+ except TypeError:
+ text_config = get_text_config()
+ if text_config is not None:
+ text_config.quantization_config = dict(original)
+ try:
+ delattr(config, UNSLOTH_FP8_TO_NF4_ATTR)
+ except AttributeError:
+ config.__dict__.pop(UNSLOTH_FP8_TO_NF4_ATTR, None)
+ return True
+
+
+def _quantize_16bit_requested() -> bool:
+ return os.environ.get(_ENV_QUANTIZE_ALL, "0").strip().lower() in ("1", "true", "on", "yes")
+
+
+def fp8_to_nf4_planner_quantization_config(config, llm_int8_skip_modules = None) -> Optional[dict]:
+ """What the device-map planner must size an armed load as: a bitsandbytes 4bit load."""
+ plan = getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) if config is not None else None
+ if plan is None:
+ return None
+ skip = list(llm_int8_skip_modules or [])
+ # Quantize-all mode quantizes the checkpoint's 16-bit Linears too: size them as 4bit.
+ for name in [] if _quantize_16bit_requested() else plan.get("modules_to_not_convert") or []:
+ if name not in skip:
+ skip.append(name)
+ return {
+ "quant_method": "bitsandbytes",
+ "load_in_4bit": True,
+ "bnb_4bit_quant_type": "nf4",
+ "bnb_4bit_use_double_quant": True,
+ "llm_int8_skip_modules": skip,
+ }
+
+
+class _LoadState:
+ """Book-keeping for the one fp8 -> NF4 load in flight. Module level, never on an op or a
+ converter: the loader deep-copies those once per target key."""
+
+ def __init__(self, plan, dtype, headers):
+ self.block = plan["weight_block_size"]
+ self.dtype = dtype
+ self.fp8_keys = {k for k, v in headers.items() if v in _FP8_SAFETENSORS}
+ # `<module>.scale` is DeepSeek-V4's name for weight_scale_inv (renamed by the fp8 quantizer too).
+ self.dot_scale_keys = {
+ k
+ for k in headers
+ if k.endswith(".scale") and (k[: -len(".scale")] + ".weight") in self.fp8_keys
+ }
+ # Activation scales (static fp8) are not needed for a weight dequant, so they are not tracked.
+ self.scale_keys = {
+ k for k in headers if k.endswith(".weight_scale_inv")
+ } | self.dot_scale_keys
+ self.native_keys = self.fp8_keys | self.scale_keys
+ # Refined once renaming is known: dropped MTP layers never reach a converter.
+ self.expected_fp8 = set(self.fp8_keys)
+ self.expected_scales = set(self.scale_keys)
+ self.headers = headers
+ self.routed_native = set()
+ self.dequantized = 0
+ self.kept_fp8 = 0
+
+
+_STATE: Optional[_LoadState] = None
+
+
+def _is_scale_pattern(pattern: str) -> bool:
+ base = pattern[:-1] if pattern.endswith("$") else pattern
+ return base.endswith(("weight_scale_inv", "activation_scale"))
+
+
+def _scale_pattern_for(pattern: str) -> str:
+ anchored = pattern.endswith("$")
+ base = pattern[:-1] if anchored else pattern
+ if base.endswith(".weight"):
+ scale = base[: -len(".weight")] + ".weight_scale_inv"
+ elif base == "weight":
+ scale = "weight_scale_inv"
+ else:
+ scale = base + "_scale_inv"
+ return scale + "$" if anchored else scale
+
+
+def _scale_as_fp32(scale: torch.Tensor) -> torch.Tensor:
+ if scale.dtype == torch.uint8:
+ return (scale.to(torch.float32) - 127.0).exp2()
+ return scale.to(torch.float32)
+
+
+def _dequantize_block_fp8(
+ weight,
+ scale,
+ block,
+ dtype,
+ out = None,
+):
+ """``(weight.float() * scale_per_block).to(dtype)``, element by element the same product as a
+ full expand of the scale grid, so the result is bit-identical to any dequant using that formula.
+ Ragged last blocks (dims not a multiple of the block) are supported. Works in row chunks to
+ bound the fp32 transient."""
+ if weight.dtype not in _FP8_DTYPES:
+ raise TypeError(f"expected an fp8 weight, got {weight.dtype}")
+ if weight.dim() > 2:
+ weight2 = weight.reshape(-1, *weight.shape[-2:])
+ if scale.numel() == 1:
+ scale2 = scale.reshape(1, 1, 1).expand(weight2.shape[0], 1, 1)
+ else:
+ scale2 = scale.reshape(-1, *scale.shape[-2:])
+ if out is None:
+ out = torch.empty(weight.shape, dtype = dtype, device = weight.device)
+ out2 = out.view(-1, *weight.shape[-2:])
+ for i in range(weight2.shape[0]):
+ _dequantize_block_fp8(weight2[i], scale2[i], block, dtype, out = out2[i])
+ return out
+ rows, cols = weight.shape
+ if scale.dim() == 0 or scale.numel() == 1:
+ scale = scale.reshape(1, 1)
+ scale = scale.reshape(scale.shape[-2], scale.shape[-1]) if scale.dim() != 2 else scale
+ scale_rows, scale_cols = scale.shape
+ if scale_rows == 1 and scale_cols == 1:
+ block_m, block_n = rows, cols
+ elif -(-rows // block[0]) == scale_rows and -(-cols // block[1]) == scale_cols:
+ block_m, block_n = block
+ elif rows % scale_rows == 0 and cols % scale_cols == 0:
+ block_m, block_n = rows // scale_rows, cols // scale_cols
+ else:
+ raise ValueError(
+ f"fp8 weight {tuple(weight.shape)} does not fit its scale grid {tuple(scale.shape)} "
+ f"with block {block}"
+ )
+ scale = _scale_as_fp32(scale).to(weight.device)
+ if out is None:
+ out = torch.empty((rows, cols), dtype = dtype, device = weight.device)
+ # About 64M fp32 elements per chunk, a whole number of row blocks.
+ chunk_blocks = max(1, (1 << 26) // max(1, block_m * cols))
+ for block_row in range(0, scale_rows, chunk_blocks):
+ row_start = block_row * block_m
+ row_end = min(rows, (block_row + chunk_blocks) * block_m)
+ piece = weight[row_start:row_end].to(torch.float32)
+ piece_scale = scale[block_row : block_row + chunk_blocks]
+ expanded = piece_scale.repeat_interleave(block_m, dim = 0)[: row_end - row_start]
+ expanded = expanded.repeat_interleave(block_n, dim = 1)[:, :cols]
+ piece.mul_(expanded)
+ out[row_start:row_end].copy_(piece)
+ del piece, expanded
+ return out
+
+
+def _output_dtype(model, full_layer_name, default):
+ """The load dtype, except fp32 for a target the model keeps in fp32 (_keep_in_fp32_modules)."""
+ if model is None or not full_layer_name:
+ return default
+ module_name, _, attr = full_layer_name.rpartition(".")
+ try:
+ module = model.get_submodule(module_name) if module_name else model
+ except AttributeError:
+ return default
+ param = getattr(module, attr, None)
+ if (
+ isinstance(param, torch.Tensor)
+ and param.dtype == torch.float32
+ and type(param).__name__ != "Params4bit"
+ and not type(module).__module__.startswith("bitsandbytes")
+ ):
+ return torch.float32
+ return default
+
+
+def _target_keeps_fp8(model, full_layer_name) -> bool:
+ """An fp8 container module (FP8Embedding) that holds its own scale: load its bytes as stored."""
+ if model is None or not full_layer_name:
+ return False
+ module_name, _, attr = full_layer_name.rpartition(".")
+ try:
+ module = model.get_submodule(module_name) if module_name else model
+ except AttributeError:
+ return False
+ param = getattr(module, attr, None)
+ if not isinstance(param, torch.Tensor) or param.dtype not in _FP8_DTYPES:
+ return False
+ return any(
+ isinstance(getattr(module, name, None), torch.Tensor)
+ for name in (attr + "_scale", attr + "_scale_inv", "weight_scale", "weight_scale_inv")
+ )
+
+
+def _merge_accepts_stacked_tensor() -> bool:
+ try:
+ from transformers.core_model_loading import MergeModulelist
+
+ stacked = torch.zeros(2, 1, 1)
+ merged = MergeModulelist(dim = 0).convert(
+ {"a": stacked}, source_patterns = ["a"], target_patterns = ["b"]
+ )
+ return next(iter(merged.values())) is stacked
+ except Exception:
+ return False
+
+
+def _build_classes():
+ from transformers.core_model_loading import ConversionOps, WeightConverter, _IdentityOp
+
+ class Fp8BlockDequantizeToHalf(ConversionOps):
+ """Dequantize fp8 sources with their scales to the load dtype; drop scale entries. `stack`
+ pre-allocates the stacked output a following MergeModulelist(dim=0) accepts as is."""
+
+ def __init__(
+ self,
+ generic = False,
+ stack = False,
+ fuse = None,
+ ):
+ self.generic = generic
+ self.stack = stack
+ # (patterns in merge order, concat dim): replaces MergeModulelist + Concatenate by writing
+ # each expert straight into the final stack, the only 16-bit tensor ever held.
+ self.fuse = fuse
+
+ def _fused(
+ self,
+ input_dict,
+ target_patterns,
+ state,
+ model = None,
+ full_layer_name = None,
+ ):
+ patterns, cat_dim = self.fuse
+ groups = []
+ for pattern in patterns:
+ weights = input_dict.get(pattern)
+ if weights is None:
+ continue
+ weights = weights if isinstance(weights, list) else [weights]
+ scales = input_dict.get(_scale_pattern_for(pattern))
+ scales = (
+ (scales if isinstance(scales, list) else [scales])
+ if scales is not None
+ else None
+ )
+ if scales is not None and len(scales) != len(weights):
+ raise ValueError(
+ f"Unsloth: {len(weights)} fp8 weights but {len(scales)} scales"
+ )
+ groups.append((weights, scales))
+ if not groups:
+ return {}
+ shapes = [(len(w), *w[0].shape) for w, _ in groups]
+ for weights, _ in groups:
+ if any(tuple(w.shape) != tuple(weights[0].shape) for w in weights):
+ raise ValueError("Unsloth: experts of one projection have different shapes")
+ for shape in shapes[1:]:
+ if any(a != b for i, (a, b) in enumerate(zip(shape, shapes[0])) if i != cat_dim):
+ raise ValueError(
+ f"Unsloth: expert stacks {shapes} cannot be concatenated on dim {cat_dim}"
+ )
+ final = list(shapes[0])
+ final[cat_dim] = sum(shape[cat_dim] for shape in shapes)
+ device = groups[0][0][0].device
+ dtype = _output_dtype(model, full_layer_name, state.dtype)
+ out = torch.empty(final, dtype = dtype, device = device)
+ offset = 0
+ for (weights, scales), shape in zip(groups, shapes):
+ part = out.narrow(cat_dim, offset, shape[cat_dim])
+ offset += shape[cat_dim]
+ for i in range(len(weights)):
+ weight = weights[i]
+ weights[i] = None
+ if weight.dtype in _FP8_DTYPES:
+ if scales is None:
+ raise RuntimeError(
+ "Unsloth: an fp8 expert weight has no weight_scale_inv"
+ )
+ _dequantize_block_fp8(weight, scales[i], state.block, dtype, out = part[i])
+ state.dequantized += 1
+ elif weight.is_floating_point() and weight.element_size() >= 2:
+ part[i].copy_(weight)
+ else:
+ raise RuntimeError(
+ f"Unsloth: an expert weight is stored as {weight.dtype}, which the fp8 -> 4bit "
+ f"load cannot dequantize. Set {_ENV}=0 to load the checkpoint as it is."
+ )
+ del weight
+ return {target_patterns[0]: out}
+
+ @torch.no_grad()
+ def convert(
+ self,
+ input_dict,
+ source_patterns = None,
+ target_patterns = None,
+ full_layer_name = None,
+ model = None,
+ **kwargs,
+ ):
+ state = _STATE
+ if state is None:
+ raise RuntimeError(
+ "Unsloth: fp8 -> 4bit dequantize ran outside an fp8 -> 4bit load"
+ )
+ block, dtype = state.block, _output_dtype(model, full_layer_name, state.dtype)
+ if _target_keeps_fp8(model, full_layer_name):
+ # modules_to_convert (Qwen3.8's n-gram table): stays fp8 with its own per-tensor scale.
+ state.kept_fp8 += sum(
+ len(v) if isinstance(v, list) else 1
+ for k, v in input_dict.items()
+ if not _is_scale_pattern(k)
+ )
+ return input_dict
+ if self.fuse is not None:
+ return self._fused(input_dict, target_patterns, state, model, full_layer_name)
+ result = {}
+ for key in list(input_dict.keys()):
+ if _is_scale_pattern(key):
+ continue
+ value = input_dict[key]
+ scale_key = _scale_pattern_for(key)
+ if scale_key not in input_dict:
+ alt = scale_key[:-1] if scale_key.endswith("$") else scale_key + "$"
+ scale_key = alt if alt in input_dict else None
+ weights = value if isinstance(value, list) else [value]
+ if scale_key is None:
+ if any(isinstance(w, torch.Tensor) and w.dtype in _FP8_DTYPES for w in weights):
+ raise RuntimeError(
+ f"Unsloth: fp8 tensor for `{full_layer_name}` has no weight_scale_inv in the checkpoint"
+ )
+ if self.generic:
+ target = full_layer_name if full_layer_name is not None else "weight"
+ result[target] = value
+ else:
+ result[key] = value
+ continue
+ scales = input_dict[scale_key]
+ scales = scales if isinstance(scales, list) else [scales]
+ if len(scales) != len(weights):
+ raise ValueError(
+ f"Unsloth: {len(weights)} fp8 weights but {len(scales)} scales for `{full_layer_name}`"
+ )
+ out = None
+ if self.stack and isinstance(value, list) and len(weights) > 0:
+ first = weights[0]
+ out = torch.empty(
+ (len(weights), *first.shape), dtype = dtype, device = first.device
+ )
+ outputs = out
+ else:
+ outputs = []
+ for i in range(len(weights)):
+ weight, scale = weights[i], scales[i]
+ weights[i] = None
+ if weight.dtype in _FP8_DTYPES:
+ converted = _dequantize_block_fp8(
+ weight, scale, block, dtype, out = out[i] if out is not None else None
+ )
+ state.dequantized += 1
+ elif weight.is_floating_point() and weight.element_size() >= 2:
+ # A 16-bit tensor that shipped a scale anyway: take it as stored.
+ converted = weight.to(dtype)
+ if out is not None:
+ out[i].copy_(converted)
+ else:
+ raise RuntimeError(
+ f"Unsloth: `{full_layer_name}` is stored as {weight.dtype} with a scale, which the "
+ f"fp8 -> 4bit load cannot dequantize. Set {_ENV}=0 to load the checkpoint as it is."
+ )
+ if out is None:
+ outputs.append(converted)
+ del weight
+ if self.generic:
+ target = full_layer_name if full_layer_name is not None else "weight"
+ result[target] = outputs if out is not None else outputs[0]
+ else:
+ result[key] = (
+ outputs if (out is not None or isinstance(value, list)) else outputs[0]
+ )
+ return result
+
+ @property
+ def reverse_op(self):
+ return _IdentityOp()
+
+ class Fp8SourceConverter(WeightConverter):
+ """A WeightConverter that loads fp8 weights and their scales at their stored dtype. An
+ on-the-fly bitsandbytes load casts every source to the model dtype, which would round the
+ fp32 scales to bf16."""
+
+ __slots__ = ()
+
+ def add_tensor(self, target_key, source_key, source_pattern, future):
+ state = _STATE
+ if state is not None and source_key in state.native_keys:
+ native = getattr(future, "_unsloth_native_materialize", None)
+ if native is None:
+ # Cast to the model dtype, a fp32 scale would round to bf16: refuse, do not guess.
+ raise RuntimeError(
+ f"Unsloth: cannot read `{source_key}` at its stored dtype during an fp8 -> 4bit load. "
+ f"Set {_ENV}=0 to load the fp8 weights as they are."
+ )
+ future = native
+ state.routed_native.add(source_key)
+ return super().add_tensor(target_key, source_key, source_pattern, future)
+
+ return Fp8BlockDequantizeToHalf, Fp8SourceConverter
+
+
+def _install_spawn_materialize_tag() -> bool:
+ """Tag each lazily materialized source with a twin that keeps the stored dtype."""
+ from transformers import core_model_loading as cml
+
+ original = cml.spawn_materialize
+ if getattr(original, "_unsloth_fp8_to_nf4", False):
+ return True
+ try:
+ signature = inspect.signature(original)
+ except (TypeError, ValueError):
+ return False
+ if not {"thread_pool", "tensor", "device", "dtype"} <= set(signature.parameters):
+ return False
+ materialize_copy = getattr(cml, "_materialize_copy", None)
+
+ @functools.wraps(original)
+ def spawn_materialize(*args, **kwargs):
+ job = original(*args, **kwargs)
+ if _STATE is None or not callable(job) or materialize_copy is None:
+ return job
+ try:
+ bound = signature.bind(*args, **kwargs)
+ except TypeError:
+ return job
+ arguments = bound.arguments
+ if (
+ arguments.get("sharding_op", None) is not None
+ or arguments.get("thread_pool", None) is not None
+ ):
+ return job
+ tensor = arguments.get("tensor")
+ device = arguments.get("device", None)
+ try:
+ job._unsloth_native_materialize = functools.partial(
+ materialize_copy, tensor, device, None
+ )
+ except Exception:
+ pass
+ return job
+
+ spawn_materialize._unsloth_fp8_to_nf4 = True
+ cml.spawn_materialize = spawn_materialize
+ return True
+
+
+def _refuse_packed_scaled_weights(headers):
+ """A scaled weight stored as int8 / uint8 is packed FP4 (or another format): it would be cast
+ to bf16 as integers. Fail before loading anything."""
+ for key, dtype in headers.items():
+ if not key.endswith((".weight_scale_inv", ".scale")):
+ continue
+ weight = headers.get(_weight_key_of_scale(key))
+ if weight is not None and weight not in _FLOAT_SAFETENSORS:
+ raise RuntimeError(
+ f"Unsloth: `{_weight_key_of_scale(key)}` is stored as {weight} with a block scale, which the "
+ f"fp8 -> 4bit load cannot dequantize. Set {_ENV}=0 to load the checkpoint as it is."
+ )
+
+
+def _keep_fp8_embeddings(model, patterns) -> int:
+ """The checkpoint's modules_to_convert are fp8 embedding tables too large to expand (Qwen3.8's
+ 51B-weight n-gram table): keep them fp8 with their per-tensor scale, as an fp8 load does."""
+ try:
+ from transformers.integrations.finegrained_fp8 import (
+ FP8Embedding,
+ replace_with_fp8_embedding,
+ )
+ except Exception:
+ return 0
+ replace_with_fp8_embedding(model, list(patterns))
+ return sum(1 for module in model.modules() if isinstance(module, FP8Embedding))
+
+
+def _read_headers(checkpoint_files) -> dict:
+ from safetensors import safe_open
+
+ headers = {}
+ for path in checkpoint_files or ():
+ path = str(path)
+ if not path.endswith(".safetensors"):
+ raise RuntimeError(
+ f"Unsloth: fp8 -> 4bit loading needs safetensors shards, got `{os.path.basename(path)}`"
+ )
+ with safe_open(path, framework = "pt") as handle:
+ for key in handle.keys():
+ headers[key] = handle.get_slice(key).get_dtype()
+ return headers
+
+
+def _generalize(name: str) -> str:
+ escaped = re.escape(name)
+ return re.sub(r"(?<=\\\.)\d+(?=\\\.|$)", r"\\d+", escaped)
+
+
+def _compress_patterns(names, others) -> list:
+ """Regexes matching every name in `names` and none in `others`: layer indices generalized
+ where that stays exact, else the exact names. Anchored, since transformers re.match()es them."""
+ families = {}
+ for name in sorted(names):
+ families.setdefault(_generalize(name), []).append(name)
+ patterns = []
+ for pattern, members in families.items():
+ regex = re.compile(pattern + "$")
+ if any(regex.match(other) for other in others):
+ patterns.extend(re.escape(member) + "$" for member in members)
+ else:
+ patterns.append(pattern + "$")
+ return patterns
+
+
+def _high_precision_linears(model, headers, weight_conversions) -> tuple:
+ """nn.Linear modules whose checkpoint weight is stored in 16/32 bits (the fp8 checkpoint's
+ modules_to_not_convert), found by renaming every checkpoint key exactly as the loader will."""
+ from copy import deepcopy
+ from transformers import core_model_loading as cml
+
+ # Copies: renaming marks a transform used, and the used ones are what saving reverses.
+ conversions = deepcopy(list(weight_conversions))
+ renamings = [c for c in conversions if isinstance(c, cml.WeightRenaming)]
+ converters = [c for c in conversions if isinstance(c, cml.WeightConverter)]
+ meta_state_dict = model.state_dict()
+ prefix = getattr(model, "base_model_prefix", None)
+ sources_fp8 = {}
+ reached = set()
+ for key, dtype in headers.items():
+ if key.endswith(_SCALE_SUFFIXES) or key.endswith(".scale"):
+ continue
+ renamed, _ = cml.rename_source_key(key, renamings, converters, prefix, meta_state_dict)
+ if renamed not in meta_state_dict and key in meta_state_dict:
+ renamed = key
+ if renamed in meta_state_dict:
+ reached.add(key)
+ sources_fp8[renamed] = sources_fp8.get(renamed, False) or dtype in _FP8_SAFETENSORS
+ keep, quantize = [], []
+ for name, module in model.named_modules():
+ if not isinstance(module, torch.nn.Linear):
+ continue
+ target = f"{name}.weight"
+ if target not in sources_fp8:
+ continue
+ (quantize if sources_fp8[target] else keep).append(name)
+ return keep, quantize, reached
+
+
+def _weight_key_of_scale(key):
+ for suffix in (".weight_scale_inv", ".scale"):
+ if key.endswith(suffix):
+ return key[: -len(suffix)] + ".weight"
+ return key
+
+
+def _restore_plain_linears(model, names, dtype) -> int:
+ """Swap bitsandbytes Linear4bit back to nn.Linear (still on meta, before any weight loads)."""
+ try:
+ import bitsandbytes as bnb
+ except Exception:
+ return 0
+ restored = 0
+ for name in names:
+ try:
+ module = model.get_submodule(name)
+ except AttributeError:
+ continue
+ if not isinstance(module, bnb.nn.Linear4bit):
+ continue
+ parent_name, _, child = name.rpartition(".")
+ parent = model.get_submodule(parent_name) if parent_name else model
+ device = module.weight.device
+ plain = torch.nn.Linear(
+ module.in_features,
+ module.out_features,
+ bias = module.bias is not None,
+ device = device,
+ dtype = dtype,
+ )
+ setattr(parent, child, plain)
+ restored += 1
+ return restored
+
+
+_installed_class = None
+
+
+def install_fp8_to_nf4_quantizer() -> bool:
+ """Register a Bnb4BitHfQuantizer subclass for `bitsandbytes_4bit` that only differs from its
+ parent on a load whose config carries the parked fp8 block. Composes with another subclass."""
+ global _installed_class
+ if _transformers_supports_fp8_to_nf4() is not None:
+ return False
+ from transformers.quantizers import auto as quantizers_auto
+ from transformers.quantizers.quantizer_bnb_4bit import Bnb4BitHfQuantizer
+ from transformers.core_model_loading import WeightConverter, WeightRenaming
+
+ mapping = quantizers_auto.AUTO_QUANTIZER_MAPPING
+ current = mapping.get("bitsandbytes_4bit")
+ if current is None or not issubclass(current, Bnb4BitHfQuantizer):
+ return False
+ if getattr(current, "_unsloth_fp8_to_nf4", False):
+ return _install_spawn_materialize_tag()
+ if not _install_spawn_materialize_tag():
+ return False
+ DequantOp, SourceConverter = _build_classes()
+ stack_ok = _merge_accepts_stacked_tensor()
+
+ def _rebuild(conv):
+ from transformers.core_model_loading import Concatenate, MergeModulelist
+
+ weight_sources = [p for p in conv.source_patterns if p.endswith(".weight")]
+ if not weight_sources:
+ return None
+ other = [p for p in conv.source_patterns if not p.endswith(".weight")]
+ sources = (
+ [p + "$" for p in weight_sources]
+ + [p[: -len(".weight")] + ".weight_scale_inv$" for p in weight_sources]
+ + other
+ )
+ ops = list(conv.operations)
+ first = ops[0] if ops else None
+ merges = isinstance(first, MergeModulelist) and getattr(first, "dim", None) == 0
+ if (
+ merges
+ and len(ops) == 2
+ and type(ops[1]) is Concatenate
+ and not other
+ and len(conv.target_patterns) == 1
+ ):
+ operations = [DequantOp(fuse = ([p + "$" for p in weight_sources], ops[1].dim % 3))]
+ else:
+ operations = [DequantOp(stack = bool(stack_ok and merges))] + ops
+ rebuilt = SourceConverter(
+ source_patterns = sources,
+ target_patterns = list(conv._original_target_patterns),
+ operations = operations,
+ force_cpu = getattr(conv, "force_cpu", False),
+ )
+ rebuilt.scope_prefix = conv.scope_prefix
+ rebuilt.base_model_prefix = conv.base_model_prefix
+ return rebuilt
+
+ class UnslothFp8ToNf4Bnb4BitHfQuantizer(current):
+ _unsloth_fp8_to_nf4 = True
+ _unsloth_fp8_plan = None
+ _unsloth_fp8_rebuilt = None
+ _unsloth_fp8_model = None
+ _unsloth_fp8_kept = 0
+
+ def __init__(self, quantization_config, **kwargs):
+ # The device-map planner builds this from the rewritten config without `pre_quantized`
+ # (transformers' own load always passes it). An armed load quantizes at runtime, so size
+ # its expert merges in the load dtype, not as packed 4bit (a 4x smaller transient).
+ if "pre_quantized" not in kwargs and _ARMED_CONFIGS.get():
+ kwargs["pre_quantized"] = False
+ super().__init__(quantization_config, **kwargs)
+
+ def _process_model_before_weight_loading(self, model, **kwargs):
+ global _STATE
+ config = getattr(model, "config", None)
+ plan = getattr(config, UNSLOTH_FP8_TO_NF4_ATTR, None) if config is not None else None
+ if plan is not None:
+ self._unsloth_fp8_plan = plan
+ # preprocess_model does not forward dtype; the meta model was built in the load dtype.
+ dtype = kwargs.get("dtype", None)
+ if not isinstance(dtype, torch.dtype):
+ dtype = getattr(model, "dtype", None)
+ if dtype not in (torch.bfloat16, torch.float16, torch.float32):
+ dtype = torch.bfloat16
+ self._unsloth_fp8_model = model
+ headers = _read_headers(kwargs.get("checkpoint_files"))
+ _refuse_packed_scaled_weights(headers)
+ _STATE = _LoadState(plan, dtype, headers)
+ if plan.get("modules_to_convert"):
+ _keep_fp8_embeddings(model, plan["modules_to_convert"])
+ return super()._process_model_before_weight_loading(model, **kwargs)
+
+ def update_weight_conversions(self, weight_conversions):
+ if self._unsloth_fp8_plan is None or _STATE is None:
+ return super().update_weight_conversions(weight_conversions)
+ model = self._unsloth_fp8_model
+ self._unsloth_fp8_model = None
+ # Here, not before loading: only now are the caller's key_mapping renamings known.
+ keep, quantize = [], []
+ quantize_all = _quantize_16bit_requested()
+ if model is not None:
+ keep, quantize, reached = _high_precision_linears(
+ model, _STATE.headers, weight_conversions
+ )
+ _STATE.expected_fp8 = _STATE.fp8_keys & reached
+ _STATE.expected_scales = {
+ key
+ for key in _STATE.scale_keys
+ if _weight_key_of_scale(key) in _STATE.expected_fp8
+ }
+ if quantize_all:
+ # UNSLOTH_FP8_TO_NF4_QUANTIZE_16BIT=1 quantizes them too, like a load of the bf16 repo.
+ keep = []
+ self._unsloth_fp8_kept = _restore_plain_linears(model, keep, _STATE.dtype)
+ if keep:
+ patterns = _compress_patterns(keep, quantize)
+ # A copy: the caller may reuse their BitsAndBytesConfig for another load.
+ quantization_config = copy.copy(self.quantization_config)
+ skip = list(quantization_config.llm_int8_skip_modules or [])
+ quantization_config.llm_int8_skip_modules = skip + patterns
+ self.quantization_config = quantization_config
+ if isinstance(getattr(self, "modules_to_not_convert", None), list):
+ self.modules_to_not_convert.extend(patterns)
+ updated = []
+ rebuilt_from = {}
+ if _STATE.dot_scale_keys:
+ updated.append(
+ WeightRenaming(
+ source_patterns = r"^(.+)\.scale$", target_patterns = r"\1.weight_scale_inv"
+ )
+ )
+ for conv in weight_conversions:
+ if isinstance(conv, WeightConverter):
+ rebuilt = _rebuild(conv)
+ if rebuilt is not None:
+ rebuilt_from[id(rebuilt)] = conv
+ updated.append(rebuilt)
+ continue
+ updated.append(conv)
+ updated.append(
+ SourceConverter(
+ source_patterns = ["weight$", "weight_scale_inv", "activation_scale"],
+ target_patterns = "weight",
+ operations = [DequantOp(generic = True)],
+ )
+ )
+ self._unsloth_fp8_rebuilt = rebuilt_from
+ return super().update_weight_conversions(updated)
+
+ def _process_model_after_weight_loading(self, model, **kwargs):
+ global _STATE
+ if self._unsloth_fp8_plan is not None:
+ state = _STATE
+ _STATE = None
+ self._unsloth_fp8_plan = None
+ self._unsloth_fp8_model = None
+ rebuilt_from = self._unsloth_fp8_rebuilt or {}
+ self._unsloth_fp8_rebuilt = None
+ _finish_fp8_to_nf4_load(
+ model, state, rebuilt_from, getattr(self, "_unsloth_fp8_kept", 0)
+ )
+ return super()._process_model_after_weight_loading(model, **kwargs)
+
+ mapping["bitsandbytes_4bit"] = UnslothFp8ToNf4Bnb4BitHfQuantizer
+ _installed_class = UnslothFp8ToNf4Bnb4BitHfQuantizer
+ return True
+
+
+def _finish_fp8_to_nf4_load(model, state, rebuilt_from, kept):
+ config = getattr(model, "config", None)
+ if config is not None:
+ try:
+ delattr(config, UNSLOTH_FP8_TO_NF4_ATTR)
+ except AttributeError:
+ config.__dict__.pop(UNSLOTH_FP8_TO_NF4_ATTR, None)
+ # Saving reverses model._weight_conversions: put the model's own converters back.
+ conversions = getattr(model, "_weight_conversions", None)
+ if isinstance(conversions, list):
+ restored = []
+ for conv in conversions:
+ original = rebuilt_from.get(id(conv))
+ if original is not None:
+ try:
+ original._was_used = True
+ except Exception:
+ pass
+ restored.append(original)
+ elif type(conv).__name__ == "Fp8SourceConverter":
+ continue
+ elif getattr(conv, "_original_source_patterns", None) == [r"^(.+)\.scale$"]:
+ continue
+ else:
+ restored.append(conv)
+ model._weight_conversions = restored
+ if state is None:
+ raise RuntimeError("Unsloth: the fp8 -> 4bit load lost its state")
+ problems = []
+ missed_scales = sorted(state.expected_scales - state.routed_native)
+ if missed_scales:
+ problems.append(
+ f"{len(missed_scales)} fp8 scales were never applied (e.g. `{missed_scales[0]}`), so their "
+ "weights would load unscaled"
+ )
+ leftover = [
+ name
+ for name, param in model.named_parameters()
+ if param.dtype in _FP8_DTYPES and not _target_keeps_fp8(model, name)
+ ]
+ if leftover:
+ problems.append(f"{len(leftover)} parameters are still fp8 (e.g. `{leftover[0]}`)")
+ if state.dequantized + state.kept_fp8 < len(state.expected_fp8):
+ problems.append(
+ f"only {state.dequantized} of {len(state.expected_fp8) - state.kept_fp8} fp8 tensors were dequantized"
+ )
+ if problems:
+ raise RuntimeError(
+ "Unsloth: loading this fp8 checkpoint in 4bit failed: " + "; ".join(problems) + ". "
+ f"Set {_ENV}=0 to load the fp8 weights as they are."
+ )
+ kept_fp8 = (
+ f" {state.kept_fp8} fp8 embedding shards (modules_to_convert) stay fp8."
+ if state.kept_fp8
+ else ""
+ )
+ kept_note = (
+ f" {kept} Linear layers the checkpoint stores in 16bit stay in 16bit "
+ f"({_ENV_QUANTIZE_ALL}=1 quantizes them too)."
+ if kept
+ else ""
+ )
+ print(
+ f"Unsloth: dequantized {state.dequantized} fp8 tensors to 4bit while loading.{kept_note}{kept_fp8}"
+ )
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index ff83c94620a..9b373ec9430 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2625,6 +2625,8 @@ class FastLlamaModel:
token = token,
model_name = model_name,
revision = revision,
+ # vLLM reads the checkpoint itself, so it cannot take the in-process fp8 -> 4bit route.
+ allow_fp8_to_nf4 = not _vllm_will_load_weights(fast_inference, num_labels),
hub_kwargs = {
"cache_dir": kwargs.get("cache_dir"),
"subfolder": kwargs.get("subfolder"),
@@ -2640,6 +2642,14 @@ class FastLlamaModel:
pop_modelopt_key_mapping,
)
+ from .fp8_to_nf4 import (
+ disarm_fp8_to_nf4,
+ fp8_to_nf4_armed,
+ fp8_to_nf4_planner_quantization_config,
+ )
+
+ # The fp8 config was parked on model_config, so the load must be handed this config.
+ _fp8_to_nf4 = fp8_to_nf4_armed(model_config)
_modelopt_rewritten = modelopt_rewritten(model_config)
if _modelopt_rewritten:
verify_fp8_support_if_applicable(model_config)
@@ -2714,7 +2724,10 @@ class FastLlamaModel:
quantization_config = kwargs.get("quantization_config", None),
rewritten_quantization_config = modelopt_planner_quantization_config(model_config)
if _modelopt_rewritten
- else None,
+ else fp8_to_nf4_planner_quantization_config(
+ model_config,
+ SKIP_QUANTIZATION_MODULES + (["out_proj"] if IS_FALCON_H1 else []),
+ ),
# The same extra the bnb config below adds.
extra_skip_modules = ["out_proj"] if IS_FALCON_H1 else None,
),
@@ -2781,16 +2794,19 @@ class FastLlamaModel:
set_task_config_attr(model_config, _cfg_key, _cfg_val)
else:
setattr(model_config, _cfg_key, _cfg_val)
- model = AutoModelForSequenceClassification.from_pretrained(
- model_name,
- config = model_config,
- device_map = device_map,
- token = token,
- trust_remote_code = trust_remote_code,
- attn_implementation = preferred_attn_impl,
- revision = revision,
- **kwargs,
- )
+ try:
+ model = AutoModelForSequenceClassification.from_pretrained(
+ model_name,
+ config = model_config,
+ device_map = device_map,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ attn_implementation = preferred_attn_impl,
+ revision = revision,
+ **kwargs,
+ )
+ finally:
+ disarm_fp8_to_nf4(model_config)
# Defensive: ensure the task head is in a floating dtype, guarding against any path leaving it
# as integer storage (#5027).
for _head_name in ("score", "classifier", "qa_outputs"):
@@ -2836,7 +2852,7 @@ class FastLlamaModel:
)
_prepare_compressed_tensors_model(model)
elif not fast_inference:
- if user_config is not None or _modelopt_rewritten:
+ if user_config is not None or _modelopt_rewritten or _fp8_to_nf4:
# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override
# on the config and pass the single config object through.
if max_position_embeddings is not None:
@@ -2844,18 +2860,22 @@ class FastLlamaModel:
_rope_scaling = kwargs.pop("rope_scaling", None)
if _rope_scaling is not None:
model_config.rope_scaling = _rope_scaling
- if _modelopt_rewritten and user_config is None:
+ if (_modelopt_rewritten or _fp8_to_nf4) and user_config is None:
move_config_overrides_onto_config(model_config, kwargs)
- model = AutoModelForCausalLM.from_pretrained(
- model_name,
- config = model_config,
- device_map = device_map,
- token = token,
- trust_remote_code = trust_remote_code,
- attn_implementation = preferred_attn_impl,
- revision = revision,
- **kwargs,
- )
+ try:
+ model = AutoModelForCausalLM.from_pretrained(
+ model_name,
+ config = model_config,
+ device_map = device_map,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ attn_implementation = preferred_attn_impl,
+ revision = revision,
+ **kwargs,
+ )
+ finally:
+ # The load deep-copied the config; give the caller's object its fp8 block back.
+ disarm_fp8_to_nf4(model_config)
else:
model = AutoModelForCausalLM.from_pretrained(
model_name,
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 44b14338867..670a469c0c0 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -33,6 +33,7 @@ from transformers import AutoConfig
from transformers import __version__ as transformers_version
from peft import PeftConfig, PeftModel
from .grouped_linear_lora import register_grouped_linear_lora_for_adapter
+from .fp8_to_nf4 import track_explicit_4bit_request
from .loader_utils import (
DEFAULT_DEVICE_MAP,
OFFLOAD_EMBEDDING_AUTO,
@@ -700,6 +701,7 @@ def _fix_rope_inv_freq(model):
class FastLanguageModel(FastLlamaModel):
@staticmethod
@_offline_aware_load
+ @track_explicit_4bit_request
def from_pretrained(
model_name = "unsloth/Llama-3.2-1B-Instruct",
max_seq_length = 2048,
@@ -1503,6 +1505,7 @@ class FastModel(FastBaseModel):
@staticmethod
@_offline_aware_load
+ @track_explicit_4bit_request
def from_pretrained(
model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",
max_seq_length = 2048,
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index d3333080a53..2478ab68f9f 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -2185,6 +2185,7 @@ def check_and_disable_bitsandbytes_loading(
model_name = None,
revision = None,
hub_kwargs = None,
+ allow_fp8_to_nf4 = True,
):
"""Disable bitsandbytes loading (load_in_4bit/load_in_8bit) when the model already carries a non-bitsandbytes quantization config. Returns ``(load_in_4bit, load_in_8bit, quant_method)``, with both flags False if they were disabled and quant_method the detected method or None. ``rewrite_modelopt`` converts ModelOpt FP8 to fp8; pass False when vLLM loads it natively."""
quant_method = get_quant_type(model_config)
@@ -2212,6 +2213,12 @@ def check_and_disable_bitsandbytes_loading(
if arm_modelopt_fp8_loading(model_config, verbose = verbose) is not None:
quant_method = "fp8"
+ # An explicit 4bit request on a block-fp8 checkpoint: dequantize each fp8 tensor and quantize it to NF4 while loading.
+ if allow_fp8_to_nf4 and str(quant_method).lower() == "fp8":
+ from .fp8_to_nf4 import maybe_arm_fp8_to_nf4
+ if maybe_arm_fp8_to_nf4(model_config, load_in_4bit, load_in_8bit, verbose = verbose):
+ return load_in_4bit, load_in_8bit, None
+
# A non-bitsandbytes quantization config (compressed-tensors, gptq, awq) means BOTH bitsandbytes loading flags must be disabled to avoid config conflicts.
if load_in_4bit or load_in_8bit:
if verbose:
@@ -2219,6 +2226,21 @@ def check_and_disable_bitsandbytes_loading(
f"Unsloth: Model already quantized with {quant_method}. "
f"Disabling `load_in_4bit` and `load_in_8bit` to avoid quantization config conflict."
)
+ if load_in_4bit and not load_in_8bit and str(quant_method).lower() == "fp8":
+ from .fp8_to_nf4 import (
+ explicit_4bit_requested,
+ fp8_block_quantization_config,
+ fp8_to_nf4_disabled,
+ )
+ if (
+ not explicit_4bit_requested()
+ and not fp8_to_nf4_disabled()
+ and fp8_block_quantization_config(model_config) is not None
+ ):
+ print(
+ "Unsloth: Pass `load_in_4bit = True` explicitly (or set UNSLOTH_FP8_TO_NF4=1) to "
+ "quantize this fp8 checkpoint to 4bit while it loads."
+ )
load_in_4bit = False
load_in_8bit = False
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index c2dc47efa59..e4e2b61ba23 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -2202,6 +2202,7 @@ class FastBaseModel:
check_and_disable_bitsandbytes_loading,
sync_unsloth_model_name_bnb_flags,
)
+ from .fp8_to_nf4 import fp8_to_nf4_planner_quantization_config
load_in_4bit, load_in_8bit, _ = check_and_disable_bitsandbytes_loading(
auto_config,
@@ -2211,6 +2212,8 @@ class FastBaseModel:
token = token,
model_name = model_name,
revision = _revision,
+ # vLLM reads the checkpoint itself, so it cannot take the in-process fp8 -> 4bit route.
+ allow_fp8_to_nf4 = not (fast_inference and is_vLLM_available()),
hub_kwargs = {
"cache_dir": kwargs.get("cache_dir"),
"subfolder": kwargs.get("subfolder"),
@@ -2239,6 +2242,13 @@ class FastBaseModel:
load_in_4bit = False
load_in_8bit = False
load_in_16bit = False
+ from .fp8_to_nf4 import disarm_fp8_to_nf4, requests_bnb_4bit
+
+ if not load_in_4bit or (
+ user_quantization_config is not None and not requests_bnb_4bit(user_quantization_config)
+ ):
+ # An fp8 -> 4bit load parked the checkpoint's fp8 config; give it back.
+ disarm_fp8_to_nf4(auto_config)
# text_only builds the bare decoder; from model_name the planner would plan the whole VLM.
_planner_skip_reason = None
@@ -2292,7 +2302,10 @@ class FastBaseModel:
auto_config, dequantize = load_in_16bit
)
if _modelopt_rewritten
- else None,
+ else fp8_to_nf4_planner_quantization_config(
+ auto_config,
+ SKIP_QUANTIZATION_MODULES + _architecture_skip_modules(model_types),
+ ),
extra_skip_modules = _architecture_skip_modules(model_types) or None,
),
)
@@ -2499,14 +2512,18 @@ class FastBaseModel:
or kwargs.get("quantization_config") is not None
)
):
- model = auto_model.from_pretrained(
- model_name,
- config = model_config,
- device_map = device_map,
- token = token,
- trust_remote_code = trust_remote_code,
- **kwargs,
- )
+ try:
+ model = auto_model.from_pretrained(
+ model_name,
+ config = model_config,
+ device_map = device_map,
+ token = token,
+ trust_remote_code = trust_remote_code,
+ **kwargs,
+ )
+ finally:
+ # The load deep-copied the config; give the caller's object its fp8 block back.
+ disarm_fp8_to_nf4(model_config)
model = _text_trainable_core(
model, text_intent = bool(text_only) if text_intent is None else bool(text_intent)
)