Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

gpt-oss bnb 4-bit: probe the checkpoint transformers loads and keep fused expert quant state on save (#14740)

Daniel Han85d14340029f
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.py
index d0482d3d102..985c8388088 100644
--- a/tests/saving/test_save_method_lora_routing.py
+++ b/tests/saving/test_save_method_lora_routing.py
@@ -40,6 +40,10 @@ def _load(*names, **env):
"raise_if_merging_mistral_format_view": lambda model, save_method: None,
"lora_relative_to_original_base": lambda model: contextlib.nullcontext(),
"nullcontext": contextlib.nullcontext,
+ # Full saves add bare Params4bit quant state (fused 4-bit experts); none in these fixtures.
+ "PeftModel": type("PeftModel", (), {}),
+ "_save_bare_params4bit_quant_state": lambda model: contextlib.nullcontext(),
+ "_with_bare_params4bit_state_dict": lambda model, original, args, kwargs: (args, kwargs),
**env,
}
for name in names:
diff --git a/tests/test_gpt_oss_bnb_checkpoint_layout.py b/tests/test_gpt_oss_bnb_checkpoint_layout.py
index cf50f0400db..73a31994e22 100644
--- a/tests/test_gpt_oss_bnb_checkpoint_layout.py
+++ b/tests/test_gpt_oss_bnb_checkpoint_layout.py
@@ -375,6 +375,7 @@ def _hub(
monkeypatch.setattr(huggingface_hub, "get_safetensors_metadata", no_metadata)
monkeypatch.setattr(huggingface_hub, "file_exists", lambda repo, f, **k: f in exists)
monkeypatch.setattr(loader_utils, "_hub_safetensors_header_dtypes", no_metadata, raising = False)
+ monkeypatch.setattr(loader_utils, "_hub_live_commit", lambda *a, **k: None, raising = False)
return calls
@@ -510,3 +511,59 @@ def test_loader_forwards_use_safetensors_to_the_probe(tmp_path, monkeypatch):
use_safetensors = False,
)
assert seen and all(v is False for v in seen)
+
+
+def test_online_probe_ignores_a_stale_cached_single_file(monkeypatch, tmp_path):
+ """Online with `subfolder`: a `model.safetensors` cached under an older commit of the branch must
+ not decide the layout once the branch moved to a sharded checkpoint (transformers refreshes it)."""
+ import huggingface_hub
+
+ monkeypatch.setenv("TMPDIR", str(tmp_path))
+ for k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"):
+ monkeypatch.delenv(k, raising = False)
+ stale = str(tmp_path / "stale_model.safetensors")
+ save_file(_tensors(SPLIT, packed = True), stale)
+ old, new = "a" * 40, "b" * 40
+ index = str(tmp_path / "index.json")
+ with open(index, "w", encoding = "utf-8") as f:
+ json.dump({"weight_map": {n: "model-00001-of-00001.safetensors" for n in FUSED_QS}}, f)
+
+ def download(
+ repo_id,
+ filename,
+ *args,
+ revision = None,
+ local_files_only = False,
+ **kwargs,
+ ):
+ if filename == "model.safetensors" and local_files_only and revision in (None, "main", old):
+ return stale # the branch's previous commit, still in the cache
+ if filename == "model.safetensors.index.json" and revision == new:
+ return index
+ raise FileNotFoundError(filename)
+
+ def header(
+ repo_id,
+ filename,
+ revision = None,
+ token = None,
+ ):
+ if filename.endswith("model-00001-of-00001.safetensors") and revision == new:
+ return {n: "U8" for n in FUSED_QS}
+ raise FileNotFoundError(filename)
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", download)
+ monkeypatch.setattr(loader_utils, "_hub_safetensors_header_dtypes", header)
+ monkeypatch.setattr(loader_utils, "_hub_live_commit", lambda repo, revision, token: new)
+ cfg = SimpleNamespace(quantization_config = BNB_QC)
+ assert (
+ loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, "someone/gpt-oss", subfolder = "sub")
+ is True
+ )
+ # Offline the cache is all there is, as in transformers.
+ assert (
+ loader_utils.gpt_oss_checkpoint_has_fused_experts(
+ cfg, "someone/gpt-oss", subfolder = "sub", local_files_only = True
+ )
+ is False
+ )
diff --git a/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py b/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py
new file mode 100644
index 00000000000..aac3c031173
--- /dev/null
+++ b/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py
@@ -0,0 +1,298 @@
+# Unsloth - 2x faster, 60% less VRAM LLM training and finetuning
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""gpt-oss fused bnb 4-bit checkpoints: the layout probe reads the file transformers loads, and a
+full save of fused 4-bit experts keeps their quantization state so the probe accepts it."""
+
+import json
+import os
+from types import SimpleNamespace
+
+import pytest
+import torch
+from safetensors.torch import load_file, save_file
+
+import unsloth # noqa: F401 (patches before transformers)
+from unsloth.models import loader_utils
+
+BNB_QC = {"quant_method": "bitsandbytes", "load_in_4bit": True}
+P = "model.layers.0.mlp"
+FUSED = [f"{P}.experts.gate_up_proj", f"{P}.experts.down_proj", f"{P}.router.weight"]
+SPLIT = [f"{P}.experts.gate_up_projs.0.weight", f"{P}.experts.down_projs.0.weight"]
+
+
+def _tensors(names):
+ return {n: torch.zeros(2, 2, dtype = torch.bfloat16) for n in names}
+
+
+def _stale_index(d, index_name, names):
+ shard = "old-00001-of-00001" + (".bin" if index_name.startswith("pytorch") else ".safetensors")
+ (d / index_name).write_text(
+ json.dumps({"metadata": {}, "weight_map": {n: shard for n in names}})
+ )
+
+
+ "single, stale_index",
+ [
+ ("model.safetensors", "model.safetensors.index.json"),
+ ("model.safetensors", "pytorch_model.bin.index.json"),
+ ("pytorch_model.bin", "pytorch_model.bin.index.json"),
+ ],
+)
+def test_lone_file_beats_a_stale_index_like_transformers(tmp_path, single, stale_index):
+ # transformers resolves model.safetensors, then its index, then pytorch_model.bin, then its index.
+ d = tmp_path / "ckpt"
+ d.mkdir()
+ if single.endswith(".safetensors"):
+ save_file(_tensors(FUSED), str(d / single))
+ else:
+ torch.save(_tensors(FUSED), str(d / single))
+ _stale_index(d, stale_index, SPLIT)
+ cfg = SimpleNamespace(quantization_config = BNB_QC)
+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, str(d)) is True
+
+
+def test_safetensors_index_still_beats_a_lone_bin(tmp_path):
+ d = tmp_path / "ckpt"
+ d.mkdir()
+ torch.save(_tensors(SPLIT), str(d / "pytorch_model.bin"))
+ shard = "model-00001-of-00001.safetensors"
+ save_file(_tensors(FUSED), str(d / shard))
+ (d / "model.safetensors.index.json").write_text(
+ json.dumps({"metadata": {}, "weight_map": {n: shard for n in FUSED}})
+ )
+ cfg = SimpleNamespace(quantization_config = BNB_QC)
+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, str(d)) is True
+
+
+def _quantize_cpu(tensor):
+ bnb = pytest.importorskip("bitsandbytes")
+ param = bnb.nn.Params4bit(tensor.detach().clone(), requires_grad = False, quant_type = "nf4").to(
+ "cpu"
+ )
+ assert param.quant_state is not None
+ return param
+
+
+def _tiny_gpt_oss():
+ from transformers import GptOssConfig, GptOssForCausalLM
+
+ config = GptOssConfig(
+ vocab_size = 128,
+ hidden_size = 64,
+ intermediate_size = 64,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 32,
+ num_local_experts = 2,
+ num_experts_per_tok = 1,
+ layer_types = ["full_attention"],
+ )
+ torch.manual_seed(0)
+ return GptOssForCausalLM(config).to(torch.bfloat16)
+
+
+def test_full_save_keeps_fused_params4bit_quant_state(tmp_path):
+ bnb = pytest.importorskip("bitsandbytes")
+ from unsloth.save import patch_saving_functions
+
+ model = _tiny_gpt_oss()
+ experts = model.model.layers[0].mlp.experts
+ for name in ("gate_up_proj", "down_proj"):
+ setattr(experts, name, _quantize_cpu(getattr(experts, name).data.normal_(0.0, 0.05)))
+ model.config.quantization_config = dict(BNB_QC, bnb_4bit_quant_type = "nf4")
+ patch_saving_functions(model)
+ out = str(tmp_path / "saved")
+ model.save_pretrained(out)
+
+ saved = load_file(os.path.join(out, "model.safetensors"))
+ for name in ("gate_up_proj", "down_proj"):
+ key = f"{P}.experts.{name}"
+ assert saved[key].dtype == torch.uint8
+ assert f"{key}.absmax" in saved and f"{key}.quant_state.bitsandbytes__nf4" in saved
+ stats = {k[len(key) + 1 :]: v for k, v in saved.items() if k.startswith(key + ".")}
+ restored = bnb.nn.Params4bit.from_prequantized(
+ data = saved[key], quantized_stats = stats, requires_grad = False, device = "cpu"
+ )
+ want = getattr(experts, name)
+ assert restored.quant_state.shape == want.quant_state.shape
+ torch.testing.assert_close(
+ bnb.functional.dequantize_4bit(restored.data, restored.quant_state),
+ bnb.functional.dequantize_4bit(want.data, want.quant_state),
+ )
+ # The hooks live only for the save.
+ assert not any(k.endswith(".absmax") or ".quant_state." in k for k in model.state_dict())
+ cfg = SimpleNamespace(quantization_config = BNB_QC)
+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, out) is True
+
+
+def test_full_save_without_bare_params4bit_is_unchanged(tmp_path):
+ from unsloth.save import patch_saving_functions
+
+ model = _tiny_gpt_oss()
+ plain = str(tmp_path / "plain")
+ model.save_pretrained(plain)
+ patch_saving_functions(model)
+ patched = str(tmp_path / "patched")
+ model.save_pretrained(patched)
+ for f in ("model.safetensors", "config.json"):
+ with open(os.path.join(plain, f), "rb") as a, open(os.path.join(patched, f), "rb") as b:
+ assert a.read() == b.read(), f
+
+
[email protected]("positional", [False, True])
+def test_full_save_with_a_supplied_state_dict_keeps_fused_quant_state(tmp_path, positional):
+ """`save_pretrained(path, state_dict = model.state_dict())` (and the Trainer's saves) pass a dict
+ built before the save installs its hooks, and transformers writes it as given."""
+ bnb = pytest.importorskip("bitsandbytes")
+ from unsloth.save import patch_saving_functions
+
+ model = _tiny_gpt_oss()
+ experts = model.model.layers[0].mlp.experts
+ for name in ("gate_up_proj", "down_proj"):
+ setattr(experts, name, _quantize_cpu(getattr(experts, name).data.normal_(0.0, 0.05)))
+ model.config.quantization_config = dict(BNB_QC, bnb_4bit_quant_type = "nf4")
+ patch_saving_functions(model)
+ supplied = model.state_dict()
+ keys = set(supplied)
+ out = str(tmp_path / "saved")
+ if positional:
+ model.save_pretrained(out, True, supplied) # save_directory, is_main_process, state_dict
+ else:
+ model.save_pretrained(out, state_dict = supplied)
+
+ saved = load_file(os.path.join(out, "model.safetensors"))
+ for name in ("gate_up_proj", "down_proj"):
+ key = f"{P}.experts.{name}"
+ assert f"{key}.absmax" in saved and f"{key}.quant_state.bitsandbytes__nf4" in saved
+ stats = {k[len(key) + 1 :]: v for k, v in saved.items() if k.startswith(key + ".")}
+ restored = bnb.nn.Params4bit.from_prequantized(
+ data = saved[key], quantized_stats = stats, requires_grad = False, device = "cpu"
+ )
+ want = getattr(experts, name)
+ torch.testing.assert_close(
+ bnb.functional.dequantize_4bit(restored.data, restored.quant_state),
+ bnb.functional.dequantize_4bit(want.data, want.quant_state),
+ )
+ cfg = SimpleNamespace(quantization_config = BNB_QC)
+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, out) is True
+ # The augmented dict is a copy (transformers may consume the dict it is given).
+ assert not any(k.endswith(".absmax") or ".quant_state." in k for k in supplied)
+
+
+def _quantized_tiny_gpt_oss():
+ from unsloth.save import patch_saving_functions
+
+ model = _tiny_gpt_oss()
+ experts = model.model.layers[0].mlp.experts
+ for name in ("gate_up_proj", "down_proj"):
+ setattr(experts, name, _quantize_cpu(getattr(experts, name).data.normal_(0.0, 0.05)))
+ model.config.quantization_config = dict(BNB_QC, bnb_4bit_quant_type = "nf4")
+ patch_saving_functions(model)
+ return model, experts
+
+
+def test_supplied_foreign_packed_bytes_without_quant_state_are_refused(tmp_path):
+ """Packed expert bytes that are not the live parameter's (an EMA snapshot, another checkpoint)
+ must not be saved with the live parameter's absmax."""
+ pytest.importorskip("bitsandbytes")
+ model, experts = _quantized_tiny_gpt_oss()
+ supplied = {k: v.clone() for k, v in model.state_dict().items()}
+ key = f"{P}.experts.gate_up_proj"
+ supplied[key] = torch.randint_like(supplied[key], 0, 255)
+ assert not torch.equal(supplied[key], experts.gate_up_proj.data)
+ with pytest.raises(ValueError, match = "not this model's parameter"):
+ model.save_pretrained(str(tmp_path / "saved"), state_dict = supplied)
+
+
+def test_supplied_quant_state_and_cloned_live_bytes_are_kept(tmp_path):
+ """A supplied dict carrying its own quant state keeps it; cloned (equal) live bytes get the live one."""
+ bnb = pytest.importorskip("bitsandbytes")
+ model, experts = _quantized_tiny_gpt_oss()
+ shape = experts.gate_up_proj.quant_state.shape
+ other = _quantize_cpu(torch.randn(shape, dtype = torch.bfloat16).mul_(0.05))
+ supplied = {k: v.clone() for k, v in model.state_dict().items()} # copies, not the live storage
+ key = f"{P}.experts.gate_up_proj"
+ supplied[key] = other.data.clone()
+ for entry, value in other.quant_state.as_dict(packed = True).items():
+ supplied[f"{key}.{entry}"] = value
+ out = str(tmp_path / "saved")
+ model.save_pretrained(out, state_dict = supplied)
+
+ saved = load_file(os.path.join(out, "model.safetensors"))
+ for name, want in (("gate_up_proj", other), ("down_proj", experts.down_proj)):
+ k = f"{P}.experts.{name}"
+ stats = {s[len(k) + 1 :]: v for s, v in saved.items() if s.startswith(k + ".")}
+ restored = bnb.nn.Params4bit.from_prequantized(
+ data = saved[k], quantized_stats = stats, requires_grad = False, device = "cpu"
+ )
+ torch.testing.assert_close(
+ bnb.functional.dequantize_4bit(restored.data, restored.quant_state),
+ bnb.functional.dequantize_4bit(want.data, want.quant_state),
+ )
+
+
+def test_live_bytes_check_compares_in_bounded_chunks(monkeypatch):
+ """Copied (not same-storage) packed bytes are compared slice by slice: no full-size copy of
+ the live tensor is ever moved to the supplied tensor's device, and a difference exits early."""
+ from unsloth import save as unsloth_save
+
+ monkeypatch.setattr(unsloth_save, "_PARAMS4BIT_COMPARE_CHUNK_BYTES", 1024)
+ live = torch.randint(0, 255, (10_000, 1), dtype = torch.uint8)
+ param = SimpleNamespace(data = live, shape = live.shape, dtype = live.dtype, device = live.device)
+ sizes = []
+ real_equal = torch.equal
+
+ def spy_equal(a, b):
+ sizes.append(max(a.numel(), b.numel()))
+ return real_equal(a, b)
+
+ monkeypatch.setattr(unsloth_save.torch, "equal", spy_equal)
+ assert unsloth_save._is_live_params4bit_data(param, live) # same storage: no comparison
+ assert sizes == []
+ assert unsloth_save._is_live_params4bit_data(param, live.clone())
+ assert sizes and max(sizes) <= 1024 and len(sizes) == 10
+ sizes.clear()
+ other = live.clone()
+ other[5] ^= 1
+ assert not unsloth_save._is_live_params4bit_data(param, other)
+ assert len(sizes) == 1 # first chunk differs: stops
+ sizes.clear()
+ noncontig = torch.randint(0, 255, (100, 100), dtype = torch.uint8).t()
+ assert not noncontig.is_contiguous()
+ param_nc = SimpleNamespace(
+ data = noncontig, shape = noncontig.shape, dtype = noncontig.dtype, device = noncontig.device
+ )
+ assert unsloth_save._is_live_params4bit_data(param_nc, noncontig.contiguous())
+ assert sizes and max(sizes) <= 1024
+
+
+def test_live_bytes_check_errors_are_not_reported_as_foreign(monkeypatch):
+ """An error while comparing (an OOM moving a slice) propagates instead of becoming the
+ misleading foreign-weights ValueError."""
+ from unsloth import save as unsloth_save
+
+ live = torch.zeros(64, 1, dtype = torch.uint8)
+ param = SimpleNamespace(data = live, shape = live.shape, dtype = live.dtype, device = live.device)
+
+ def boom(a, b):
+ raise torch.OutOfMemoryError("CUDA out of memory")
+
+ monkeypatch.setattr(unsloth_save.torch, "equal", boom)
+ with pytest.raises(torch.OutOfMemoryError):
+ unsloth_save._is_live_params4bit_data(param, live.clone())
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index be375da3a0e..6c8a88d961c 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -4096,6 +4096,23 @@ def _hub_safetensors_header_dtypes(
return {k: v.get("dtype") for k, v in header.items() if k != "__metadata__"}
+def _hub_live_commit(
+ repo_id,
+ revision = None,
+ token = None,
+):
+ """The commit `revision` names on the Hub now, or None when it cannot be resolved (offline,
+ network error). A branch or tag can move past what the cache holds; transformers' online
+ `hf_hub_download` refreshes it, so cache lookups must use the live commit too."""
+ if isinstance(revision, str) and re.fullmatch(r"[0-9a-f]{40}", revision):
+ return revision
+ try:
+ from huggingface_hub import HfApi
+ return HfApi(token = token).model_info(repo_id, revision = revision).sha or None
+ except Exception:
+ return None
+
+
def _checkpoint_tensor_dtypes(
model_name,
token = None,
@@ -4145,24 +4162,23 @@ def _checkpoint_tensor_dtypes(
try:
if os.path.isdir(model_name):
root = os.path.join(model_name, subfolder) if subfolder else model_name
- for index_file in index_files:
- index_path = os.path.join(root, index_file)
- if not os.path.exists(index_path):
+ # transformers' order: a lone file beats an index of the same format, so a stale index
+ # left beside a re-saved model.safetensors must not decide the layout.
+ for filename in (st_single, st_index, bin_single, bin_index):
+ path = os.path.join(root, filename) if filename else None
+ if not path or not os.path.isfile(path):
continue
- weight_map = _index_weight_map(index_path)
- return _fill_fused_dtypes(
- weight_map,
- lambda shard, names: _local_shard_dtypes(os.path.join(root, shard), names),
- )
- single_path = os.path.join(root, st_single) if st_single else None
- if single_path and os.path.exists(single_path):
- return _safetensors_dtypes(single_path)
- single_path = os.path.join(root, bin_single) if bin_single else None
- if single_path and os.path.exists(single_path):
+ if filename in index_files:
+ return _fill_fused_dtypes(
+ _index_weight_map(path),
+ lambda shard, names: _local_shard_dtypes(os.path.join(root, shard), names),
+ )
+ if filename == st_single:
+ return _safetensors_dtypes(path)
try:
- return _bin_dtypes(single_path)
+ return _bin_dtypes(path)
except Exception:
- raise _UnreadableCheckpointNames(single_path)
+ raise _UnreadableCheckpointNames(path)
return None
if not local_files_only and not subfolder and not variant and st_single:
@@ -4180,13 +4196,20 @@ def _checkpoint_tensor_dtypes(
from huggingface_hub import hf_hub_download
hub = dict(revision = revision, subfolder = subfolder, cache_dir = cache_dir, token = token)
- for index_file in index_files:
+ if not local_files_only and not _env_says_offline():
+ # Cached files of a moved branch are stale: look them up under the commit it names now
+ # (a miss falls through to the live header read), as transformers' online resolver does.
+ live_commit = _hub_live_commit(model_name, revision, token)
+ if live_commit is not None:
+ hub["revision"] = live_commit
+
+ def _hub_index_dtypes(index_file):
try:
index_path = hf_hub_download(
model_name, index_file, local_files_only = local_files_only, **hub
)
except Exception:
- continue
+ return None
weight_map = _index_weight_map(index_path)
def _hub_shard_dtypes(shard, names):
@@ -4199,9 +4222,11 @@ def _checkpoint_tensor_dtypes(
if local_files_only or not shard.endswith(".safetensors"):
return {}
remote = f"{subfolder}/{shard}" if subfolder else shard
- return _hub_safetensors_header_dtypes(model_name, remote, revision, token)
+ return _hub_safetensors_header_dtypes(model_name, remote, hub["revision"], token)
return _fill_fused_dtypes(weight_map, _hub_shard_dtypes)
+
+ # Same order as transformers: model.safetensors, its index, pytorch_model.bin, its index.
if st_single:
try:
return _safetensors_dtypes(
@@ -4212,11 +4237,17 @@ def _checkpoint_tensor_dtypes(
if not local_files_only:
remote = f"{subfolder}/{st_single}" if subfolder else st_single
try:
- return _hub_safetensors_header_dtypes(model_name, remote, revision, token)
+ return _hub_safetensors_header_dtypes(
+ model_name, remote, hub["revision"], token
+ )
except Exception:
pass
+ if st_index:
+ found = _hub_index_dtypes(st_index)
+ if found is not None:
+ return found
if not bin_single:
- return None
+ return _hub_index_dtypes(bin_index) if bin_index else None
try:
cached_bin = hf_hub_download(model_name, bin_single, local_files_only = True, **hub)
except Exception:
@@ -4231,12 +4262,12 @@ def _checkpoint_tensor_dtypes(
remote = f"{subfolder}/{bin_single}" if subfolder else bin_single
try:
- exists = file_exists(model_name, remote, revision = revision, token = token)
+ exists = file_exists(model_name, remote, revision = hub["revision"], token = token)
except Exception:
exists = False
if exists:
raise _UnreadableCheckpointNames(f"{model_name}/{remote}")
- return None
+ return _hub_index_dtypes(bin_index) if bin_index else None
except _UnreadableCheckpointNames:
raise
except Exception:
diff --git a/unsloth/save.py b/unsloth/save.py
index 5cda23858bc..9c60b7668c3 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -254,6 +254,188 @@ def _normalize_safe_serialization(safe_serialization):
return True if safe_serialization is None else safe_serialization
+@contextmanager
+def _save_bare_params4bit_quant_state(model):
+ """Write the bitsandbytes quant state of bare `Params4bit` parameters (fused MoE experts).
+
+ bitsandbytes only serializes a QuantState from `Linear4bit._save_to_state_dict`, so a
+ fused expert stack quantized in place (gpt-oss `experts.gate_up_proj`) is saved as
+ packed bytes alone and cannot be reloaded. This adds the same `<name>.absmax`,
+ `<name>.quant_map`, `<name>.quant_state.bitsandbytes__nf4`, ... entries Linear4bit
+ writes, for the duration of one save. Models without such parameters are untouched.
+ """
+ try:
+ from bitsandbytes.nn import Linear4bit, Params4bit
+ except Exception:
+ yield
+ return
+ if not callable(getattr(model, "modules", None)):
+ yield
+ return
+ handles = []
+ try:
+ for module in model.modules():
+ if isinstance(module, Linear4bit):
+ continue
+ names = [
+ name
+ for name, param in module.named_parameters(recurse = False)
+ if isinstance(param, Params4bit) and getattr(param, "quant_state", None) is not None
+ ]
+ if not names:
+ continue
+
+ def _hook(
+ module,
+ state_dict,
+ prefix,
+ local_metadata,
+ _names = tuple(names),
+ ):
+ for name in _names:
+ if prefix + name not in state_dict:
+ continue
+ quant_state = getattr(module, name).quant_state
+ for key, value in quant_state.as_dict(packed = True).items():
+ state_dict[f"{prefix}{name}.{key}"] = value.detach()
+
+ handles.append(module._register_state_dict_hook(_hook))
+ yield
+ finally:
+ for handle in handles:
+ handle.remove()
+
+
+def _add_bare_params4bit_quant_state(model, state_dict):
+ """A caller-supplied `state_dict` (`save_pretrained(path, state_dict = model.state_dict())`, the
+ Trainer's saves) was built before `_save_bare_params4bit_quant_state` installed its hooks, and
+ transformers writes it as given. Returns a copy with the same quant state entries added next to
+ each bare `Params4bit` it holds, or the dict itself when there is nothing to add. The live quant
+ state is only attached to the live parameter's own packed bytes; other packed bytes need their
+ quant state in the dict (or on a `Params4bit` value), else this raises."""
+ if not isinstance(state_dict, dict) or not callable(getattr(model, "named_modules", None)):
+ return state_dict
+ try:
+ from bitsandbytes.nn import Linear4bit, Params4bit
+ except Exception:
+ return state_dict
+ added = {}
+ for module_name, module in model.named_modules():
+ if isinstance(module, Linear4bit):
+ continue
+ for name, param in module.named_parameters(recurse = False):
+ if not isinstance(param, Params4bit) or getattr(param, "quant_state", None) is None:
+ continue
+ key = f"{module_name}.{name}" if module_name else name
+ if key not in state_dict:
+ continue
+ supplied = state_dict[key]
+ quant_state = _supplied_params4bit_quant_state(param, supplied)
+ if quant_state is None:
+ continue
+ missing = {
+ f"{key}.{entry}": value
+ for entry, value in quant_state.as_dict(packed = True).items()
+ if f"{key}.{entry}" not in state_dict
+ }
+ if not missing:
+ continue
+ if quant_state is param.quant_state and not _is_live_params4bit_data(param, supplied):
+ # Packed bytes from elsewhere (an EMA snapshot, another checkpoint): the live
+ # parameter's absmax would describe different codes and silently corrupt the save.
+ raise ValueError(
+ f"Unsloth: the `state_dict` passed to `save_pretrained` holds 4-bit packed "
+ f"`{key}` that is not this model's parameter, and has no quantization state "
+ f"for it ({', '.join(sorted(missing))}). Include those entries in the "
+ f"`state_dict`, or save without passing one."
+ )
+ added.update({k: v.detach() for k, v in missing.items()})
+ if not added:
+ return state_dict
+ out = dict(state_dict)
+ out.update(added)
+ return out
+
+
+_PARAMS4BIT_COMPARE_CHUNK_BYTES = 64 * 1024 * 1024
+
+
+def _is_live_params4bit_data(param, value):
+ """True when `value` is `param`'s packed data: the same storage, or equal bytes. The byte
+ comparison moves bounded slices of the live data to `value`'s device (a CPU-offloaded
+ `state_dict` never pulls a full fused expert tensor back onto the GPU) and stops at the first
+ differing slice. Errors during the comparison (OOM, device faults) propagate."""
+ if not isinstance(value, torch.Tensor):
+ return False
+ if value.shape != param.shape or value.dtype != param.dtype:
+ return False
+ live = param.data.detach()
+ value = value.detach()
+ if value.device == live.device and value.data_ptr() == live.data_ptr():
+ return True
+ if value.device.type == "meta" or live.device.type == "meta":
+ return False
+ if value.numel() == 0:
+ return True
+ chunk_bytes = _PARAMS4BIT_COMPARE_CHUNK_BYTES
+ if live.is_contiguous() and value.is_contiguous():
+ live_flat, value_flat = live.reshape(-1), value.reshape(-1)
+ step = max(1, chunk_bytes // max(1, live.element_size()))
+ for start in range(0, live_flat.numel(), step):
+ live_chunk = live_flat[start : start + step].to(value.device)
+ if not torch.equal(live_chunk, value_flat[start : start + step]):
+ return False
+ return True
+ # Non-contiguous: slice along dim 0 (each slice copies at most its own rows).
+ if live.dim() == 0:
+ return bool(torch.equal(live.to(value.device), value))
+ row_bytes = max(1, live[0].numel() * live.element_size())
+ step = max(1, chunk_bytes // row_bytes)
+ for start in range(0, live.shape[0], step):
+ live_chunk = live[start : start + step].to(value.device)
+ if not torch.equal(live_chunk, value[start : start + step]):
+ return False
+ return True
+
+
+def _supplied_params4bit_quant_state(param, value):
+ """The QuantState describing a supplied `value` for the bare `Params4bit` `param`: the value's
+ own (a `Params4bit` from another model), else the live one when `value` is packed like `param`
+ (whether it is the live data is checked by the caller), else None (not 4-bit packed: leave it)."""
+ own = getattr(value, "quant_state", None)
+ if own is not None and hasattr(own, "as_dict"):
+ return own
+ if not isinstance(value, torch.Tensor):
+ return None
+ if value.shape != param.shape or value.dtype != param.dtype:
+ return None
+ return param.quant_state
+
+
+def _with_bare_params4bit_state_dict(model, original, args, kwargs):
+ """`args`, `kwargs` of a full save, with a caller-supplied `state_dict` (keyword or positional,
+ located by binding `original`'s signature) given its bare `Params4bit` quant state."""
+ if "state_dict" in kwargs:
+ kwargs = dict(
+ kwargs, state_dict = _add_bare_params4bit_quant_state(model, kwargs["state_dict"])
+ )
+ return args, kwargs
+ if not args:
+ return args, kwargs
+ import inspect
+
+ try:
+ bound = inspect.signature(original).bind_partial(*args, **kwargs)
+ except (TypeError, ValueError):
+ return args, kwargs
+ if bound.arguments.get("state_dict") is None:
+ return args, kwargs
+ bound.arguments["state_dict"] = _add_bare_params4bit_quant_state(
+ model, bound.arguments["state_dict"]
+ )
+ return bound.args, bound.kwargs
+
+
def _filter_push_to_hub_kwargs(push_fn, kwargs):
"""Keep only the keywords `push_fn` actually accepts.
@@ -8450,6 +8632,12 @@ def patch_saving_functions(model, vision = False):
import inspect
original = self.original_model_save_pretrained
+ # Adapter saves never write base weights; full saves also need fused 4-bit experts' quant state.
+ if isinstance(self, PeftModel):
+ quant_state = nullcontext()
+ else:
+ quant_state = _save_bare_params4bit_quant_state(self)
+ args, kwargs = _with_bare_params4bit_state_dict(self, original, args, kwargs)
if kwargs.get("safe_serialization", True) is None:
kwargs["safe_serialization"] = _normalize_safe_serialization(None)
elif args:
@@ -8459,8 +8647,10 @@ def patch_saving_functions(model, vision = False):
bound = None
if bound is not None and bound.arguments.get("safe_serialization", True) is None:
bound.arguments["safe_serialization"] = _normalize_safe_serialization(None)
- return original(*bound.args, **bound.kwargs)
- return original(*args, **kwargs)
+ with quant_state:
+ return original(*bound.args, **bound.kwargs)
+ with quant_state:
+ return original(*args, **kwargs)
if isinstance(model, PreTrainedTokenizerBase):
if model.save_pretrained.__name__ != "unsloth_tokenizer_save_pretrained":