Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
gpt-oss bnb 4-bit: probe the checkpoint transformers loads and keep fused expert quant state on save (#14740)
Daniel Han85d14340029f
diff --git a/tests/saving/test_save_method_lora_routing.py b/tests/saving/test_save_method_lora_routing.pyindex d0482d3d102..985c8388088 100644--- a/tests/saving/test_save_method_lora_routing.py+++ b/tests/saving/test_save_method_lora_routing.py@@ -40,6 +40,10 @@ def _load(*names, **env):"raise_if_merging_mistral_format_view": lambda model, save_method: None,"lora_relative_to_original_base": lambda model: contextlib.nullcontext(),"nullcontext": contextlib.nullcontext,+ # Full saves add bare Params4bit quant state (fused 4-bit experts); none in these fixtures.+ "PeftModel": type("PeftModel", (), {}),+ "_save_bare_params4bit_quant_state": lambda model: contextlib.nullcontext(),+ "_with_bare_params4bit_state_dict": lambda model, original, args, kwargs: (args, kwargs),**env,}for name in names:diff --git a/tests/test_gpt_oss_bnb_checkpoint_layout.py b/tests/test_gpt_oss_bnb_checkpoint_layout.pyindex cf50f0400db..73a31994e22 100644--- a/tests/test_gpt_oss_bnb_checkpoint_layout.py+++ b/tests/test_gpt_oss_bnb_checkpoint_layout.py@@ -375,6 +375,7 @@ def _hub(monkeypatch.setattr(huggingface_hub, "get_safetensors_metadata", no_metadata)monkeypatch.setattr(huggingface_hub, "file_exists", lambda repo, f, **k: f in exists)monkeypatch.setattr(loader_utils, "_hub_safetensors_header_dtypes", no_metadata, raising = False)+ monkeypatch.setattr(loader_utils, "_hub_live_commit", lambda *a, **k: None, raising = False)return calls@@ -510,3 +511,59 @@ def test_loader_forwards_use_safetensors_to_the_probe(tmp_path, monkeypatch):use_safetensors = False,)assert seen and all(v is False for v in seen)+++def test_online_probe_ignores_a_stale_cached_single_file(monkeypatch, tmp_path):+ """Online with `subfolder`: a `model.safetensors` cached under an older commit of the branch must+ not decide the layout once the branch moved to a sharded checkpoint (transformers refreshes it)."""+ import huggingface_hub++ monkeypatch.setenv("TMPDIR", str(tmp_path))+ for k in ("HF_HUB_OFFLINE", "TRANSFORMERS_OFFLINE"):+ monkeypatch.delenv(k, raising = False)+ stale = str(tmp_path / "stale_model.safetensors")+ save_file(_tensors(SPLIT, packed = True), stale)+ old, new = "a" * 40, "b" * 40+ index = str(tmp_path / "index.json")+ with open(index, "w", encoding = "utf-8") as f:+ json.dump({"weight_map": {n: "model-00001-of-00001.safetensors" for n in FUSED_QS}}, f)++ def download(+ repo_id,+ filename,+ *args,+ revision = None,+ local_files_only = False,+ **kwargs,+ ):+ if filename == "model.safetensors" and local_files_only and revision in (None, "main", old):+ return stale # the branch's previous commit, still in the cache+ if filename == "model.safetensors.index.json" and revision == new:+ return index+ raise FileNotFoundError(filename)++ def header(+ repo_id,+ filename,+ revision = None,+ token = None,+ ):+ if filename.endswith("model-00001-of-00001.safetensors") and revision == new:+ return {n: "U8" for n in FUSED_QS}+ raise FileNotFoundError(filename)++ monkeypatch.setattr(huggingface_hub, "hf_hub_download", download)+ monkeypatch.setattr(loader_utils, "_hub_safetensors_header_dtypes", header)+ monkeypatch.setattr(loader_utils, "_hub_live_commit", lambda repo, revision, token: new)+ cfg = SimpleNamespace(quantization_config = BNB_QC)+ assert (+ loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, "someone/gpt-oss", subfolder = "sub")+ is True+ )+ # Offline the cache is all there is, as in transformers.+ assert (+ loader_utils.gpt_oss_checkpoint_has_fused_experts(+ cfg, "someone/gpt-oss", subfolder = "sub", local_files_only = True+ )+ is False+ )diff --git a/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py b/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.pynew file mode 100644index 00000000000..aac3c031173--- /dev/null+++ b/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py@@ -0,0 +1,298 @@+# Unsloth - 2x faster, 60% less VRAM LLM training and finetuning+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""gpt-oss fused bnb 4-bit checkpoints: the layout probe reads the file transformers loads, and a+full save of fused 4-bit experts keeps their quantization state so the probe accepts it."""++import json+import os+from types import SimpleNamespace++import pytest+import torch+from safetensors.torch import load_file, save_file++import unsloth # noqa: F401 (patches before transformers)+from unsloth.models import loader_utils++BNB_QC = {"quant_method": "bitsandbytes", "load_in_4bit": True}+P = "model.layers.0.mlp"+FUSED = [f"{P}.experts.gate_up_proj", f"{P}.experts.down_proj", f"{P}.router.weight"]+SPLIT = [f"{P}.experts.gate_up_projs.0.weight", f"{P}.experts.down_projs.0.weight"]+++def _tensors(names):+ return {n: torch.zeros(2, 2, dtype = torch.bfloat16) for n in names}+++def _stale_index(d, index_name, names):+ shard = "old-00001-of-00001" + (".bin" if index_name.startswith("pytorch") else ".safetensors")+ (d / index_name).write_text(+ json.dumps({"metadata": {}, "weight_map": {n: shard for n in names}})+ )+++ "single, stale_index",+ [+ ("model.safetensors", "model.safetensors.index.json"),+ ("model.safetensors", "pytorch_model.bin.index.json"),+ ("pytorch_model.bin", "pytorch_model.bin.index.json"),+ ],+)+def test_lone_file_beats_a_stale_index_like_transformers(tmp_path, single, stale_index):+ # transformers resolves model.safetensors, then its index, then pytorch_model.bin, then its index.+ d = tmp_path / "ckpt"+ d.mkdir()+ if single.endswith(".safetensors"):+ save_file(_tensors(FUSED), str(d / single))+ else:+ torch.save(_tensors(FUSED), str(d / single))+ _stale_index(d, stale_index, SPLIT)+ cfg = SimpleNamespace(quantization_config = BNB_QC)+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, str(d)) is True+++def test_safetensors_index_still_beats_a_lone_bin(tmp_path):+ d = tmp_path / "ckpt"+ d.mkdir()+ torch.save(_tensors(SPLIT), str(d / "pytorch_model.bin"))+ shard = "model-00001-of-00001.safetensors"+ save_file(_tensors(FUSED), str(d / shard))+ (d / "model.safetensors.index.json").write_text(+ json.dumps({"metadata": {}, "weight_map": {n: shard for n in FUSED}})+ )+ cfg = SimpleNamespace(quantization_config = BNB_QC)+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, str(d)) is True+++def _quantize_cpu(tensor):+ bnb = pytest.importorskip("bitsandbytes")+ param = bnb.nn.Params4bit(tensor.detach().clone(), requires_grad = False, quant_type = "nf4").to(+ "cpu"+ )+ assert param.quant_state is not None+ return param+++def _tiny_gpt_oss():+ from transformers import GptOssConfig, GptOssForCausalLM++ config = GptOssConfig(+ vocab_size = 128,+ hidden_size = 64,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 2,+ num_key_value_heads = 1,+ head_dim = 32,+ num_local_experts = 2,+ num_experts_per_tok = 1,+ layer_types = ["full_attention"],+ )+ torch.manual_seed(0)+ return GptOssForCausalLM(config).to(torch.bfloat16)+++def test_full_save_keeps_fused_params4bit_quant_state(tmp_path):+ bnb = pytest.importorskip("bitsandbytes")+ from unsloth.save import patch_saving_functions++ model = _tiny_gpt_oss()+ experts = model.model.layers[0].mlp.experts+ for name in ("gate_up_proj", "down_proj"):+ setattr(experts, name, _quantize_cpu(getattr(experts, name).data.normal_(0.0, 0.05)))+ model.config.quantization_config = dict(BNB_QC, bnb_4bit_quant_type = "nf4")+ patch_saving_functions(model)+ out = str(tmp_path / "saved")+ model.save_pretrained(out)++ saved = load_file(os.path.join(out, "model.safetensors"))+ for name in ("gate_up_proj", "down_proj"):+ key = f"{P}.experts.{name}"+ assert saved[key].dtype == torch.uint8+ assert f"{key}.absmax" in saved and f"{key}.quant_state.bitsandbytes__nf4" in saved+ stats = {k[len(key) + 1 :]: v for k, v in saved.items() if k.startswith(key + ".")}+ restored = bnb.nn.Params4bit.from_prequantized(+ data = saved[key], quantized_stats = stats, requires_grad = False, device = "cpu"+ )+ want = getattr(experts, name)+ assert restored.quant_state.shape == want.quant_state.shape+ torch.testing.assert_close(+ bnb.functional.dequantize_4bit(restored.data, restored.quant_state),+ bnb.functional.dequantize_4bit(want.data, want.quant_state),+ )+ # The hooks live only for the save.+ assert not any(k.endswith(".absmax") or ".quant_state." in k for k in model.state_dict())+ cfg = SimpleNamespace(quantization_config = BNB_QC)+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, out) is True+++def test_full_save_without_bare_params4bit_is_unchanged(tmp_path):+ from unsloth.save import patch_saving_functions++ model = _tiny_gpt_oss()+ plain = str(tmp_path / "plain")+ model.save_pretrained(plain)+ patch_saving_functions(model)+ patched = str(tmp_path / "patched")+ model.save_pretrained(patched)+ for f in ("model.safetensors", "config.json"):+ with open(os.path.join(plain, f), "rb") as a, open(os.path.join(patched, f), "rb") as b:+ assert a.read() == b.read(), f++[email protected]("positional", [False, True])+def test_full_save_with_a_supplied_state_dict_keeps_fused_quant_state(tmp_path, positional):+ """`save_pretrained(path, state_dict = model.state_dict())` (and the Trainer's saves) pass a dict+ built before the save installs its hooks, and transformers writes it as given."""+ bnb = pytest.importorskip("bitsandbytes")+ from unsloth.save import patch_saving_functions++ model = _tiny_gpt_oss()+ experts = model.model.layers[0].mlp.experts+ for name in ("gate_up_proj", "down_proj"):+ setattr(experts, name, _quantize_cpu(getattr(experts, name).data.normal_(0.0, 0.05)))+ model.config.quantization_config = dict(BNB_QC, bnb_4bit_quant_type = "nf4")+ patch_saving_functions(model)+ supplied = model.state_dict()+ keys = set(supplied)+ out = str(tmp_path / "saved")+ if positional:+ model.save_pretrained(out, True, supplied) # save_directory, is_main_process, state_dict+ else:+ model.save_pretrained(out, state_dict = supplied)++ saved = load_file(os.path.join(out, "model.safetensors"))+ for name in ("gate_up_proj", "down_proj"):+ key = f"{P}.experts.{name}"+ assert f"{key}.absmax" in saved and f"{key}.quant_state.bitsandbytes__nf4" in saved+ stats = {k[len(key) + 1 :]: v for k, v in saved.items() if k.startswith(key + ".")}+ restored = bnb.nn.Params4bit.from_prequantized(+ data = saved[key], quantized_stats = stats, requires_grad = False, device = "cpu"+ )+ want = getattr(experts, name)+ torch.testing.assert_close(+ bnb.functional.dequantize_4bit(restored.data, restored.quant_state),+ bnb.functional.dequantize_4bit(want.data, want.quant_state),+ )+ cfg = SimpleNamespace(quantization_config = BNB_QC)+ assert loader_utils.gpt_oss_checkpoint_has_fused_experts(cfg, out) is True+ # The augmented dict is a copy (transformers may consume the dict it is given).+ assert not any(k.endswith(".absmax") or ".quant_state." in k for k in supplied)+++def _quantized_tiny_gpt_oss():+ from unsloth.save import patch_saving_functions++ model = _tiny_gpt_oss()+ experts = model.model.layers[0].mlp.experts+ for name in ("gate_up_proj", "down_proj"):+ setattr(experts, name, _quantize_cpu(getattr(experts, name).data.normal_(0.0, 0.05)))+ model.config.quantization_config = dict(BNB_QC, bnb_4bit_quant_type = "nf4")+ patch_saving_functions(model)+ return model, experts+++def test_supplied_foreign_packed_bytes_without_quant_state_are_refused(tmp_path):+ """Packed expert bytes that are not the live parameter's (an EMA snapshot, another checkpoint)+ must not be saved with the live parameter's absmax."""+ pytest.importorskip("bitsandbytes")+ model, experts = _quantized_tiny_gpt_oss()+ supplied = {k: v.clone() for k, v in model.state_dict().items()}+ key = f"{P}.experts.gate_up_proj"+ supplied[key] = torch.randint_like(supplied[key], 0, 255)+ assert not torch.equal(supplied[key], experts.gate_up_proj.data)+ with pytest.raises(ValueError, match = "not this model's parameter"):+ model.save_pretrained(str(tmp_path / "saved"), state_dict = supplied)+++def test_supplied_quant_state_and_cloned_live_bytes_are_kept(tmp_path):+ """A supplied dict carrying its own quant state keeps it; cloned (equal) live bytes get the live one."""+ bnb = pytest.importorskip("bitsandbytes")+ model, experts = _quantized_tiny_gpt_oss()+ shape = experts.gate_up_proj.quant_state.shape+ other = _quantize_cpu(torch.randn(shape, dtype = torch.bfloat16).mul_(0.05))+ supplied = {k: v.clone() for k, v in model.state_dict().items()} # copies, not the live storage+ key = f"{P}.experts.gate_up_proj"+ supplied[key] = other.data.clone()+ for entry, value in other.quant_state.as_dict(packed = True).items():+ supplied[f"{key}.{entry}"] = value+ out = str(tmp_path / "saved")+ model.save_pretrained(out, state_dict = supplied)++ saved = load_file(os.path.join(out, "model.safetensors"))+ for name, want in (("gate_up_proj", other), ("down_proj", experts.down_proj)):+ k = f"{P}.experts.{name}"+ stats = {s[len(k) + 1 :]: v for s, v in saved.items() if s.startswith(k + ".")}+ restored = bnb.nn.Params4bit.from_prequantized(+ data = saved[k], quantized_stats = stats, requires_grad = False, device = "cpu"+ )+ torch.testing.assert_close(+ bnb.functional.dequantize_4bit(restored.data, restored.quant_state),+ bnb.functional.dequantize_4bit(want.data, want.quant_state),+ )+++def test_live_bytes_check_compares_in_bounded_chunks(monkeypatch):+ """Copied (not same-storage) packed bytes are compared slice by slice: no full-size copy of+ the live tensor is ever moved to the supplied tensor's device, and a difference exits early."""+ from unsloth import save as unsloth_save++ monkeypatch.setattr(unsloth_save, "_PARAMS4BIT_COMPARE_CHUNK_BYTES", 1024)+ live = torch.randint(0, 255, (10_000, 1), dtype = torch.uint8)+ param = SimpleNamespace(data = live, shape = live.shape, dtype = live.dtype, device = live.device)+ sizes = []+ real_equal = torch.equal++ def spy_equal(a, b):+ sizes.append(max(a.numel(), b.numel()))+ return real_equal(a, b)++ monkeypatch.setattr(unsloth_save.torch, "equal", spy_equal)+ assert unsloth_save._is_live_params4bit_data(param, live) # same storage: no comparison+ assert sizes == []+ assert unsloth_save._is_live_params4bit_data(param, live.clone())+ assert sizes and max(sizes) <= 1024 and len(sizes) == 10+ sizes.clear()+ other = live.clone()+ other[5] ^= 1+ assert not unsloth_save._is_live_params4bit_data(param, other)+ assert len(sizes) == 1 # first chunk differs: stops+ sizes.clear()+ noncontig = torch.randint(0, 255, (100, 100), dtype = torch.uint8).t()+ assert not noncontig.is_contiguous()+ param_nc = SimpleNamespace(+ data = noncontig, shape = noncontig.shape, dtype = noncontig.dtype, device = noncontig.device+ )+ assert unsloth_save._is_live_params4bit_data(param_nc, noncontig.contiguous())+ assert sizes and max(sizes) <= 1024+++def test_live_bytes_check_errors_are_not_reported_as_foreign(monkeypatch):+ """An error while comparing (an OOM moving a slice) propagates instead of becoming the+ misleading foreign-weights ValueError."""+ from unsloth import save as unsloth_save++ live = torch.zeros(64, 1, dtype = torch.uint8)+ param = SimpleNamespace(data = live, shape = live.shape, dtype = live.dtype, device = live.device)++ def boom(a, b):+ raise torch.OutOfMemoryError("CUDA out of memory")++ monkeypatch.setattr(unsloth_save.torch, "equal", boom)+ with pytest.raises(torch.OutOfMemoryError):+ unsloth_save._is_live_params4bit_data(param, live.clone())diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex be375da3a0e..6c8a88d961c 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -4096,6 +4096,23 @@ def _hub_safetensors_header_dtypes(return {k: v.get("dtype") for k, v in header.items() if k != "__metadata__"}+def _hub_live_commit(+ repo_id,+ revision = None,+ token = None,+):+ """The commit `revision` names on the Hub now, or None when it cannot be resolved (offline,+ network error). A branch or tag can move past what the cache holds; transformers' online+ `hf_hub_download` refreshes it, so cache lookups must use the live commit too."""+ if isinstance(revision, str) and re.fullmatch(r"[0-9a-f]{40}", revision):+ return revision+ try:+ from huggingface_hub import HfApi+ return HfApi(token = token).model_info(repo_id, revision = revision).sha or None+ except Exception:+ return None++def _checkpoint_tensor_dtypes(model_name,token = None,@@ -4145,24 +4162,23 @@ def _checkpoint_tensor_dtypes(try:if os.path.isdir(model_name):root = os.path.join(model_name, subfolder) if subfolder else model_name- for index_file in index_files:- index_path = os.path.join(root, index_file)- if not os.path.exists(index_path):+ # transformers' order: a lone file beats an index of the same format, so a stale index+ # left beside a re-saved model.safetensors must not decide the layout.+ for filename in (st_single, st_index, bin_single, bin_index):+ path = os.path.join(root, filename) if filename else None+ if not path or not os.path.isfile(path):continue- weight_map = _index_weight_map(index_path)- return _fill_fused_dtypes(- weight_map,- lambda shard, names: _local_shard_dtypes(os.path.join(root, shard), names),- )- single_path = os.path.join(root, st_single) if st_single else None- if single_path and os.path.exists(single_path):- return _safetensors_dtypes(single_path)- single_path = os.path.join(root, bin_single) if bin_single else None- if single_path and os.path.exists(single_path):+ if filename in index_files:+ return _fill_fused_dtypes(+ _index_weight_map(path),+ lambda shard, names: _local_shard_dtypes(os.path.join(root, shard), names),+ )+ if filename == st_single:+ return _safetensors_dtypes(path)try:- return _bin_dtypes(single_path)+ return _bin_dtypes(path)except Exception:- raise _UnreadableCheckpointNames(single_path)+ raise _UnreadableCheckpointNames(path)return Noneif not local_files_only and not subfolder and not variant and st_single:@@ -4180,13 +4196,20 @@ def _checkpoint_tensor_dtypes(from huggingface_hub import hf_hub_downloadhub = dict(revision = revision, subfolder = subfolder, cache_dir = cache_dir, token = token)- for index_file in index_files:+ if not local_files_only and not _env_says_offline():+ # Cached files of a moved branch are stale: look them up under the commit it names now+ # (a miss falls through to the live header read), as transformers' online resolver does.+ live_commit = _hub_live_commit(model_name, revision, token)+ if live_commit is not None:+ hub["revision"] = live_commit++ def _hub_index_dtypes(index_file):try:index_path = hf_hub_download(model_name, index_file, local_files_only = local_files_only, **hub)except Exception:- continue+ return Noneweight_map = _index_weight_map(index_path)def _hub_shard_dtypes(shard, names):@@ -4199,9 +4222,11 @@ def _checkpoint_tensor_dtypes(if local_files_only or not shard.endswith(".safetensors"):return {}remote = f"{subfolder}/{shard}" if subfolder else shard- return _hub_safetensors_header_dtypes(model_name, remote, revision, token)+ return _hub_safetensors_header_dtypes(model_name, remote, hub["revision"], token)return _fill_fused_dtypes(weight_map, _hub_shard_dtypes)++ # Same order as transformers: model.safetensors, its index, pytorch_model.bin, its index.if st_single:try:return _safetensors_dtypes(@@ -4212,11 +4237,17 @@ def _checkpoint_tensor_dtypes(if not local_files_only:remote = f"{subfolder}/{st_single}" if subfolder else st_singletry:- return _hub_safetensors_header_dtypes(model_name, remote, revision, token)+ return _hub_safetensors_header_dtypes(+ model_name, remote, hub["revision"], token+ )except Exception:pass+ if st_index:+ found = _hub_index_dtypes(st_index)+ if found is not None:+ return foundif not bin_single:- return None+ return _hub_index_dtypes(bin_index) if bin_index else Nonetry:cached_bin = hf_hub_download(model_name, bin_single, local_files_only = True, **hub)except Exception:@@ -4231,12 +4262,12 @@ def _checkpoint_tensor_dtypes(remote = f"{subfolder}/{bin_single}" if subfolder else bin_singletry:- exists = file_exists(model_name, remote, revision = revision, token = token)+ exists = file_exists(model_name, remote, revision = hub["revision"], token = token)except Exception:exists = Falseif exists:raise _UnreadableCheckpointNames(f"{model_name}/{remote}")- return None+ return _hub_index_dtypes(bin_index) if bin_index else Noneexcept _UnreadableCheckpointNames:raiseexcept Exception:diff --git a/unsloth/save.py b/unsloth/save.pyindex 5cda23858bc..9c60b7668c3 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -254,6 +254,188 @@ def _normalize_safe_serialization(safe_serialization):return True if safe_serialization is None else safe_serialization+@contextmanager+def _save_bare_params4bit_quant_state(model):+ """Write the bitsandbytes quant state of bare `Params4bit` parameters (fused MoE experts).++ bitsandbytes only serializes a QuantState from `Linear4bit._save_to_state_dict`, so a+ fused expert stack quantized in place (gpt-oss `experts.gate_up_proj`) is saved as+ packed bytes alone and cannot be reloaded. This adds the same `<name>.absmax`,+ `<name>.quant_map`, `<name>.quant_state.bitsandbytes__nf4`, ... entries Linear4bit+ writes, for the duration of one save. Models without such parameters are untouched.+ """+ try:+ from bitsandbytes.nn import Linear4bit, Params4bit+ except Exception:+ yield+ return+ if not callable(getattr(model, "modules", None)):+ yield+ return+ handles = []+ try:+ for module in model.modules():+ if isinstance(module, Linear4bit):+ continue+ names = [+ name+ for name, param in module.named_parameters(recurse = False)+ if isinstance(param, Params4bit) and getattr(param, "quant_state", None) is not None+ ]+ if not names:+ continue++ def _hook(+ module,+ state_dict,+ prefix,+ local_metadata,+ _names = tuple(names),+ ):+ for name in _names:+ if prefix + name not in state_dict:+ continue+ quant_state = getattr(module, name).quant_state+ for key, value in quant_state.as_dict(packed = True).items():+ state_dict[f"{prefix}{name}.{key}"] = value.detach()++ handles.append(module._register_state_dict_hook(_hook))+ yield+ finally:+ for handle in handles:+ handle.remove()+++def _add_bare_params4bit_quant_state(model, state_dict):+ """A caller-supplied `state_dict` (`save_pretrained(path, state_dict = model.state_dict())`, the+ Trainer's saves) was built before `_save_bare_params4bit_quant_state` installed its hooks, and+ transformers writes it as given. Returns a copy with the same quant state entries added next to+ each bare `Params4bit` it holds, or the dict itself when there is nothing to add. The live quant+ state is only attached to the live parameter's own packed bytes; other packed bytes need their+ quant state in the dict (or on a `Params4bit` value), else this raises."""+ if not isinstance(state_dict, dict) or not callable(getattr(model, "named_modules", None)):+ return state_dict+ try:+ from bitsandbytes.nn import Linear4bit, Params4bit+ except Exception:+ return state_dict+ added = {}+ for module_name, module in model.named_modules():+ if isinstance(module, Linear4bit):+ continue+ for name, param in module.named_parameters(recurse = False):+ if not isinstance(param, Params4bit) or getattr(param, "quant_state", None) is None:+ continue+ key = f"{module_name}.{name}" if module_name else name+ if key not in state_dict:+ continue+ supplied = state_dict[key]+ quant_state = _supplied_params4bit_quant_state(param, supplied)+ if quant_state is None:+ continue+ missing = {+ f"{key}.{entry}": value+ for entry, value in quant_state.as_dict(packed = True).items()+ if f"{key}.{entry}" not in state_dict+ }+ if not missing:+ continue+ if quant_state is param.quant_state and not _is_live_params4bit_data(param, supplied):+ # Packed bytes from elsewhere (an EMA snapshot, another checkpoint): the live+ # parameter's absmax would describe different codes and silently corrupt the save.+ raise ValueError(+ f"Unsloth: the `state_dict` passed to `save_pretrained` holds 4-bit packed "+ f"`{key}` that is not this model's parameter, and has no quantization state "+ f"for it ({', '.join(sorted(missing))}). Include those entries in the "+ f"`state_dict`, or save without passing one."+ )+ added.update({k: v.detach() for k, v in missing.items()})+ if not added:+ return state_dict+ out = dict(state_dict)+ out.update(added)+ return out+++_PARAMS4BIT_COMPARE_CHUNK_BYTES = 64 * 1024 * 1024+++def _is_live_params4bit_data(param, value):+ """True when `value` is `param`'s packed data: the same storage, or equal bytes. The byte+ comparison moves bounded slices of the live data to `value`'s device (a CPU-offloaded+ `state_dict` never pulls a full fused expert tensor back onto the GPU) and stops at the first+ differing slice. Errors during the comparison (OOM, device faults) propagate."""+ if not isinstance(value, torch.Tensor):+ return False+ if value.shape != param.shape or value.dtype != param.dtype:+ return False+ live = param.data.detach()+ value = value.detach()+ if value.device == live.device and value.data_ptr() == live.data_ptr():+ return True+ if value.device.type == "meta" or live.device.type == "meta":+ return False+ if value.numel() == 0:+ return True+ chunk_bytes = _PARAMS4BIT_COMPARE_CHUNK_BYTES+ if live.is_contiguous() and value.is_contiguous():+ live_flat, value_flat = live.reshape(-1), value.reshape(-1)+ step = max(1, chunk_bytes // max(1, live.element_size()))+ for start in range(0, live_flat.numel(), step):+ live_chunk = live_flat[start : start + step].to(value.device)+ if not torch.equal(live_chunk, value_flat[start : start + step]):+ return False+ return True+ # Non-contiguous: slice along dim 0 (each slice copies at most its own rows).+ if live.dim() == 0:+ return bool(torch.equal(live.to(value.device), value))+ row_bytes = max(1, live[0].numel() * live.element_size())+ step = max(1, chunk_bytes // row_bytes)+ for start in range(0, live.shape[0], step):+ live_chunk = live[start : start + step].to(value.device)+ if not torch.equal(live_chunk, value[start : start + step]):+ return False+ return True+++def _supplied_params4bit_quant_state(param, value):+ """The QuantState describing a supplied `value` for the bare `Params4bit` `param`: the value's+ own (a `Params4bit` from another model), else the live one when `value` is packed like `param`+ (whether it is the live data is checked by the caller), else None (not 4-bit packed: leave it)."""+ own = getattr(value, "quant_state", None)+ if own is not None and hasattr(own, "as_dict"):+ return own+ if not isinstance(value, torch.Tensor):+ return None+ if value.shape != param.shape or value.dtype != param.dtype:+ return None+ return param.quant_state+++def _with_bare_params4bit_state_dict(model, original, args, kwargs):+ """`args`, `kwargs` of a full save, with a caller-supplied `state_dict` (keyword or positional,+ located by binding `original`'s signature) given its bare `Params4bit` quant state."""+ if "state_dict" in kwargs:+ kwargs = dict(+ kwargs, state_dict = _add_bare_params4bit_quant_state(model, kwargs["state_dict"])+ )+ return args, kwargs+ if not args:+ return args, kwargs+ import inspect++ try:+ bound = inspect.signature(original).bind_partial(*args, **kwargs)+ except (TypeError, ValueError):+ return args, kwargs+ if bound.arguments.get("state_dict") is None:+ return args, kwargs+ bound.arguments["state_dict"] = _add_bare_params4bit_quant_state(+ model, bound.arguments["state_dict"]+ )+ return bound.args, bound.kwargs++def _filter_push_to_hub_kwargs(push_fn, kwargs):"""Keep only the keywords `push_fn` actually accepts.@@ -8450,6 +8632,12 @@ def patch_saving_functions(model, vision = False):import inspectoriginal = self.original_model_save_pretrained+ # Adapter saves never write base weights; full saves also need fused 4-bit experts' quant state.+ if isinstance(self, PeftModel):+ quant_state = nullcontext()+ else:+ quant_state = _save_bare_params4bit_quant_state(self)+ args, kwargs = _with_bare_params4bit_state_dict(self, original, args, kwargs)if kwargs.get("safe_serialization", True) is None:kwargs["safe_serialization"] = _normalize_safe_serialization(None)elif args:@@ -8459,8 +8647,10 @@ def patch_saving_functions(model, vision = False):bound = Noneif bound is not None and bound.arguments.get("safe_serialization", True) is None:bound.arguments["safe_serialization"] = _normalize_safe_serialization(None)- return original(*bound.args, **bound.kwargs)- return original(*args, **kwargs)+ with quant_state:+ return original(*bound.args, **bound.kwargs)+ with quant_state:+ return original(*args, **kwargs)if isinstance(model, PreTrainedTokenizerBase):if model.save_pretrained.__name__ != "unsloth_tokenizer_save_pretrained":