Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Save: compare fused 4-bit expert bytes bitwise for float quant_storage (#15378)

Daniel Hanbf5448b8fb98
diff --git a/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py b/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py
index aac3c031173..deb1b47a387 100644
--- a/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py
+++ b/tests/test_gpt_oss_fused_4bit_checkpoint_roundtrip.py
@@ -296,3 +296,60 @@ def test_live_bytes_check_errors_are_not_reported_as_foreign(monkeypatch):
monkeypatch.setattr(unsloth_save.torch, "equal", boom)
with pytest.raises(torch.OutOfMemoryError):
unsloth_save._is_live_params4bit_data(param, live.clone())
+
+
[email protected]("dtype", [torch.float16, torch.bfloat16, torch.float32])
+def test_live_bytes_check_is_bitwise_for_float_quant_storage(monkeypatch, dtype):
+ from unsloth import save as unsloth_save
+
+ monkeypatch.setattr(unsloth_save, "_PARAMS4BIT_COMPARE_CHUNK_BYTES", 64)
+ raw = torch.randint(0, 256, (257 * 4 * 4,), dtype = torch.uint8)
+ live = raw.view(dtype).reshape(-1, 4)
+ live[3, 1] = float("nan")
+ live[7, 2] = 0.0
+ assert torch.isnan(live).any()
+
+ def param_of(t):
+ return SimpleNamespace(data = t, shape = t.shape, dtype = t.dtype, device = t.device)
+
+ param = param_of(live)
+ assert unsloth_save._is_live_params4bit_data(param, live.clone())
+ flipped = live.clone()
+ flipped[7, 2] = -0.0 # different bytes, equal under ==
+ assert not unsloth_save._is_live_params4bit_data(param, flipped)
+ param_nc = param_of(live.t())
+ assert unsloth_save._is_live_params4bit_data(param_nc, live.t().contiguous())
+ assert not unsloth_save._is_live_params4bit_data(param_nc, flipped.t().contiguous())
+ scalar = torch.tensor(float("nan"), dtype = dtype)
+ assert unsloth_save._is_live_params4bit_data(param_of(scalar), scalar.clone())
+
+
[email protected]("quant_storage", [torch.float16, torch.bfloat16, torch.float32])
+def test_supplied_cloned_float_quant_storage_bytes_are_kept(tmp_path, quant_storage):
+ bnb = pytest.importorskip("bitsandbytes")
+ from unsloth.save import patch_saving_functions
+
+ model = _tiny_gpt_oss()
+ experts = model.model.layers[0].mlp.experts
+ for name in ("gate_up_proj", "down_proj"):
+ weight = getattr(experts, name).data.normal_(0.0, 0.05)
+ param = bnb.nn.Params4bit(
+ weight.detach().clone(),
+ requires_grad = False,
+ quant_type = "nf4",
+ quant_storage = quant_storage,
+ ).to("cpu")
+ setattr(experts, name, param)
+ assert any(torch.isnan(getattr(experts, n).data).any() for n in ("gate_up_proj", "down_proj"))
+ model.config.quantization_config = dict(BNB_QC, bnb_4bit_quant_type = "nf4")
+ patch_saving_functions(model)
+ supplied = {k: v.clone() for k, v in model.state_dict().items()}
+ out = str(tmp_path / "saved")
+ model.save_pretrained(out, state_dict = supplied)
+
+ saved = load_file(os.path.join(out, "model.safetensors"))
+ for name in ("gate_up_proj", "down_proj"):
+ key = f"{P}.experts.{name}"
+ assert f"{key}.absmax" in saved and f"{key}.quant_state.bitsandbytes__nf4" in saved
+ want = getattr(experts, name)
+ assert torch.equal(saved[key].view(torch.uint8), want.data.view(torch.uint8))
diff --git a/unsloth/save.py b/unsloth/save.py
index 57145b170a5..b75fb20d75c 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -362,6 +362,14 @@ def _add_bare_params4bit_quant_state(model, state_dict):
_PARAMS4BIT_COMPARE_CHUNK_BYTES = 64 * 1024 * 1024
+def _packed_bytes_equal(a, b):
+ # Float quant_storage packs NaN / +-0 byte patterns: torch.equal would misjudge them.
+ if a.is_floating_point() or a.is_complex():
+ a = a.contiguous().reshape(-1).view(torch.uint8)
+ b = b.contiguous().reshape(-1).view(torch.uint8)
+ return torch.equal(a, b)
+
+
def _is_live_params4bit_data(param, value):
"""True when `value` is `param`'s packed data: the same storage, or equal bytes. The byte
comparison moves bounded slices of the live data to `value`'s device (a CPU-offloaded
@@ -385,17 +393,17 @@ def _is_live_params4bit_data(param, value):
step = max(1, chunk_bytes // max(1, live.element_size()))
for start in range(0, live_flat.numel(), step):
live_chunk = live_flat[start : start + step].to(value.device)
- if not torch.equal(live_chunk, value_flat[start : start + step]):
+ if not _packed_bytes_equal(live_chunk, value_flat[start : start + step]):
return False
return True
# Non-contiguous: slice along dim 0 (each slice copies at most its own rows).
if live.dim() == 0:
- return bool(torch.equal(live.to(value.device), value))
+ return bool(_packed_bytes_equal(live.to(value.device), value))
row_bytes = max(1, live[0].numel() * live.element_size())
step = max(1, chunk_bytes // row_bytes)
for start in range(0, live.shape[0], step):
live_chunk = live[start : start + step].to(value.device)
- if not torch.equal(live_chunk, value[start : start + step]):
+ if not _packed_bytes_equal(live_chunk, value[start : start + step]):
return False
return True