Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Finish a 16bit load of a static per-tensor fp8 checkpoint (Mistral-Small-4) (#11531)
Daniel Hanbda4023c0182
diff --git a/tests/test_fp8_leftover_dequant_16bit.py b/tests/test_fp8_leftover_dequant_16bit.pynew file mode 100644index 00000000000..68a0f9e6565--- /dev/null+++ b/tests/test_fp8_leftover_dequant_16bit.py@@ -0,0 +1,422 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""`_dequantize_leftover_fp8_params` on synthetic fp8 checkpoints, offline on CPU."""++import json+import os+import tempfile++import pytest+from real_accelerator import (+ has_real_cuda,+) # tests/_shared, on sys.path via tests/conftest.py+import torch+from torch import nn+from safetensors.torch import save_file++import unsloth+from unsloth.models.loader_utils import (+ _dequantize_leftover_fp8_params,+ _fp8_scale_grid_dequant,+ _FP8_DTYPES,+)++_SHARD = "model-00001-of-00001.safetensors"+_FP8 = _FP8_DTYPES[0] if _FP8_DTYPES else None++pytestmark = pytest.mark.skipif(_FP8 is None, reason = "torch has no float8 dtype")+++def _write_checkpoint(path, tensors):+ save_file(tensors, os.path.join(path, _SHARD))+ with open(os.path.join(path, "model.safetensors.index.json"), "w") as f:+ json.dump({"weight_map": {name: _SHARD for name in tensors}}, f)+++class _Experts(nn.Module):+ def __init__(self, E, M, N, dtype):+ super().__init__()+ self.gate_up_proj = nn.Parameter(torch.empty(E, 2 * M, N, dtype = dtype), requires_grad = False)+ self.down_proj = nn.Parameter(torch.empty(E, N, M, dtype = dtype), requires_grad = False)+++class _Model(nn.Module):+ def __init__(+ self,+ E = 4,+ M = 8,+ N = 16,+ dtype = torch.bfloat16,+ ):+ super().__init__()+ self.q_proj = nn.Linear(N, N, bias = False, dtype = dtype)+ self.experts = _Experts(E, M, N, dtype)+ self.config = None+++def _quantize(t):+ scale = t.abs().amax().float().clamp(min = 1e-8) / 448.0+ return (t.float() / scale).to(_FP8), scale+++def _build(+ E = 4,+ M = 8,+ N = 16,+ hidden_state = True,+):+ torch.manual_seed(0)+ model = _Model(E, M, N)+ gate_up = torch.randn(E, 2 * M, N)+ down = torch.randn(E, N, M)+ q_gate_up, s_gate_up = _quantize(gate_up)+ q_down, s_down = _quantize(down)+ model.experts.gate_up_proj = nn.Parameter(q_gate_up, requires_grad = False)+ model.experts.down_proj = nn.Parameter(q_down, requires_grad = False)+ q_w, s_w = _quantize(model.q_proj.weight.detach())+ tensors = {+ "experts.gate_up_proj": q_gate_up,+ "experts.gate_up_proj_scale_inv": s_gate_up.reshape(1),+ "experts.gate_up_proj_activation_scale": torch.tensor([0.5]),+ "experts.down_proj": q_down,+ "experts.down_proj_scale_inv": s_down.reshape(1),+ "q_proj.weight": q_w,+ "q_proj.weight_scale_inv": s_w.reshape(1),+ }+ expected = {+ "experts.gate_up_proj": (q_gate_up.float() * s_gate_up).to(torch.bfloat16),+ "experts.down_proj": (q_down.float() * s_down).to(torch.bfloat16),+ }+ return model, tensors, expected+++def test_leftover_expert_stacks_are_dequantized_from_checkpoint_scale():+ model, tensors, expected = _build()+ q_proj_before = model.q_proj.weight.detach().clone()+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 2+ assert model.experts.gate_up_proj.dtype == torch.bfloat16+ assert model.experts.down_proj.dtype == torch.bfloat16+ assert torch.equal(model.experts.gate_up_proj.detach(), expected["experts.gate_up_proj"])+ assert torch.equal(model.experts.down_proj.detach(), expected["experts.down_proj"])+ assert torch.equal(model.q_proj.weight.detach(), q_proj_before)+ assert not any(p.dtype in _FP8_DTYPES for p in model.parameters())+++def test_module_with_live_scale_is_left_in_fp8():+ """A converted fp8 module keeps its own scale and its fp8 forward: never rewrite it."""+ model, tensors, _ = _build()+ model.experts.gate_up_proj_scale_inv = nn.Parameter(+ tensors["experts.gate_up_proj_scale_inv"].clone(), requires_grad = False+ )+ model.experts.down_proj_scale_inv = nn.Parameter(+ tensors["experts.down_proj_scale_inv"].clone(), requires_grad = False+ )+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 0+ assert skipped == 2+ assert model.experts.gate_up_proj.dtype == _FP8+++def test_no_fp8_params_is_a_noop_without_reading_the_checkpoint():+ model = _Model()+ done, skipped = _dequantize_leftover_fp8_params(model, "/nonexistent/path", torch.bfloat16)+ assert (done, skipped) == (0, 0)+++def test_vlm_key_remap_resolves_language_model_prefix():+ class _Inner(nn.Module):+ def __init__(self):+ super().__init__()+ self.layers = nn.ModuleList([nn.Module()])+ self.layers[0].mlp = nn.Module()+ self.layers[0].mlp.experts = _Experts(2, 4, 8, torch.bfloat16)++ class _VLM(nn.Module):+ def __init__(self):+ super().__init__()+ self.model = nn.Module()+ self.model.language_model = _Inner()++ torch.manual_seed(1)+ model = _VLM()+ experts = model.model.language_model.layers[0].mlp.experts+ gate_up = torch.randn(2, 8, 8)+ q, s = _quantize(gate_up)+ experts.gate_up_proj = nn.Parameter(q, requires_grad = False)+ tensors = {+ "language_model.model.layers.0.mlp.experts.gate_up_proj": q,+ "language_model.model.layers.0.mlp.experts.gate_up_proj_scale_inv": s.reshape(1),+ }+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, _ = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 1+ assert torch.equal(experts.gate_up_proj.detach(), (q.float() * s).to(torch.bfloat16))+++ "shape, scale_shape",+ [+ ((6, 8), ()),+ ((6, 8), (1,)),+ ((6, 8), (1, 1)),+ ((6, 8), (3, 2)),+ ((6, 8), (6,)),+ ((3, 6, 8), (3,)),+ ((3, 6, 8), (3, 1, 1)),+ ((3, 6, 8), (3, 3, 4)),+ ],+)+def test_scale_grid_dequant_matches_expanded_reference(shape, scale_shape):+ torch.manual_seed(2)+ q = torch.randn(*shape).to(_FP8)+ s = torch.rand(scale_shape) + 0.5+ out = _fp8_scale_grid_dequant(q, s, torch.float32)+ assert out is not None+ s_full = s.reshape(-1) if s.numel() == 1 else s+ if s.numel() == 1:+ ref = q.float() * s_full+ elif len(shape) == 2:+ s2 = s_full.view(-1, 1) if s_full.ndim == 1 else s_full+ p, qq = s2.shape+ ref = (+ s2.repeat_interleave(shape[0] // p, 0).repeat_interleave(shape[1] // qq, 1) * q.float()+ )+ else:+ s3 = s_full.view(shape[0], 1, 1) if s_full.ndim == 1 else s_full+ p, qq = s3.shape[1], s3.shape[2]+ ref = (+ s3.repeat_interleave(shape[1] // p, 1).repeat_interleave(shape[2] // qq, 2) * q.float()+ )+ assert torch.equal(out, ref)+++def test_scale_grid_that_does_not_tile_is_refused():+ q = torch.randn(6, 8).to(_FP8)+ assert _fp8_scale_grid_dequant(q, torch.rand(4, 2), torch.float32) is None+ assert (+ _fp8_scale_grid_dequant(torch.randn(3, 6, 8).to(_FP8), torch.rand(2, 1, 1), torch.float32)+ is None+ )+++# has_real_cuda(): another test spoofs torch.cuda.is_available() process-wide.[email protected](not has_real_cuda(), reason = "needs CUDA")+def test_out_of_memory_on_the_device_is_finished_through_the_cpu(monkeypatch):+ from unsloth.models import loader_utils++ model, tensors, expected = _build()+ model = model.to("cuda")+ real = loader_utils._fp8_scale_grid_dequant+ calls = {"cuda": 0, "cpu": 0}++ def flaky(quantized, scale, out_dtype):+ calls[quantized.device.type] += 1+ if quantized.device.type == "cuda":+ raise torch.OutOfMemoryError("CUDA out of memory (simulated)")+ return real(quantized, scale, out_dtype)++ monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = loader_utils._dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 2+ assert calls == {"cuda": 2, "cpu": 2}+ assert model.experts.gate_up_proj.device.type == "cuda"+ assert model.experts.gate_up_proj.dtype == torch.bfloat16+ assert torch.equal(model.experts.gate_up_proj.detach().cpu(), expected["experts.gate_up_proj"])+ assert torch.equal(model.experts.down_proj.detach().cpu(), expected["experts.down_proj"])+++def test_standard_weight_and_weight_scale_inv_pair_is_dequantized():+ model, tensors, expected = _build()+ q_w = tensors["q_proj.weight"]+ s_w = tensors["q_proj.weight_scale_inv"]+ model.q_proj.weight = nn.Parameter(q_w, requires_grad = False)+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 3+ assert model.q_proj.weight.dtype == torch.bfloat16+ assert torch.equal(model.q_proj.weight.detach(), (q_w.float() * s_w).to(torch.bfloat16))+ assert not any(p.dtype in _FP8_DTYPES for p in model.parameters())+++def test_no_reference_to_the_fp8_parameter_survives_into_the_cpu_pass(monkeypatch):+ import gc+ import weakref+ from unsloth.models import loader_utils++ model, tensors, expected = _build()+ original = weakref.ref(model.experts.gate_up_proj)+ real = loader_utils._fp8_scale_grid_dequant+ state = {"raised": False, "alive_in_pass_2": None}++ def flaky(quantized, scale, out_dtype):+ if not state["raised"]:+ state["raised"] = True+ raise torch.OutOfMemoryError("out of memory (simulated)")+ if state["alive_in_pass_2"] is None:+ gc.collect()+ state["alive_in_pass_2"] = original() is not None+ return real(quantized, scale, out_dtype)++ monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = loader_utils._dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 2+ assert state["alive_in_pass_2"] is False+ assert torch.equal(model.experts.gate_up_proj.detach(), expected["experts.gate_up_proj"])+++def test_disk_offloaded_leftover_is_refused_with_an_instruction():+ model, tensors, expected = _build()+ model.experts.gate_up_proj = nn.Parameter(+ torch.empty_like(model.experts.gate_up_proj, device = "meta"), requires_grad = False+ )+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ with pytest.raises(RuntimeError, match = "offloaded to disk"):+ _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+++def test_activation_scale_survives_on_a_module_that_kept_its_fp8_weight():+ model, tensors, expected = _build()+ # q_proj keeps its own scale, so it stays fp8 and must keep its activation scale.+ model.q_proj.weight = nn.Parameter(tensors["q_proj.weight"], requires_grad = False)+ model.q_proj.weight_scale_inv = nn.Parameter(+ tensors["q_proj.weight_scale_inv"], requires_grad = False+ )+ model.q_proj.register_buffer("input_activation_scale", torch.ones(1))+ model.experts.register_buffer("input_activation_scale", torch.ones(1))+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 2 and skipped >= 1+ assert model.q_proj.weight.dtype in _FP8_DTYPES+ assert hasattr(model.q_proj, "input_activation_scale")+ assert not hasattr(model.experts, "input_activation_scale")+++def test_per_tensor_scale_on_a_3d_stack_is_chunked(monkeypatch):+ from unsloth.models import loader_utils++ E, M, N = 8, 32, 32+ # Budget of exactly two experts, so a chunked pass is visibly different from one that is not.+ monkeypatch.setattr(loader_utils, "_FP8_LEFTOVER_MAX_CHUNK", 2 * M * N)+ torch.manual_seed(4)+ q = torch.randn(E, M, N).to(_FP8)+ seen = []+ real_to = torch.Tensor.to++ def spy(self, *args, **kwargs):+ if args and args[0] is torch.float32 and self.dtype == _FP8:+ seen.append(self.shape[0])+ return real_to(self, *args, **kwargs)++ monkeypatch.setattr(torch.Tensor, "to", spy, raising = True)+ out = loader_utils._fp8_scale_grid_dequant(q, torch.tensor([0.25]), torch.bfloat16)+ monkeypatch.undo()+ assert torch.equal(out, (q.float() * 0.25).to(torch.bfloat16))+ assert seen, "no fp32 cast observed"+ assert max(seen) == 2, (seen, E)+++def test_activation_scale_cleanup_is_per_attribute():+ model, tensors, expected = _build()+ model.experts.down_proj_scale_inv = nn.Parameter(+ tensors["experts.down_proj_scale_inv"], requires_grad = False+ )+ model.experts.register_buffer("gate_up_proj_activation_scale", torch.ones(1))+ model.experts.register_buffer("down_proj_activation_scale", torch.ones(1))+ model.experts.register_buffer("input_activation_scale", torch.ones(1))+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert model.experts.gate_up_proj.dtype == torch.bfloat16+ assert model.experts.down_proj.dtype in _FP8_DTYPES+ assert not hasattr(model.experts, "gate_up_proj_activation_scale")+ assert hasattr(model.experts, "down_proj_activation_scale")+ assert hasattr(model.experts, "input_activation_scale")+++def test_a_trainable_fp8_parameter_stays_trainable_after_dequantization():+ model, tensors, expected = _build()+ model.experts.gate_up_proj.requires_grad_(True)+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert model.experts.gate_up_proj.dtype == torch.bfloat16+ assert model.experts.gate_up_proj.requires_grad is True+ assert model.experts.down_proj.requires_grad is False+++def test_a_transposed_block_grid_is_turned_around_by_the_configured_block_size():+ from unsloth.models.loader_utils import _fp8_scale_grid_dequant, _orient_block_scale++ raw = (torch.arange(8, dtype = torch.float32).reshape(4, 2) + 1).to(_FP8_DTYPES[0])+ scale = torch.tensor([[2.0], [4.0]]) # canonical (2, 1): rows blocks x col blocks+ expected = _fp8_scale_grid_dequant(raw, scale, torch.float32, block_size = (2, 2))+ stored_transposed = scale.t().contiguous() # (1, 2)+ assert torch.equal(_orient_block_scale(stored_transposed, 4, 2, (2, 2)), scale)+ out = _fp8_scale_grid_dequant(raw, stored_transposed, torch.float32, block_size = (2, 2))+ assert torch.equal(out, expected)+ assert not torch.equal(_fp8_scale_grid_dequant(raw, stored_transposed, torch.float32), expected)+++def test_generic_out_of_memory_runtime_errors_defer_to_the_cpu(monkeypatch):+ from unsloth.models import loader_utils++ model, tensors, expected = _build()+ calls = {"n": 0}+ original = loader_utils._fp8_scale_grid_dequant++ def flaky(+ quantized,+ scale,+ dtype,+ block_size = None,+ ):+ calls["n"] += 1+ if calls["n"] == 1:+ raise RuntimeError("HIP out of memory. Tried to allocate 2 GiB")+ return original(quantized, scale, dtype, block_size = block_size)++ monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, tensors)+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)+ assert done == 2+ assert model.experts.gate_up_proj.dtype == torch.bfloat16+++def test_a_variant_index_uses_transformers_naming(tmp_path):+ import json+ from unsloth.models.loader_utils import _load_fp8_weight_map++ (tmp_path / "model.safetensors.index.fp8.json").write_text(+ json.dumps({"weight_map": {"a.weight": "model-fp8-00001.safetensors"}})+ )+ assert _load_fp8_weight_map(str(tmp_path), True, None, variant = "fp8") == {+ "a.weight": "model-fp8-00001.safetensors"+ }diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex fb5464c36c5..addce0e59d6 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -1205,8 +1205,9 @@ def _load_fp8_weight_map(revision = None,subfolder = None,cache_dir = None,+ variant = None,):- """The checkpoint's tensor->file map, using the same snapshot the load used. Prefers the sharded `model.safetensors.index.json`, falling back to a single `model.safetensors` so unsharded checkpoints are covered too."""+ """Tensor->file map from the load's snapshot; sharded index, else single `model.safetensors`."""def _local_path(filename):return (@@ -1227,8 +1228,10 @@ def _load_fp8_weight_map(token = token,)- index_file = "model.safetensors.index.json"- single_file = "model.safetensors"+ index_file = (+ f"model.safetensors.index.{variant}.json" if variant else "model.safetensors.index.json"+ )+ single_file = f"model.{variant}.safetensors" if variant else "model.safetensors"is_local = os.path.isdir(model_name)if is_local and os.path.exists(_local_path(index_file)):@@ -1547,6 +1550,345 @@ def _restore_dropped_fp8_scales(return (0, 0)+_FP8_SCALE_SUFFIXES = (".weight_scale_inv", "_scale_inv", ".weight_scale", "_scale")+_FP8_LEFTOVER_MAX_CHUNK = 1 << 26+++def _empty_device_cache(device):+ try:+ backend = getattr(torch, getattr(device, "type", "cuda"), None)+ empty = getattr(backend, "empty_cache", None)+ if callable(empty):+ empty()+ except Exception:+ pass+++def _orient_block_scale(scale, rows, cols, block_size):+ """Fix a grid stored `(in_blocks, out_blocks)`; block size decides when both orientations tile."""+ if block_size is None or scale.ndim < 2:+ return scale+ try:+ bm, bn = int(block_size[0]), int(block_size[1])+ except Exception:+ return scale+ if bm <= 0 or bn <= 0 or rows % bm or cols % bn:+ return scale+ canonical = (rows // bm, cols // bn)+ grid = tuple(scale.shape[-2:])+ if grid == canonical:+ return scale+ if grid == canonical[::-1] and canonical[0] != canonical[1]:+ return scale.transpose(-1, -2).contiguous()+ return scale+++def _fp8_scale_grid_dequant(+ quantized,+ scale,+ out_dtype,+ block_size = None,+):+ """Returns None when the scale grid does not tile the weight."""+ # MXFP8 stores E8M0 exponents as uint8: the scale is 2 ** (byte - 127).+ if scale.dtype == torch.uint8:+ scale = (scale.to(torch.float32) - 127.0).exp2()+ else:+ scale = scale.to(torch.float32)+ q_shape = tuple(quantized.shape)+ if scale.numel() == 1:+ if quantized.ndim == 3:+ scale = scale.reshape(()).expand(q_shape[0], 1, 1)+ else:+ return (quantized.to(torch.float32) * scale.reshape(())).to(out_dtype)+ if quantized.ndim == 2:+ if scale.ndim != 2:+ if scale.ndim == 1 and scale.numel() == q_shape[0]:+ scale = scale.view(-1, 1)+ else:+ return None+ rows, cols = q_shape+ scale = _orient_block_scale(scale, rows, cols, block_size)+ p, q = scale.shape+ if rows % p or cols % q:+ return None+ bm, bn = rows // p, cols // q+ out = quantized.to(torch.float32).view(p, bm, q, bn) * scale[:, None, :, None]+ return out.reshape(rows, cols).to(out_dtype)+ if quantized.ndim == 3:+ E, rows, cols = q_shape+ if scale.ndim == 1 and scale.numel() == E:+ scale = scale.view(E, 1, 1)+ elif scale.ndim == 2 and scale.shape[0] == E and scale.shape[1] == 1:+ scale = scale.view(E, 1, 1)+ if scale.ndim != 3 or scale.shape[0] != E:+ return None+ scale = _orient_block_scale(scale, rows, cols, block_size)+ p, q = scale.shape[1], scale.shape[2]+ if rows % p or cols % q:+ return None+ bm, bn = rows // p, cols // q+ out = torch.empty((E, rows, cols), dtype = out_dtype, device = quantized.device)+ step = max(1, _FP8_LEFTOVER_MAX_CHUNK // max(1, rows * cols))+ for start in range(0, E, step):+ stop = min(E, start + step)+ chunk = quantized[start:stop].to(torch.float32).view(stop - start, p, bm, q, bn)+ out[start:stop] = (+ (chunk * scale[start:stop, :, None, :, None])+ .reshape(stop - start, rows, cols)+ .to(out_dtype)+ )+ return out+ return None+++class FP8LeftoverOffloadedError(RuntimeError):+ """A leftover fp8 tensor is disk-offloaded, so the 16bit load cannot be finished in place."""+++def _restore_parked_fp8(module, attr, device):+ try:+ stranded = module._parameters.get(attr)+ if (+ isinstance(stranded, torch.Tensor)+ and stranded.device != device+ and stranded.device.type == "cpu"+ ):+ module._parameters[attr] = torch.nn.Parameter(+ stranded.data.to(device), requires_grad = bool(stranded.requires_grad)+ )+ except Exception:+ pass+++def _dequantize_leftover_fp8_params(+ model,+ model_name,+ dtype,+ *,+ local_files_only = False,+ token = None,+ revision = None,+ subfolder = None,+ cache_dir = None,+ variant = None,+):+ """Dequantize fp8 params transformers left on a 16bit load. Returns (dequantized, skipped)."""+ try:+ if not _FP8_DTYPES:+ return (0, 0)+ leftover = [+ name+ for name, param in model.named_parameters()+ if param.dtype in _FP8_DTYPES and param.ndim in (2, 3)+ ]+ block_size = None+ try:+ _qc = getattr(getattr(model, "config", None), "quantization_config", None)+ _bs = (+ _qc.get("weight_block_size")+ if isinstance(_qc, dict)+ else getattr(_qc, "weight_block_size", None)+ )+ if _bs is not None and len(_bs) == 2:+ block_size = (int(_bs[0]), int(_bs[1]))+ except Exception:+ block_size = None+ _grid_kwargs = {"block_size": block_size} if block_size is not None else {}+ if not leftover:+ return (0, 0)+ weight_map = _load_fp8_weight_map(+ model_name,+ local_files_only,+ token,+ revision,+ subfolder,+ cache_dir,+ variant = variant,+ )+ if not weight_map:+ return (0, 0)+ scale_by_weight_key = {}+ for key, shard in weight_map.items():+ for suffix in _FP8_SCALE_SUFFIXES:+ if key.endswith(suffix):+ weight_key = key[: -len(suffix)]+ # `layer.weight_scale_inv` belongs to `layer.weight`, not to `layer`.+ if suffix.startswith(".weight"):+ weight_key += ".weight"+ scale_by_weight_key[weight_key] = (key, shard)+ break+ if not scale_by_weight_key:+ return (0, 0)+ module_by_name = dict(model.named_modules())+ # Names only: holding the fp8 Parameter would defeat the OOM fallback.+ params_by_module = {}+ for name in leftover:+ module_name, _, attr = name.rpartition(".")+ params_by_module.setdefault(module_name, []).append(attr)++ target_of_ckpt = {}+ for weight_key in scale_by_weight_key:+ module_part, _, attr = weight_key.rpartition(".")+ if attr == "weight":+ ckpt_module, ckpt_attr = module_part, "weight"+ else:+ ckpt_module, ckpt_attr = module_part, attr+ module = _match_fp8_module(module_by_name, ckpt_module)+ if module is None:+ continue+ live_name = None+ for candidate, mod in module_by_name.items():+ if mod is module:+ live_name = candidate+ break+ if live_name is None or live_name not in params_by_module:+ continue+ target_of_ckpt[weight_key] = (module, live_name, ckpt_attr)++ dequantized = 0+ skipped = 0+ failed = 0+ last_error = None+ shard_cache = {}++ def _scale_for(weight_key, device):+ scale_key, shard = scale_by_weight_key[weight_key]+ if shard not in shard_cache:+ from safetensors import safe_open+ shard_path = _resolve_fp8_shard(+ model_name, shard, local_files_only, token, revision, subfolder, cache_dir+ )+ shard_cache[shard] = safe_open(shard_path, framework = "pt")+ return shard_cache[shard].get_tensor(scale_key).to(device)++ def _is_oom(error):+ # Some backends raise a plain RuntimeError for OOM.+ return (+ hasattr(torch, "OutOfMemoryError") and isinstance(error, torch.OutOfMemoryError)+ ) or (isinstance(error, RuntimeError) and "out of memory" in str(error).lower())++ deferred = []+ converted = set()+ for weight_key, (module, live_name, attr) in target_of_ckpt.items():+ param = getattr(module, attr, None)+ if not isinstance(param, torch.Tensor) or param.dtype not in _FP8_DTYPES:+ continue+ trainable = bool(getattr(param, "requires_grad", False))+ # A module with its own scale runs the fp8 forward.+ if any(+ isinstance(getattr(module, s, None), torch.Tensor)+ for s in (attr + "_scale_inv", attr + "_scale", "weight_scale_inv", "weight_scale")+ ):+ skipped += 1+ continue+ if param.device.type == "meta":+ # Disk-offloaded: the hook would restore raw fp8 bytes at forward time.+ raise FP8LeftoverOffloadedError(+ f"Unsloth: `{weight_key}` is an fp8 tensor transformers left quantized on a 16bit "+ "load, and it is offloaded to disk, so it cannot be dequantized in place. Load with "+ "enough GPU or CPU memory to keep the model resident (no disk offload), or use "+ "`load_in_4bit = True`."+ )+ try:+ scale = _scale_for(weight_key, param.device)+ with torch.no_grad():+ out = _fp8_scale_grid_dequant(param.data, scale, dtype, **_grid_kwargs)+ if out is None:+ failed += 1+ last_error = f"{weight_key}: scale {tuple(scale.shape)} does not tile {tuple(param.shape)}"+ continue+ module._parameters[attr] = torch.nn.Parameter(out, requires_grad = trainable)+ converted.add((module, attr))+ dequantized += 1+ except Exception as e:+ if _is_oom(e):+ device = param.device+ quantized = param.data.to("cpu")+ module._parameters[attr] = torch.nn.Parameter(+ quantized, requires_grad = trainable+ )+ del param+ deferred.append((weight_key, module, attr, device))+ _empty_device_cache(device)+ continue+ failed += 1+ last_error = f"{weight_key}: {type(e).__name__}: {e}"+ continue++ for weight_key, module, attr, device in deferred:+ try:+ param = getattr(module, attr)+ trainable = bool(getattr(param, "requires_grad", False))+ scale = _scale_for(weight_key, "cpu")+ with torch.no_grad():+ out = _fp8_scale_grid_dequant(param.data, scale, dtype, **_grid_kwargs)+ if out is None:+ failed += 1+ last_error = f"{weight_key}: scale {tuple(scale.shape)} does not tile {tuple(param.shape)}"+ _restore_parked_fp8(module, attr, device)+ continue+ _empty_device_cache(device)+ module._parameters[attr] = torch.nn.Parameter(+ out.to(device), requires_grad = trainable+ )+ del param, out+ converted.add((module, attr))+ dequantized += 1+ except Exception as e:+ failed += 1+ last_error = f"{weight_key}: {type(e).__name__}: {e}"+ _restore_parked_fp8(module, attr, device)+ continue+ for shard_file in shard_cache.values():+ close = getattr(shard_file, "__exit__", None)+ if close is not None:+ try:+ close(None, None, None)+ except Exception:+ pass++ if dequantized > 0:+ by_module = {}+ for module, attr in converted:+ by_module.setdefault(module, set()).add(attr)+ for module, attrs in by_module.items():+ still_fp8 = [+ n+ for n, p in module._parameters.items()+ if isinstance(p, torch.Tensor) and p.dtype in _FP8_DTYPES+ ]+ for stale in [+ n+ for n in list(module._parameters) + list(module._buffers)+ if n.endswith("activation_scale")+ ]:+ owner = next(+ (a for a in list(attrs) + still_fp8 if stale.startswith(a + "_")), None+ )+ if owner is None and still_fp8:+ continue+ if owner is not None and owner not in attrs:+ continue+ if stale in module._parameters:+ del module._parameters[stale]+ else:+ del module._buffers[stale]+ print(+ f"Unsloth: Dequantized {dequantized} FP8 tensor(s) transformers left quantized on a 16bit load."+ )+ if failed > 0:+ print(+ f"Unsloth: {failed} FP8 tensor(s) could not be dequantized on the 16bit load and stay fp8 "+ f"(last: {last_error})."+ )+ return (dequantized, skipped)+ except FP8LeftoverOffloadedError:+ raise+ except Exception:+ return (0, 0)++_BNB_QUANTIZED_TYPES = ("Params4bit", "Int8Params", "Linear4bit", "Linear8bitLt")diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex cc222675200..84371b0b6eb 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -97,6 +97,7 @@ from .loader_utils import (planner_kwargs_with_max_memory,_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,+ _dequantize_leftover_fp8_params,_restore_dropped_fp8_scales,planner_class_mismatch_reason,planner_model_class,@@ -2075,6 +2076,18 @@ class FastBaseModel:variant = kwargs.get("variant"),dtype = torch_dtype,)+ if load_in_16bit and not load_in_4bit and not load_in_8bit:+ _dequantize_leftover_fp8_params(+ model,+ model_name,+ torch_dtype,+ local_files_only = local_files_only,+ token = token,+ revision = kwargs.get("revision"),+ subfolder = kwargs.get("subfolder"),+ cache_dir = kwargs.get("cache_dir"),+ variant = kwargs.get("variant"),+ )if hasattr(model, "generate"):model.fast_generate = make_fast_generate_wrapper(model.generate)model.fast_generate_batches = error_out_no_vllm