Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Finish a 16bit load of a static per-tensor fp8 checkpoint (Mistral-Small-4) (#11531)

Daniel Hanbda4023c0182
diff --git a/tests/test_fp8_leftover_dequant_16bit.py b/tests/test_fp8_leftover_dequant_16bit.py
new file mode 100644
index 00000000000..68a0f9e6565
--- /dev/null
+++ b/tests/test_fp8_leftover_dequant_16bit.py
@@ -0,0 +1,422 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""`_dequantize_leftover_fp8_params` on synthetic fp8 checkpoints, offline on CPU."""
+
+import json
+import os
+import tempfile
+
+import pytest
+from real_accelerator import (
+ has_real_cuda,
+) # tests/_shared, on sys.path via tests/conftest.py
+import torch
+from torch import nn
+from safetensors.torch import save_file
+
+import unsloth
+from unsloth.models.loader_utils import (
+ _dequantize_leftover_fp8_params,
+ _fp8_scale_grid_dequant,
+ _FP8_DTYPES,
+)
+
+_SHARD = "model-00001-of-00001.safetensors"
+_FP8 = _FP8_DTYPES[0] if _FP8_DTYPES else None
+
+pytestmark = pytest.mark.skipif(_FP8 is None, reason = "torch has no float8 dtype")
+
+
+def _write_checkpoint(path, tensors):
+ save_file(tensors, os.path.join(path, _SHARD))
+ with open(os.path.join(path, "model.safetensors.index.json"), "w") as f:
+ json.dump({"weight_map": {name: _SHARD for name in tensors}}, f)
+
+
+class _Experts(nn.Module):
+ def __init__(self, E, M, N, dtype):
+ super().__init__()
+ self.gate_up_proj = nn.Parameter(torch.empty(E, 2 * M, N, dtype = dtype), requires_grad = False)
+ self.down_proj = nn.Parameter(torch.empty(E, N, M, dtype = dtype), requires_grad = False)
+
+
+class _Model(nn.Module):
+ def __init__(
+ self,
+ E = 4,
+ M = 8,
+ N = 16,
+ dtype = torch.bfloat16,
+ ):
+ super().__init__()
+ self.q_proj = nn.Linear(N, N, bias = False, dtype = dtype)
+ self.experts = _Experts(E, M, N, dtype)
+ self.config = None
+
+
+def _quantize(t):
+ scale = t.abs().amax().float().clamp(min = 1e-8) / 448.0
+ return (t.float() / scale).to(_FP8), scale
+
+
+def _build(
+ E = 4,
+ M = 8,
+ N = 16,
+ hidden_state = True,
+):
+ torch.manual_seed(0)
+ model = _Model(E, M, N)
+ gate_up = torch.randn(E, 2 * M, N)
+ down = torch.randn(E, N, M)
+ q_gate_up, s_gate_up = _quantize(gate_up)
+ q_down, s_down = _quantize(down)
+ model.experts.gate_up_proj = nn.Parameter(q_gate_up, requires_grad = False)
+ model.experts.down_proj = nn.Parameter(q_down, requires_grad = False)
+ q_w, s_w = _quantize(model.q_proj.weight.detach())
+ tensors = {
+ "experts.gate_up_proj": q_gate_up,
+ "experts.gate_up_proj_scale_inv": s_gate_up.reshape(1),
+ "experts.gate_up_proj_activation_scale": torch.tensor([0.5]),
+ "experts.down_proj": q_down,
+ "experts.down_proj_scale_inv": s_down.reshape(1),
+ "q_proj.weight": q_w,
+ "q_proj.weight_scale_inv": s_w.reshape(1),
+ }
+ expected = {
+ "experts.gate_up_proj": (q_gate_up.float() * s_gate_up).to(torch.bfloat16),
+ "experts.down_proj": (q_down.float() * s_down).to(torch.bfloat16),
+ }
+ return model, tensors, expected
+
+
+def test_leftover_expert_stacks_are_dequantized_from_checkpoint_scale():
+ model, tensors, expected = _build()
+ q_proj_before = model.q_proj.weight.detach().clone()
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 2
+ assert model.experts.gate_up_proj.dtype == torch.bfloat16
+ assert model.experts.down_proj.dtype == torch.bfloat16
+ assert torch.equal(model.experts.gate_up_proj.detach(), expected["experts.gate_up_proj"])
+ assert torch.equal(model.experts.down_proj.detach(), expected["experts.down_proj"])
+ assert torch.equal(model.q_proj.weight.detach(), q_proj_before)
+ assert not any(p.dtype in _FP8_DTYPES for p in model.parameters())
+
+
+def test_module_with_live_scale_is_left_in_fp8():
+ """A converted fp8 module keeps its own scale and its fp8 forward: never rewrite it."""
+ model, tensors, _ = _build()
+ model.experts.gate_up_proj_scale_inv = nn.Parameter(
+ tensors["experts.gate_up_proj_scale_inv"].clone(), requires_grad = False
+ )
+ model.experts.down_proj_scale_inv = nn.Parameter(
+ tensors["experts.down_proj_scale_inv"].clone(), requires_grad = False
+ )
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 0
+ assert skipped == 2
+ assert model.experts.gate_up_proj.dtype == _FP8
+
+
+def test_no_fp8_params_is_a_noop_without_reading_the_checkpoint():
+ model = _Model()
+ done, skipped = _dequantize_leftover_fp8_params(model, "/nonexistent/path", torch.bfloat16)
+ assert (done, skipped) == (0, 0)
+
+
+def test_vlm_key_remap_resolves_language_model_prefix():
+ class _Inner(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.layers = nn.ModuleList([nn.Module()])
+ self.layers[0].mlp = nn.Module()
+ self.layers[0].mlp.experts = _Experts(2, 4, 8, torch.bfloat16)
+
+ class _VLM(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.model = nn.Module()
+ self.model.language_model = _Inner()
+
+ torch.manual_seed(1)
+ model = _VLM()
+ experts = model.model.language_model.layers[0].mlp.experts
+ gate_up = torch.randn(2, 8, 8)
+ q, s = _quantize(gate_up)
+ experts.gate_up_proj = nn.Parameter(q, requires_grad = False)
+ tensors = {
+ "language_model.model.layers.0.mlp.experts.gate_up_proj": q,
+ "language_model.model.layers.0.mlp.experts.gate_up_proj_scale_inv": s.reshape(1),
+ }
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, _ = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 1
+ assert torch.equal(experts.gate_up_proj.detach(), (q.float() * s).to(torch.bfloat16))
+
+
+ "shape, scale_shape",
+ [
+ ((6, 8), ()),
+ ((6, 8), (1,)),
+ ((6, 8), (1, 1)),
+ ((6, 8), (3, 2)),
+ ((6, 8), (6,)),
+ ((3, 6, 8), (3,)),
+ ((3, 6, 8), (3, 1, 1)),
+ ((3, 6, 8), (3, 3, 4)),
+ ],
+)
+def test_scale_grid_dequant_matches_expanded_reference(shape, scale_shape):
+ torch.manual_seed(2)
+ q = torch.randn(*shape).to(_FP8)
+ s = torch.rand(scale_shape) + 0.5
+ out = _fp8_scale_grid_dequant(q, s, torch.float32)
+ assert out is not None
+ s_full = s.reshape(-1) if s.numel() == 1 else s
+ if s.numel() == 1:
+ ref = q.float() * s_full
+ elif len(shape) == 2:
+ s2 = s_full.view(-1, 1) if s_full.ndim == 1 else s_full
+ p, qq = s2.shape
+ ref = (
+ s2.repeat_interleave(shape[0] // p, 0).repeat_interleave(shape[1] // qq, 1) * q.float()
+ )
+ else:
+ s3 = s_full.view(shape[0], 1, 1) if s_full.ndim == 1 else s_full
+ p, qq = s3.shape[1], s3.shape[2]
+ ref = (
+ s3.repeat_interleave(shape[1] // p, 1).repeat_interleave(shape[2] // qq, 2) * q.float()
+ )
+ assert torch.equal(out, ref)
+
+
+def test_scale_grid_that_does_not_tile_is_refused():
+ q = torch.randn(6, 8).to(_FP8)
+ assert _fp8_scale_grid_dequant(q, torch.rand(4, 2), torch.float32) is None
+ assert (
+ _fp8_scale_grid_dequant(torch.randn(3, 6, 8).to(_FP8), torch.rand(2, 1, 1), torch.float32)
+ is None
+ )
+
+
+# has_real_cuda(): another test spoofs torch.cuda.is_available() process-wide.
[email protected](not has_real_cuda(), reason = "needs CUDA")
+def test_out_of_memory_on_the_device_is_finished_through_the_cpu(monkeypatch):
+ from unsloth.models import loader_utils
+
+ model, tensors, expected = _build()
+ model = model.to("cuda")
+ real = loader_utils._fp8_scale_grid_dequant
+ calls = {"cuda": 0, "cpu": 0}
+
+ def flaky(quantized, scale, out_dtype):
+ calls[quantized.device.type] += 1
+ if quantized.device.type == "cuda":
+ raise torch.OutOfMemoryError("CUDA out of memory (simulated)")
+ return real(quantized, scale, out_dtype)
+
+ monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = loader_utils._dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 2
+ assert calls == {"cuda": 2, "cpu": 2}
+ assert model.experts.gate_up_proj.device.type == "cuda"
+ assert model.experts.gate_up_proj.dtype == torch.bfloat16
+ assert torch.equal(model.experts.gate_up_proj.detach().cpu(), expected["experts.gate_up_proj"])
+ assert torch.equal(model.experts.down_proj.detach().cpu(), expected["experts.down_proj"])
+
+
+def test_standard_weight_and_weight_scale_inv_pair_is_dequantized():
+ model, tensors, expected = _build()
+ q_w = tensors["q_proj.weight"]
+ s_w = tensors["q_proj.weight_scale_inv"]
+ model.q_proj.weight = nn.Parameter(q_w, requires_grad = False)
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 3
+ assert model.q_proj.weight.dtype == torch.bfloat16
+ assert torch.equal(model.q_proj.weight.detach(), (q_w.float() * s_w).to(torch.bfloat16))
+ assert not any(p.dtype in _FP8_DTYPES for p in model.parameters())
+
+
+def test_no_reference_to_the_fp8_parameter_survives_into_the_cpu_pass(monkeypatch):
+ import gc
+ import weakref
+ from unsloth.models import loader_utils
+
+ model, tensors, expected = _build()
+ original = weakref.ref(model.experts.gate_up_proj)
+ real = loader_utils._fp8_scale_grid_dequant
+ state = {"raised": False, "alive_in_pass_2": None}
+
+ def flaky(quantized, scale, out_dtype):
+ if not state["raised"]:
+ state["raised"] = True
+ raise torch.OutOfMemoryError("out of memory (simulated)")
+ if state["alive_in_pass_2"] is None:
+ gc.collect()
+ state["alive_in_pass_2"] = original() is not None
+ return real(quantized, scale, out_dtype)
+
+ monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = loader_utils._dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 2
+ assert state["alive_in_pass_2"] is False
+ assert torch.equal(model.experts.gate_up_proj.detach(), expected["experts.gate_up_proj"])
+
+
+def test_disk_offloaded_leftover_is_refused_with_an_instruction():
+ model, tensors, expected = _build()
+ model.experts.gate_up_proj = nn.Parameter(
+ torch.empty_like(model.experts.gate_up_proj, device = "meta"), requires_grad = False
+ )
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ with pytest.raises(RuntimeError, match = "offloaded to disk"):
+ _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+
+
+def test_activation_scale_survives_on_a_module_that_kept_its_fp8_weight():
+ model, tensors, expected = _build()
+ # q_proj keeps its own scale, so it stays fp8 and must keep its activation scale.
+ model.q_proj.weight = nn.Parameter(tensors["q_proj.weight"], requires_grad = False)
+ model.q_proj.weight_scale_inv = nn.Parameter(
+ tensors["q_proj.weight_scale_inv"], requires_grad = False
+ )
+ model.q_proj.register_buffer("input_activation_scale", torch.ones(1))
+ model.experts.register_buffer("input_activation_scale", torch.ones(1))
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 2 and skipped >= 1
+ assert model.q_proj.weight.dtype in _FP8_DTYPES
+ assert hasattr(model.q_proj, "input_activation_scale")
+ assert not hasattr(model.experts, "input_activation_scale")
+
+
+def test_per_tensor_scale_on_a_3d_stack_is_chunked(monkeypatch):
+ from unsloth.models import loader_utils
+
+ E, M, N = 8, 32, 32
+ # Budget of exactly two experts, so a chunked pass is visibly different from one that is not.
+ monkeypatch.setattr(loader_utils, "_FP8_LEFTOVER_MAX_CHUNK", 2 * M * N)
+ torch.manual_seed(4)
+ q = torch.randn(E, M, N).to(_FP8)
+ seen = []
+ real_to = torch.Tensor.to
+
+ def spy(self, *args, **kwargs):
+ if args and args[0] is torch.float32 and self.dtype == _FP8:
+ seen.append(self.shape[0])
+ return real_to(self, *args, **kwargs)
+
+ monkeypatch.setattr(torch.Tensor, "to", spy, raising = True)
+ out = loader_utils._fp8_scale_grid_dequant(q, torch.tensor([0.25]), torch.bfloat16)
+ monkeypatch.undo()
+ assert torch.equal(out, (q.float() * 0.25).to(torch.bfloat16))
+ assert seen, "no fp32 cast observed"
+ assert max(seen) == 2, (seen, E)
+
+
+def test_activation_scale_cleanup_is_per_attribute():
+ model, tensors, expected = _build()
+ model.experts.down_proj_scale_inv = nn.Parameter(
+ tensors["experts.down_proj_scale_inv"], requires_grad = False
+ )
+ model.experts.register_buffer("gate_up_proj_activation_scale", torch.ones(1))
+ model.experts.register_buffer("down_proj_activation_scale", torch.ones(1))
+ model.experts.register_buffer("input_activation_scale", torch.ones(1))
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert model.experts.gate_up_proj.dtype == torch.bfloat16
+ assert model.experts.down_proj.dtype in _FP8_DTYPES
+ assert not hasattr(model.experts, "gate_up_proj_activation_scale")
+ assert hasattr(model.experts, "down_proj_activation_scale")
+ assert hasattr(model.experts, "input_activation_scale")
+
+
+def test_a_trainable_fp8_parameter_stays_trainable_after_dequantization():
+ model, tensors, expected = _build()
+ model.experts.gate_up_proj.requires_grad_(True)
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert model.experts.gate_up_proj.dtype == torch.bfloat16
+ assert model.experts.gate_up_proj.requires_grad is True
+ assert model.experts.down_proj.requires_grad is False
+
+
+def test_a_transposed_block_grid_is_turned_around_by_the_configured_block_size():
+ from unsloth.models.loader_utils import _fp8_scale_grid_dequant, _orient_block_scale
+
+ raw = (torch.arange(8, dtype = torch.float32).reshape(4, 2) + 1).to(_FP8_DTYPES[0])
+ scale = torch.tensor([[2.0], [4.0]]) # canonical (2, 1): rows blocks x col blocks
+ expected = _fp8_scale_grid_dequant(raw, scale, torch.float32, block_size = (2, 2))
+ stored_transposed = scale.t().contiguous() # (1, 2)
+ assert torch.equal(_orient_block_scale(stored_transposed, 4, 2, (2, 2)), scale)
+ out = _fp8_scale_grid_dequant(raw, stored_transposed, torch.float32, block_size = (2, 2))
+ assert torch.equal(out, expected)
+ assert not torch.equal(_fp8_scale_grid_dequant(raw, stored_transposed, torch.float32), expected)
+
+
+def test_generic_out_of_memory_runtime_errors_defer_to_the_cpu(monkeypatch):
+ from unsloth.models import loader_utils
+
+ model, tensors, expected = _build()
+ calls = {"n": 0}
+ original = loader_utils._fp8_scale_grid_dequant
+
+ def flaky(
+ quantized,
+ scale,
+ dtype,
+ block_size = None,
+ ):
+ calls["n"] += 1
+ if calls["n"] == 1:
+ raise RuntimeError("HIP out of memory. Tried to allocate 2 GiB")
+ return original(quantized, scale, dtype, block_size = block_size)
+
+ monkeypatch.setattr(loader_utils, "_fp8_scale_grid_dequant", flaky)
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, tensors)
+ done, skipped = _dequantize_leftover_fp8_params(model, d, torch.bfloat16)
+ assert done == 2
+ assert model.experts.gate_up_proj.dtype == torch.bfloat16
+
+
+def test_a_variant_index_uses_transformers_naming(tmp_path):
+ import json
+ from unsloth.models.loader_utils import _load_fp8_weight_map
+
+ (tmp_path / "model.safetensors.index.fp8.json").write_text(
+ json.dumps({"weight_map": {"a.weight": "model-fp8-00001.safetensors"}})
+ )
+ assert _load_fp8_weight_map(str(tmp_path), True, None, variant = "fp8") == {
+ "a.weight": "model-fp8-00001.safetensors"
+ }
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index fb5464c36c5..addce0e59d6 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -1205,8 +1205,9 @@ def _load_fp8_weight_map(
revision = None,
subfolder = None,
cache_dir = None,
+ variant = None,
):
- """The checkpoint's tensor->file map, using the same snapshot the load used. Prefers the sharded `model.safetensors.index.json`, falling back to a single `model.safetensors` so unsharded checkpoints are covered too."""
+ """Tensor->file map from the load's snapshot; sharded index, else single `model.safetensors`."""
def _local_path(filename):
return (
@@ -1227,8 +1228,10 @@ def _load_fp8_weight_map(
token = token,
)
- index_file = "model.safetensors.index.json"
- single_file = "model.safetensors"
+ index_file = (
+ f"model.safetensors.index.{variant}.json" if variant else "model.safetensors.index.json"
+ )
+ single_file = f"model.{variant}.safetensors" if variant else "model.safetensors"
is_local = os.path.isdir(model_name)
if is_local and os.path.exists(_local_path(index_file)):
@@ -1547,6 +1550,345 @@ def _restore_dropped_fp8_scales(
return (0, 0)
+_FP8_SCALE_SUFFIXES = (".weight_scale_inv", "_scale_inv", ".weight_scale", "_scale")
+_FP8_LEFTOVER_MAX_CHUNK = 1 << 26
+
+
+def _empty_device_cache(device):
+ try:
+ backend = getattr(torch, getattr(device, "type", "cuda"), None)
+ empty = getattr(backend, "empty_cache", None)
+ if callable(empty):
+ empty()
+ except Exception:
+ pass
+
+
+def _orient_block_scale(scale, rows, cols, block_size):
+ """Fix a grid stored `(in_blocks, out_blocks)`; block size decides when both orientations tile."""
+ if block_size is None or scale.ndim < 2:
+ return scale
+ try:
+ bm, bn = int(block_size[0]), int(block_size[1])
+ except Exception:
+ return scale
+ if bm <= 0 or bn <= 0 or rows % bm or cols % bn:
+ return scale
+ canonical = (rows // bm, cols // bn)
+ grid = tuple(scale.shape[-2:])
+ if grid == canonical:
+ return scale
+ if grid == canonical[::-1] and canonical[0] != canonical[1]:
+ return scale.transpose(-1, -2).contiguous()
+ return scale
+
+
+def _fp8_scale_grid_dequant(
+ quantized,
+ scale,
+ out_dtype,
+ block_size = None,
+):
+ """Returns None when the scale grid does not tile the weight."""
+ # MXFP8 stores E8M0 exponents as uint8: the scale is 2 ** (byte - 127).
+ if scale.dtype == torch.uint8:
+ scale = (scale.to(torch.float32) - 127.0).exp2()
+ else:
+ scale = scale.to(torch.float32)
+ q_shape = tuple(quantized.shape)
+ if scale.numel() == 1:
+ if quantized.ndim == 3:
+ scale = scale.reshape(()).expand(q_shape[0], 1, 1)
+ else:
+ return (quantized.to(torch.float32) * scale.reshape(())).to(out_dtype)
+ if quantized.ndim == 2:
+ if scale.ndim != 2:
+ if scale.ndim == 1 and scale.numel() == q_shape[0]:
+ scale = scale.view(-1, 1)
+ else:
+ return None
+ rows, cols = q_shape
+ scale = _orient_block_scale(scale, rows, cols, block_size)
+ p, q = scale.shape
+ if rows % p or cols % q:
+ return None
+ bm, bn = rows // p, cols // q
+ out = quantized.to(torch.float32).view(p, bm, q, bn) * scale[:, None, :, None]
+ return out.reshape(rows, cols).to(out_dtype)
+ if quantized.ndim == 3:
+ E, rows, cols = q_shape
+ if scale.ndim == 1 and scale.numel() == E:
+ scale = scale.view(E, 1, 1)
+ elif scale.ndim == 2 and scale.shape[0] == E and scale.shape[1] == 1:
+ scale = scale.view(E, 1, 1)
+ if scale.ndim != 3 or scale.shape[0] != E:
+ return None
+ scale = _orient_block_scale(scale, rows, cols, block_size)
+ p, q = scale.shape[1], scale.shape[2]
+ if rows % p or cols % q:
+ return None
+ bm, bn = rows // p, cols // q
+ out = torch.empty((E, rows, cols), dtype = out_dtype, device = quantized.device)
+ step = max(1, _FP8_LEFTOVER_MAX_CHUNK // max(1, rows * cols))
+ for start in range(0, E, step):
+ stop = min(E, start + step)
+ chunk = quantized[start:stop].to(torch.float32).view(stop - start, p, bm, q, bn)
+ out[start:stop] = (
+ (chunk * scale[start:stop, :, None, :, None])
+ .reshape(stop - start, rows, cols)
+ .to(out_dtype)
+ )
+ return out
+ return None
+
+
+class FP8LeftoverOffloadedError(RuntimeError):
+ """A leftover fp8 tensor is disk-offloaded, so the 16bit load cannot be finished in place."""
+
+
+def _restore_parked_fp8(module, attr, device):
+ try:
+ stranded = module._parameters.get(attr)
+ if (
+ isinstance(stranded, torch.Tensor)
+ and stranded.device != device
+ and stranded.device.type == "cpu"
+ ):
+ module._parameters[attr] = torch.nn.Parameter(
+ stranded.data.to(device), requires_grad = bool(stranded.requires_grad)
+ )
+ except Exception:
+ pass
+
+
+def _dequantize_leftover_fp8_params(
+ model,
+ model_name,
+ dtype,
+ *,
+ local_files_only = False,
+ token = None,
+ revision = None,
+ subfolder = None,
+ cache_dir = None,
+ variant = None,
+):
+ """Dequantize fp8 params transformers left on a 16bit load. Returns (dequantized, skipped)."""
+ try:
+ if not _FP8_DTYPES:
+ return (0, 0)
+ leftover = [
+ name
+ for name, param in model.named_parameters()
+ if param.dtype in _FP8_DTYPES and param.ndim in (2, 3)
+ ]
+ block_size = None
+ try:
+ _qc = getattr(getattr(model, "config", None), "quantization_config", None)
+ _bs = (
+ _qc.get("weight_block_size")
+ if isinstance(_qc, dict)
+ else getattr(_qc, "weight_block_size", None)
+ )
+ if _bs is not None and len(_bs) == 2:
+ block_size = (int(_bs[0]), int(_bs[1]))
+ except Exception:
+ block_size = None
+ _grid_kwargs = {"block_size": block_size} if block_size is not None else {}
+ if not leftover:
+ return (0, 0)
+ weight_map = _load_fp8_weight_map(
+ model_name,
+ local_files_only,
+ token,
+ revision,
+ subfolder,
+ cache_dir,
+ variant = variant,
+ )
+ if not weight_map:
+ return (0, 0)
+ scale_by_weight_key = {}
+ for key, shard in weight_map.items():
+ for suffix in _FP8_SCALE_SUFFIXES:
+ if key.endswith(suffix):
+ weight_key = key[: -len(suffix)]
+ # `layer.weight_scale_inv` belongs to `layer.weight`, not to `layer`.
+ if suffix.startswith(".weight"):
+ weight_key += ".weight"
+ scale_by_weight_key[weight_key] = (key, shard)
+ break
+ if not scale_by_weight_key:
+ return (0, 0)
+ module_by_name = dict(model.named_modules())
+ # Names only: holding the fp8 Parameter would defeat the OOM fallback.
+ params_by_module = {}
+ for name in leftover:
+ module_name, _, attr = name.rpartition(".")
+ params_by_module.setdefault(module_name, []).append(attr)
+
+ target_of_ckpt = {}
+ for weight_key in scale_by_weight_key:
+ module_part, _, attr = weight_key.rpartition(".")
+ if attr == "weight":
+ ckpt_module, ckpt_attr = module_part, "weight"
+ else:
+ ckpt_module, ckpt_attr = module_part, attr
+ module = _match_fp8_module(module_by_name, ckpt_module)
+ if module is None:
+ continue
+ live_name = None
+ for candidate, mod in module_by_name.items():
+ if mod is module:
+ live_name = candidate
+ break
+ if live_name is None or live_name not in params_by_module:
+ continue
+ target_of_ckpt[weight_key] = (module, live_name, ckpt_attr)
+
+ dequantized = 0
+ skipped = 0
+ failed = 0
+ last_error = None
+ shard_cache = {}
+
+ def _scale_for(weight_key, device):
+ scale_key, shard = scale_by_weight_key[weight_key]
+ if shard not in shard_cache:
+ from safetensors import safe_open
+ shard_path = _resolve_fp8_shard(
+ model_name, shard, local_files_only, token, revision, subfolder, cache_dir
+ )
+ shard_cache[shard] = safe_open(shard_path, framework = "pt")
+ return shard_cache[shard].get_tensor(scale_key).to(device)
+
+ def _is_oom(error):
+ # Some backends raise a plain RuntimeError for OOM.
+ return (
+ hasattr(torch, "OutOfMemoryError") and isinstance(error, torch.OutOfMemoryError)
+ ) or (isinstance(error, RuntimeError) and "out of memory" in str(error).lower())
+
+ deferred = []
+ converted = set()
+ for weight_key, (module, live_name, attr) in target_of_ckpt.items():
+ param = getattr(module, attr, None)
+ if not isinstance(param, torch.Tensor) or param.dtype not in _FP8_DTYPES:
+ continue
+ trainable = bool(getattr(param, "requires_grad", False))
+ # A module with its own scale runs the fp8 forward.
+ if any(
+ isinstance(getattr(module, s, None), torch.Tensor)
+ for s in (attr + "_scale_inv", attr + "_scale", "weight_scale_inv", "weight_scale")
+ ):
+ skipped += 1
+ continue
+ if param.device.type == "meta":
+ # Disk-offloaded: the hook would restore raw fp8 bytes at forward time.
+ raise FP8LeftoverOffloadedError(
+ f"Unsloth: `{weight_key}` is an fp8 tensor transformers left quantized on a 16bit "
+ "load, and it is offloaded to disk, so it cannot be dequantized in place. Load with "
+ "enough GPU or CPU memory to keep the model resident (no disk offload), or use "
+ "`load_in_4bit = True`."
+ )
+ try:
+ scale = _scale_for(weight_key, param.device)
+ with torch.no_grad():
+ out = _fp8_scale_grid_dequant(param.data, scale, dtype, **_grid_kwargs)
+ if out is None:
+ failed += 1
+ last_error = f"{weight_key}: scale {tuple(scale.shape)} does not tile {tuple(param.shape)}"
+ continue
+ module._parameters[attr] = torch.nn.Parameter(out, requires_grad = trainable)
+ converted.add((module, attr))
+ dequantized += 1
+ except Exception as e:
+ if _is_oom(e):
+ device = param.device
+ quantized = param.data.to("cpu")
+ module._parameters[attr] = torch.nn.Parameter(
+ quantized, requires_grad = trainable
+ )
+ del param
+ deferred.append((weight_key, module, attr, device))
+ _empty_device_cache(device)
+ continue
+ failed += 1
+ last_error = f"{weight_key}: {type(e).__name__}: {e}"
+ continue
+
+ for weight_key, module, attr, device in deferred:
+ try:
+ param = getattr(module, attr)
+ trainable = bool(getattr(param, "requires_grad", False))
+ scale = _scale_for(weight_key, "cpu")
+ with torch.no_grad():
+ out = _fp8_scale_grid_dequant(param.data, scale, dtype, **_grid_kwargs)
+ if out is None:
+ failed += 1
+ last_error = f"{weight_key}: scale {tuple(scale.shape)} does not tile {tuple(param.shape)}"
+ _restore_parked_fp8(module, attr, device)
+ continue
+ _empty_device_cache(device)
+ module._parameters[attr] = torch.nn.Parameter(
+ out.to(device), requires_grad = trainable
+ )
+ del param, out
+ converted.add((module, attr))
+ dequantized += 1
+ except Exception as e:
+ failed += 1
+ last_error = f"{weight_key}: {type(e).__name__}: {e}"
+ _restore_parked_fp8(module, attr, device)
+ continue
+ for shard_file in shard_cache.values():
+ close = getattr(shard_file, "__exit__", None)
+ if close is not None:
+ try:
+ close(None, None, None)
+ except Exception:
+ pass
+
+ if dequantized > 0:
+ by_module = {}
+ for module, attr in converted:
+ by_module.setdefault(module, set()).add(attr)
+ for module, attrs in by_module.items():
+ still_fp8 = [
+ n
+ for n, p in module._parameters.items()
+ if isinstance(p, torch.Tensor) and p.dtype in _FP8_DTYPES
+ ]
+ for stale in [
+ n
+ for n in list(module._parameters) + list(module._buffers)
+ if n.endswith("activation_scale")
+ ]:
+ owner = next(
+ (a for a in list(attrs) + still_fp8 if stale.startswith(a + "_")), None
+ )
+ if owner is None and still_fp8:
+ continue
+ if owner is not None and owner not in attrs:
+ continue
+ if stale in module._parameters:
+ del module._parameters[stale]
+ else:
+ del module._buffers[stale]
+ print(
+ f"Unsloth: Dequantized {dequantized} FP8 tensor(s) transformers left quantized on a 16bit load."
+ )
+ if failed > 0:
+ print(
+ f"Unsloth: {failed} FP8 tensor(s) could not be dequantized on the 16bit load and stay fp8 "
+ f"(last: {last_error})."
+ )
+ return (dequantized, skipped)
+ except FP8LeftoverOffloadedError:
+ raise
+ except Exception:
+ return (0, 0)
+
+
_BNB_QUANTIZED_TYPES = ("Params4bit", "Int8Params", "Linear4bit", "Linear8bitLt")
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index cc222675200..84371b0b6eb 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -97,6 +97,7 @@ from .loader_utils import (
planner_kwargs_with_max_memory,
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
+ _dequantize_leftover_fp8_params,
_restore_dropped_fp8_scales,
planner_class_mismatch_reason,
planner_model_class,
@@ -2075,6 +2076,18 @@ class FastBaseModel:
variant = kwargs.get("variant"),
dtype = torch_dtype,
)
+ if load_in_16bit and not load_in_4bit and not load_in_8bit:
+ _dequantize_leftover_fp8_params(
+ model,
+ model_name,
+ torch_dtype,
+ local_files_only = local_files_only,
+ token = token,
+ revision = kwargs.get("revision"),
+ subfolder = kwargs.get("subfolder"),
+ cache_dir = kwargs.get("cache_dir"),
+ variant = kwargs.get("variant"),
+ )
if hasattr(model, "generate"):
model.fast_generate = make_fast_generate_wrapper(model.generate)
model.fast_generate_batches = error_out_no_vllm