Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Dequantize FP8 weights left raw by text_only and offloaded loads (#11841)

Daniel Hana20dcb2bc800
diff --git a/tests/test_fp8_restore_dropped_scale.py b/tests/test_fp8_restore_dropped_scale.py
index 3ae45e9ecdb..aa684a89298 100644
--- a/tests/test_fp8_restore_dropped_scale.py
+++ b/tests/test_fp8_restore_dropped_scale.py
@@ -85,7 +85,7 @@ def test_restore_dequantizes_orphaned_scale():
if _FP8 is None:
return
torch.manual_seed(0)
- raw = torch.randn(4, 4, dtype = torch.bfloat16)
+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)
scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
model = nn.Module()
@@ -109,7 +109,7 @@ def test_restore_dequantizes_orphaned_scale():
def test_skips_already_fp8_weight():
- """A correctly converted fp8 weight is skipped, never double-scaled."""
+ """A correctly converted fp8 module (fp8 weight plus its own scale, as FP8Linear) is skipped, never double-scaled."""
if _FP8 is None:
return
weight = torch.randn(4, 4).to(_FP8)
@@ -119,6 +119,7 @@ def test_skips_already_fp8_weight():
model.config = _fp8_config((2, 2))
model.layer = nn.Linear(4, 4, bias = False)
model.layer.weight = nn.Parameter(weight, requires_grad = False)
+ model.layer.weight_scale_inv = nn.Parameter(torch.ones(2, 2), requires_grad = False)
with tempfile.TemporaryDirectory() as d:
_write_checkpoint(d, {"layer.weight_scale_inv": torch.rand(2, 2)})
@@ -132,7 +133,7 @@ def test_skips_offloaded_meta_weight():
"""A disk-offloaded layer (weight on the meta device) is skipped without error or restore."""
if _FP8 is None:
return
- raw = torch.randn(4, 4, dtype = torch.bfloat16)
+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)
scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
model = nn.Module()
@@ -179,7 +180,7 @@ def test_non_block_divisible_shape():
"""Block scale is expanded then sliced to a non-divisible weight shape."""
if _FP8 is None:
return
- raw = torch.randn(3, 4, dtype = torch.bfloat16)
+ raw = torch.randn(3, 4).to(_FP8).to(torch.bfloat16)
scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
model = nn.Module()
@@ -200,7 +201,7 @@ def test_transposed_scale_layout():
"""A scale stored in the transposed block grid is transposed before use."""
if _FP8 is None:
return
- raw = torch.randn(4, 2, dtype = torch.bfloat16) # weight [4, 2] -> grid (2, 1)
+ raw = torch.randn(4, 2).to(_FP8).to(torch.bfloat16)
scale_correct = torch.rand(2, 1, dtype = torch.float32) + 0.1
scale_stored = scale_correct.t().contiguous() # stored transposed as (1, 2)
@@ -222,7 +223,7 @@ def test_single_file_checkpoint_without_index():
"""Unsharded model.safetensors (no index) is still scanned for dropped scales."""
if _FP8 is None:
return
- raw = torch.randn(4, 4, dtype = torch.bfloat16)
+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)
scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
model = nn.Module()
@@ -245,7 +246,7 @@ def test_scalar_block_size_config():
"""A scalar weight_block_size (not a list) is handled without error."""
if _FP8 is None:
return
- raw = torch.randn(4, 4, dtype = torch.bfloat16)
+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)
scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
model = nn.Module()
@@ -266,7 +267,7 @@ def test_text_only_prefix_mapping():
"""Checkpoint keys with a language_model prefix match the stripped text-only module names."""
if _FP8 is None:
return
- raw = torch.randn(2, 2, dtype = torch.bfloat16)
+ raw = torch.randn(2, 2).to(_FP8).to(torch.bfloat16)
scale = torch.rand(1, 1, dtype = torch.float32) + 0.1
model = nn.Module()
@@ -289,7 +290,7 @@ def test_skips_variant_load():
"""A variant load (variant="fp8") is skipped to avoid applying default-checkpoint scales."""
if _FP8 is None:
return
- raw = torch.randn(4, 4, dtype = torch.bfloat16)
+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)
scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
model = nn.Module()
model.config = _fp8_config((2, 2))
@@ -306,7 +307,7 @@ def test_vlm_language_model_model_alias():
"""A checkpoint key language_model.model.* matches a model.language_model.* module."""
if _FP8 is None:
return
- raw = torch.randn(2, 2, dtype = torch.bfloat16)
+ raw = torch.randn(2, 2).to(_FP8).to(torch.bfloat16)
scale = torch.rand(1, 1, dtype = torch.float32) + 0.1
model = nn.Module()
model.config = _fp8_config((2, 2))
@@ -354,3 +355,365 @@ def test_noop_when_not_block_fp8():
with tempfile.TemporaryDirectory() as d:
_write_checkpoint(d, {"layer.weight_scale_inv": scale})
assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (0, 0)
+
+
+def _fp8_linear(out_f, in_f, raw_fp8):
+ """A plain Linear holding raw fp8 values and no scale (unconverted text_only key)."""
+ m = nn.Linear(in_f, out_f, bias = False)
+ m.weight = nn.Parameter(raw_fp8, requires_grad = False)
+ return m
+
+
+def test_text_only_orphaned_fp8_weight_is_dequantized():
+ """A text_only fp8 orphan is dequantized into the load dtype, matching the unrenamed load."""
+ if _FP8 is None:
+ return
+ torch.manual_seed(0)
+ raw_fp8 = (torch.randn(4, 4) * 100).to(_FP8)
+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+
+ text_only = nn.Module()
+ text_only.config = _fp8_config((2, 2))
+ text_only.model = nn.Module()
+ text_only.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())
+ full = nn.Module()
+ full.config = _fp8_config((2, 2))
+ full.anchor = _fp8_anchor()
+ full.model = nn.Module()
+ full.model.language_model = nn.Module()
+ full.model.language_model.gate_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))
+
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})
+ restored, skipped = _restore_dropped_fp8_scales(
+ text_only, d, local_files_only = True, dtype = torch.bfloat16
+ )
+ assert _restore_dropped_fp8_scales(full, d, local_files_only = True) == (1, 0)
+
+ assert (restored, skipped) == (1, 0)
+ got = text_only.model.gate_proj.weight
+ assert isinstance(got, nn.Parameter) and got.dtype == torch.bfloat16
+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)
+ assert torch.equal(got.data, expected)
+ assert torch.equal(got.data, full.model.language_model.gate_proj.weight.data)
+ # Used to raise `BFloat16 != Float8_e4m3fn`.
+ x = torch.randn(3, 4, dtype = torch.bfloat16)
+ assert text_only.model.gate_proj(x).dtype == torch.bfloat16
+
+
+def test_orphaned_fp8_weight_uses_requested_dtype():
+ """The dequant target follows the load dtype (fp16 on T4 / V100), then config.dtype, then bfloat16."""
+ if _FP8 is None:
+ return
+ raw_fp8 = torch.randn(4, 4).to(_FP8)
+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+ for kwargs, config_dtype, want in (
+ ({"dtype": torch.float16}, None, torch.float16),
+ ({}, "float16", torch.float16),
+ ({}, torch.float32, torch.float32),
+ ({}, None, torch.bfloat16),
+ ):
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.config.dtype = config_dtype
+ model.layer = _fp8_linear(4, 4, raw_fp8.clone())
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, {"layer.weight_scale_inv": scale})
+ restored, _ = _restore_dropped_fp8_scales(model, d, local_files_only = True, **kwargs)
+ assert restored == 1
+ assert model.layer.weight.dtype == want, (kwargs, config_dtype, model.layer.weight.dtype)
+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(want)
+ assert torch.equal(model.layer.weight.data, expected)
+
+
+def test_fp8_module_with_scale_attr_untouched_next_to_orphan():
+ """A real fp8 module keeps its weight and scale; only the scale-less orphan is dequantized."""
+ if _FP8 is None:
+ return
+ raw_fp8 = torch.randn(4, 4).to(_FP8)
+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())
+ try:
+ from transformers.integrations.finegrained_fp8 import FP8Linear
+ up = FP8Linear(4, 4, block_size = (2, 2))
+ except Exception:
+ up = nn.Linear(4, 4, bias = False)
+ up.weight_scale_inv = nn.Parameter(torch.ones(2, 2), requires_grad = False)
+ up.weight = nn.Parameter(raw_fp8.clone(), requires_grad = False)
+ up.weight_scale_inv.data.copy_(scale)
+ model.up_proj = up
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(
+ d,
+ {
+ "gate_proj.weight_scale_inv": scale.clone(),
+ "up_proj.weight_scale_inv": scale.clone(),
+ },
+ )
+ restored, skipped = _restore_dropped_fp8_scales(
+ model, d, local_files_only = True, dtype = torch.bfloat16
+ )
+ assert (restored, skipped) == (1, 1)
+ assert model.gate_proj.weight.dtype == torch.bfloat16
+ assert model.up_proj.weight.dtype == _FP8
+ assert torch.equal(model.up_proj.weight.data.float(), raw_fp8.float())
+ assert torch.equal(model.up_proj.weight_scale_inv.data, scale)
+
+
+def test_load_call_sites_pass_dtype():
+ """Every loader call passes the load dtype, so an orphan is dequantized into what the load asked for."""
+ import ast
+ import inspect
+ import unsloth.models.llama as llama_mod
+ import unsloth.models.vision as vision_mod
+
+ for mod, want in ((llama_mod, 2), (vision_mod, 1)):
+ calls = [
+ node
+ for node in ast.walk(ast.parse(inspect.getsource(mod)))
+ if isinstance(node, ast.Call)
+ and getattr(node.func, "id", None) == "_restore_dropped_fp8_scales"
+ ]
+ assert len(calls) == want, (mod.__name__, len(calls))
+ for call in calls:
+ assert "dtype" in {kw.arg for kw in call.keywords}, mod.__name__
+
+
+def _offload(
+ model,
+ module_name,
+ disk_dir = None,
+):
+ """Offload one submodule through accelerate like a sequential device map."""
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ from accelerate.utils import OffloadedWeightsLoader, PrefixedDataset, offload_state_dict
+
+ module = dict(model.named_modules())[module_name]
+ state = {f"{module_name}.{k}": v.detach().clone() for k, v in module.state_dict().items()}
+ if disk_dir is None:
+ store = OffloadedWeightsLoader(state_dict = state)
+ else:
+ offload_state_dict(disk_dir, state)
+ store = OffloadedWeightsLoader(save_folder = disk_dir)
+ hook = AlignDevicesHook(
+ execution_device = "cpu",
+ offload = True,
+ weights_map = PrefixedDataset(store, f"{module_name}."),
+ )
+ add_hook_to_module(module, hook)
+ assert module.weight.device.type == "meta"
+ return store
+
+
+def _offloaded_case(raw, placeholder_dtype):
+ raw_fp8 = raw.to(_FP8)
+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.anchor = _fp8_anchor()
+ model.model = nn.Module()
+ if placeholder_dtype == _FP8:
+ model.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())
+ else:
+ model.model.gate_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))
+ return model, raw_fp8, scale
+
+
+def test_offloaded_orphans_are_restored_through_weights_map():
+ """An offloaded orphan (cpu or disk, fp8 or bf16) is dequantized in the weights map."""
+ if _FP8 is None:
+ return
+ import pytest
+
+ pytest.importorskip("accelerate")
+ torch.manual_seed(0)
+ raw = torch.randn(4, 4) * 100
+ for placeholder_dtype in (_FP8, torch.bfloat16):
+ for disk in (False, True):
+ model, raw_fp8, scale = _offloaded_case(raw, placeholder_dtype)
+ with tempfile.TemporaryDirectory() as d, tempfile.TemporaryDirectory() as off:
+ _offload(model, "model.gate_proj", off if disk else None)
+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})
+ restored, skipped = _restore_dropped_fp8_scales(
+ model, d, local_files_only = True, dtype = torch.bfloat16
+ )
+ assert (restored, skipped) == (1, 0), (placeholder_dtype, disk)
+ gate = model.model.gate_proj
+ assert gate.weight.device.type == "meta" and gate.weight.dtype == torch.bfloat16
+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(
+ torch.bfloat16
+ )
+ x = torch.randn(3, 4, dtype = torch.bfloat16)
+ out = gate(x)
+ assert torch.equal(out, torch.nn.functional.linear(x, expected)), (
+ placeholder_dtype,
+ disk,
+ )
+
+
+def test_offloaded_fp8_module_with_scale_is_skipped_not_counted_offloaded():
+ """A converted fp8 module that is offloaded still carries its scale placeholder: skipped, store untouched."""
+ if _FP8 is None:
+ return
+ import pytest
+
+ pytest.importorskip("accelerate")
+ raw_fp8 = torch.randn(4, 4).to(_FP8)
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.up_proj = _fp8_linear(4, 4, raw_fp8.clone())
+ model.up_proj.weight_scale_inv = nn.Parameter(torch.ones(2, 2), requires_grad = False)
+ store = _offload(model, "up_proj")
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, {"up_proj.weight_scale_inv": torch.rand(2, 2)})
+ assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (0, 1)
+ assert store["up_proj.weight"].dtype == _FP8
+ assert torch.equal(store["up_proj.weight"].float(), raw_fp8.float())
+
+
+def _raw_and_folded(scale):
+ """Raw fp8 values, and the same values with the block scale folded in, in bf16."""
+ torch.manual_seed(0)
+ raw_fp8 = (torch.randn(4, 4) * 100).to(_FP8)
+ folded = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)
+ return raw_fp8, folded
+
+
+def test_orphan_is_scaled_exactly_once():
+ """Orphans are scaled once; a second pass leaves them alone."""
+ if _FP8 is None:
+ return
+ scale = torch.tensor([[0.0123, 0.0456], [0.0789, 0.0321]])
+ raw_fp8, expected = _raw_and_folded(scale)
+ for holds in ("fp8", "bf16"):
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.anchor = _fp8_anchor()
+ model.model = nn.Module()
+ if holds == "fp8":
+ model.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())
+ else:
+ model.model.gate_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})
+ first = _restore_dropped_fp8_scales(
+ model, d, local_files_only = True, dtype = torch.bfloat16
+ )
+ second = _restore_dropped_fp8_scales(
+ model, d, local_files_only = True, dtype = torch.bfloat16
+ )
+ assert first == (1, 0) and second == (0, 1), (holds, first, second)
+ assert torch.equal(model.model.gate_proj.weight.data, expected), holds
+
+
+def test_already_dequantized_weight_is_not_scaled_again():
+ """A bf16 weight with its scale already folded in is skipped, never double-scaled."""
+ if _FP8 is None:
+ return
+ scale = torch.tensor([[0.0123, 0.0456], [0.0789, 0.0321]])
+ raw_fp8, folded = _raw_and_folded(scale)
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.anchor = _fp8_anchor()
+ model.model = nn.Module()
+ model.model.gate_proj = _bf16_linear(4, 4, folded.clone())
+ model.model.up_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))
+ with tempfile.TemporaryDirectory() as d:
+ _write_checkpoint(
+ d,
+ {
+ "model.language_model.gate_proj.weight_scale_inv": scale,
+ "model.language_model.up_proj.weight_scale_inv": scale.clone(),
+ },
+ )
+ assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (1, 1)
+ assert torch.equal(model.model.gate_proj.weight.data, folded)
+ assert torch.equal(model.model.up_proj.weight.data, folded)
+
+
+def test_offloaded_already_dequantized_weight_is_not_scaled_again():
+ """Offload path: folded stored value left as is, raw one scaled once, repeat pass a no-op."""
+ if _FP8 is None:
+ return
+ import pytest
+
+ pytest.importorskip("accelerate")
+ scale = torch.tensor([[0.0123, 0.0456], [0.0789, 0.0321]])
+ raw_fp8, folded = _raw_and_folded(scale)
+ for stored, disk in (("folded", False), ("folded", True), ("raw", False), ("raw", True)):
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.anchor = _fp8_anchor()
+ model.model = nn.Module()
+ model.model.gate_proj = _bf16_linear(
+ 4, 4, folded.clone() if stored == "folded" else raw_fp8.to(torch.bfloat16)
+ )
+ with tempfile.TemporaryDirectory() as d, tempfile.TemporaryDirectory() as off:
+ store = _offload(model, "model.gate_proj", off if disk else None)
+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})
+ first = _restore_dropped_fp8_scales(
+ model, d, local_files_only = True, dtype = torch.bfloat16
+ )
+ second = _restore_dropped_fp8_scales(
+ model, d, local_files_only = True, dtype = torch.bfloat16
+ )
+ assert first == ((0, 1) if stored == "folded" else (1, 0)), (stored, disk, first)
+ assert second == (0, 1), (stored, disk, second)
+ assert torch.equal(store["model.gate_proj.weight"].to(torch.bfloat16), folded), (
+ stored,
+ disk,
+ )
+ x = torch.randn(3, 4, dtype = torch.bfloat16)
+ assert torch.equal(model.model.gate_proj(x), torch.nn.functional.linear(x, folded)), (
+ stored,
+ disk,
+ )
+
+
+def test_disk_offloaded_orphan_stays_on_disk():
+ """A disk-offloaded orphan goes to a new .dat in the offload folder, not RAM; the checkpoint is untouched."""
+ if _FP8 is None:
+ return
+ import pytest
+
+ pytest.importorskip("accelerate")
+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module
+ from accelerate.utils import OffloadedWeightsLoader, PrefixedDataset
+
+ torch.manual_seed(0)
+ raw_fp8 = (torch.randn(4, 4) * 100).to(_FP8)
+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1
+ model = nn.Module()
+ model.config = _fp8_config((2, 2))
+ model.anchor = _fp8_anchor()
+ model.model = nn.Module()
+ model.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())
+ key = "model.gate_proj.weight"
+ with tempfile.TemporaryDirectory() as ck, tempfile.TemporaryDirectory() as off:
+ ck_file = os.path.join(ck, "weights.safetensors")
+ save_file({key: raw_fp8}, ck_file)
+ before = open(ck_file, "rb").read()
+ index = {key: {"safetensors_file": ck_file, "weight_name": key}}
+ store = OffloadedWeightsLoader(save_folder = off, index = index)
+ add_hook_to_module(
+ model.model.gate_proj,
+ AlignDevicesHook(
+ execution_device = "cpu",
+ offload = True,
+ weights_map = PrefixedDataset(store, "model.gate_proj."),
+ ),
+ )
+ _write_checkpoint(ck, {"model.language_model.gate_proj.weight_scale_inv": scale})
+ assert _restore_dropped_fp8_scales(
+ model, ck, local_files_only = True, dtype = torch.bfloat16
+ ) == (1, 0)
+ assert key not in store.state_dict
+ assert store.index[key] == {"dtype": "bfloat16", "shape": [4, 4]}
+ assert os.path.exists(os.path.join(off, key + ".dat"))
+ assert open(ck_file, "rb").read() == before
+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)
+ assert torch.equal(store[key], expected)
+ x = torch.randn(3, 4, dtype = torch.bfloat16)
+ assert torch.equal(model.model.gate_proj(x), torch.nn.functional.linear(x, expected))
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index e9f2dbba3a0..bb0f77fa9f9 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2778,6 +2778,7 @@ class FastLlamaModel:
subfolder = kwargs.get("subfolder"),
cache_dir = kwargs.get("cache_dir"),
variant = kwargs.get("variant"),
+ dtype = dtype,
)
elif not fast_inference:
if user_config is not None:
@@ -2831,6 +2832,7 @@ class FastLlamaModel:
subfolder = kwargs.get("subfolder"),
cache_dir = kwargs.get("cache_dir"),
variant = kwargs.get("variant"),
+ dtype = dtype,
)
model.fast_generate = make_fast_generate_wrapper(model.generate)
model.fast_generate_batches = None
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index d2f84ac67f1..b87f12a0341 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -1306,6 +1306,110 @@ def _match_fp8_module(module_by_name, base):
return None
+_FP8_DEQUANT_DTYPES = (torch.bfloat16, torch.float16, torch.float32)
+
+
+def _has_fp8_scale_attr(module):
+ return any(
+ isinstance(getattr(module, name, None), torch.Tensor)
+ for name in ("weight_scale_inv", "weight_scale")
+ )
+
+
+def _fp8_dequant_target_dtype(model, dtype = None):
+ candidates = [dtype]
+ config = getattr(model, "config", None)
+ candidates += [getattr(config, "dtype", None), getattr(config, "torch_dtype", None)]
+ try:
+ candidates.append(model.get_input_embeddings().weight.dtype)
+ except Exception:
+ pass
+ for candidate in candidates:
+ if isinstance(candidate, str):
+ candidate = getattr(torch, candidate.replace("torch.", ""), None)
+ if candidate in _FP8_DEQUANT_DTYPES:
+ return candidate
+ return torch.bfloat16
+
+
+def _holds_raw_fp8_values(tensor, chunk_elements = 1 << 24):
+ """True if every value lies on the e4m3/e5m2 grid; a folded-in scale falls off it, so never scale twice."""
+ if tensor.dtype in _FP8_DTYPES:
+ return True
+ if tensor.dtype not in _FP8_DEQUANT_DTYPES:
+ return False
+ flat = tensor.detach().reshape(-1)
+ for fp8 in _FP8_DTYPES:
+ for start in range(0, flat.numel(), chunk_elements):
+ chunk = flat[start : start + chunk_elements]
+ if not torch.equal(chunk.to(fp8).to(chunk.dtype), chunk):
+ break
+ else:
+ return True
+ return False
+
+
+def _apply_fp8_block_scale(weight, scale, bs0, bs1):
+ out_features, in_features = weight.shape
+ out_blocks = (out_features + bs0 - 1) // bs0
+ in_blocks = (in_features + bs1 - 1) // bs1
+ if out_features % bs0 == 0 and in_features % bs1 == 0:
+ # In place on block views: a full expanded fp32 scale + copy can OOM.
+ weight.view(out_blocks, bs0, in_blocks, bs1).mul_(scale[:, None, :, None])
+ return weight
+ scale_expanded = scale.repeat_interleave(bs0, dim = 0).repeat_interleave(bs1, dim = 1)[
+ :out_features, :in_features
+ ]
+ return (weight.to(torch.float32) * scale_expanded).to(weight.dtype)
+
+
+def _accelerate_offload_store(model):
+ for module in model.modules():
+ hook = getattr(module, "_hf_hook", None)
+ if hook is None:
+ continue
+ for h in getattr(hook, "hooks", None) or (hook,):
+ weights_map = getattr(h, "weights_map", None)
+ if weights_map is None:
+ continue
+ return getattr(weights_map, "dataset", weights_map)
+ return None
+
+
+def _offload_store_has(store, key):
+ # `key in OffloadedWeightsLoader` would read the disk tensor.
+ keys = getattr(store, "all_keys", None)
+ if keys is not None:
+ return key in keys
+ try:
+ return key in store
+ except Exception:
+ return False
+
+
+def _offload_store_set(store, key, value):
+ state_dict = getattr(store, "state_dict", None)
+ index = getattr(store, "index", None)
+ save_folder = getattr(store, "save_folder", None)
+ if (
+ save_folder
+ and isinstance(index, dict)
+ and key in index
+ and not (isinstance(state_dict, dict) and key in state_dict)
+ ):
+ # Disk-offloaded: write a new .dat (the index entry may point at the read-only checkpoint), never RAM.
+ from accelerate.utils import offload_weight
+ offload_weight(value.detach().cpu(), key, save_folder, index = index)
+ return
+ # OffloadedWeightsLoader checks state_dict before the disk index, so this overrides cpu tensors.
+ if isinstance(state_dict, dict):
+ state_dict[key] = value
+ elif isinstance(store, dict):
+ store[key] = value
+ else:
+ raise TypeError(f"Unsloth: cannot write offloaded tensor into {type(store).__name__}")
+
+
def _restore_dropped_fp8_scales(
model,
model_name,
@@ -1316,8 +1420,9 @@ def _restore_dropped_fp8_scales(
subfolder = None,
cache_dir = None,
variant = None,
+ dtype = None,
):
- """Re-apply block-fp8 `weight_scale_inv` tensors that transformers dropped on load. On some block-scale fp8 checkpoints (e.g. Qwen3.6-27B-FP8, issue #6200) transformers fails to convert a Linear such as `mlp.gate_proj` to an fp8 module, loading the raw quantized values into a plain bf16 weight and discarding its `weight_scale_inv` as an unexpected key, so the weight is used un-scaled and the model is garbage. For every checkpoint scale whose live weight is not fp8, dequantize the orphaned weight in place; correctly converted modules keep an fp8 weight and are skipped, so a healthy checkpoint is a no-op. Returns (restored, skipped)."""
+ """Re-apply block-fp8 `weight_scale_inv` tensors that transformers dropped on load. On some block-scale fp8 checkpoints (e.g. Qwen3.6-27B-FP8, issue #6200) transformers fails to convert a Linear such as `mlp.gate_proj` to an fp8 module, loading the raw quantized values into a plain bf16 weight and discarding its `weight_scale_inv` as an unexpected key, so the weight is used un-scaled and the model is garbage. For every checkpoint scale whose live weight is not fp8, dequantize the orphaned weight in place; correctly converted modules keep an fp8 weight and are skipped, so a healthy checkpoint is a no-op. A text_only-renamed key can instead leave raw fp8 values with no scale attribute; those are dequantized into `dtype`. Returns (restored, skipped)."""
try:
block = _fp8_block_size_from_config(model)
if block is None or not _FP8_DTYPES:
@@ -1339,6 +1444,9 @@ def _restore_dropped_fp8_scales(
return (0, 0)
module_by_name = dict(model.named_modules())
+ name_by_module = {id(m): n for n, m in module_by_name.items()}
+ _unset = object()
+ offload_store = _unset
bs0, bs1 = block
restored = 0
skipped = 0
@@ -1353,14 +1461,18 @@ def _restore_dropped_fp8_scales(
weight = getattr(module, "weight", None)
if not isinstance(weight, torch.Tensor) or weight.ndim != 2:
continue
- if weight.device.type == "meta":
- # Disk-offloaded layer: the weight lives on meta until forward, so it cannot be scaled in place here. Count and warn rather than silently leave it unscaled.
- offloaded += 1
- continue
- if weight.dtype in _FP8_DTYPES:
- # Correctly converted fp8 module: the fp8 path already handles the scale.
+ if weight.dtype in _FP8_DTYPES and _has_fp8_scale_attr(module):
skipped += 1
continue
+ store_key = None
+ if weight.device.type == "meta":
+ if offload_store is _unset:
+ offload_store = _accelerate_offload_store(model)
+ store_key = f"{name_by_module.get(id(module), '')}.weight"
+ if offload_store is None or not _offload_store_has(offload_store, store_key):
+ offloaded += 1
+ continue
+ orphan_fp8 = weight.dtype in _FP8_DTYPES
# Errors after this point are per-tensor: warn and continue, never abort or hide them.
try:
@@ -1389,19 +1501,32 @@ def _restore_dropped_fp8_scales(
else:
# Shape does not match the block grid: skip rather than apply a wrong scale.
continue
- scale = scale.to(weight.device)
+ stored = None
+ if store_key is not None:
+ stored = offload_store[store_key]
+ if tuple(stored.shape) != tuple(weight.shape):
+ continue
+ orphan_fp8 = stored.dtype in _FP8_DTYPES
+ source = stored if stored is not None else weight
+ if source.is_floating_point() and not _holds_raw_fp8_values(source):
+ # Scale already folded in: never double-scale.
+ skipped += 1
+ continue
+ target = _fp8_dequant_target_dtype(model, dtype) if orphan_fp8 else weight.dtype
with torch.no_grad():
- if out_features % bs0 == 0 and in_features % bs1 == 0:
- # Memory-frugal path: multiply block views in place against the broadcast fp32 scale, avoiding a full expanded scale and fp32 copy that could OOM.
- module.weight.data.view(out_blocks, bs0, in_blocks, bs1).mul_(
- scale[:, None, :, None]
- )
+ if store_key is not None:
+ # Copy: the stored tensor can be a read-only disk memmap.
+ value = stored.to(dtype = target, copy = True)
+ value = _apply_fp8_block_scale(value, scale.to(value.device), bs0, bs1)
+ _offload_store_set(offload_store, store_key, value)
+ if weight.dtype != target:
+ # accelerate casts the stored value to the placeholder dtype.
+ module.weight.data = torch.empty_like(weight, dtype = target)
else:
- scale_expanded = scale.repeat_interleave(bs0, dim = 0).repeat_interleave(
- bs1, dim = 1
- )[:out_features, :in_features]
- module.weight.data = (weight.to(torch.float32) * scale_expanded).to(
- weight.dtype
+ # Swap in once so an OOM leaves the weight untouched, not widened but unscaled.
+ value = weight.data.to(target) if orphan_fp8 else weight.data
+ module.weight.data = _apply_fp8_block_scale(
+ value, scale.to(weight.device), bs0, bs1
)
restored += 1
except Exception:
@@ -1415,7 +1540,7 @@ def _restore_dropped_fp8_scales(
if offloaded > 0:
print(
f"Unsloth: {offloaded} dropped FP8 weight_scale_inv tensor(s) skipped because the "
- "layer is disk-offloaded; load without disk offload so the scales can be restored"
+ "layer is offloaded outside accelerate's weights map; load without offload so the scales can be restored"
)
return (restored, skipped)
except Exception:
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 18547b33eff..38d7716d050 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1783,6 +1783,7 @@ class FastBaseModel:
subfolder = kwargs.get("subfolder"),
cache_dir = kwargs.get("cache_dir"),
variant = kwargs.get("variant"),
+ dtype = torch_dtype,
)
if hasattr(model, "generate"):
model.fast_generate = make_fast_generate_wrapper(model.generate)