Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Dequantize FP8 weights left raw by text_only and offloaded loads (#11841)
Daniel Hana20dcb2bc800
diff --git a/tests/test_fp8_restore_dropped_scale.py b/tests/test_fp8_restore_dropped_scale.pyindex 3ae45e9ecdb..aa684a89298 100644--- a/tests/test_fp8_restore_dropped_scale.py+++ b/tests/test_fp8_restore_dropped_scale.py@@ -85,7 +85,7 @@ def test_restore_dequantizes_orphaned_scale():if _FP8 is None:returntorch.manual_seed(0)- raw = torch.randn(4, 4, dtype = torch.bfloat16)+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)scale = torch.rand(2, 2, dtype = torch.float32) + 0.1model = nn.Module()@@ -109,7 +109,7 @@ def test_restore_dequantizes_orphaned_scale():def test_skips_already_fp8_weight():- """A correctly converted fp8 weight is skipped, never double-scaled."""+ """A correctly converted fp8 module (fp8 weight plus its own scale, as FP8Linear) is skipped, never double-scaled."""if _FP8 is None:returnweight = torch.randn(4, 4).to(_FP8)@@ -119,6 +119,7 @@ def test_skips_already_fp8_weight():model.config = _fp8_config((2, 2))model.layer = nn.Linear(4, 4, bias = False)model.layer.weight = nn.Parameter(weight, requires_grad = False)+ model.layer.weight_scale_inv = nn.Parameter(torch.ones(2, 2), requires_grad = False)with tempfile.TemporaryDirectory() as d:_write_checkpoint(d, {"layer.weight_scale_inv": torch.rand(2, 2)})@@ -132,7 +133,7 @@ def test_skips_offloaded_meta_weight():"""A disk-offloaded layer (weight on the meta device) is skipped without error or restore."""if _FP8 is None:return- raw = torch.randn(4, 4, dtype = torch.bfloat16)+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)scale = torch.rand(2, 2, dtype = torch.float32) + 0.1model = nn.Module()@@ -179,7 +180,7 @@ def test_non_block_divisible_shape():"""Block scale is expanded then sliced to a non-divisible weight shape."""if _FP8 is None:return- raw = torch.randn(3, 4, dtype = torch.bfloat16)+ raw = torch.randn(3, 4).to(_FP8).to(torch.bfloat16)scale = torch.rand(2, 2, dtype = torch.float32) + 0.1model = nn.Module()@@ -200,7 +201,7 @@ def test_transposed_scale_layout():"""A scale stored in the transposed block grid is transposed before use."""if _FP8 is None:return- raw = torch.randn(4, 2, dtype = torch.bfloat16) # weight [4, 2] -> grid (2, 1)+ raw = torch.randn(4, 2).to(_FP8).to(torch.bfloat16)scale_correct = torch.rand(2, 1, dtype = torch.float32) + 0.1scale_stored = scale_correct.t().contiguous() # stored transposed as (1, 2)@@ -222,7 +223,7 @@ def test_single_file_checkpoint_without_index():"""Unsharded model.safetensors (no index) is still scanned for dropped scales."""if _FP8 is None:return- raw = torch.randn(4, 4, dtype = torch.bfloat16)+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)scale = torch.rand(2, 2, dtype = torch.float32) + 0.1model = nn.Module()@@ -245,7 +246,7 @@ def test_scalar_block_size_config():"""A scalar weight_block_size (not a list) is handled without error."""if _FP8 is None:return- raw = torch.randn(4, 4, dtype = torch.bfloat16)+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)scale = torch.rand(2, 2, dtype = torch.float32) + 0.1model = nn.Module()@@ -266,7 +267,7 @@ def test_text_only_prefix_mapping():"""Checkpoint keys with a language_model prefix match the stripped text-only module names."""if _FP8 is None:return- raw = torch.randn(2, 2, dtype = torch.bfloat16)+ raw = torch.randn(2, 2).to(_FP8).to(torch.bfloat16)scale = torch.rand(1, 1, dtype = torch.float32) + 0.1model = nn.Module()@@ -289,7 +290,7 @@ def test_skips_variant_load():"""A variant load (variant="fp8") is skipped to avoid applying default-checkpoint scales."""if _FP8 is None:return- raw = torch.randn(4, 4, dtype = torch.bfloat16)+ raw = torch.randn(4, 4).to(_FP8).to(torch.bfloat16)scale = torch.rand(2, 2, dtype = torch.float32) + 0.1model = nn.Module()model.config = _fp8_config((2, 2))@@ -306,7 +307,7 @@ def test_vlm_language_model_model_alias():"""A checkpoint key language_model.model.* matches a model.language_model.* module."""if _FP8 is None:return- raw = torch.randn(2, 2, dtype = torch.bfloat16)+ raw = torch.randn(2, 2).to(_FP8).to(torch.bfloat16)scale = torch.rand(1, 1, dtype = torch.float32) + 0.1model = nn.Module()model.config = _fp8_config((2, 2))@@ -354,3 +355,365 @@ def test_noop_when_not_block_fp8():with tempfile.TemporaryDirectory() as d:_write_checkpoint(d, {"layer.weight_scale_inv": scale})assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (0, 0)+++def _fp8_linear(out_f, in_f, raw_fp8):+ """A plain Linear holding raw fp8 values and no scale (unconverted text_only key)."""+ m = nn.Linear(in_f, out_f, bias = False)+ m.weight = nn.Parameter(raw_fp8, requires_grad = False)+ return m+++def test_text_only_orphaned_fp8_weight_is_dequantized():+ """A text_only fp8 orphan is dequantized into the load dtype, matching the unrenamed load."""+ if _FP8 is None:+ return+ torch.manual_seed(0)+ raw_fp8 = (torch.randn(4, 4) * 100).to(_FP8)+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1++ text_only = nn.Module()+ text_only.config = _fp8_config((2, 2))+ text_only.model = nn.Module()+ text_only.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())+ full = nn.Module()+ full.config = _fp8_config((2, 2))+ full.anchor = _fp8_anchor()+ full.model = nn.Module()+ full.model.language_model = nn.Module()+ full.model.language_model.gate_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))++ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})+ restored, skipped = _restore_dropped_fp8_scales(+ text_only, d, local_files_only = True, dtype = torch.bfloat16+ )+ assert _restore_dropped_fp8_scales(full, d, local_files_only = True) == (1, 0)++ assert (restored, skipped) == (1, 0)+ got = text_only.model.gate_proj.weight+ assert isinstance(got, nn.Parameter) and got.dtype == torch.bfloat16+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)+ assert torch.equal(got.data, expected)+ assert torch.equal(got.data, full.model.language_model.gate_proj.weight.data)+ # Used to raise `BFloat16 != Float8_e4m3fn`.+ x = torch.randn(3, 4, dtype = torch.bfloat16)+ assert text_only.model.gate_proj(x).dtype == torch.bfloat16+++def test_orphaned_fp8_weight_uses_requested_dtype():+ """The dequant target follows the load dtype (fp16 on T4 / V100), then config.dtype, then bfloat16."""+ if _FP8 is None:+ return+ raw_fp8 = torch.randn(4, 4).to(_FP8)+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1+ for kwargs, config_dtype, want in (+ ({"dtype": torch.float16}, None, torch.float16),+ ({}, "float16", torch.float16),+ ({}, torch.float32, torch.float32),+ ({}, None, torch.bfloat16),+ ):+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.config.dtype = config_dtype+ model.layer = _fp8_linear(4, 4, raw_fp8.clone())+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, {"layer.weight_scale_inv": scale})+ restored, _ = _restore_dropped_fp8_scales(model, d, local_files_only = True, **kwargs)+ assert restored == 1+ assert model.layer.weight.dtype == want, (kwargs, config_dtype, model.layer.weight.dtype)+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(want)+ assert torch.equal(model.layer.weight.data, expected)+++def test_fp8_module_with_scale_attr_untouched_next_to_orphan():+ """A real fp8 module keeps its weight and scale; only the scale-less orphan is dequantized."""+ if _FP8 is None:+ return+ raw_fp8 = torch.randn(4, 4).to(_FP8)+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())+ try:+ from transformers.integrations.finegrained_fp8 import FP8Linear+ up = FP8Linear(4, 4, block_size = (2, 2))+ except Exception:+ up = nn.Linear(4, 4, bias = False)+ up.weight_scale_inv = nn.Parameter(torch.ones(2, 2), requires_grad = False)+ up.weight = nn.Parameter(raw_fp8.clone(), requires_grad = False)+ up.weight_scale_inv.data.copy_(scale)+ model.up_proj = up+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(+ d,+ {+ "gate_proj.weight_scale_inv": scale.clone(),+ "up_proj.weight_scale_inv": scale.clone(),+ },+ )+ restored, skipped = _restore_dropped_fp8_scales(+ model, d, local_files_only = True, dtype = torch.bfloat16+ )+ assert (restored, skipped) == (1, 1)+ assert model.gate_proj.weight.dtype == torch.bfloat16+ assert model.up_proj.weight.dtype == _FP8+ assert torch.equal(model.up_proj.weight.data.float(), raw_fp8.float())+ assert torch.equal(model.up_proj.weight_scale_inv.data, scale)+++def test_load_call_sites_pass_dtype():+ """Every loader call passes the load dtype, so an orphan is dequantized into what the load asked for."""+ import ast+ import inspect+ import unsloth.models.llama as llama_mod+ import unsloth.models.vision as vision_mod++ for mod, want in ((llama_mod, 2), (vision_mod, 1)):+ calls = [+ node+ for node in ast.walk(ast.parse(inspect.getsource(mod)))+ if isinstance(node, ast.Call)+ and getattr(node.func, "id", None) == "_restore_dropped_fp8_scales"+ ]+ assert len(calls) == want, (mod.__name__, len(calls))+ for call in calls:+ assert "dtype" in {kw.arg for kw in call.keywords}, mod.__name__+++def _offload(+ model,+ module_name,+ disk_dir = None,+):+ """Offload one submodule through accelerate like a sequential device map."""+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module+ from accelerate.utils import OffloadedWeightsLoader, PrefixedDataset, offload_state_dict++ module = dict(model.named_modules())[module_name]+ state = {f"{module_name}.{k}": v.detach().clone() for k, v in module.state_dict().items()}+ if disk_dir is None:+ store = OffloadedWeightsLoader(state_dict = state)+ else:+ offload_state_dict(disk_dir, state)+ store = OffloadedWeightsLoader(save_folder = disk_dir)+ hook = AlignDevicesHook(+ execution_device = "cpu",+ offload = True,+ weights_map = PrefixedDataset(store, f"{module_name}."),+ )+ add_hook_to_module(module, hook)+ assert module.weight.device.type == "meta"+ return store+++def _offloaded_case(raw, placeholder_dtype):+ raw_fp8 = raw.to(_FP8)+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.anchor = _fp8_anchor()+ model.model = nn.Module()+ if placeholder_dtype == _FP8:+ model.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())+ else:+ model.model.gate_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))+ return model, raw_fp8, scale+++def test_offloaded_orphans_are_restored_through_weights_map():+ """An offloaded orphan (cpu or disk, fp8 or bf16) is dequantized in the weights map."""+ if _FP8 is None:+ return+ import pytest++ pytest.importorskip("accelerate")+ torch.manual_seed(0)+ raw = torch.randn(4, 4) * 100+ for placeholder_dtype in (_FP8, torch.bfloat16):+ for disk in (False, True):+ model, raw_fp8, scale = _offloaded_case(raw, placeholder_dtype)+ with tempfile.TemporaryDirectory() as d, tempfile.TemporaryDirectory() as off:+ _offload(model, "model.gate_proj", off if disk else None)+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})+ restored, skipped = _restore_dropped_fp8_scales(+ model, d, local_files_only = True, dtype = torch.bfloat16+ )+ assert (restored, skipped) == (1, 0), (placeholder_dtype, disk)+ gate = model.model.gate_proj+ assert gate.weight.device.type == "meta" and gate.weight.dtype == torch.bfloat16+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(+ torch.bfloat16+ )+ x = torch.randn(3, 4, dtype = torch.bfloat16)+ out = gate(x)+ assert torch.equal(out, torch.nn.functional.linear(x, expected)), (+ placeholder_dtype,+ disk,+ )+++def test_offloaded_fp8_module_with_scale_is_skipped_not_counted_offloaded():+ """A converted fp8 module that is offloaded still carries its scale placeholder: skipped, store untouched."""+ if _FP8 is None:+ return+ import pytest++ pytest.importorskip("accelerate")+ raw_fp8 = torch.randn(4, 4).to(_FP8)+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.up_proj = _fp8_linear(4, 4, raw_fp8.clone())+ model.up_proj.weight_scale_inv = nn.Parameter(torch.ones(2, 2), requires_grad = False)+ store = _offload(model, "up_proj")+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, {"up_proj.weight_scale_inv": torch.rand(2, 2)})+ assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (0, 1)+ assert store["up_proj.weight"].dtype == _FP8+ assert torch.equal(store["up_proj.weight"].float(), raw_fp8.float())+++def _raw_and_folded(scale):+ """Raw fp8 values, and the same values with the block scale folded in, in bf16."""+ torch.manual_seed(0)+ raw_fp8 = (torch.randn(4, 4) * 100).to(_FP8)+ folded = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)+ return raw_fp8, folded+++def test_orphan_is_scaled_exactly_once():+ """Orphans are scaled once; a second pass leaves them alone."""+ if _FP8 is None:+ return+ scale = torch.tensor([[0.0123, 0.0456], [0.0789, 0.0321]])+ raw_fp8, expected = _raw_and_folded(scale)+ for holds in ("fp8", "bf16"):+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.anchor = _fp8_anchor()+ model.model = nn.Module()+ if holds == "fp8":+ model.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())+ else:+ model.model.gate_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})+ first = _restore_dropped_fp8_scales(+ model, d, local_files_only = True, dtype = torch.bfloat16+ )+ second = _restore_dropped_fp8_scales(+ model, d, local_files_only = True, dtype = torch.bfloat16+ )+ assert first == (1, 0) and second == (0, 1), (holds, first, second)+ assert torch.equal(model.model.gate_proj.weight.data, expected), holds+++def test_already_dequantized_weight_is_not_scaled_again():+ """A bf16 weight with its scale already folded in is skipped, never double-scaled."""+ if _FP8 is None:+ return+ scale = torch.tensor([[0.0123, 0.0456], [0.0789, 0.0321]])+ raw_fp8, folded = _raw_and_folded(scale)+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.anchor = _fp8_anchor()+ model.model = nn.Module()+ model.model.gate_proj = _bf16_linear(4, 4, folded.clone())+ model.model.up_proj = _bf16_linear(4, 4, raw_fp8.to(torch.bfloat16))+ with tempfile.TemporaryDirectory() as d:+ _write_checkpoint(+ d,+ {+ "model.language_model.gate_proj.weight_scale_inv": scale,+ "model.language_model.up_proj.weight_scale_inv": scale.clone(),+ },+ )+ assert _restore_dropped_fp8_scales(model, d, local_files_only = True) == (1, 1)+ assert torch.equal(model.model.gate_proj.weight.data, folded)+ assert torch.equal(model.model.up_proj.weight.data, folded)+++def test_offloaded_already_dequantized_weight_is_not_scaled_again():+ """Offload path: folded stored value left as is, raw one scaled once, repeat pass a no-op."""+ if _FP8 is None:+ return+ import pytest++ pytest.importorskip("accelerate")+ scale = torch.tensor([[0.0123, 0.0456], [0.0789, 0.0321]])+ raw_fp8, folded = _raw_and_folded(scale)+ for stored, disk in (("folded", False), ("folded", True), ("raw", False), ("raw", True)):+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.anchor = _fp8_anchor()+ model.model = nn.Module()+ model.model.gate_proj = _bf16_linear(+ 4, 4, folded.clone() if stored == "folded" else raw_fp8.to(torch.bfloat16)+ )+ with tempfile.TemporaryDirectory() as d, tempfile.TemporaryDirectory() as off:+ store = _offload(model, "model.gate_proj", off if disk else None)+ _write_checkpoint(d, {"model.language_model.gate_proj.weight_scale_inv": scale})+ first = _restore_dropped_fp8_scales(+ model, d, local_files_only = True, dtype = torch.bfloat16+ )+ second = _restore_dropped_fp8_scales(+ model, d, local_files_only = True, dtype = torch.bfloat16+ )+ assert first == ((0, 1) if stored == "folded" else (1, 0)), (stored, disk, first)+ assert second == (0, 1), (stored, disk, second)+ assert torch.equal(store["model.gate_proj.weight"].to(torch.bfloat16), folded), (+ stored,+ disk,+ )+ x = torch.randn(3, 4, dtype = torch.bfloat16)+ assert torch.equal(model.model.gate_proj(x), torch.nn.functional.linear(x, folded)), (+ stored,+ disk,+ )+++def test_disk_offloaded_orphan_stays_on_disk():+ """A disk-offloaded orphan goes to a new .dat in the offload folder, not RAM; the checkpoint is untouched."""+ if _FP8 is None:+ return+ import pytest++ pytest.importorskip("accelerate")+ from accelerate.hooks import AlignDevicesHook, add_hook_to_module+ from accelerate.utils import OffloadedWeightsLoader, PrefixedDataset++ torch.manual_seed(0)+ raw_fp8 = (torch.randn(4, 4) * 100).to(_FP8)+ scale = torch.rand(2, 2, dtype = torch.float32) + 0.1+ model = nn.Module()+ model.config = _fp8_config((2, 2))+ model.anchor = _fp8_anchor()+ model.model = nn.Module()+ model.model.gate_proj = _fp8_linear(4, 4, raw_fp8.clone())+ key = "model.gate_proj.weight"+ with tempfile.TemporaryDirectory() as ck, tempfile.TemporaryDirectory() as off:+ ck_file = os.path.join(ck, "weights.safetensors")+ save_file({key: raw_fp8}, ck_file)+ before = open(ck_file, "rb").read()+ index = {key: {"safetensors_file": ck_file, "weight_name": key}}+ store = OffloadedWeightsLoader(save_folder = off, index = index)+ add_hook_to_module(+ model.model.gate_proj,+ AlignDevicesHook(+ execution_device = "cpu",+ offload = True,+ weights_map = PrefixedDataset(store, "model.gate_proj."),+ ),+ )+ _write_checkpoint(ck, {"model.language_model.gate_proj.weight_scale_inv": scale})+ assert _restore_dropped_fp8_scales(+ model, ck, local_files_only = True, dtype = torch.bfloat16+ ) == (1, 0)+ assert key not in store.state_dict+ assert store.index[key] == {"dtype": "bfloat16", "shape": [4, 4]}+ assert os.path.exists(os.path.join(off, key + ".dat"))+ assert open(ck_file, "rb").read() == before+ expected = (raw_fp8.to(torch.float32) * _expand(scale, (2, 2), (4, 4))).to(torch.bfloat16)+ assert torch.equal(store[key], expected)+ x = torch.randn(3, 4, dtype = torch.bfloat16)+ assert torch.equal(model.model.gate_proj(x), torch.nn.functional.linear(x, expected))diff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex e9f2dbba3a0..bb0f77fa9f9 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2778,6 +2778,7 @@ class FastLlamaModel:subfolder = kwargs.get("subfolder"),cache_dir = kwargs.get("cache_dir"),variant = kwargs.get("variant"),+ dtype = dtype,)elif not fast_inference:if user_config is not None:@@ -2831,6 +2832,7 @@ class FastLlamaModel:subfolder = kwargs.get("subfolder"),cache_dir = kwargs.get("cache_dir"),variant = kwargs.get("variant"),+ dtype = dtype,)model.fast_generate = make_fast_generate_wrapper(model.generate)model.fast_generate_batches = Nonediff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex d2f84ac67f1..b87f12a0341 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -1306,6 +1306,110 @@ def _match_fp8_module(module_by_name, base):return None+_FP8_DEQUANT_DTYPES = (torch.bfloat16, torch.float16, torch.float32)+++def _has_fp8_scale_attr(module):+ return any(+ isinstance(getattr(module, name, None), torch.Tensor)+ for name in ("weight_scale_inv", "weight_scale")+ )+++def _fp8_dequant_target_dtype(model, dtype = None):+ candidates = [dtype]+ config = getattr(model, "config", None)+ candidates += [getattr(config, "dtype", None), getattr(config, "torch_dtype", None)]+ try:+ candidates.append(model.get_input_embeddings().weight.dtype)+ except Exception:+ pass+ for candidate in candidates:+ if isinstance(candidate, str):+ candidate = getattr(torch, candidate.replace("torch.", ""), None)+ if candidate in _FP8_DEQUANT_DTYPES:+ return candidate+ return torch.bfloat16+++def _holds_raw_fp8_values(tensor, chunk_elements = 1 << 24):+ """True if every value lies on the e4m3/e5m2 grid; a folded-in scale falls off it, so never scale twice."""+ if tensor.dtype in _FP8_DTYPES:+ return True+ if tensor.dtype not in _FP8_DEQUANT_DTYPES:+ return False+ flat = tensor.detach().reshape(-1)+ for fp8 in _FP8_DTYPES:+ for start in range(0, flat.numel(), chunk_elements):+ chunk = flat[start : start + chunk_elements]+ if not torch.equal(chunk.to(fp8).to(chunk.dtype), chunk):+ break+ else:+ return True+ return False+++def _apply_fp8_block_scale(weight, scale, bs0, bs1):+ out_features, in_features = weight.shape+ out_blocks = (out_features + bs0 - 1) // bs0+ in_blocks = (in_features + bs1 - 1) // bs1+ if out_features % bs0 == 0 and in_features % bs1 == 0:+ # In place on block views: a full expanded fp32 scale + copy can OOM.+ weight.view(out_blocks, bs0, in_blocks, bs1).mul_(scale[:, None, :, None])+ return weight+ scale_expanded = scale.repeat_interleave(bs0, dim = 0).repeat_interleave(bs1, dim = 1)[+ :out_features, :in_features+ ]+ return (weight.to(torch.float32) * scale_expanded).to(weight.dtype)+++def _accelerate_offload_store(model):+ for module in model.modules():+ hook = getattr(module, "_hf_hook", None)+ if hook is None:+ continue+ for h in getattr(hook, "hooks", None) or (hook,):+ weights_map = getattr(h, "weights_map", None)+ if weights_map is None:+ continue+ return getattr(weights_map, "dataset", weights_map)+ return None+++def _offload_store_has(store, key):+ # `key in OffloadedWeightsLoader` would read the disk tensor.+ keys = getattr(store, "all_keys", None)+ if keys is not None:+ return key in keys+ try:+ return key in store+ except Exception:+ return False+++def _offload_store_set(store, key, value):+ state_dict = getattr(store, "state_dict", None)+ index = getattr(store, "index", None)+ save_folder = getattr(store, "save_folder", None)+ if (+ save_folder+ and isinstance(index, dict)+ and key in index+ and not (isinstance(state_dict, dict) and key in state_dict)+ ):+ # Disk-offloaded: write a new .dat (the index entry may point at the read-only checkpoint), never RAM.+ from accelerate.utils import offload_weight+ offload_weight(value.detach().cpu(), key, save_folder, index = index)+ return+ # OffloadedWeightsLoader checks state_dict before the disk index, so this overrides cpu tensors.+ if isinstance(state_dict, dict):+ state_dict[key] = value+ elif isinstance(store, dict):+ store[key] = value+ else:+ raise TypeError(f"Unsloth: cannot write offloaded tensor into {type(store).__name__}")++def _restore_dropped_fp8_scales(model,model_name,@@ -1316,8 +1420,9 @@ def _restore_dropped_fp8_scales(subfolder = None,cache_dir = None,variant = None,+ dtype = None,):- """Re-apply block-fp8 `weight_scale_inv` tensors that transformers dropped on load. On some block-scale fp8 checkpoints (e.g. Qwen3.6-27B-FP8, issue #6200) transformers fails to convert a Linear such as `mlp.gate_proj` to an fp8 module, loading the raw quantized values into a plain bf16 weight and discarding its `weight_scale_inv` as an unexpected key, so the weight is used un-scaled and the model is garbage. For every checkpoint scale whose live weight is not fp8, dequantize the orphaned weight in place; correctly converted modules keep an fp8 weight and are skipped, so a healthy checkpoint is a no-op. Returns (restored, skipped)."""+ """Re-apply block-fp8 `weight_scale_inv` tensors that transformers dropped on load. On some block-scale fp8 checkpoints (e.g. Qwen3.6-27B-FP8, issue #6200) transformers fails to convert a Linear such as `mlp.gate_proj` to an fp8 module, loading the raw quantized values into a plain bf16 weight and discarding its `weight_scale_inv` as an unexpected key, so the weight is used un-scaled and the model is garbage. For every checkpoint scale whose live weight is not fp8, dequantize the orphaned weight in place; correctly converted modules keep an fp8 weight and are skipped, so a healthy checkpoint is a no-op. A text_only-renamed key can instead leave raw fp8 values with no scale attribute; those are dequantized into `dtype`. Returns (restored, skipped)."""try:block = _fp8_block_size_from_config(model)if block is None or not _FP8_DTYPES:@@ -1339,6 +1444,9 @@ def _restore_dropped_fp8_scales(return (0, 0)module_by_name = dict(model.named_modules())+ name_by_module = {id(m): n for n, m in module_by_name.items()}+ _unset = object()+ offload_store = _unsetbs0, bs1 = blockrestored = 0skipped = 0@@ -1353,14 +1461,18 @@ def _restore_dropped_fp8_scales(weight = getattr(module, "weight", None)if not isinstance(weight, torch.Tensor) or weight.ndim != 2:continue- if weight.device.type == "meta":- # Disk-offloaded layer: the weight lives on meta until forward, so it cannot be scaled in place here. Count and warn rather than silently leave it unscaled.- offloaded += 1- continue- if weight.dtype in _FP8_DTYPES:- # Correctly converted fp8 module: the fp8 path already handles the scale.+ if weight.dtype in _FP8_DTYPES and _has_fp8_scale_attr(module):skipped += 1continue+ store_key = None+ if weight.device.type == "meta":+ if offload_store is _unset:+ offload_store = _accelerate_offload_store(model)+ store_key = f"{name_by_module.get(id(module), '')}.weight"+ if offload_store is None or not _offload_store_has(offload_store, store_key):+ offloaded += 1+ continue+ orphan_fp8 = weight.dtype in _FP8_DTYPES# Errors after this point are per-tensor: warn and continue, never abort or hide them.try:@@ -1389,19 +1501,32 @@ def _restore_dropped_fp8_scales(else:# Shape does not match the block grid: skip rather than apply a wrong scale.continue- scale = scale.to(weight.device)+ stored = None+ if store_key is not None:+ stored = offload_store[store_key]+ if tuple(stored.shape) != tuple(weight.shape):+ continue+ orphan_fp8 = stored.dtype in _FP8_DTYPES+ source = stored if stored is not None else weight+ if source.is_floating_point() and not _holds_raw_fp8_values(source):+ # Scale already folded in: never double-scale.+ skipped += 1+ continue+ target = _fp8_dequant_target_dtype(model, dtype) if orphan_fp8 else weight.dtypewith torch.no_grad():- if out_features % bs0 == 0 and in_features % bs1 == 0:- # Memory-frugal path: multiply block views in place against the broadcast fp32 scale, avoiding a full expanded scale and fp32 copy that could OOM.- module.weight.data.view(out_blocks, bs0, in_blocks, bs1).mul_(- scale[:, None, :, None]- )+ if store_key is not None:+ # Copy: the stored tensor can be a read-only disk memmap.+ value = stored.to(dtype = target, copy = True)+ value = _apply_fp8_block_scale(value, scale.to(value.device), bs0, bs1)+ _offload_store_set(offload_store, store_key, value)+ if weight.dtype != target:+ # accelerate casts the stored value to the placeholder dtype.+ module.weight.data = torch.empty_like(weight, dtype = target)else:- scale_expanded = scale.repeat_interleave(bs0, dim = 0).repeat_interleave(- bs1, dim = 1- )[:out_features, :in_features]- module.weight.data = (weight.to(torch.float32) * scale_expanded).to(- weight.dtype+ # Swap in once so an OOM leaves the weight untouched, not widened but unscaled.+ value = weight.data.to(target) if orphan_fp8 else weight.data+ module.weight.data = _apply_fp8_block_scale(+ value, scale.to(weight.device), bs0, bs1)restored += 1except Exception:@@ -1415,7 +1540,7 @@ def _restore_dropped_fp8_scales(if offloaded > 0:print(f"Unsloth: {offloaded} dropped FP8 weight_scale_inv tensor(s) skipped because the "- "layer is disk-offloaded; load without disk offload so the scales can be restored"+ "layer is offloaded outside accelerate's weights map; load without offload so the scales can be restored")return (restored, skipped)except Exception:diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 18547b33eff..38d7716d050 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -1783,6 +1783,7 @@ class FastBaseModel:subfolder = kwargs.get("subfolder"),cache_dir = kwargs.get("cache_dir"),variant = kwargs.get("variant"),+ dtype = torch_dtype,)if hasattr(model, "generate"):model.fast_generate = make_fast_generate_wrapper(model.generate)