Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Block Swap blocker: Qwen3.8-Flash-Next-FP8 can't train with offload_layers, its FP8 n-gram table loads without its weight_scale (#15632)
Leo Borcherding2ed0e9864ee7
diff --git a/tests/test_fp8_modules_to_convert_backfill.py b/tests/test_fp8_modules_to_convert_backfill.pynew file mode 100644index 00000000000..c2987a8b813--- /dev/null+++ b/tests/test_fp8_modules_to_convert_backfill.py@@ -0,0 +1,136 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""`backfill_fp8_modules_to_convert`: unsloth/Qwen3.8-Flash-Next-FP8's config omits the n-gram table from+`modules_to_convert`, so transformers 5.21 loaded it as raw fp8 and dropped its `weight_scale`."""++import ast, os, re, types++HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+SRC = open(os.path.join(HERE, "unsloth", "models", "loader_utils.py"), encoding = "utf-8").read()++# The real index keys of unsloth/Qwen3.8-Flash-Next-FP8, trimmed: one per-tensor scale, block scales elsewhere.+TABLE = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"+WEIGHT_MAP = {+ f"{TABLE}.weight": "a",+ f"{TABLE}.weight_scale": "a",+ "model.language_model.layers.1.ple.key_proj.weight": "a",+ "model.language_model.layers.0.self_attn.q_proj.weight": "b",+ "model.language_model.layers.0.self_attn.q_proj.weight_scale_inv": "b",+}+++def _backfill(weight_map):+ node = next(+ n+ for n in ast.parse(SRC).body+ if isinstance(n, ast.FunctionDef) and n.name == "backfill_fp8_modules_to_convert"+ )+ ns = {"re": re, "_load_fp8_weight_map": lambda *a, **k: weight_map}+ exec(ast.get_source_segment(SRC, node), ns)+ return ns["backfill_fp8_modules_to_convert"]+++def _config(**quant):+ return types.SimpleNamespace(quantization_config = {"quant_method": "fp8", **quant})+++def test_adds_the_table_the_unsloth_repo_left_out():+ config = _config(modules_to_not_convert = ["lm_head"])+ found = ["layers.1.ple.ple_embedding.ngram_embedding"]+ assert _backfill(WEIGHT_MAP)(config, "unsloth/Qwen3.8-Flash-Next-FP8") == found+ assert config.quantization_config["modules_to_convert"] == found+ # The quantizer matches by suffix: the runtime name must match, another layer's table must not.+ assert "model.language_model.layers.1.ple.ple_embedding.ngram_embedding".endswith(found[0])+ assert not "model.language_model.layers.11.ple.ple_embedding.ngram_embedding".endswith(found[0])+++def test_mixed_precision_tables_keep_their_layer():+ weight_map = {+ "model.layers.0.table.weight": "a",+ "model.layers.0.table.weight_scale": "a",+ "model.layers.1.table.weight": "a",+ "model.embed_fp8.weight_scale": "a",+ }+ assert _backfill(weight_map)(_config(), "x") == ["layers.0.table", "model.embed_fp8"]+++def test_leaves_configs_that_already_list_it_or_are_not_fp8_alone():+ listed = _config(modules_to_convert = ["ple.ple_embedding.ngram_embedding"])+ assert _backfill(WEIGHT_MAP)(listed, "Qwen/Qwen3.8-Flash-Next-FP8") is None+ bnb = types.SimpleNamespace(quantization_config = {"quant_method": "bitsandbytes"})+ assert _backfill(WEIGHT_MAP)(bnb, "x") is None+ assert "modules_to_convert" not in bnb.quantization_config+ assert _backfill(WEIGHT_MAP)(types.SimpleNamespace(), "x") is None+++def test_block_fp8_only_and_skipped_tables_add_nothing():+ block_only = {k: v for k, v in WEIGHT_MAP.items() if not k.endswith(".weight_scale")}+ config = _config()+ assert (+ _backfill(block_only)(config, "x") is None+ and "modules_to_convert" not in config.quantization_config+ )+ skipped = _config(modules_to_not_convert = [TABLE])+ assert _backfill(WEIGHT_MAP)(skipped, "x") is None+++def test_forwards_the_checkpoint_locator_options():+ calls = []+ node = next(+ n+ for n in ast.parse(SRC).body+ if isinstance(n, ast.FunctionDef) and n.name == "backfill_fp8_modules_to_convert"+ )+ ns = {"re": re, "_load_fp8_weight_map": lambda *a, **k: calls.append((a, k)) or WEIGHT_MAP}+ exec(ast.get_source_segment(SRC, node), ns)+ ns["backfill_fp8_modules_to_convert"](+ _config(), "repo", "tok", "rev", subfolder = "sub", cache_dir = "/c", variant = "fp8"+ )+ # subfolder / cache_dir / variant pick which index and shards are probed.+ ((args, kwargs),) = calls+ assert args == ("repo", False, "tok", "rev", "sub", "/c") and kwargs == {+ "variant": "fp8",+ "index_only": True,+ }+++def test_index_only_never_downloads_a_single_file(monkeypatch):+ import sys++ node = next(+ n+ for n in ast.parse(SRC).body+ if isinstance(n, ast.FunctionDef) and n.name == "_load_fp8_weight_map"+ )+ ns = {"os": os}+ exec(ast.get_source_segment(SRC, node), ns)+ fetched = []++ def hf_hub_download(repo, filename, **k):+ fetched.append(filename)+ raise FileNotFoundError(filename) # no index on this repo++ monkeypatch.setitem(+ sys.modules, "huggingface_hub", types.SimpleNamespace(hf_hub_download = hf_hub_download)+ )+ assert ns["_load_fp8_weight_map"]("org/single-file-fp8", False, None, index_only = True) is None+ assert fetched == ["model.safetensors.index.json"]+++def test_both_loaders_backfill_before_loading():+ # FastLanguageModel sends llama/mistral/gemma/qwen2/qwen3 through llama.py, everything else through vision.py.+ for name in ("llama.py", "vision.py"):+ src = open(os.path.join(HERE, "unsloth", "models", name), encoding = "utf-8").read()+ assert "backfill_fp8_modules_to_convert(" in src, name+ call = src.index("backfill_fp8_modules_to_convert(", src.index("def from_pretrained"))+ assert call < src.index(".from_pretrained(", call), name+++def test_llama_backfill_reaches_the_load():+ # The plain llama.py load re-reads config.json unless it passes config=; a backfill must force that branch.+ src = open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()+ assert "_fp8_backfilled = backfill_fp8_modules_to_convert(" in src+ routing = src[src.index("or _block_swap_saved is not None") :]+ routing = routing[: routing.index("):")]+ assert "or _fp8_backfilled" in routingdiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex 57405afb428..65e7e1aeef2 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -66,6 +66,7 @@ from .loader_utils import (_exclude_rope_inv_freq_from_ddp,_get_fp8_mode_and_check_settings,_restore_dropped_fp8_scales,+ backfill_fp8_modules_to_convert,_prepare_compressed_tensors_model,fsdp_will_wrap,fsdp2_keeps_fused_lora,@@ -3774,6 +3775,16 @@ class FastLlamaModel:revision = revision,)model_config.model_name = model_name+ _fp8_backfilled = backfill_fp8_modules_to_convert(+ model_config,+ model_name,+ token,+ revision,+ local_files_only = kwargs.get("local_files_only", False),+ subfolder = kwargs.get("subfolder"),+ cache_dir = kwargs.get("cache_dir"),+ variant = kwargs.get("variant"),+ )model_max_seq_length = model_config.max_position_embeddingsverify_fp8_support_if_applicable(@@ -4252,6 +4263,8 @@ class FastLlamaModel:or _ct_requantor _fp8_to_nf4or _block_swap_saved is not None+ # Backfilled modules_to_convert lives only on model_config.+ or _fp8_backfilled):# Transformers 5.x @strict init rejects extra kwargs alongside config=, so set overrides on the config instead.if max_position_embeddings is not None:@@ -4259,7 +4272,9 @@ class FastLlamaModel:_rope_scaling = kwargs.pop("rope_scaling", None)if _rope_scaling is not None:model_config.rope_scaling = _rope_scaling- if (_modelopt_rewritten or _fp8_to_nf4) and user_config is None:+ if (+ _modelopt_rewritten or _fp8_to_nf4 or _fp8_backfilled+ ) and user_config is None:move_config_overrides_onto_config(model_config, kwargs)try:with sync_load_when_quantizing(diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.pyindex 56414a025e2..dfcf9b18a36 100644--- a/unsloth/models/loader_utils.py+++ b/unsloth/models/loader_utils.py@@ -2375,6 +2375,50 @@ def _fp8_block_size_from_config(model):return [int(block[0]), int(block[1])]+def backfill_fp8_modules_to_convert(+ config,+ model_name,+ token = None,+ revision = None,+ local_files_only = False,+ subfolder = None,+ cache_dir = None,+ variant = None,+):+ """List per-tensor fp8 tables (`.weight_scale`) a config leaves out of `modules_to_convert` (Qwen3.8-Flash-Next-FP8)."""+ quant = getattr(config, "quantization_config", None)+ if (+ not isinstance(quant, dict)+ or quant.get("quant_method") != "fp8"+ or quant.get("modules_to_convert")+ ):+ return None+ try:+ weight_map = _load_fp8_weight_map(+ model_name,+ local_files_only,+ token,+ revision,+ subfolder,+ cache_dir,+ variant = variant,+ index_only = True, # runs before the guarded weight download: never fetch the weights here+ )+ except Exception:+ return None+ skip = set(quant.get("modules_to_not_convert") or [])+ modules = {k[: -len(".weight_scale")] for k in weight_map or {} if k.endswith(".weight_scale")}+ # Linear blocks carry weight_scale_inv; only embeddings match, and the quantizer converts nn.Embedding only.+ # Keep the layer index: the quantizer matches by suffix, so a bare name would also convert another layer's dense table.+ found = sorted({re.sub(r"^.*\.(?=layers\.\d+\.)", "", m) for m in modules - skip})+ if found:+ quant["modules_to_convert"] = found+ print(+ f"Unsloth: {model_name} ships fp8 scales for {found} without listing them in modules_to_convert; added."+ )+ return found or None++def _load_fp8_weight_map(model_name,local_files_only,@@ -2383,8 +2427,10 @@ def _load_fp8_weight_map(subfolder = None,cache_dir = None,variant = None,+ index_only = False,):- """Tensor->file map from the load's snapshot; sharded index, else single `model.safetensors`."""+ """Tensor->file map from the load's snapshot; sharded index, else single `model.safetensors`.+ `index_only` never downloads a remote single file just to list its keys."""def _local_path(filename):return (@@ -2429,7 +2475,7 @@ def _load_fp8_weight_map(try:if is_local and os.path.exists(_local_path(single_file)):single_path = _local_path(single_file)- elif not is_local:+ elif not is_local and not index_only:single_path = _remote_path(single_file)else:return Nonediff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 70539820213..8ab51e690d5 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -315,6 +315,7 @@ from .loader_utils import (planner_class_mismatch_reason,planner_model_class,exclude_no_placement_params,+ backfill_fp8_modules_to_convert,planner_quantization_kwargs,raise_if_bnb_cpu_spill,requested_device_map,@@ -4081,6 +4082,16 @@ class FastBaseModel:local_files_only = local_files_only,revision = _revision,)+ backfill_fp8_modules_to_convert(+ auto_config,+ model_name,+ token,+ _revision,+ local_files_only = local_files_only,+ subfolder = kwargs.get("subfolder"),+ cache_dir = kwargs.get("cache_dir"),+ variant = kwargs.get("variant"),+ )# Remote classes shadowing a native name can differ in attention flags._builds_remote_class, _remote_class = resolve_remote_code_model_class(auto_model,