Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Block Swap blocker: Qwen3.8-Flash-Next-FP8 can't train with offload_layers, its FP8 n-gram table loads without its weight_scale (#15632)

Leo Borcherding2ed0e9864ee7
diff --git a/tests/test_fp8_modules_to_convert_backfill.py b/tests/test_fp8_modules_to_convert_backfill.py
new file mode 100644
index 00000000000..c2987a8b813
--- /dev/null
+++ b/tests/test_fp8_modules_to_convert_backfill.py
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""`backfill_fp8_modules_to_convert`: unsloth/Qwen3.8-Flash-Next-FP8's config omits the n-gram table from
+`modules_to_convert`, so transformers 5.21 loaded it as raw fp8 and dropped its `weight_scale`."""
+
+import ast, os, re, types
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+SRC = open(os.path.join(HERE, "unsloth", "models", "loader_utils.py"), encoding = "utf-8").read()
+
+# The real index keys of unsloth/Qwen3.8-Flash-Next-FP8, trimmed: one per-tensor scale, block scales elsewhere.
+TABLE = "model.language_model.layers.1.ple.ple_embedding.ngram_embedding"
+WEIGHT_MAP = {
+ f"{TABLE}.weight": "a",
+ f"{TABLE}.weight_scale": "a",
+ "model.language_model.layers.1.ple.key_proj.weight": "a",
+ "model.language_model.layers.0.self_attn.q_proj.weight": "b",
+ "model.language_model.layers.0.self_attn.q_proj.weight_scale_inv": "b",
+}
+
+
+def _backfill(weight_map):
+ node = next(
+ n
+ for n in ast.parse(SRC).body
+ if isinstance(n, ast.FunctionDef) and n.name == "backfill_fp8_modules_to_convert"
+ )
+ ns = {"re": re, "_load_fp8_weight_map": lambda *a, **k: weight_map}
+ exec(ast.get_source_segment(SRC, node), ns)
+ return ns["backfill_fp8_modules_to_convert"]
+
+
+def _config(**quant):
+ return types.SimpleNamespace(quantization_config = {"quant_method": "fp8", **quant})
+
+
+def test_adds_the_table_the_unsloth_repo_left_out():
+ config = _config(modules_to_not_convert = ["lm_head"])
+ found = ["layers.1.ple.ple_embedding.ngram_embedding"]
+ assert _backfill(WEIGHT_MAP)(config, "unsloth/Qwen3.8-Flash-Next-FP8") == found
+ assert config.quantization_config["modules_to_convert"] == found
+ # The quantizer matches by suffix: the runtime name must match, another layer's table must not.
+ assert "model.language_model.layers.1.ple.ple_embedding.ngram_embedding".endswith(found[0])
+ assert not "model.language_model.layers.11.ple.ple_embedding.ngram_embedding".endswith(found[0])
+
+
+def test_mixed_precision_tables_keep_their_layer():
+ weight_map = {
+ "model.layers.0.table.weight": "a",
+ "model.layers.0.table.weight_scale": "a",
+ "model.layers.1.table.weight": "a",
+ "model.embed_fp8.weight_scale": "a",
+ }
+ assert _backfill(weight_map)(_config(), "x") == ["layers.0.table", "model.embed_fp8"]
+
+
+def test_leaves_configs_that_already_list_it_or_are_not_fp8_alone():
+ listed = _config(modules_to_convert = ["ple.ple_embedding.ngram_embedding"])
+ assert _backfill(WEIGHT_MAP)(listed, "Qwen/Qwen3.8-Flash-Next-FP8") is None
+ bnb = types.SimpleNamespace(quantization_config = {"quant_method": "bitsandbytes"})
+ assert _backfill(WEIGHT_MAP)(bnb, "x") is None
+ assert "modules_to_convert" not in bnb.quantization_config
+ assert _backfill(WEIGHT_MAP)(types.SimpleNamespace(), "x") is None
+
+
+def test_block_fp8_only_and_skipped_tables_add_nothing():
+ block_only = {k: v for k, v in WEIGHT_MAP.items() if not k.endswith(".weight_scale")}
+ config = _config()
+ assert (
+ _backfill(block_only)(config, "x") is None
+ and "modules_to_convert" not in config.quantization_config
+ )
+ skipped = _config(modules_to_not_convert = [TABLE])
+ assert _backfill(WEIGHT_MAP)(skipped, "x") is None
+
+
+def test_forwards_the_checkpoint_locator_options():
+ calls = []
+ node = next(
+ n
+ for n in ast.parse(SRC).body
+ if isinstance(n, ast.FunctionDef) and n.name == "backfill_fp8_modules_to_convert"
+ )
+ ns = {"re": re, "_load_fp8_weight_map": lambda *a, **k: calls.append((a, k)) or WEIGHT_MAP}
+ exec(ast.get_source_segment(SRC, node), ns)
+ ns["backfill_fp8_modules_to_convert"](
+ _config(), "repo", "tok", "rev", subfolder = "sub", cache_dir = "/c", variant = "fp8"
+ )
+ # subfolder / cache_dir / variant pick which index and shards are probed.
+ ((args, kwargs),) = calls
+ assert args == ("repo", False, "tok", "rev", "sub", "/c") and kwargs == {
+ "variant": "fp8",
+ "index_only": True,
+ }
+
+
+def test_index_only_never_downloads_a_single_file(monkeypatch):
+ import sys
+
+ node = next(
+ n
+ for n in ast.parse(SRC).body
+ if isinstance(n, ast.FunctionDef) and n.name == "_load_fp8_weight_map"
+ )
+ ns = {"os": os}
+ exec(ast.get_source_segment(SRC, node), ns)
+ fetched = []
+
+ def hf_hub_download(repo, filename, **k):
+ fetched.append(filename)
+ raise FileNotFoundError(filename) # no index on this repo
+
+ monkeypatch.setitem(
+ sys.modules, "huggingface_hub", types.SimpleNamespace(hf_hub_download = hf_hub_download)
+ )
+ assert ns["_load_fp8_weight_map"]("org/single-file-fp8", False, None, index_only = True) is None
+ assert fetched == ["model.safetensors.index.json"]
+
+
+def test_both_loaders_backfill_before_loading():
+ # FastLanguageModel sends llama/mistral/gemma/qwen2/qwen3 through llama.py, everything else through vision.py.
+ for name in ("llama.py", "vision.py"):
+ src = open(os.path.join(HERE, "unsloth", "models", name), encoding = "utf-8").read()
+ assert "backfill_fp8_modules_to_convert(" in src, name
+ call = src.index("backfill_fp8_modules_to_convert(", src.index("def from_pretrained"))
+ assert call < src.index(".from_pretrained(", call), name
+
+
+def test_llama_backfill_reaches_the_load():
+ # The plain llama.py load re-reads config.json unless it passes config=; a backfill must force that branch.
+ src = open(os.path.join(HERE, "unsloth", "models", "llama.py"), encoding = "utf-8").read()
+ assert "_fp8_backfilled = backfill_fp8_modules_to_convert(" in src
+ routing = src[src.index("or _block_swap_saved is not None") :]
+ routing = routing[: routing.index("):")]
+ assert "or _fp8_backfilled" in routing
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 57405afb428..65e7e1aeef2 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -66,6 +66,7 @@ from .loader_utils import (
_exclude_rope_inv_freq_from_ddp,
_get_fp8_mode_and_check_settings,
_restore_dropped_fp8_scales,
+ backfill_fp8_modules_to_convert,
_prepare_compressed_tensors_model,
fsdp_will_wrap,
fsdp2_keeps_fused_lora,
@@ -3774,6 +3775,16 @@ class FastLlamaModel:
revision = revision,
)
model_config.model_name = model_name
+ _fp8_backfilled = backfill_fp8_modules_to_convert(
+ model_config,
+ model_name,
+ token,
+ revision,
+ local_files_only = kwargs.get("local_files_only", False),
+ subfolder = kwargs.get("subfolder"),
+ cache_dir = kwargs.get("cache_dir"),
+ variant = kwargs.get("variant"),
+ )
model_max_seq_length = model_config.max_position_embeddings
verify_fp8_support_if_applicable(
@@ -4252,6 +4263,8 @@ class FastLlamaModel:
or _ct_requant
or _fp8_to_nf4
or _block_swap_saved is not None
+ # Backfilled modules_to_convert lives only on model_config.
+ or _fp8_backfilled
):
# Transformers 5.x @strict init rejects extra kwargs alongside config=, so set overrides on the config instead.
if max_position_embeddings is not None:
@@ -4259,7 +4272,9 @@ class FastLlamaModel:
_rope_scaling = kwargs.pop("rope_scaling", None)
if _rope_scaling is not None:
model_config.rope_scaling = _rope_scaling
- if (_modelopt_rewritten or _fp8_to_nf4) and user_config is None:
+ if (
+ _modelopt_rewritten or _fp8_to_nf4 or _fp8_backfilled
+ ) and user_config is None:
move_config_overrides_onto_config(model_config, kwargs)
try:
with sync_load_when_quantizing(
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 56414a025e2..dfcf9b18a36 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -2375,6 +2375,50 @@ def _fp8_block_size_from_config(model):
return [int(block[0]), int(block[1])]
+def backfill_fp8_modules_to_convert(
+ config,
+ model_name,
+ token = None,
+ revision = None,
+ local_files_only = False,
+ subfolder = None,
+ cache_dir = None,
+ variant = None,
+):
+ """List per-tensor fp8 tables (`.weight_scale`) a config leaves out of `modules_to_convert` (Qwen3.8-Flash-Next-FP8)."""
+ quant = getattr(config, "quantization_config", None)
+ if (
+ not isinstance(quant, dict)
+ or quant.get("quant_method") != "fp8"
+ or quant.get("modules_to_convert")
+ ):
+ return None
+ try:
+ weight_map = _load_fp8_weight_map(
+ model_name,
+ local_files_only,
+ token,
+ revision,
+ subfolder,
+ cache_dir,
+ variant = variant,
+ index_only = True, # runs before the guarded weight download: never fetch the weights here
+ )
+ except Exception:
+ return None
+ skip = set(quant.get("modules_to_not_convert") or [])
+ modules = {k[: -len(".weight_scale")] for k in weight_map or {} if k.endswith(".weight_scale")}
+ # Linear blocks carry weight_scale_inv; only embeddings match, and the quantizer converts nn.Embedding only.
+ # Keep the layer index: the quantizer matches by suffix, so a bare name would also convert another layer's dense table.
+ found = sorted({re.sub(r"^.*\.(?=layers\.\d+\.)", "", m) for m in modules - skip})
+ if found:
+ quant["modules_to_convert"] = found
+ print(
+ f"Unsloth: {model_name} ships fp8 scales for {found} without listing them in modules_to_convert; added."
+ )
+ return found or None
+
+
def _load_fp8_weight_map(
model_name,
local_files_only,
@@ -2383,8 +2427,10 @@ def _load_fp8_weight_map(
subfolder = None,
cache_dir = None,
variant = None,
+ index_only = False,
):
- """Tensor->file map from the load's snapshot; sharded index, else single `model.safetensors`."""
+ """Tensor->file map from the load's snapshot; sharded index, else single `model.safetensors`.
+ `index_only` never downloads a remote single file just to list its keys."""
def _local_path(filename):
return (
@@ -2429,7 +2475,7 @@ def _load_fp8_weight_map(
try:
if is_local and os.path.exists(_local_path(single_file)):
single_path = _local_path(single_file)
- elif not is_local:
+ elif not is_local and not index_only:
single_path = _remote_path(single_file)
else:
return None
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 70539820213..8ab51e690d5 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -315,6 +315,7 @@ from .loader_utils import (
planner_class_mismatch_reason,
planner_model_class,
exclude_no_placement_params,
+ backfill_fp8_modules_to_convert,
planner_quantization_kwargs,
raise_if_bnb_cpu_spill,
requested_device_map,
@@ -4081,6 +4082,16 @@ class FastBaseModel:
local_files_only = local_files_only,
revision = _revision,
)
+ backfill_fp8_modules_to_convert(
+ auto_config,
+ model_name,
+ token,
+ _revision,
+ local_files_only = local_files_only,
+ subfolder = kwargs.get("subfolder"),
+ cache_dir = kwargs.get("cache_dir"),
+ variant = kwargs.get("variant"),
+ )
# Remote classes shadowing a native name can differ in attention flags.
_builds_remote_class, _remote_class = resolve_remote_code_model_class(
auto_model,