Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load NVIDIA ModelOpt FP8 checkpoints through the transformers fp8 quantizer (#11592)

Daniel Han614f2d5af15f
diff --git a/tests/test_modelopt_fp8_loading.py b/tests/test_modelopt_fp8_loading.py
new file mode 100644
index 00000000000..fa9569dc7f3
--- /dev/null
+++ b/tests/test_modelopt_fp8_loading.py
@@ -0,0 +1,1138 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""ModelOpt FP8 checkpoints (sarvam-105b-fp8) load via the transformers fp8 quantizer."""
+
+import inspect
+import json
+import os
+import re
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn as nn
+from real_accelerator import has_real_cuda
+
+import unsloth # noqa: F401
+from unsloth.models.modelopt_fp8 import (
+ MODELOPT_FP8_KEY_MAPPING,
+ _transformers_accepts_fp8_plan,
+ UNSLOTH_MODELOPT_KEY_MAPPING_ATTR,
+ arm_modelopt_fp8_loading,
+ modelopt_fp8_plan,
+ pop_modelopt_key_mapping,
+)
+from unsloth.models.loader_utils import check_and_disable_bitsandbytes_loading
+
+
+def _sarvam_quant(**overrides):
+ quant = {
+ "config_groups": {
+ "group_0": {
+ "input_activations": {"dynamic": False, "num_bits": 8, "type": "float"},
+ "weights": {"dynamic": False, "num_bits": 8, "type": "float"},
+ "targets": ["Linear"],
+ }
+ },
+ "ignore": ["lm_head"],
+ "quant_algo": "FP8",
+ "kv_cache_scheme": {"dynamic": False, "num_bits": 8, "type": "float"},
+ "producer": {"name": "modelopt", "version": "0.42.0"},
+ "quant_method": "modelopt",
+ }
+ quant.update(overrides)
+ return quant
+
+
+# transformers 4.x only has dynamic block fp8, so the rewrite is skipped there.
+needs_per_tensor_fp8 = pytest.mark.skipif(
+ not _transformers_accepts_fp8_plan(
+ modelopt_fp8_plan(SimpleNamespace(quantization_config = _sarvam_quant()))
+ ),
+ reason = "this transformers has no per-tensor fp8, so ModelOpt configs are left as is",
+)
+
+
+def test_plan_maps_sarvam_block_to_static_per_tensor_fp8():
+ from transformers.quantizers.quantizers_utils import should_convert_module
+
+ plan = modelopt_fp8_plan(SimpleNamespace(quantization_config = _sarvam_quant()))
+ skip = plan.pop("modules_to_not_convert")
+ assert plan == {"quant_method": "fp8", "weight_block_size": None, "activation_scheme": "static"}
+ assert not should_convert_module("lm_head", skip)
+ assert should_convert_module("model.layers.0.self_attn.q_proj", skip)
+
+
+def test_plan_accepts_hf_quant_config_spelling_and_weight_only():
+ legacy = {
+ "quant_method": "modelopt",
+ "quantization": {"quant_algo": "FP8", "exclude_modules": ["lm_head", "mlp.gate"]},
+ }
+ from transformers.quantizers.quantizers_utils import should_convert_module
+
+ plan = modelopt_fp8_plan(SimpleNamespace(quantization_config = legacy))
+ skip = plan["modules_to_not_convert"]
+ assert not should_convert_module("lm_head", skip)
+ assert not should_convert_module("model.layers.3.mlp.gate", skip)
+ assert should_convert_module("model.layers.3.mlp.gate_proj", skip)
+ assert plan["activation_scheme"] == "static"
+
+ weight_only = _sarvam_quant()
+ weight_only["config_groups"]["group_0"]["input_activations"] = None
+ plan = modelopt_fp8_plan(SimpleNamespace(quantization_config = weight_only))
+ assert plan["activation_scheme"] == "dynamic"
+
+
+def test_plan_declines_everything_else():
+ cases = {
+ "none": None,
+ "native fp8": {"quant_method": "fp8", "weight_block_size": [128, 128]},
+ "nvfp4": _sarvam_quant(quant_algo = "NVFP4"),
+ "int4 awq": _sarvam_quant(quant_algo = "W4A8_AWQ"),
+ "fp4 weights": _sarvam_quant(
+ config_groups = {
+ "g": {"weights": {"num_bits": 4, "type": "float"}, "input_activations": None}
+ }
+ ),
+ "block strategy": _sarvam_quant(
+ config_groups = {
+ "g": {
+ "weights": {"num_bits": 8, "type": "float", "strategy": "block"},
+ "input_activations": None,
+ }
+ }
+ ),
+ "no groups": _sarvam_quant(config_groups = {}),
+ "targets narrower than every Linear": _sarvam_quant(
+ config_groups = {
+ "g": {
+ "weights": {"num_bits": 8, "type": "float"},
+ "input_activations": {"num_bits": 8, "type": "float"},
+ "targets": ["re:.*mlp.*"],
+ }
+ }
+ ),
+ "static channel activations": _sarvam_quant(
+ config_groups = {
+ "g": {
+ "weights": {"num_bits": 8, "type": "float"},
+ "input_activations": {
+ "num_bits": 8,
+ "type": "float",
+ "dynamic": False,
+ "strategy": "channel",
+ },
+ }
+ }
+ ),
+ **{
+ f"dynamic {strategy} activations": _sarvam_quant(
+ config_groups = {
+ "g": {
+ "weights": {"num_bits": 8, "type": "float"},
+ "input_activations": {
+ "num_bits": 8,
+ "type": "float",
+ "dynamic": True,
+ "strategy": strategy,
+ },
+ }
+ }
+ )
+ for strategy in ("channel", "group", "block")
+ },
+ "int8 activations": _sarvam_quant(
+ config_groups = {
+ "g": {
+ "weights": {"num_bits": 8, "type": "float"},
+ "input_activations": {"num_bits": 8, "type": "int", "dynamic": False},
+ }
+ }
+ ),
+ }
+ for name, quant in cases.items():
+ assert modelopt_fp8_plan(SimpleNamespace(quantization_config = quant)) is None, name
+
+
+@needs_per_tensor_fp8
+def test_arm_rewrites_config_and_hands_mapping_to_kwargs():
+ config = SimpleNamespace(quantization_config = _sarvam_quant())
+ plan = arm_modelopt_fp8_loading(config, verbose = False)
+ assert config.quantization_config == plan
+ assert config.quantization_config["quant_method"] == "fp8"
+
+ kwargs = {"key_mapping": {r"\.weight_scale$": "user_wins", r"^old\.": "new."}}
+ pop_modelopt_key_mapping(config, kwargs)
+ assert not hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+ assert kwargs["key_mapping"] == {
+ r"\.weight_scale$": "user_wins",
+ r"^old\.": "new.",
+ r"\.input_scale$": ".activation_scale",
+ }
+ before = dict(kwargs)
+ pop_modelopt_key_mapping(config, kwargs)
+ assert kwargs == before
+
+
+def _pin_vlm_names(monkeypatch, names):
+ """Point both places transformers has kept its VLM name list at ``names``."""
+ import importlib
+ for module_name in ("transformers.conversion_mapping", "transformers.modeling_utils"):
+ try:
+ module = importlib.import_module(module_name)
+ except Exception:
+ continue
+ monkeypatch.setattr(module, "VLMS", list(names), raising = False)
+
+
+def test_key_mapping_keeps_the_vlm_checkpoint_renames(monkeypatch):
+ """transformers 5.3 applies a VLM's class renames only without a caller key_mapping."""
+ _pin_vlm_names(monkeypatch, ["llava"])
+
+ class LlavaForConditionalGeneration:
+ _checkpoint_conversion_mapping = {r"^language_model\.model": "model.language_model"}
+
+ config = SimpleNamespace(quantization_config = _sarvam_quant())
+ if arm_modelopt_fp8_loading(config, verbose = False) is None:
+ pytest.skip("this transformers has no per-tensor fp8")
+ kwargs = {}
+ pop_modelopt_key_mapping(config, kwargs, LlavaForConditionalGeneration)
+ assert kwargs["key_mapping"] == {
+ r"^language_model\.model": "model.language_model",
+ r"\.weight_scale$": ".weight_scale_inv",
+ r"\.input_scale$": ".activation_scale",
+ }
+
+
+def test_key_mapping_adds_no_class_renames_transformers_would_not_apply(monkeypatch):
+ _pin_vlm_names(monkeypatch, ["llava"])
+ renames = {r"^language_model\.model": "model.language_model"}
+
+ class MistralForCausalLM:
+ _checkpoint_conversion_mapping = dict(renames)
+
+ class LlavaForConditionalGeneration:
+ _checkpoint_conversion_mapping = dict(renames)
+
+ scale_only = {
+ r"\.weight_scale$": ".weight_scale_inv",
+ r"\.input_scale$": ".activation_scale",
+ }
+ for model_class, kwargs in (
+ (MistralForCausalLM, {}), # not a VLM
+ (None, {}), # class not resolved
+ (LlavaForConditionalGeneration, {"key_mapping": {}}), # the caller's mapping wins
+ ):
+ config = SimpleNamespace(quantization_config = _sarvam_quant())
+ if arm_modelopt_fp8_loading(config, verbose = False) is None:
+ pytest.skip("this transformers has no per-tensor fp8")
+ pop_modelopt_key_mapping(config, kwargs, model_class)
+ assert kwargs["key_mapping"] == scale_only, model_class
+
+
+def test_transformers_without_per_tensor_fp8_keep_the_config():
+ if _transformers_accepts_fp8_plan({"quant_method": "fp8", "weight_block_size": None}):
+ pytest.skip("this transformers loads per-tensor fp8")
+ config = SimpleNamespace(quantization_config = _sarvam_quant())
+ assert arm_modelopt_fp8_loading(config, verbose = False) is None
+ assert config.quantization_config == _sarvam_quant()
+ assert not hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+
+
+def test_key_mapping_is_anchored():
+ renamed = {}
+ for key in (
+ "model.layers.0.mlp.experts.3.down_proj.weight_scale",
+ "model.layers.0.mlp.experts.3.down_proj.input_scale",
+ "model.layers.0.mlp.experts.3.down_proj.weight_scale_inv",
+ "model.layers.0.mlp.experts.3.down_proj.weight_scale_2",
+ "model.layers.0.mlp.experts.3.down_proj.weight",
+ ):
+ new = key
+ for pattern, target in MODELOPT_FP8_KEY_MAPPING.items():
+ new = re.sub(pattern, target, new)
+ renamed[key] = new
+ assert list(renamed.values()) == [
+ "model.layers.0.mlp.experts.3.down_proj.weight_scale_inv",
+ "model.layers.0.mlp.experts.3.down_proj.activation_scale",
+ "model.layers.0.mlp.experts.3.down_proj.weight_scale_inv",
+ "model.layers.0.mlp.experts.3.down_proj.weight_scale_2",
+ "model.layers.0.mlp.experts.3.down_proj.weight",
+ ]
+
+
+@needs_per_tensor_fp8
+def test_check_and_disable_rewrites_only_when_asked():
+ config = SimpleNamespace(quantization_config = _sarvam_quant())
+ load_in_4bit, load_in_8bit, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = True, verbose = False, rewrite_modelopt = False
+ )
+ assert method == "modelopt" and config.quantization_config["quant_method"] == "modelopt"
+ assert not hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+
+ load_in_4bit, load_in_8bit, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = True, verbose = False
+ )
+ assert method == "fp8" and (load_in_4bit, load_in_8bit) == (False, False)
+ assert config.quantization_config["quant_method"] == "fp8"
+ assert hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+
+
+def _fp8_quantizer(**config_kwargs):
+ try:
+ from transformers import FineGrainedFP8Config
+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer
+ except Exception as e:
+ pytest.skip(f"transformers fp8 quantizer unavailable: {e}")
+ config = FineGrainedFP8Config(
+ activation_scheme = "static", weight_block_size = None, **config_kwargs
+ )
+ quantizer = FineGrainedFP8HfQuantizer(config)
+ quantizer.pre_quantized = True
+ return quantizer
+
+
+def _tiny_remote_moe():
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ config = LlamaConfig(
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 128,
+ )
+ with torch.device("meta"):
+ model = LlamaForCausalLM(config)
+ for layer in model.model.layers:
+ mlp = nn.Module()
+ mlp.gate = nn.Linear(64, 4, bias = False, device = "meta")
+ mlp.experts = nn.ModuleList()
+ for _ in range(4):
+ expert = nn.Module()
+ expert.gate_proj = nn.Linear(64, 32, bias = False, device = "meta")
+ expert.up_proj = nn.Linear(64, 32, bias = False, device = "meta")
+ expert.down_proj = nn.Linear(32, 64, bias = False, device = "meta")
+ mlp.experts.append(expert)
+ layer.mlp = mlp
+ return model
+
+
+@needs_per_tensor_fp8
+def test_modulelist_experts_become_fp8_linears():
+ quantizer = _fp8_quantizer()
+ from transformers.integrations.finegrained_fp8 import FP8Linear
+
+ model = _tiny_remote_moe()
+ quantizer.quantization_config.modules_to_not_convert = [
+ "lm_head",
+ "model.layers.1.mlp.experts.2.down_proj",
+ ]
+ quantizer._process_model_before_weight_loading(model)
+
+ names = [name for name, _ in model.named_modules()]
+ assert not any("_unsloth_modulelist" in name for name in names)
+ for i, layer in enumerate(model.model.layers):
+ assert isinstance(layer.mlp.experts, nn.ModuleList)
+ assert len(layer.mlp.experts) == 4
+ for j, expert in enumerate(layer.mlp.experts):
+ for proj in ("gate_proj", "up_proj", "down_proj"):
+ module = getattr(expert, proj)
+ excluded = (i, j, proj) == (1, 2, "down_proj")
+ assert isinstance(module, FP8Linear) != excluded, (i, j, proj)
+ assert type(model.lm_head) is nn.Linear
+ assert list(model.model.layers[0].mlp._modules) == ["gate", "experts"]
+
+
+@needs_per_tensor_fp8
+def test_wrapper_hides_only_modulelist_experts():
+ from unsloth.import_fixes import _wrap_fp8_replace_for_modulelist_experts
+
+ model = _tiny_remote_moe()
+
+ class StackedExperts(nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.gate_up_proj = nn.Parameter(torch.empty(4, 64, 64, device = "meta"))
+
+ model.model.layers[0].mlp.experts = StackedExperts()
+ seen = {}
+
+ def original(
+ model,
+ modules_to_not_convert = None,
+ quantization_config = None,
+ pre_quantized = False,
+ ):
+ seen["experts"] = [n for n, _ in model.named_modules() if n.endswith(".experts")]
+ seen["patterns"] = modules_to_not_convert
+ return model
+
+ wrapped = _wrap_fp8_replace_for_modulelist_experts(original)
+ assert _wrap_fp8_replace_for_modulelist_experts(wrapped) is wrapped
+ wrapped(model, modules_to_not_convert = ["lm_head", "model.layers.1.mlp.experts.0.up_proj"])
+ # The stacked module still reaches the by-name FP8Experts branch; the list does not.
+ assert seen["experts"] == ["model.layers.0.mlp.experts"]
+ assert seen["patterns"][0] == "lm_head"
+ assert any("experts_unsloth_modulelist\\.0\\.up_proj$" in p for p in seen["patterns"])
+ assert isinstance(model.model.layers[1].mlp.experts, nn.ModuleList)
+ assert list(model.model.layers[1].mlp._modules) == ["gate", "experts"]
+
+ def boom(
+ model,
+ modules_to_not_convert = None,
+ quantization_config = None,
+ pre_quantized = False,
+ ):
+ raise RuntimeError("boom")
+
+ with pytest.raises(RuntimeError):
+ _wrap_fp8_replace_for_modulelist_experts(boom)(model)
+ assert list(model.model.layers[1].mlp._modules) == ["gate", "experts"]
+
+
+def _write_tiny_modelopt_llama(path):
+ from safetensors.torch import save_file
+ from transformers import LlamaConfig, LlamaForCausalLM
+
+ torch.manual_seed(0)
+ config = LlamaConfig(
+ hidden_size = 256,
+ intermediate_size = 512,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 512,
+ )
+ reference = LlamaForCausalLM(config).to(torch.bfloat16)
+ tensors, dequantized = {}, {}
+ for key, value in reference.state_dict().items():
+ if key.endswith("proj.weight"):
+ scale = value.float().abs().amax() / 448.0
+ packed = (value.float() / scale).to(torch.float8_e4m3fn)
+ tensors[key] = packed
+ tensors[key.replace(".weight", ".weight_scale")] = scale.reshape(()).float()
+ tensors[key.replace(".weight", ".input_scale")] = torch.tensor(0.02)
+ dequantized[key] = (packed.float() * scale).to(torch.bfloat16)
+ else:
+ tensors[key] = value.contiguous()
+ dequantized[key] = value
+ save_file(tensors, os.path.join(path, "model.safetensors"))
+ reference.load_state_dict(dequantized)
+ raw = config.to_dict()
+ raw["architectures"] = ["LlamaForCausalLM"]
+ raw["quantization_config"] = _sarvam_quant()
+ with open(os.path.join(path, "config.json"), "w") as f:
+ json.dump(raw, f)
+ return reference
+
+
+@needs_per_tensor_fp8
[email protected](not has_real_cuda(), reason = "fp8 kernels need CUDA")
[email protected]("dequantize", [False, True])
+def test_tiny_modelopt_llama_round_trip(tmp_path, dequantize):
+ from transformers import AutoConfig, AutoModelForCausalLM, FineGrainedFP8Config
+
+ if torch.cuda.get_device_capability()[0] < 9 and not dequantize:
+ pytest.skip("fp8 matmul needs sm_89+")
+ reference = _write_tiny_modelopt_llama(str(tmp_path)).cuda()
+ config = AutoConfig.from_pretrained(str(tmp_path))
+ plan = arm_modelopt_fp8_loading(config, verbose = False)
+ kwargs = {}
+ pop_modelopt_key_mapping(config, kwargs)
+ extra = {"dequantize": True} if dequantize else {}
+ kwargs["quantization_config"] = FineGrainedFP8Config.from_dict(dict(plan), **extra)
+ model = AutoModelForCausalLM.from_pretrained(
+ str(tmp_path), config = config, dtype = torch.bfloat16, device_map = "cuda", **kwargs
+ )
+ q_proj = model.model.layers[0].self_attn.q_proj
+ if dequantize:
+ assert q_proj.weight.dtype == torch.bfloat16
+ assert torch.equal(q_proj.weight, reference.model.layers[0].self_attn.q_proj.weight)
+ else:
+ assert q_proj.weight.dtype == torch.float8_e4m3fn
+ assert float(q_proj.activation_scale) == pytest.approx(0.02)
+ x = torch.randint(0, 512, (2, 32), device = "cuda")
+ with torch.no_grad():
+ got = model(x).logits.float()
+ want = reference(x).logits.float()
+ rel = ((got - want).norm() / want.norm()).item()
+ # Dequantized weights are exact; the fp8 path also rounds activations to e4m3.
+ assert rel < (0.01 if dequantize else 0.08), rel
+
+
+def test_rewrite_follows_who_loads_the_weights():
+ import inspect
+ from unsloth.models import llama, vision
+
+ llama_source = inspect.getsource(llama.FastLlamaModel.from_pretrained)
+ assert (
+ "rewrite_modelopt = not _vllm_will_load_weights(fast_inference, num_labels)" in llama_source
+ )
+ assert llama._vllm_will_load_weights(True, num_labels = 2) is False
+ vision_source = inspect.getsource(vision.FastBaseModel.from_pretrained)
+ assert "rewrite_modelopt = not (fast_inference and is_vLLM_available())" in vision_source
+ # The un-rewritten ModelOpt config must not be looked up in transformers' quantizer map.
+ assert "AUTO_QUANTIZATION_CONFIG_MAPPING.get(quant_method)" in vision_source
+ assert "AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]" not in vision_source
+
+
[email protected](not has_real_cuda(), reason = "FastModel loads need an accelerator")
+def test_a_declined_modelopt_format_still_refuses_to_load_in_process(tmp_path):
+ from transformers import LlamaConfig, LlamaForCausalLM
+ from unsloth import FastModel
+
+ config = LlamaConfig(
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 1,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 128,
+ )
+ LlamaForCausalLM(config).save_pretrained(tmp_path)
+ raw = json.loads((tmp_path / "config.json").read_text())
+ raw["quantization_config"] = {"quant_method": "modelopt", "quant_algo": "NVFP4"}
+ (tmp_path / "config.json").write_text(json.dumps(raw))
+ with pytest.raises(KeyError, match = "cannot load this `modelopt` checkpoint"):
+ FastModel.from_pretrained(str(tmp_path), load_in_4bit = False, load_in_16bit = True)
+
+
+@needs_per_tensor_fp8
[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need an accelerator")
+def test_fast_llama_checks_fp8_hardware_on_the_rewritten_config(tmp_path):
+ """Subprocess: FastLanguageModel patches the Llama classes process-wide."""
+ import subprocess
+ import sys
+
+ _write_tiny_modelopt_llama(str(tmp_path))
+ code = f"""
+import unsloth
+from unsloth import FastLanguageModel
+from unsloth.models import llama
+from unsloth.models._utils import get_quant_type
+seen = []
+llama.verify_fp8_support_if_applicable = lambda config: seen.append(get_quant_type(config))
+try:
+ FastLanguageModel.from_pretrained({str(tmp_path)!r}, load_in_4bit = False, max_seq_length = 64)
+except Exception:
+ pass # the tiny checkpoint ships no tokenizer; the checks run before that
+print("SEEN", seen)
+"""
+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 600)
+ seen = [line for line in out.stdout.splitlines() if line.startswith("SEEN")]
+ assert seen and "'fp8'" in seen[-1], (out.stdout[-2000:], out.stderr[-2000:])
+
+
+def test_config_branch_moves_rope_extension_onto_the_config():
+ import inspect
+ from unsloth.models import llama
+
+ source = inspect.getsource(llama.FastLlamaModel.from_pretrained)
+ branch = source.split("if user_config is not None or _modelopt_rewritten:", 1)[1]
+ branch = branch.split("AutoModelForCausalLM.from_pretrained(", 1)[0]
+ assert 'kwargs.pop("rope_scaling", None)' in branch
+
+
+def test_task_heads_stay_out_of_the_rewritten_plan_only_for_task_loads():
+ from transformers import (
+ AutoModelForCausalLM,
+ AutoModelForSequenceClassification,
+ LlamaForCausalLM,
+ LlamaForSequenceClassification,
+ )
+ from unsloth.models.modelopt_fp8 import keep_task_heads_unquantized
+
+ def rewritten():
+ return SimpleNamespace(
+ quantization_config = {"quant_method": "fp8", "modules_to_not_convert": ["lm_head"]}
+ )
+
+ config = rewritten()
+ assert keep_task_heads_unquantized(config, AutoModelForSequenceClassification)
+ assert config.quantization_config["modules_to_not_convert"] == [
+ "lm_head",
+ "score",
+ "classifier",
+ "classification_head",
+ "qa_outputs",
+ ]
+ assert keep_task_heads_unquantized(config, None, LlamaForSequenceClassification)
+ assert len(config.quantization_config["modules_to_not_convert"]) == 5
+
+ for causal in (AutoModelForCausalLM, LlamaForCausalLM):
+ config = rewritten()
+ assert not keep_task_heads_unquantized(config, causal)
+ assert config.quantization_config["modules_to_not_convert"] == ["lm_head"]
+ other = SimpleNamespace(quantization_config = {"quant_method": "gptq"})
+ assert not keep_task_heads_unquantized(other, AutoModelForSequenceClassification)
+ assert "modules_to_not_convert" not in other.quantization_config
+
+
+ "task",
+ [
+ "AutoModelForMultipleChoice",
+ "AutoModelForImageClassification",
+ "AutoModelForAudioClassification",
+ ],
+)
+def test_other_task_auto_classes_keep_their_head_out(task):
+ import transformers
+ from unsloth.models.modelopt_fp8 import keep_task_heads_unquantized
+
+ if not hasattr(transformers, task):
+ pytest.skip(f"no {task}")
+ config = SimpleNamespace(
+ quantization_config = {"quant_method": "fp8", "modules_to_not_convert": ["lm_head"]}
+ )
+ assert keep_task_heads_unquantized(config, getattr(transformers, task))
+ assert {"score", "classifier"} <= set(config.quantization_config["modules_to_not_convert"])
+
+
+def test_every_transformers_task_head_name_is_kept_out():
+ """Bart-style sequence classification names its fresh head `classification_head`."""
+ from transformers import BartConfig, BartForSequenceClassification
+ from unsloth.models.modelopt_fp8 import _TASK_HEAD_MODULES
+
+ model = BartForSequenceClassification(
+ BartConfig(
+ d_model = 16,
+ encoder_layers = 1,
+ decoder_layers = 1,
+ encoder_attention_heads = 2,
+ decoder_attention_heads = 2,
+ encoder_ffn_dim = 32,
+ decoder_ffn_dim = 32,
+ vocab_size = 64,
+ )
+ )
+ fresh = {name.split(".")[0] for name, _ in model.named_children()} - {"model"}
+ assert fresh <= set(_TASK_HEAD_MODULES), fresh
+
+
+def test_both_loaders_keep_task_heads_out_of_the_rewrite():
+ import inspect
+ from unsloth.models import llama, vision
+
+ llama_source = inspect.getsource(llama.FastLlamaModel.from_pretrained)
+ assert (
+ "keep_task_heads_unquantized(model_config, AutoModelForSequenceClassification)"
+ in llama_source
+ )
+ vision_source = inspect.getsource(vision.FastBaseModel.from_pretrained)
+ assert "keep_task_heads_unquantized(auto_config, auto_model, model_class)" in vision_source
+
+
+@needs_per_tensor_fp8
[email protected](not has_real_cuda(), reason = "fp8 kernels need CUDA")
+def test_tiny_modelopt_llama_loads_a_classification_head(tmp_path):
+ from transformers import AutoConfig, AutoModelForSequenceClassification
+ from unsloth.models.modelopt_fp8 import keep_task_heads_unquantized
+
+ if torch.cuda.get_device_capability()[0] < 9:
+ pytest.skip("fp8 matmul needs sm_89+")
+ _write_tiny_modelopt_llama(str(tmp_path))
+ config = AutoConfig.from_pretrained(str(tmp_path), num_labels = 2, pad_token_id = 0)
+ arm_modelopt_fp8_loading(config, verbose = False)
+ assert keep_task_heads_unquantized(config, AutoModelForSequenceClassification)
+ kwargs = {}
+ pop_modelopt_key_mapping(config, kwargs)
+ model = AutoModelForSequenceClassification.from_pretrained(
+ str(tmp_path), config = config, dtype = torch.bfloat16, device_map = "cuda", **kwargs
+ )
+ assert type(model.score) is nn.Linear
+ assert model.score.weight.dtype == torch.bfloat16
+ assert model.model.layers[0].self_attn.q_proj.weight.dtype == torch.float8_e4m3fn
+ x = torch.randint(1, 512, (2, 16), device = "cuda")
+ out = model(x, labels = torch.tensor([0, 1], device = "cuda"))
+ out.loss.backward()
+ assert torch.isfinite(out.loss)
+ assert model.score.weight.grad is not None and model.score.weight.grad.abs().sum() > 0
+
+
+def test_both_loaders_hand_the_planner_the_rewritten_plan():
+ import inspect
+ from unsloth.models import llama, vision
+
+ llama_source = inspect.getsource(llama.FastLlamaModel.from_pretrained)
+ assert (
+ "rewritten_quantization_config = modelopt_planner_quantization_config(model_config)"
+ in llama_source
+ )
+ vision_source = inspect.getsource(vision.FastBaseModel.from_pretrained)
+ # A 16-bit load dequantizes the fp8 weights, so the planner must size them at bf16.
+ assert "rewritten_quantization_config = modelopt_planner_quantization_config(" in vision_source
+ assert "auto_config, dequantize = load_in_16bit" in vision_source
+
+
+@needs_per_tensor_fp8
+ not has_real_cuda(), reason = "transformers dequantizes fp8 to bf16 without an accelerator"
+)
+def test_the_planner_sizes_a_modelopt_checkpoint_from_the_rewritten_plan(tmp_path):
+ from transformers import AutoConfig, LlamaConfig
+ from unsloth.models.loader_utils import planner_quantization_kwargs
+ from unsloth.models.modelopt_fp8 import modelopt_planner_quantization_config
+
+ planner = pytest.importorskip("unsloth_zoo.device_map_planner")
+ if "rewritten_quantization_config" not in inspect.getsource(planner.build_meta_model):
+ pytest.skip("this unsloth_zoo planner cannot size a rewritten quantization method")
+ config = LlamaConfig(
+ hidden_size = 64,
+ intermediate_size = 128,
+ num_hidden_layers = 2,
+ num_attention_heads = 4,
+ num_key_value_heads = 2,
+ vocab_size = 256,
+ )
+ config.quantization_config = _sarvam_quant()
+ config.save_pretrained(tmp_path)
+ loaded = AutoConfig.from_pretrained(str(tmp_path))
+ plan = arm_modelopt_fp8_loading(loaded, verbose = False)
+ fp8 = modelopt_planner_quantization_config(loaded)
+ assert fp8 == plan and fp8 is not loaded.quantization_config
+ kwargs = planner_quantization_kwargs(rewritten_quantization_config = fp8)
+ assert kwargs["rewritten_quantization_config"] is fp8 and "quantization_config" not in kwargs
+ model, hf_quantizer, _ = planner.build_meta_model(str(tmp_path), **kwargs)
+ assert type(hf_quantizer).__name__ == "FineGrainedFP8HfQuantizer"
+ assert type(model.model.layers[0].self_attn.q_proj).__name__ != "Linear"
+ assert type(model.lm_head).__name__ == "Linear"
+
+ # A 16-bit load dequantizes: the planner then sees bf16 Linear layers, not fp8 ones.
+ bf16 = modelopt_planner_quantization_config(loaded, dequantize = True)
+ assert bf16["dequantize"] is True and "dequantize" not in loaded.quantization_config
+ kwargs = planner_quantization_kwargs(rewritten_quantization_config = bf16)
+ model, _, _ = planner.build_meta_model(str(tmp_path), **kwargs)
+ assert type(model.model.layers[0].self_attn.q_proj).__name__ == "Linear"
+
+
+def test_dynamic_token_and_tensor_activations_map_onto_per_token_fp8():
+ # transformers' dynamic fp8 scales activations per token.
+ for strategy in ("token", "tensor", None):
+ inputs = {"num_bits": 8, "type": "float", "dynamic": True}
+ if strategy is not None:
+ inputs["strategy"] = strategy
+ quant = _sarvam_quant(
+ config_groups = {
+ "g": {"weights": {"num_bits": 8, "type": "float"}, "input_activations": inputs}
+ }
+ )
+ plan = modelopt_fp8_plan(SimpleNamespace(quantization_config = quant))
+ assert plan is not None and plan["activation_scheme"] == "dynamic", strategy
+
+
+def test_mixed_static_and_dynamic_activation_groups_are_declined():
+ # One scheme for the whole model would drop the static groups' calibrated input scales.
+ static = {"num_bits": 8, "type": "float", "dynamic": False}
+ dynamic = {"num_bits": 8, "type": "float", "dynamic": True, "strategy": "token"}
+ weights = {"num_bits": 8, "type": "float"}
+ quant = _sarvam_quant(
+ config_groups = {
+ "a": {"weights": dict(weights), "input_activations": static},
+ "b": {"weights": dict(weights), "input_activations": dynamic},
+ }
+ )
+ assert modelopt_fp8_plan(SimpleNamespace(quantization_config = quant)) is None
+ for inputs, scheme in ((static, "static"), (dynamic, "dynamic")):
+ quant = _sarvam_quant(
+ config_groups = {
+ "a": {"weights": dict(weights), "input_activations": dict(inputs)},
+ "b": {"weights": dict(weights), "input_activations": dict(inputs)},
+ }
+ )
+ plan = modelopt_fp8_plan(SimpleNamespace(quantization_config = quant))
+ assert plan is not None and plan["activation_scheme"] == scheme
+
+
+def test_a_task_checkpoint_keeps_its_quantized_head():
+ # Its head is on disk as fp8 with scales; excluding it would load fp8 bytes into a Linear.
+ from transformers import AutoModelForSequenceClassification
+
+ from unsloth.models.modelopt_fp8 import keep_task_heads_unquantized
+
+ config = SimpleNamespace(
+ architectures = ["LlamaForSequenceClassification"],
+ quantization_config = {"quant_method": "fp8", "modules_to_not_convert": ["lm_head"]},
+ )
+ assert not keep_task_heads_unquantized(config, AutoModelForSequenceClassification)
+ assert config.quantization_config["modules_to_not_convert"] == ["lm_head"]
+ config.architectures = ["LlamaForCausalLM"]
+ assert keep_task_heads_unquantized(config, AutoModelForSequenceClassification)
+ assert "score" in config.quantization_config["modules_to_not_convert"]
+
+
+def test_a_reused_config_keeps_the_scale_renaming():
+ from transformers import LlamaConfig
+
+ from unsloth.models.modelopt_fp8 import modelopt_rewritten
+
+ config = LlamaConfig(hidden_size = 8, num_hidden_layers = 1, num_attention_heads = 2, vocab_size = 16)
+ config.quantization_config = _sarvam_quant()
+ if arm_modelopt_fp8_loading(config, verbose = False) is None:
+ pytest.skip("this transformers has no per-tensor fp8")
+ first, second = {}, {}
+ pop_modelopt_key_mapping(config, first)
+ assert not hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+ assert arm_modelopt_fp8_loading(config, verbose = False) is None # already fp8
+ assert modelopt_rewritten(config)
+ pop_modelopt_key_mapping(config, second)
+ assert second["key_mapping"] == first["key_mapping"]
+ assert UNSLOTH_MODELOPT_KEY_MAPPING_ATTR not in config.to_dict()
+
+ fresh = LlamaConfig(hidden_size = 8, num_hidden_layers = 1, num_attention_heads = 2, vocab_size = 16)
+ assert not modelopt_rewritten(fresh)
+ kwargs = {}
+ pop_modelopt_key_mapping(fresh, kwargs)
+ assert kwargs == {}
+
+
+@needs_per_tensor_fp8
+def test_modelopt_ignore_globs_keep_fnmatch_meaning():
+ from transformers.quantizers.quantizers_utils import should_convert_module
+
+ quant = _sarvam_quant()
+ quant["ignore"] = ["lm_head", "backbone.layers.16*", "*embed_tokens*", "visual*"]
+ patterns = modelopt_fp8_plan(SimpleNamespace(quantization_config = quant))[
+ "modules_to_not_convert"
+ ]
+ assert not should_convert_module("lm_head", patterns)
+ assert should_convert_module("backbone.layers.1.mixer.in_proj", patterns)
+ assert should_convert_module("backbone.layers.10.mixer.in_proj", patterns)
+ assert not should_convert_module("backbone.layers.16.mixer.in_proj", patterns)
+ assert not should_convert_module("model.embed_tokens", patterns)
+ assert not should_convert_module("visual.blocks.0.attn.qkv", patterns)
+ assert not should_convert_module("lm_head", patterns)
+ assert should_convert_module("model.layers.0.self_attn.q_proj", patterns)
+
+
+@needs_per_tensor_fp8
+def test_merged_save_detects_a_rewritten_modelopt_checkpoint_as_fp8(tmp_path, monkeypatch):
+ zoo_saving = pytest.importorskip("unsloth_zoo.saving_utils")
+ original = getattr(
+ zoo_saving._is_fp8_quant_config, "__wrapped__", zoo_saving._is_fp8_quant_config
+ )
+ if original(_sarvam_quant()):
+ pytest.skip("this unsloth_zoo already dequantizes ModelOpt FP8 on a merged save")
+ monkeypatch.setattr(zoo_saving, "_is_fp8_quant_config", original)
+ dirs = {}
+ for name, quant in (("fp8", _sarvam_quant()), ("nvfp4", _sarvam_quant(quant_algo = "NVFP4"))):
+ dirs[name] = tmp_path / name
+ dirs[name].mkdir()
+ (dirs[name] / "config.json").write_text(
+ json.dumps({"model_type": "llama", "quantization_config": quant})
+ )
+ status = zoo_saving.check_model_quantization_status
+ assert status(str(dirs["fp8"])) == (False, None)
+ arm_modelopt_fp8_loading(SimpleNamespace(quantization_config = _sarvam_quant()), verbose = False)
+ assert status(str(dirs["fp8"])) == (True, "fp8")
+ assert status(str(dirs["nvfp4"])) == (False, None)
+
+
+@needs_per_tensor_fp8
[email protected](not has_real_cuda(), reason = "FastLanguageModel loads need an accelerator")
+def test_merged_16bit_save_of_a_modelopt_lora_reloads_without_unsloth(tmp_path):
+ """Subprocess: FastLanguageModel patches the Llama classes process-wide."""
+ import subprocess
+ import sys
+
+ from tokenizers import Tokenizer, models, pre_tokenizers
+ from transformers import PreTrainedTokenizerFast
+
+ if torch.cuda.get_device_capability()[0] < 9:
+ pytest.skip("fp8 matmul needs sm_89+")
+ ckpt, merged = tmp_path / "ckpt", tmp_path / "merged"
+ ckpt.mkdir()
+ _write_tiny_modelopt_llama(str(ckpt))
+ vocab = {f"t{i}": i for i in range(512)}
+ raw = Tokenizer(models.WordLevel(vocab, unk_token = "t0"))
+ raw.pre_tokenizer = pre_tokenizers.Whitespace()
+ PreTrainedTokenizerFast(
+ tokenizer_object = raw, unk_token = "t0", pad_token = "t1", eos_token = "t2"
+ ).save_pretrained(str(ckpt))
+ code = f"""
+import torch
+from unsloth import FastLanguageModel
+model, tok = FastLanguageModel.from_pretrained({str(ckpt)!r}, load_in_4bit = False, max_seq_length = 64, dtype = torch.bfloat16)
+model = FastLanguageModel.get_peft_model(model, r = 8, lora_alpha = 16, target_modules = ["q_proj", "v_proj", "down_proj"])
+torch.manual_seed(0)
+with torch.no_grad():
+ for name, p in model.named_parameters():
+ if "lora_B" in name:
+ p.normal_(0, 0.02)
+x = torch.randint(0, 512, (2, 32), device = "cuda")
+model.eval()
+with torch.no_grad():
+ lora = model.base_model.model.model.layers[0].self_attn.q_proj
+ base = lora.base_layer
+ expected = base.weight.float() * base.weight_scale_inv.float() + (
+ lora.lora_B["default"].weight.float() @ lora.lora_A["default"].weight.float()
+ ) * lora.scaling["default"]
+ torch.save((x.cpu(), model(x).logits.float().cpu(), expected.cpu()), {str(tmp_path / "want.pt")!r})
+model.save_pretrained_merged({str(merged)!r}, tok, save_method = "merged_16bit")
+"""
+ out = subprocess.run([sys.executable, "-c", code], capture_output = True, text = True, timeout = 900)
+ assert out.returncode == 0, (out.stdout[-2000:], out.stderr[-3000:])
+ check = f"""
+import sys, torch
+from transformers import AutoModelForCausalLM
+x, want, expected = torch.load({str(tmp_path / "want.pt")!r})
+model = AutoModelForCausalLM.from_pretrained({str(merged)!r}, dtype = torch.bfloat16).cuda()
+assert "unsloth" not in sys.modules
+q = model.model.layers[0].self_attn.q_proj.weight
+with torch.no_grad():
+ got = model(x.cuda()).logits.float().cpu()
+w_rel = float((q.float().cpu() - expected).norm() / expected.norm())
+print("CHECK", q.dtype, w_rel, float((got - want).norm() / want.norm()))
+"""
+ out = subprocess.run([sys.executable, "-c", check], capture_output = True, text = True, timeout = 600)
+ line = [l for l in out.stdout.splitlines() if l.startswith("CHECK")]
+ assert line, (out.stdout[-2000:], out.stderr[-3000:])
+ _, dtype, w_rel, rel = line[-1].split()
+ assert dtype == "torch.bfloat16"
+ # dequant(W) + B @ A * scaling, up to bf16 rounding; raw e4m3 bytes are off by ~4 orders.
+ assert float(w_rel) < 1e-2, w_rel
+ # Loose: the in-memory model also rounds activations to e4m3 (raw bytes give ~1.4).
+ assert float(rel) < 0.2, rel
+
+
+def test_fp8_linear_forward_patch_adds_the_bias():
+ from unsloth.kernels.fp8 import module_forward_patch
+
+ forward = module_forward_patch(lambda X, weight, scale: X @ weight.t(), "weight_scale_inv")
+ biased, plain = nn.Linear(4, 3), nn.Linear(4, 3, bias = False)
+ for module in (biased, plain):
+ module.weight_scale_inv = torch.ones(())
+ # fbgemm keeps its bias in fp32; the output must stay in the activation dtype.
+ biased.bias.data = biased.bias.data.float()
+ X = torch.randn(2, 4, dtype = torch.bfloat16)
+ biased.weight.data, plain.weight.data = (m.weight.data.bfloat16() for m in (biased, plain))
+ out = forward(biased, X)
+ assert out.dtype == torch.bfloat16
+ torch.testing.assert_close(out, X @ biased.weight.t() + biased.bias.bfloat16())
+ assert torch.equal(forward(plain, X), X @ plain.weight.t())
+
+
+def test_save_keeps_transformers_fp8_scale_names():
+ core = pytest.importorskip("transformers.core_model_loading")
+ from unsloth.models.modelopt_fp8 import keep_fp8_scale_names_on_save
+
+ ours = [
+ core.WeightRenaming(source_patterns = k, target_patterns = v)
+ for k, v in MODELOPT_FP8_KEY_MAPPING.items()
+ ]
+ other = core.WeightRenaming(source_patterns = r"^model\.old\.", target_patterns = "model.new.")
+ model = nn.Module()
+ model._weight_conversions = [other, *ours]
+ keep_fp8_scale_names_on_save(model)
+ assert model._weight_conversions == [other]
+
+
+def test_exact_ignore_names_keep_their_module_boundary():
+ from transformers.quantizers.quantizers_utils import should_convert_module
+
+ skip = modelopt_fp8_plan(
+ SimpleNamespace(quantization_config = _sarvam_quant(ignore = ["model.layers.1", "lm_head"]))
+ )["modules_to_not_convert"]
+ assert not should_convert_module("model.layers.1.self_attn.q_proj", skip)
+ assert should_convert_module("model.layers.10.self_attn.q_proj", skip)
+ assert should_convert_module("model.layers.11.mlp.down_proj", skip)
+ assert not should_convert_module("lm_head", skip)
+
+
+def test_vlm_ignore_globs_follow_the_instantiated_names():
+ from transformers.quantizers.quantizers_utils import should_convert_module
+
+ config = SimpleNamespace(
+ quantization_config = _sarvam_quant(ignore = ["visual*", "lm_head"]),
+ model_type = "qwen2_5_vl",
+ architectures = ["Qwen2_5_VLForConditionalGeneration"],
+ )
+ skip = modelopt_fp8_plan(config)["modules_to_not_convert"]
+ # Checkpoint `visual.*` is instantiated as `model.visual.*`; its bf16 weights have no fp8 scales.
+ assert not should_convert_module("model.visual.blocks.0.attn.qkv", skip)
+ assert not should_convert_module("model.visual.merger.mlp.0", skip)
+ assert should_convert_module("model.language_model.layers.0.self_attn.q_proj", skip)
+
+
+def _hf_quant_config_checkpoint(path, quant_algo = "FP8"):
+ from transformers import LlamaConfig
+
+ LlamaConfig(hidden_size = 64, num_hidden_layers = 1, num_attention_heads = 4).save_pretrained(path)
+ hf_quant = {
+ "producer": {"name": "modelopt", "version": "0.23.0"},
+ "quantization": {
+ "quant_algo": quant_algo,
+ "kv_cache_quant_algo": None,
+ "exclude_modules": ["lm_head"],
+ },
+ }
+ (path / "hf_quant_config.json").write_text(json.dumps(hf_quant))
+
+
+@needs_per_tensor_fp8
+def test_standalone_hf_quant_config_is_rewritten(tmp_path):
+ from transformers import AutoConfig
+
+ fp8, nvfp4, plain = tmp_path / "fp8", tmp_path / "nvfp4", tmp_path / "plain"
+ _hf_quant_config_checkpoint(fp8)
+ _hf_quant_config_checkpoint(nvfp4, quant_algo = "NVFP4")
+ from transformers import LlamaConfig
+
+ LlamaConfig(hidden_size = 64, num_hidden_layers = 1, num_attention_heads = 4).save_pretrained(plain)
+
+ config = AutoConfig.from_pretrained(str(fp8))
+ assert getattr(config, "quantization_config", None) is None
+ _, _, method = check_and_disable_bitsandbytes_loading(config, load_in_4bit = False, verbose = False)
+ assert method == "fp8" and config.quantization_config["quant_method"] == "fp8"
+ assert hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+
+ for other in (nvfp4, plain):
+ config = AutoConfig.from_pretrained(str(other))
+ _, _, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = False, verbose = False
+ )
+ assert method is None and getattr(config, "quantization_config", None) is None
+
+ # Under vLLM the block is attached but not rewritten (vLLM reads ModelOpt itself); the
+ # default load_in_4bit must still drop so vLLM is not asked for bitsandbytes.
+ config = AutoConfig.from_pretrained(str(fp8))
+ load_in_4bit, _, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = True, verbose = False, rewrite_modelopt = False
+ )
+ assert method == "modelopt" and load_in_4bit is False
+ assert config.quantization_config["quant_method"] == "modelopt"
+ assert not hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+
+ # A caller-built config carries no checkpoint path; the loader's model name finds the file.
+ from transformers import LlamaConfig
+
+ config = LlamaConfig(hidden_size = 64, num_hidden_layers = 1, num_attention_heads = 4)
+ _, _, method = check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = False, verbose = False, model_name = str(fp8)
+ )
+ assert method == "fp8"
+
+
+@needs_per_tensor_fp8
+def test_merged_save_detects_a_standalone_hf_quant_config(tmp_path, monkeypatch):
+ zoo_saving = pytest.importorskip("unsloth_zoo.saving_utils")
+ for name in ("_is_fp8_quant_config", "check_model_quantization_status"):
+ fn = getattr(zoo_saving, name)
+ monkeypatch.setattr(zoo_saving, name, getattr(fn, "__wrapped__", fn))
+ fp8, nvfp4 = tmp_path / "fp8", tmp_path / "nvfp4"
+ _hf_quant_config_checkpoint(fp8)
+ _hf_quant_config_checkpoint(nvfp4, quant_algo = "NVFP4")
+ status = lambda p: zoo_saving.check_model_quantization_status(str(p))
+ if status(fp8) == (True, "fp8"):
+ pytest.skip("this unsloth_zoo already reads hf_quant_config.json")
+ arm_modelopt_fp8_loading(SimpleNamespace(quantization_config = _sarvam_quant()), verbose = False)
+ assert status(fp8) == (True, "fp8")
+ assert status(nvfp4) == (False, None)
+
+
+def test_config_overrides_move_onto_the_rewritten_config():
+ from transformers import LlamaConfig
+ from unsloth.models.modelopt_fp8 import move_config_overrides_onto_config
+
+ config = LlamaConfig(hidden_size = 64, num_hidden_layers = 1, num_attention_heads = 4)
+ kwargs = {
+ "use_cache": False,
+ "pad_token_id": 7,
+ "dtype": torch.bfloat16,
+ "key_mapping": {},
+ "subfolder": "x",
+ }
+ move_config_overrides_onto_config(config, kwargs)
+ assert config.use_cache is False and config.pad_token_id == 7
+ # from_pretrained's own arguments stay load arguments.
+ assert set(kwargs) == {"dtype", "key_mapping", "subfolder"}
+
+
+def test_save_keeps_fp8_scale_names_without_original_pattern_copies():
+ from unsloth.models.modelopt_fp8 import keep_fp8_scale_names_on_save
+
+ class WeightRenaming(SimpleNamespace): # transformers 5.3 / 5.5: live patterns only
+ pass
+
+ renames = [
+ WeightRenaming(source_patterns = [k], target_patterns = [v])
+ for k, v in MODELOPT_FP8_KEY_MAPPING.items()
+ ]
+ other = WeightRenaming(source_patterns = ["^old"], target_patterns = ["new"])
+ model = nn.Module()
+ model._weight_conversions = [other, *renames]
+ keep_fp8_scale_names_on_save(model)
+ assert model._weight_conversions == [other]
+
+
+@needs_per_tensor_fp8
+def test_merged_save_is_armed_on_the_vllm_path(tmp_path, monkeypatch):
+ zoo_saving = pytest.importorskip("unsloth_zoo.saving_utils")
+ for name in ("_is_fp8_quant_config", "check_model_quantization_status"):
+ fn = getattr(zoo_saving, name)
+ monkeypatch.setattr(zoo_saving, name, getattr(fn, "__wrapped__", fn))
+ if zoo_saving._is_fp8_quant_config(_sarvam_quant()):
+ pytest.skip("this unsloth_zoo already dequantizes ModelOpt FP8 on a merged save")
+ (tmp_path / "config.json").write_text(
+ json.dumps({"model_type": "llama", "quantization_config": _sarvam_quant()})
+ )
+ assert zoo_saving.check_model_quantization_status(str(tmp_path)) == (False, None)
+ config = SimpleNamespace(quantization_config = _sarvam_quant())
+ check_and_disable_bitsandbytes_loading(
+ config, load_in_4bit = True, verbose = False, rewrite_modelopt = False
+ )
+ assert config.quantization_config["quant_method"] == "modelopt"
+ assert zoo_saving.check_model_quantization_status(str(tmp_path)) == (True, "fp8")
+
+
+@needs_per_tensor_fp8
+def test_hf_quant_config_lookup_follows_the_load_location(tmp_path, monkeypatch):
+ import huggingface_hub
+ from transformers import LlamaConfig
+ from unsloth.models.modelopt_fp8 import attach_hf_quant_config
+
+ _hf_quant_config_checkpoint(tmp_path / "repo" / "sub")
+ config = LlamaConfig(hidden_size = 64, num_hidden_layers = 1, num_attention_heads = 4)
+ assert attach_hf_quant_config(
+ config, model_name = str(tmp_path / "repo"), hub_kwargs = {"subfolder": "sub"}
+ )
+
+ seen = {}
+
+ def fake_download(repo, filename, **kwargs):
+ seen.update(kwargs)
+ raise huggingface_hub.errors.LocalEntryNotFoundError("offline")
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
+ config = LlamaConfig(hidden_size = 64, num_hidden_layers = 1, num_attention_heads = 4)
+ hub = {"local_files_only": True, "cache_dir": str(tmp_path / "cache")}
+ assert not attach_hf_quant_config(config, model_name = "org/not-cached", hub_kwargs = hub)
+ assert seen["local_files_only"] is True and seen["cache_dir"] == hub["cache_dir"]
+
+
+@needs_per_tensor_fp8
+def test_merged_save_lookup_keeps_a_positional_token(tmp_path, monkeypatch):
+ import unsloth.models.modelopt_fp8 as modelopt
+
+ zoo_saving = pytest.importorskip("unsloth_zoo.saving_utils")
+ for name in ("_is_fp8_quant_config", "check_model_quantization_status"):
+ fn = getattr(zoo_saving, name)
+ monkeypatch.setattr(zoo_saving, name, getattr(fn, "__wrapped__", fn))
+ if zoo_saving._is_fp8_quant_config(_sarvam_quant()):
+ pytest.skip("this unsloth_zoo already dequantizes ModelOpt FP8 on a merged save")
+ _hf_quant_config_checkpoint(tmp_path)
+ seen = []
+ real = modelopt._hf_quant_config_path
+ monkeypatch.setattr(
+ modelopt,
+ "_hf_quant_config_path",
+ lambda name, revision = None, token = None, hub_kwargs = None: seen.append(token)
+ or real(name, revision, token, hub_kwargs),
+ )
+ arm_modelopt_fp8_loading(SimpleNamespace(quantization_config = _sarvam_quant()), verbose = False)
+ # As unsloth_zoo calls it: check_model_quantization_status(model_name, token, ...).
+ assert zoo_saving.check_model_quantization_status(str(tmp_path), "hf_secret") == (True, "fp8")
+ assert seen == ["hf_secret"]
diff --git a/tests/test_omni_text_only_load.py b/tests/test_omni_text_only_load.py
index 0dd16cf98ce..1e3782d6b09 100644
--- a/tests/test_omni_text_only_load.py
+++ b/tests/test_omni_text_only_load.py
@@ -521,7 +521,10 @@ def test_a_kept_wrapper_resolves_its_layer_count_through_the_thinker():
model = _text_trainable_core(_tiny_omni(), text_intent = False)
assert _get_total_transformer_layers(model) is None
- assert _get_total_transformer_layers(model.thinker) == model.thinker.config.text_config.num_hidden_layers
+ assert (
+ _get_total_transformer_layers(model.thinker)
+ == model.thinker.config.text_config.num_hidden_layers
+ )
def test_an_explicit_leaf_list_on_a_kept_wrapper_is_scoped_to_the_thinker(monkeypatch):
@@ -556,7 +559,9 @@ def test_a_kept_wrapper_resizes_its_vocabulary_through_the_thinker():
def _kept_wrapper_peft(monkeypatch, **peft_kwargs):
from unsloth.models import vision
- monkeypatch.setattr(vision.FastBaseModel, "post_patch_model", staticmethod(lambda model, *a, **k: model))
+ monkeypatch.setattr(
+ vision.FastBaseModel, "post_patch_model", staticmethod(lambda model, *a, **k: model)
+ )
model = vision._text_trainable_core(_tiny_omni(), text_intent = False)
model.max_seq_length = 64
return vision.FastBaseModel.get_peft_model(model, r = 2, **peft_kwargs)
@@ -572,7 +577,9 @@ def test_a_qualified_target_on_a_kept_wrapper_reaches_peft_unchanged(monkeypatch
def test_saved_modules_on_a_kept_wrapper_skip_the_talker(monkeypatch):
"""An unqualified "lm_head" also suffix-matched talker.code_predictor.lm_head, a ModuleList PEFT refuses."""
- peft_model = _kept_wrapper_peft(monkeypatch, target_modules = ["q_proj"], modules_to_save = ["lm_head"])
+ peft_model = _kept_wrapper_peft(
+ monkeypatch, target_modules = ["q_proj"], modules_to_save = ["lm_head"]
+ )
trained = [n for n, p in peft_model.named_parameters() if p.requires_grad]
assert any("thinker.lm_head" in n for n in trained)
assert not any(".talker." in n for n in trained)
diff --git a/tests/test_root_hook_follows_input_embedding.py b/tests/test_root_hook_follows_input_embedding.py
index ccb7bc0530f..f1fe0a1c6f2 100644
--- a/tests/test_root_hook_follows_input_embedding.py
+++ b/tests/test_root_hook_follows_input_embedding.py
@@ -112,11 +112,15 @@ def test_omni_config_with_nested_towers_is_untouched():
assert omni._hf_hook.execution_device == 0
[email protected]("tower", ["vision_encoder_config", "audio_encoder_config", "encoder_config"])
+ "tower", ["vision_encoder_config", "audio_encoder_config", "encoder_config"]
+)
def test_encoder_spelled_tower_configs_are_untouched(tower):
"""The same spellings _uses_flash_attention_for_generation already treats as non-language."""
align = _helper()
split = {"model.layers.1": 2, "model.embeddings": 1, "lm_head": 2, "model.layers.0": 0}
- multimodal = _model(split, 0, torch.device("cuda", 1), config = SimpleNamespace(**{tower: SimpleNamespace()}))
+ multimodal = _model(
+ split, 0, torch.device("cuda", 1), config = SimpleNamespace(**{tower: SimpleNamespace()})
+ )
assert align(multimodal) is None
assert multimodal._hf_hook.execution_device == 0
diff --git a/unsloth/_gpu_init.py b/unsloth/_gpu_init.py
index 53f30b15998..28b21c14f2a 100644
--- a/unsloth/_gpu_init.py
+++ b/unsloth/_gpu_init.py
@@ -256,6 +256,7 @@ from .import_fixes import (
fix_transformers_chunked_mask_block_sequence_ids,
fix_transformers_longcat_lsa_config,
fix_transformers_rope_scaling_drops_theta,
+ fix_transformers_fp8_modulelist_experts,
fix_transformers_validate_rope_ignore_keys,
fix_transformers5_remote_code_legacy_defaults,
fix_transformers_config_only_remote_code,
@@ -320,6 +321,7 @@ del check_transformers_prequantized_vlm_quant_state
# RoPE base frequency. Ordered here, before any config is built, so the object-style delegation
# retry in models/llama.py sees a config that kept its base (#2405).
fix_transformers_rope_scaling_drops_theta()
+fix_transformers_fp8_modulelist_experts()
fix_transformers_validate_rope_ignore_keys()
fix_transformers5_remote_code_legacy_defaults()
fix_transformers_config_only_remote_code()
@@ -382,6 +384,7 @@ patch_accelerate_recursively_apply()
del fix_transformers5_bare_annotation_configs
del fix_transformers_rope_scaling_drops_theta
+del fix_transformers_fp8_modulelist_experts
del fix_transformers_validate_rope_ignore_keys
del fix_transformers_longcat_lsa_config
del fix_transformers_remote_rope_scaling_none
diff --git a/unsloth/import_fixes.py b/unsloth/import_fixes.py
index f0d5ea35f2e..cd9799726b5 100644
--- a/unsloth/import_fixes.py
+++ b/unsloth/import_fixes.py
@@ -2459,6 +2459,138 @@ def _transformers_rope_scaling_assignment_drops_theta():
return False
+def _fp8_replace_swaps_named_experts(fn) -> bool:
+ try:
+ return 'endswith(".experts")' in inspect.getsource(fn)
+ except Exception:
+ return False
+
+
+def _wrap_fp8_replace_for_modulelist_experts(original):
+ if getattr(original, "_unsloth_modulelist_experts", False):
+ return original
+ try:
+ from transformers.quantizers.quantizers_utils import should_convert_module
+ signature = inspect.signature(original)
+ except Exception:
+ return original
+
+ @functools.wraps(original)
+ def replace_with_fp8_linear(model, *args, **kwargs):
+ import torch.nn as nn
+
+ try:
+ bound = signature.bind(model, *args, **kwargs)
+ except TypeError:
+ return original(model, *args, **kwargs)
+ patterns = bound.arguments.get("modules_to_not_convert", None)
+ hidden = [
+ (name, module)
+ for name, module in model.named_modules()
+ if name.endswith(".experts") and isinstance(module, nn.ModuleList)
+ ]
+ if not hidden:
+ return original(model, *args, **kwargs)
+
+ # Rename off `.experts` so children take the FP8Linear branch; keep exclusions under the new name.
+ parked_suffix = "_unsloth_modulelist"
+ extra_patterns = []
+ renames = []
+ for name, module in hidden:
+ parent_name, _, child = name.rpartition(".")
+ parent = model.get_submodule(parent_name)
+ parked = child + parked_suffix
+ if parked in parent._modules:
+ return original(model, *args, **kwargs)
+ for sub_name, _ in module.named_modules():
+ if sub_name and not should_convert_module(f"{name}.{sub_name}", patterns):
+ extra_patterns.append(re.escape(f"{parent_name}.{parked}.{sub_name}") + "$")
+ renames.append((parent, child, parked))
+
+ def _rename(parent, old, new):
+ items = list(parent._modules.items())
+ parent._modules.clear()
+ for key, value in items:
+ parent._modules[new if key == old else key] = value
+
+ for parent, child, parked in renames:
+ _rename(parent, child, parked)
+ try:
+ if extra_patterns:
+ bound.arguments["modules_to_not_convert"] = list(patterns or []) + extra_patterns
+ return original(*bound.args, **bound.kwargs)
+ finally:
+ for parent, child, parked in renames:
+ _rename(parent, parked, child)
+
+ replace_with_fp8_linear._unsloth_modulelist_experts = True
+ return replace_with_fp8_linear
+
+
+def fix_transformers_fp8_modulelist_experts():
+ """transformers 5.x swaps any `*.experts` for FP8Experts, breaking remote-code ModuleList experts (sarvam)."""
+ try:
+ from transformers.quantizers import quantizer_finegrained_fp8
+ except Exception:
+ return
+ quantizer_cls = getattr(quantizer_finegrained_fp8, "FineGrainedFP8HfQuantizer", None)
+ method = getattr(quantizer_cls, "_process_model_before_weight_loading", None)
+ if method is None or getattr(method, "_unsloth_modulelist_experts", False):
+ return
+
+ @functools.wraps(method)
+ def _process_model_before_weight_loading(self, model, *args, **kwargs):
+ try:
+ import transformers.integrations.finegrained_fp8 as fp8_integration
+ current = getattr(fp8_integration, "replace_with_fp8_linear", None)
+ if current is not None and _fp8_replace_swaps_named_experts(current):
+ fp8_integration.replace_with_fp8_linear = _wrap_fp8_replace_for_modulelist_experts(
+ current
+ )
+ except Exception:
+ pass
+ try:
+ import transformers.integrations.finegrained_fp8 as fp8_integration
+ _cast_fp8_dequantize_to_model_dtype(getattr(fp8_integration, "Fp8Dequantize", None))
+ except Exception:
+ pass
+ return method(self, model, *args, **kwargs)
+
+ _process_model_before_weight_loading._unsloth_modulelist_experts = True
+ quantizer_cls._process_model_before_weight_loading = _process_model_before_weight_loading
+
+
+def _cast_fp8_dequantize_to_model_dtype(op_cls):
+ # transformers 5.4 dequantizes to the scale's fp32; cast to the replaced parameter's dtype.
+ convert = getattr(op_cls, "convert", None)
+ if convert is None or getattr(convert, "_unsloth_model_dtype", False):
+ return
+
+ @functools.wraps(convert)
+ def cast_convert(self, input_dict, *args, **kwargs):
+ import torch
+
+ out = convert(self, input_dict, *args, **kwargs)
+ model = kwargs.get("model")
+ if model is None or not isinstance(out, dict):
+ return out
+ for name, value in out.items():
+ tensor = value[0] if isinstance(value, list) and value else value
+ if not isinstance(tensor, torch.Tensor) or not tensor.is_floating_point():
+ continue
+ try:
+ target = model.get_parameter(name).dtype
+ except Exception:
+ continue
+ if target.is_floating_point and target != tensor.dtype:
+ tensor = tensor.to(target)
+ out[name] = [tensor] if isinstance(value, list) else tensor
+ return out
+
+ cast_convert._unsloth_model_dtype = True
+ op_cls.convert = cast_convert
+
+
def fix_transformers_is_torch_fx_available():
"""Restore ``is_torch_fx_available`` (removed in 5.0) for 4.x-era remote code."""
try:
diff --git a/unsloth/kernels/fp8.py b/unsloth/kernels/fp8.py
index 80912efc56d..db90408ca28 100644
--- a/unsloth/kernels/fp8.py
+++ b/unsloth/kernels/fp8.py
@@ -761,7 +761,10 @@ def fp8_linear(
def module_forward_patch(forward_function, scale_attr = "weight_scale"):
def patched_forward(self, X):
- return forward_function(X, self.weight, getattr(self, scale_attr))
+ out = forward_function(X, self.weight, getattr(self, scale_attr))
+ # The kernels take no bias, so a biased Linear (Qwen2-style q/k/v) adds it here; fbgemm keeps it fp32.
+ bias = self._parameters.get("bias")
+ return out if bias is None else out + bias.to(out.dtype)
return patched_forward
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index 6779bba4d87..021ee7a3aba 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2601,8 +2601,34 @@ class FastLlamaModel:
# Disable bitsandbytes loading if the model has non-bitsandbytes quantization.
load_in_4bit, load_in_8bit, _ckpt_quant_method = check_and_disable_bitsandbytes_loading(
- model_config, load_in_4bit = load_in_4bit, load_in_8bit = load_in_8bit
- )
+ model_config,
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = load_in_8bit,
+ rewrite_modelopt = not _vllm_will_load_weights(fast_inference, num_labels),
+ token = token,
+ model_name = model_name,
+ revision = revision,
+ hub_kwargs = {
+ "cache_dir": kwargs.get("cache_dir"),
+ "subfolder": kwargs.get("subfolder"),
+ "local_files_only": kwargs.get("local_files_only", False),
+ },
+ )
+ from .modelopt_fp8 import (
+ keep_fp8_scale_names_on_save,
+ move_config_overrides_onto_config,
+ keep_task_heads_unquantized,
+ modelopt_planner_quantization_config,
+ modelopt_rewritten,
+ pop_modelopt_key_mapping,
+ )
+
+ _modelopt_rewritten = modelopt_rewritten(model_config)
+ if _modelopt_rewritten:
+ verify_fp8_support_if_applicable(model_config)
+ if _modelopt_rewritten and num_labels is not None:
+ keep_task_heads_unquantized(model_config, AutoModelForSequenceClassification)
+ pop_modelopt_key_mapping(model_config, kwargs)
# Correct UNSLOTH_MODEL_NAME's bnb tokens now the effective bnb state is known (the per-load env
# was built before remap/disable). gpt-oss only.
sync_unsloth_model_name_bnb_flags(load_in_4bit, load_in_8bit)
@@ -2669,6 +2695,9 @@ class FastLlamaModel:
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
quantization_config = kwargs.get("quantization_config", None),
+ rewritten_quantization_config = modelopt_planner_quantization_config(model_config)
+ if _modelopt_rewritten
+ else None,
# The same extra the bnb config below adds.
extra_skip_modules = ["out_proj"] if IS_FALCON_H1 else None,
),
@@ -2789,11 +2818,16 @@ class FastLlamaModel:
dtype = dtype,
)
elif not fast_inference:
- if user_config is not None:
+ if user_config is not None or _modelopt_rewritten:
# Transformers 5.x @strict model init rejects extra kwargs next to config=, so set the override
# on the config and pass the single config object through.
if max_position_embeddings is not None:
model_config.max_position_embeddings = max_position_embeddings
+ _rope_scaling = kwargs.pop("rope_scaling", None)
+ if _rope_scaling is not None:
+ model_config.rope_scaling = _rope_scaling
+ if _modelopt_rewritten and user_config is None:
+ move_config_overrides_onto_config(model_config, kwargs)
model = AutoModelForCausalLM.from_pretrained(
model_name,
config = model_config,
@@ -2899,6 +2933,8 @@ class FastLlamaModel:
finally:
raise_handler.remove()
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer
+ if _modelopt_rewritten:
+ keep_fp8_scale_names_on_save(model)
# Counteract saved tokenizers.
tokenizer_name = model_name if tokenizer_name is None else tokenizer_name
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 8d8c8e2d59e..abc9f5bc109 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -388,11 +388,19 @@ def _adapter_targets_text_core(
return isinstance(targets, str) and not any(child in targets for child in children)
-def _is_forwardless_composition(model_config, trust_remote_code = None, **hub_kwargs):
+def _is_forwardless_composition(
+ model_config,
+ trust_remote_code = None,
+ **hub_kwargs,
+):
# Only a wrapper with no forward (Qwen3-Omni) can have a thinker-trained adapter; VLM adapters never.
- auto_class = _resolve_omni_auto_model(model_config, trust_remote_code = trust_remote_code, **hub_kwargs)
+ auto_class = _resolve_omni_auto_model(
+ model_config, trust_remote_code = trust_remote_code, **hub_kwargs
+ )
model_class = (
- resolve_model_class(auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs)
+ resolve_model_class(
+ auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs
+ )
if auto_class is not None
else None
)
@@ -2127,7 +2135,8 @@ class FastModel(FastBaseModel):
text_only_decoder = True
elif (
is_vlm
- and resolve_model_class(AutoModelForCausalLM, model_config, **_probe_hub_kwargs) is None
+ and resolve_model_class(AutoModelForCausalLM, model_config, **_probe_hub_kwargs)
+ is None
and _resolve_omni_auto_model(model_config, **_probe_hub_kwargs) is not None
):
# Qwen3-Omni has no causal-LM class; load the composition, text_intent picks the thinker.
diff --git a/unsloth/models/loader_utils.py b/unsloth/models/loader_utils.py
index 5088dc11766..59212e7fb49 100644
--- a/unsloth/models/loader_utils.py
+++ b/unsloth/models/loader_utils.py
@@ -236,11 +236,17 @@ def planner_quantization_kwargs(
load_in_8bit = False,
quantization_config = None,
extra_skip_modules = None,
+ rewritten_quantization_config = None,
):
"""The quantization the planner must size for, as the load will really apply it. The config or the flags, never both, since transformers refuses both and loader.py clears the flags whenever it forwards a config; bare flags would describe a full-precision load and raise `DeviceMapInfeasible` on one that would have fit. The skip list travels with the flags: SKIP_QUANTIZATION_MODULES stays in compute dtype as `modules_to_not_convert`, and sizing it at 4bit understates the head device by GiBs on a large-vocab VLM. A pre-quantized checkpoint carries its own list in config.json."""
+ rewritten = (
+ {}
+ if rewritten_quantization_config is None
+ else {"rewritten_quantization_config": rewritten_quantization_config}
+ )
if quantization_config is not None:
- return {"quantization_config": quantization_config}
- kwargs = {"load_in_4bit": load_in_4bit, "load_in_8bit": load_in_8bit}
+ return {"quantization_config": quantization_config, **rewritten}
+ kwargs = {"load_in_4bit": load_in_4bit, "load_in_8bit": load_in_8bit, **rewritten}
if load_in_4bit or load_in_8bit:
try:
from unsloth_zoo.peft_utils import SKIP_QUANTIZATION_MODULES
@@ -1944,13 +1950,38 @@ def check_and_disable_bitsandbytes_loading(
load_in_4bit = True,
load_in_8bit = False,
verbose = True,
+ rewrite_modelopt = True,
+ token = None,
+ model_name = None,
+ revision = None,
+ hub_kwargs = None,
):
- """Disable bitsandbytes loading (load_in_4bit/load_in_8bit) when the model already carries a non-bitsandbytes quantization config. Returns ``(load_in_4bit, load_in_8bit, quant_method)``, with both flags False if they were disabled and quant_method the detected method or None."""
+ """Disable bitsandbytes loading (load_in_4bit/load_in_8bit) when the model already carries a non-bitsandbytes quantization config. Returns ``(load_in_4bit, load_in_8bit, quant_method)``, with both flags False if they were disabled and quant_method the detected method or None. ``rewrite_modelopt`` converts ModelOpt FP8 to fp8; pass False when vLLM loads it natively."""
quant_method = get_quant_type(model_config)
+ if quant_method is None:
+ # Also under vLLM: it reads the file itself, but the bitsandbytes flags must still drop.
+ from .modelopt_fp8 import attach_hf_quant_config
+ if attach_hf_quant_config(
+ model_config,
+ token = token,
+ model_name = model_name,
+ revision = revision,
+ hub_kwargs = hub_kwargs,
+ ):
+ quant_method = get_quant_type(model_config)
if quant_method is None or quant_method == "bitsandbytes":
return load_in_4bit, load_in_8bit, quant_method
+ if str(quant_method).lower() == "modelopt":
+ # Whoever loads the weights (vLLM included), a merged_16bit save must dequantize them.
+ from .modelopt_fp8 import enable_modelopt_merged_save
+ enable_modelopt_merged_save(model_config)
+ if rewrite_modelopt and str(quant_method).lower() == "modelopt":
+ from .modelopt_fp8 import arm_modelopt_fp8_loading
+ if arm_modelopt_fp8_loading(model_config, verbose = verbose) is not None:
+ quant_method = "fp8"
+
# A non-bitsandbytes quantization config (compressed-tensors, gptq, awq) means BOTH bitsandbytes loading flags must be disabled to avoid config conflicts.
if load_in_4bit or load_in_8bit:
if verbose:
diff --git a/unsloth/models/modelopt_fp8.py b/unsloth/models/modelopt_fp8.py
new file mode 100644
index 00000000000..2bdc387d34a
--- /dev/null
+++ b/unsloth/models/modelopt_fp8.py
@@ -0,0 +1,553 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+"""Load NVIDIA ModelOpt FP8 checkpoints through transformers' fp8 quantizer.
+
+ModelOpt per-tensor FP8 equals transformers' static fp8 format except for scale names
+(``weight_scale`` / ``input_scale`` vs ``weight_scale_inv`` / ``activation_scale``), so the
+config block is rewritten to ``quant_method: fp8`` and the scales renamed via ``key_mapping``.
+"""
+
+import fnmatch
+import functools
+import inspect
+import json
+import os
+import re
+import weakref
+from types import SimpleNamespace
+from typing import Optional
+
+__all__ = [
+ "modelopt_fp8_plan",
+ "arm_modelopt_fp8_loading",
+ "MODELOPT_FP8_KEY_MAPPING",
+ "UNSLOTH_MODELOPT_KEY_MAPPING_ATTR",
+ "pop_modelopt_key_mapping",
+ "modelopt_rewritten",
+ "keep_task_heads_unquantized",
+ "modelopt_planner_quantization_config",
+ "keep_fp8_scale_names_on_save",
+ "attach_hf_quant_config",
+ "move_config_overrides_onto_config",
+ "enable_modelopt_merged_save",
+]
+
+UNSLOTH_MODELOPT_KEY_MAPPING_ATTR = "_unsloth_modelopt_key_mapping"
+
+# Anchored at the end of the key, so `weight_scale_inv` or `weight_scale_2` never match.
+MODELOPT_FP8_KEY_MAPPING = {
+ r"\.weight_scale$": ".weight_scale_inv",
+ r"\.input_scale$": ".activation_scale",
+}
+
+_FP8_ALGOS = ("FP8", "FP8_PER_TENSOR")
+
+
+def _modelopt_pattern(name) -> str:
+ # ModelOpt ignore entries are fnmatch globs; as regexes `layers.16*` would also skip layers 1, 10-19.
+ name = str(name)
+ if any(ch in name for ch in "*?["):
+ return fnmatch.translate(name)
+ # transformers matches unanchored, so a bare `model.layers.1` would also skip layers 10-19.
+ return r"(?:.*\.)?" + re.escape(name) + r"(?:\.|$)"
+
+
+def _checkpoint_renames(config) -> list:
+ """Checkpoint renames (VLM `visual.*` -> `model.visual.*`); ignore lists use checkpoint names."""
+ try:
+ from transformers.conversion_mapping import get_checkpoint_conversion_mapping
+ except Exception:
+ return []
+ keys = list(getattr(config, "architectures", None) or []) + [
+ getattr(config, "model_type", None)
+ ]
+ renames = []
+ for key in keys:
+ if not key:
+ continue
+ try:
+ conversions = get_checkpoint_conversion_mapping(key) or []
+ except Exception:
+ continue
+ for conversion in conversions:
+ if type(conversion).__name__ != "WeightRenaming":
+ continue
+ # 5.5 has no `_original_*` copies; later releases rewrite the live patterns in place.
+ src = (
+ getattr(conversion, "_original_source_patterns", None)
+ or getattr(conversion, "source_patterns", None)
+ or []
+ )
+ tgt = (
+ getattr(conversion, "_original_target_patterns", None)
+ or getattr(conversion, "target_patterns", None)
+ or []
+ )
+ if len(src) == 1 and len(tgt) == 1 and (src[0], tgt[0]) not in renames:
+ renames.append((src[0], tgt[0]))
+ return renames
+
+
+def _renamed(name: str, renames) -> str:
+ for src, tgt in renames:
+ try:
+ name = re.sub(src, tgt, name, count = 1)
+ except re.error:
+ pass
+ return name
+
+
+def _as_dict(quant) -> Optional[dict]:
+ if quant is None:
+ return None
+ if isinstance(quant, dict):
+ return quant
+ to_dict = getattr(quant, "to_dict", None)
+ if callable(to_dict):
+ try:
+ out = to_dict()
+ except Exception:
+ return None
+ return out if isinstance(out, dict) else None
+ return None
+
+
+def _group_is_fp8(group) -> bool:
+ if not isinstance(group, dict):
+ return False
+ weights = group.get("weights")
+ if not isinstance(weights, dict):
+ return False
+ if str(weights.get("type", "")).lower() != "float" or int(weights.get("num_bits", 0) or 0) != 8:
+ return False
+ if weights.get("strategy", "tensor") not in (None, "tensor"):
+ return False
+ if weights.get("dynamic", False):
+ return False
+ return True
+
+
+def _activations_map_onto_fp8(inputs) -> bool:
+ # transformers' dynamic fp8 is per token, so dynamic "token" maps; static must be per tensor.
+ if not isinstance(inputs, dict):
+ return False
+ if str(inputs.get("type", "")).lower() != "float" or int(inputs.get("num_bits", 0) or 0) != 8:
+ return False
+ strategy = inputs.get("strategy", "tensor")
+ if inputs.get("dynamic", False):
+ return strategy in (None, "tensor", "token")
+ return strategy in (None, "tensor")
+
+
+def _exclusion_patterns(ignore, renames) -> list:
+ patterns = []
+ for name in ignore:
+ variants = [str(name), _renamed(str(name), renames)]
+ # transformers also prefixes an unprefixed checkpoint with the base model (no 5.5 rename).
+ variants += ["model." + v for v in variants if not v.startswith(("*", "model."))]
+ for variant in variants:
+ pattern = _modelopt_pattern(variant)
+ if pattern not in patterns:
+ patterns.append(pattern)
+ return patterns
+
+
+def modelopt_fp8_plan(config) -> Optional[dict]:
+ quant = _as_dict(getattr(config, "quantization_config", None))
+ if quant is None:
+ return None
+ if str(quant.get("quant_method", "")).lower() != "modelopt":
+ return None
+ algo = quant.get("quant_algo")
+ if algo is None and isinstance(quant.get("quantization"), dict):
+ algo = quant["quantization"].get("quant_algo")
+ if str(algo).upper() not in _FP8_ALGOS:
+ return None
+ groups = quant.get("config_groups")
+ activation_scheme = "static"
+ if groups is not None:
+ if not isinstance(groups, dict) or len(groups) == 0:
+ return None
+ if not all(_group_is_fp8(group) for group in groups.values()):
+ return None
+ # fp8 converts every Linear not ignored; narrower targets would convert unscaled layers.
+ if not all(
+ list(group.get("targets") or ["Linear"]) == ["Linear"] for group in groups.values()
+ ):
+ return None
+ inputs = [group.get("input_activations") for group in groups.values()]
+ if all(x is None for x in inputs):
+ activation_scheme = None
+ elif any(x is None for x in inputs):
+ return None
+ elif not all(_activations_map_onto_fp8(x) for x in inputs):
+ return None
+ elif len({bool(x.get("dynamic", False)) for x in inputs}) > 1:
+ return None
+ elif inputs[0].get("dynamic", False):
+ activation_scheme = "dynamic"
+ ignore = quant.get("ignore")
+ if ignore is None:
+ ignore = quant.get("exclude_modules")
+ if ignore is None and isinstance(quant.get("quantization"), dict):
+ ignore = quant["quantization"].get("exclude_modules")
+ plan = {
+ "quant_method": "fp8",
+ "weight_block_size": None,
+ "modules_to_not_convert": _exclusion_patterns(ignore or [], _checkpoint_renames(config)),
+ }
+ if activation_scheme is not None:
+ plan["activation_scheme"] = activation_scheme
+ else:
+ plan["activation_scheme"] = "dynamic"
+ return plan
+
+
+def _hf_quant_config_path(
+ name,
+ revision = None,
+ token = None,
+ hub_kwargs = None,
+) -> Optional[str]:
+ if not name:
+ return None
+ hub = dict(hub_kwargs or {})
+ subfolder = hub.get("subfolder") or ""
+ filename = "hf_quant_config.json"
+ if os.path.isdir(name):
+ path = os.path.join(name, subfolder, filename)
+ return path if os.path.isfile(path) else None
+ try:
+ from huggingface_hub import hf_hub_download, try_to_load_from_cache
+
+ cached = try_to_load_from_cache(
+ name,
+ f"{subfolder}/{filename}" if subfolder else filename,
+ cache_dir = hub.get("cache_dir"),
+ revision = revision,
+ )
+ if isinstance(cached, str):
+ return cached
+ if cached is not None:
+ # Cached as absent for this revision.
+ return None
+ return hf_hub_download(
+ name,
+ filename,
+ subfolder = subfolder or None,
+ revision = revision,
+ token = token,
+ cache_dir = hub.get("cache_dir"),
+ local_files_only = bool(hub.get("local_files_only", False)),
+ )
+ except Exception:
+ return None
+
+
+def _modelopt_block_from_hf_quant_config(path) -> Optional[dict]:
+ try:
+ with open(path, "r", encoding = "utf-8") as f:
+ data = json.load(f)
+ except Exception:
+ return None
+ if not isinstance(data, dict):
+ return None
+ producer = data.get("producer")
+ if not (isinstance(producer, dict) and str(producer.get("name", "")).lower() == "modelopt"):
+ return None
+ return {"quant_method": "modelopt", **data}
+
+
+def attach_hf_quant_config(
+ config,
+ token = None,
+ model_name = None,
+ revision = None,
+ hub_kwargs = None,
+) -> bool:
+ """Older ModelOpt exports (nvidia/Llama-3.1-8B-Instruct-FP8) keep quantization only in hf_quant_config.json."""
+ if getattr(config, "quantization_config", None) is not None:
+ return False
+ name = model_name or getattr(config, "_name_or_path", None)
+ rev = revision if model_name else getattr(config, "_commit_hash", None)
+ path = _hf_quant_config_path(name, rev, token, hub_kwargs)
+ quant = _modelopt_block_from_hf_quant_config(path) if path else None
+ if quant is None:
+ return False
+ probe = SimpleNamespace(
+ quantization_config = quant,
+ model_type = getattr(config, "model_type", None),
+ architectures = getattr(config, "architectures", None),
+ )
+ if modelopt_fp8_plan(probe) is None:
+ return False
+ try:
+ config.quantization_config = quant
+ except Exception:
+ return False
+ return True
+
+
+def _transformers_accepts_fp8_plan(plan) -> bool:
+ # transformers 4.x only knows dynamic block fp8 and rejects a per-tensor plan.
+ try:
+ from transformers.utils.quantization_config import FineGrainedFP8Config
+ FineGrainedFP8Config(**{k: v for k, v in plan.items() if k != "quant_method"})
+ except Exception:
+ return False
+ return True
+
+
+# A retried / second load reuses the config (now native fp8, marker moved off): keep the renaming by id.
+_REWRITTEN_CONFIGS: dict = {}
+
+
+def _remember_rewrite(config, mapping) -> None:
+ key = id(config)
+ entry = _REWRITTEN_CONFIGS.get(key)
+ if entry is not None and entry[0]() is config:
+ return
+ try:
+ ref = weakref.ref(config)
+ weakref.finalize(config, _REWRITTEN_CONFIGS.pop, key, None)
+ except TypeError:
+ return
+ _REWRITTEN_CONFIGS[key] = (ref, dict(mapping))
+
+
+def _remembered_mapping(config) -> Optional[dict]:
+ entry = _REWRITTEN_CONFIGS.get(id(config))
+ if entry is None or entry[0]() is not config:
+ return None
+ return dict(entry[1])
+
+
+def modelopt_rewritten(config) -> bool:
+ return (
+ hasattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+ or _remembered_mapping(config) is not None
+ )
+
+
+def arm_modelopt_fp8_loading(config, verbose: bool = True) -> Optional[dict]:
+ plan = modelopt_fp8_plan(config)
+ if plan is None or not _transformers_accepts_fp8_plan(plan):
+ return None
+ try:
+ config.quantization_config = dict(plan)
+ setattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR, dict(MODELOPT_FP8_KEY_MAPPING))
+ except Exception:
+ return None
+ _remember_rewrite(config, MODELOPT_FP8_KEY_MAPPING)
+ _dequantize_modelopt_on_merged_save()
+ if verbose:
+ print(
+ "Unsloth: NVIDIA ModelOpt FP8 checkpoint detected; loading it as a static per-tensor "
+ "fp8 checkpoint (weight_scale -> weight_scale_inv, input_scale -> activation_scale)."
+ )
+ return plan
+
+
+def enable_modelopt_merged_save(config) -> bool:
+ """Arm the merged-save dequantization, also when vLLM reads the weights and no rewrite runs."""
+ if modelopt_fp8_plan(config) is None:
+ return False
+ _dequantize_modelopt_on_merged_save()
+ return True
+
+
+def _dequantize_modelopt_on_merged_save() -> None:
+ """unsloth_zoo's merged_16bit save reads the on-disk config and only dequantizes methods it
+ knows as fp8; a ModelOpt block was skipped, writing raw fp8 bytes beside a config without
+ quantization_config. Its per-tensor dequant already handles ModelOpt scales."""
+ try:
+ import unsloth_zoo.saving_utils as zoo_saving
+ except Exception:
+ return
+ original = getattr(zoo_saving, "_is_fp8_quant_config", None)
+ if original is None or getattr(original, "_unsloth_modelopt", False):
+ return
+ try:
+ if original({"quant_method": "modelopt", "quant_algo": "FP8"}):
+ return
+ except Exception:
+ return
+
+ @functools.wraps(original)
+ def _is_fp8_quant_config(quant_config):
+ if original(quant_config):
+ return True
+ return modelopt_fp8_plan(SimpleNamespace(quantization_config = quant_config)) is not None
+
+ _is_fp8_quant_config._unsloth_modelopt = True
+ zoo_saving._is_fp8_quant_config = _is_fp8_quant_config
+
+ status = getattr(zoo_saving, "check_model_quantization_status", None)
+ if status is None or getattr(status, "_unsloth_modelopt", False):
+ return
+
+ try:
+ signature = inspect.signature(status)
+ except Exception:
+ signature = None
+
+ @functools.wraps(status)
+ def check_model_quantization_status(*args, **kwargs):
+ result = status(*args, **kwargs)
+ if result != (False, None) or signature is None:
+ return result
+ # zoo passes the token positionally: check_model_quantization_status(model_name, token, ...).
+ try:
+ bound = signature.bind(*args, **kwargs).arguments
+ except TypeError:
+ return result
+ path = _hf_quant_config_path(bound.get("model_name_or_path"), token = bound.get("token"))
+ quant = _modelopt_block_from_hf_quant_config(path) if path else None
+ if quant is not None and _is_fp8_quant_config(quant):
+ return (True, "fp8")
+ return result
+
+ check_model_quantization_status._unsloth_modelopt = True
+ zoo_saving.check_model_quantization_status = check_model_quantization_status
+
+
+def _is_modelopt_rename(conversion, ours) -> bool:
+ if type(conversion).__name__ != "WeightRenaming":
+ return False
+ sources = getattr(conversion, "_original_source_patterns", None) or getattr(
+ conversion, "source_patterns", None
+ )
+ if isinstance(sources, str):
+ sources = [sources]
+ return bool(sources) and set(sources) <= ours
+
+
+def keep_fp8_scale_names_on_save(model) -> None:
+ """save_pretrained reverses key_mapping; drop the scale renames so names match the fp8 config."""
+ ours = set(MODELOPT_FP8_KEY_MAPPING)
+ for module in model.modules():
+ conversions = getattr(module, "_weight_conversions", None)
+ if not isinstance(conversions, list):
+ continue
+ kept = [c for c in conversions if not _is_modelopt_rename(c, ours)]
+ if len(kept) != len(conversions):
+ module._weight_conversions = kept
+
+
[email protected]_cache(maxsize = 1)
+def _from_pretrained_own_kwargs() -> frozenset:
+ from transformers import PreTrainedModel
+
+ fn = PreTrainedModel.from_pretrained
+ names = set(inspect.signature(fn).parameters)
+ try:
+ names |= set(re.findall(r"kwargs\.pop\(\s*[\"'](\w+)[\"']", inspect.getsource(fn)))
+ except Exception:
+ pass
+ return frozenset(names | {"quantization_config", "key_mapping", "attn_implementation"})
+
+
+def move_config_overrides_onto_config(config, kwargs: dict) -> None:
+ """Apply config kwargs (use_cache=False) that a config= load would pass to the strict model init."""
+ own = _from_pretrained_own_kwargs()
+ for key in list(kwargs):
+ if key not in own and hasattr(config, key):
+ setattr(config, key, kwargs.pop(key))
+
+
+def _class_checkpoint_mapping(model_class) -> dict:
+ # transformers 5.3 applies VLM class renames only when no caller key_mapping is passed.
+ mapping = getattr(model_class, "_checkpoint_conversion_mapping", None)
+ if not isinstance(mapping, dict) or not mapping:
+ return {}
+ try:
+ from transformers.conversion_mapping import VLMS
+ except Exception:
+ try:
+ from transformers.modeling_utils import VLMS
+ except Exception:
+ return {}
+ mro = getattr(model_class, "__mro__", ())[:-1]
+ if not any(name in cls.__name__.lower() for cls in mro for name in VLMS):
+ return {}
+ return dict(mapping)
+
+
+def pop_modelopt_key_mapping(
+ config,
+ kwargs: dict,
+ model_class = None,
+) -> None:
+ mapping = getattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR, None)
+ if mapping is None:
+ mapping = _remembered_mapping(config)
+ if mapping is None:
+ return
+ try:
+ delattr(config, UNSLOTH_MODELOPT_KEY_MAPPING_ATTR)
+ except Exception:
+ pass
+ user = kwargs.get("key_mapping", None)
+ if isinstance(user, dict):
+ merged = dict(user)
+ else:
+ merged = _class_checkpoint_mapping(model_class)
+ for pattern, target in mapping.items():
+ merged.setdefault(pattern, target)
+ kwargs["key_mapping"] = merged
+
+
+# Fresh task heads with no fp8 weight on disk.
+_TASK_HEAD_MODULES = ("score", "classifier", "classification_head", "qa_outputs")
+_TASK_CLASS_SUFFIXES = (
+ "ForSequenceClassification",
+ "ForTokenClassification",
+ "ForQuestionAnswering",
+ "ForMultipleChoice",
+ "ForImageClassification",
+ "ForAudioClassification",
+ "ForAudioFrameClassification",
+ "ForVideoClassification",
+)
+
+
+def keep_task_heads_unquantized(config, *model_classes) -> bool:
+ names = [getattr(cls, "__name__", "") for cls in model_classes if cls is not None]
+ if not any(name.endswith(_TASK_CLASS_SUFFIXES) for name in names):
+ return False
+ # A task checkpoint already ships its head, quantized unless the ignore list names it.
+ architectures = getattr(config, "architectures", None) or []
+ if any(str(name).endswith(_TASK_CLASS_SUFFIXES) for name in architectures):
+ return False
+ quant = getattr(config, "quantization_config", None)
+ if not isinstance(quant, dict) or quant.get("quant_method") != "fp8":
+ return False
+ skip = list(quant.get("modules_to_not_convert") or [])
+ skip.extend(name for name in _TASK_HEAD_MODULES if name not in skip)
+ quant["modules_to_not_convert"] = skip
+ return True
+
+
+def modelopt_planner_quantization_config(config, dequantize: bool = False) -> Optional[dict]:
+ quant = getattr(config, "quantization_config", None)
+ if not isinstance(quant, dict):
+ return None
+ plan = dict(quant)
+ if dequantize:
+ try:
+ from transformers.utils.quantization_config import FineGrainedFP8Config
+ if "dequantize" in inspect.signature(FineGrainedFP8Config).parameters:
+ plan["dequantize"] = True
+ except Exception:
+ pass
+ return plan
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 0e13f5ad968..e8e78fec518 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -2028,8 +2028,31 @@ class FastBaseModel:
)
load_in_4bit, load_in_8bit, _ = check_and_disable_bitsandbytes_loading(
- auto_config, load_in_4bit = load_in_4bit, load_in_8bit = load_in_8bit
+ auto_config,
+ load_in_4bit = load_in_4bit,
+ load_in_8bit = load_in_8bit,
+ rewrite_modelopt = not (fast_inference and is_vLLM_available()),
+ token = token,
+ model_name = model_name,
+ revision = _revision,
+ hub_kwargs = {
+ "cache_dir": kwargs.get("cache_dir"),
+ "subfolder": kwargs.get("subfolder"),
+ "local_files_only": local_files_only,
+ },
+ )
+ from .modelopt_fp8 import (
+ keep_fp8_scale_names_on_save,
+ keep_task_heads_unquantized,
+ modelopt_planner_quantization_config,
+ modelopt_rewritten,
+ pop_modelopt_key_mapping,
)
+
+ _modelopt_rewritten = modelopt_rewritten(auto_config)
+ if _modelopt_rewritten:
+ keep_task_heads_unquantized(auto_config, auto_model, model_class)
+ pop_modelopt_key_mapping(auto_config, kwargs, model_class)
# Correct UNSLOTH_MODEL_NAME's bnb tokens now the effective bnb state is known (the per-load env was built before remap/disable). gpt-oss only.
sync_unsloth_model_name_bnb_flags(load_in_4bit, load_in_8bit)
@@ -2089,6 +2112,11 @@ class FastBaseModel:
load_in_4bit = load_in_4bit,
load_in_8bit = load_in_8bit,
quantization_config = user_quantization_config,
+ rewritten_quantization_config = modelopt_planner_quantization_config(
+ auto_config, dequantize = load_in_16bit
+ )
+ if _modelopt_rewritten
+ else None,
extra_skip_modules = _architecture_skip_modules(model_types) or None,
),
)
@@ -2224,9 +2252,14 @@ class FastBaseModel:
)
quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING["bitsandbytes_4bit"]
else:
- quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING[quant_method]
+ quantizer = AUTO_QUANTIZATION_CONFIG_MAPPING.get(quant_method)
+ # In process, an unknown method would load quantized tensors unscaled; only vLLM may.
+ if quantizer is None and not (fast_inference and is_vLLM_available()):
+ raise KeyError(
+ f"Unsloth: transformers cannot load this `{quant_method}` checkpoint in process."
+ )
quantizer_kwargs = {}
- if quant_method == "compressed-tensors":
+ if quant_method == "compressed-tensors" or quantizer is None:
pass
else:
# Cannot dequantize, since gpt-oss-20b MXFP4 would become gpt-oss-20b-BF16.
@@ -2452,6 +2485,8 @@ class FastBaseModel:
finally:
raise_handler.remove()
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = old_hf_transfer
+ if _modelopt_rewritten:
+ keep_fp8_scale_names_on_save(model)
if os.environ.get("UNSLOTH_HIGH_PRECISION_LAYERNORM", "0") == "1":
for jj, (name, module) in enumerate(model.named_modules()):