Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Load Mistral-format checkpoints (params.json only) through transformers' Mistral4 (Mistral-Large-3) (#12144)

Daniel Han78afe4445e5f
diff --git a/tests/test_fp8_to_nf4_load.py b/tests/test_fp8_to_nf4_load.py
index 34437b0c565..09a526b640c 100644
--- a/tests/test_fp8_to_nf4_load.py
+++ b/tests/test_fp8_to_nf4_load.py
@@ -625,3 +625,23 @@ def test_planner_sizes_an_armed_load_expert_merge_in_the_load_dtype():
assert measured[False][0] and not measured[True][0]
assert measured[True][1] == measured[False][1] # weights still sized as 4bit
assert measured[True][2] == 4 * measured[False][2] # bf16 stack, not packed nibbles
+
+
+def test_mistral_format_views_are_not_armed(monkeypatch):
+ # A Mistral-format view renames its fp8 scales itself; fp8 -> 4bit keeps the fp8 load there.
+ from unsloth.models import fp8_to_nf4, mistral_format
+
+ config = SimpleNamespace(
+ quantization_config = {
+ "quant_method": "fp8",
+ "weight_block_size": [128, 128],
+ "activation_scheme": "dynamic",
+ }
+ )
+ monkeypatch.setattr(mistral_format, "_active_conversions", 1)
+ token = fp8_to_nf4._EXPLICIT_4BIT.set(True)
+ try:
+ assert not fp8_to_nf4.maybe_arm_fp8_to_nf4(config, True, False, verbose = False)
+ finally:
+ fp8_to_nf4._EXPLICIT_4BIT.reset(token)
+ assert config.quantization_config["quant_method"] == "fp8"
diff --git a/tests/test_mistral_format_mistral4.py b/tests/test_mistral_format_mistral4.py
new file mode 100644
index 00000000000..8b715a4d07b
--- /dev/null
+++ b/tests/test_mistral_format_mistral4.py
@@ -0,0 +1,818 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Tiny checkpoints written under Mistral's tensor names (as in vLLM's MistralLarge3 mapper
+and the real Large-3 shard headers) load through the view and match the source model.
+The module is exec'd from its file so `import unsloth` (needs an accelerator) is avoided.
+"""
+
+import importlib.util
+import json
+import os
+import re
+
+import pytest
+
+torch = pytest.importorskip("torch")
+transformers = pytest.importorskip("transformers")
+if not hasattr(transformers, "Mistral4ForCausalLM"):
+ pytest.skip("this transformers has no mistral4", allow_module_level = True)
+
+from safetensors.torch import save_file # noqa: E402
+from transformers import Mistral4Config, Mistral4ForCausalLM # noqa: E402
+
+_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models", "mistral_format.py")
+_spec = importlib.util.spec_from_file_location("_unsloth_mistral_format", _PATH)
+mf = importlib.util.module_from_spec(_spec)
+_spec.loader.exec_module(mf)
+
+# params.json of mistralai/Mistral-Large-3-675B-Instruct-2512 without vision_encoder.
+LARGE3_PARAMS = {
+ "dim": 7168,
+ "head_dim": 192,
+ "hidden_dim": 16384,
+ "kv_lora_rank": 512,
+ "llama_4_scaling": {"beta": 0.1, "original_max_position_embeddings": 8192},
+ "max_position_embeddings": 294912,
+ "moe": {
+ "expert_hidden_dim": 4096,
+ "expert_model_parallel": 1,
+ "expert_parallel": 1,
+ "first_k_dense_replace": 3,
+ "num_expert_groups": 1,
+ "num_expert_groups_per_tok": 1,
+ "num_experts": 128,
+ "num_experts_per_tok": 4,
+ "num_shared_experts": 1,
+ "route_every_n": 1,
+ "routed_scale": 1.0,
+ },
+ "n_heads": 128,
+ "n_kv_heads": 128,
+ "n_layers": 61,
+ "norm_eps": 1e-06,
+ "q_lora_rank": 1536,
+ "qk_nope_head_dim": 128,
+ "qk_rope_head_dim": 64,
+ "quantization_config": {
+ "config_groups": {
+ "FP8_BLOCK": {
+ "format": "float-quantized",
+ "input_activations": {
+ "dynamic": True,
+ "group_size": 128,
+ "num_bits": 8,
+ "strategy": "group",
+ "symmetric": True,
+ "type": "float",
+ },
+ "output_activations": None,
+ "targets": ["Linear"],
+ "weights": {
+ "block_structure": [128, 128],
+ "dynamic": False,
+ "num_bits": 8,
+ "strategy": "block",
+ "symmetric": True,
+ "type": "float",
+ },
+ }
+ },
+ "format": "float-quantized",
+ "ignore": [
+ "model.embed_tokens",
+ "re:patch_merger.*",
+ "re:vision_encoder.*",
+ "re:vision_language_adapter.*",
+ "re:.*kv_a_proj_with_mqa$",
+ "re:.*q_a_proj$",
+ "re:.*gate$",
+ "lm_head",
+ ],
+ "quant_method": "compressed-tensors",
+ "quantization_status": "compressed",
+ },
+ "rope_theta": 10000.0,
+ "tied_embeddings": False,
+ "v_head_dim": 128,
+ "vocab_size": 131072,
+ "yarn": {
+ "alpha": 1,
+ "apply_scale": False,
+ "beta": 32,
+ "factor": 36,
+ "original_max_position_embeddings": 8192,
+ },
+}
+
+# Text tensor names of Large-3 FP8 dense layer 0 and MoE layer 3, from the shard headers.
+LARGE3_TEXT_KEYS = {
+ "layers.L.attention.kv_a_norm.weight",
+ "layers.L.attention.q_a_norm.weight",
+ "layers.L.attention.wkv_a_with_mqa.weight",
+ "layers.L.attention.wkv_b.weight",
+ "layers.L.attention.wkv_b.weight_scale",
+ "layers.L.attention.wo.weight",
+ "layers.L.attention.wo.weight_scale",
+ "layers.L.attention.wq_a.weight",
+ "layers.L.attention.wq_b.weight",
+ "layers.L.attention.wq_b.weight_scale",
+ "layers.L.attention_norm.weight",
+ "layers.L.experts.N.w1.weight",
+ "layers.L.experts.N.w1.weight_scale",
+ "layers.L.experts.N.w2.weight",
+ "layers.L.experts.N.w2.weight_scale",
+ "layers.L.experts.N.w3.weight",
+ "layers.L.experts.N.w3.weight_scale",
+ "layers.L.feed_forward.w1.weight",
+ "layers.L.feed_forward.w1.weight_scale",
+ "layers.L.feed_forward.w2.weight",
+ "layers.L.feed_forward.w2.weight_scale",
+ "layers.L.feed_forward.w3.weight",
+ "layers.L.feed_forward.w3.weight_scale",
+ "layers.L.ffn_norm.weight",
+ "layers.L.gate.weight",
+ "layers.L.shared_experts.w1.weight",
+ "layers.L.shared_experts.w1.weight_scale",
+ "layers.L.shared_experts.w2.weight",
+ "layers.L.shared_experts.w2.weight_scale",
+ "layers.L.shared_experts.w3.weight",
+ "layers.L.shared_experts.w3.weight_scale",
+ "norm.weight",
+ "output.weight",
+ "tok_embeddings.weight",
+}
+
+BLOCK = 128
+TINY_PARAMS = {
+ "dim": 256,
+ "head_dim": 128,
+ "hidden_dim": 256,
+ "kv_lora_rank": 128,
+ "llama_4_scaling": {"beta": 0.1, "original_max_position_embeddings": 16},
+ "max_position_embeddings": 4096,
+ "moe": {
+ "expert_hidden_dim": 128,
+ "expert_model_parallel": 1,
+ "expert_parallel": 1,
+ "first_k_dense_replace": 1,
+ "num_expert_groups": 1,
+ "num_expert_groups_per_tok": 1,
+ "num_experts": 4,
+ "num_experts_per_tok": 2,
+ "num_shared_experts": 1,
+ "route_every_n": 1,
+ "routed_scale": 1.0,
+ },
+ "n_heads": 2,
+ "n_kv_heads": 2,
+ "n_layers": 3,
+ "norm_eps": 1e-06,
+ "q_lora_rank": 128,
+ "qk_nope_head_dim": 64,
+ "qk_rope_head_dim": 64,
+ "rope_theta": 10000.0,
+ "tied_embeddings": False,
+ "v_head_dim": 128,
+ "vocab_size": 512,
+ "yarn": {
+ "alpha": 1,
+ "apply_scale": False,
+ "beta": 32,
+ "factor": 36,
+ "original_max_position_embeddings": 16,
+ },
+}
+
+# Mistral name -> transformers name, written out independently of the module under test.
+_ATTN = {
+ "attention.wq_a": "self_attn.q_a_proj",
+ "attention.q_a_norm": "self_attn.q_a_layernorm",
+ "attention.wq_b": "self_attn.q_b_proj",
+ "attention.wkv_a_with_mqa": "self_attn.kv_a_proj_with_mqa",
+ "attention.kv_a_norm": "self_attn.kv_a_layernorm",
+ "attention.wkv_b": "self_attn.kv_b_proj",
+ "attention.wo": "self_attn.o_proj",
+ "attention_norm": "input_layernorm",
+ "ffn_norm": "post_attention_layernorm",
+}
+_W = {"w1": "gate_proj", "w2": "down_proj", "w3": "up_proj"}
+
+
+def _reference(params, seed = 0):
+ torch.manual_seed(seed)
+ kwargs = dict(mf.mistral_params_to_mistral4_config(params))
+ kwargs.pop("architectures", None)
+ kwargs.pop("quantization_config", None)
+ model = Mistral4ForCausalLM(Mistral4Config(**kwargs)).float().eval()
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if "norm" in name and p.ndim == 1:
+ p.copy_(1 + 0.1 * torch.randn_like(p))
+ else:
+ p.copy_(torch.randn_like(p) * 0.05)
+ return model
+
+
+def _mistral_tensors(model, params):
+ """The reference's weights under Mistral's names (per-expert w1 / w2 / w3)."""
+ sd = model.state_dict()
+ inter = params["moe"]["expert_hidden_dim"]
+ out = {
+ "tok_embeddings.weight": sd["model.embed_tokens.weight"],
+ "norm.weight": sd["model.norm.weight"],
+ "output.weight": sd["lm_head.weight"],
+ }
+ for layer in range(params["n_layers"]):
+ hf = f"model.layers.{layer}"
+ for mistral, name in _ATTN.items():
+ out[f"layers.{layer}.{mistral}.weight"] = sd[f"{hf}.{name}.weight"]
+ if layer < params["moe"]["first_k_dense_replace"]:
+ for w, name in _W.items():
+ out[f"layers.{layer}.feed_forward.{w}.weight"] = sd[f"{hf}.mlp.{name}.weight"]
+ continue
+ out[f"layers.{layer}.gate.weight"] = sd[f"{hf}.mlp.gate.weight"]
+ for w, name in _W.items():
+ out[f"layers.{layer}.shared_experts.{w}.weight"] = sd[
+ f"{hf}.mlp.shared_experts.{name}.weight"
+ ]
+ gate_up, down = sd[f"{hf}.mlp.experts.gate_up_proj"], sd[f"{hf}.mlp.experts.down_proj"]
+ for e in range(params["moe"]["num_experts"]):
+ out[f"layers.{layer}.experts.{e}.w1.weight"] = gate_up[e, :inter]
+ out[f"layers.{layer}.experts.{e}.w3.weight"] = gate_up[e, inter:]
+ out[f"layers.{layer}.experts.{e}.w2.weight"] = down[e]
+ return {k: v.contiguous().clone() for k, v in out.items()}
+
+
+def _write(path, tensors, params):
+ os.makedirs(path, exist_ok = True)
+ save_file(tensors, os.path.join(path, "consolidated.safetensors"))
+ with open(os.path.join(path, "params.json"), "w") as f:
+ json.dump(params, f)
+
+
+def hub_cache(tmp_path, monkeypatch):
+ from huggingface_hub import constants
+
+ cache = tmp_path / "hub"
+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(cache))
+ return cache
+
+
+def test_large3_params_translate_to_mistral4():
+ cfg = mf.mistral_params_to_mistral4_config(LARGE3_PARAMS)
+ assert cfg["model_type"] == "mistral4"
+ assert cfg["architectures"] == ["Mistral4ForCausalLM"]
+ expect = {
+ "hidden_size": 7168,
+ "num_hidden_layers": 61,
+ "num_attention_heads": 128,
+ "q_lora_rank": 1536,
+ "kv_lora_rank": 512,
+ "qk_nope_head_dim": 128,
+ "qk_rope_head_dim": 64,
+ "v_head_dim": 128,
+ "n_routed_experts": 128,
+ "num_experts_per_tok": 4,
+ "moe_intermediate_size": 4096,
+ "n_shared_experts": 1,
+ "first_k_dense_replace": 3,
+ "intermediate_size": 16384,
+ "vocab_size": 131072,
+ "norm_topk_prob": True,
+ "rope_interleave": True,
+ "tie_word_embeddings": False,
+ }
+ assert {k: cfg[k] for k in expect} == expect
+ rope = cfg["rope_parameters"]
+ assert rope["rope_type"] == "yarn" and rope["factor"] == 36
+ assert rope["original_max_position_embeddings"] == 8192
+ assert rope["llama_4_scaling_beta"] == 0.1
+ assert cfg["quantization_config"] == {
+ "quant_method": "fp8",
+ "activation_scheme": "dynamic",
+ "weight_block_size": [128, 128],
+ "modules_to_not_convert": [
+ "embed_tokens",
+ "gate",
+ "kv_a_proj_with_mqa",
+ "lm_head",
+ "q_a_proj",
+ ],
+ }
+ # transformers accepts it and derives the MLA head sizes Mistral lists.
+ config = Mistral4Config(**{k: v for k, v in cfg.items() if k != "quantization_config"})
+ assert config.head_dim == LARGE3_PARAMS["head_dim"] == config.qk_head_dim
+
+
+ "edit",
+ [
+ lambda p: p.pop("q_lora_rank"), # no MLA
+ lambda p: p["moe"].update(num_shared_experts = 0),
+ lambda p: p["moe"].update(route_every_n = 2),
+ lambda p: p.update(sliding_window = 4096),
+ lambda p: p.update(
+ quantization = {"qformat_weight": "fp8_e4m3"}
+ ), # per-tensor, Small-4 style
+ lambda p: p["quantization_config"]["config_groups"]["FP8_BLOCK"]["weights"].update(
+ num_bits = 4
+ ),
+ lambda p: p["yarn"].update(apply_scale = True),
+ lambda p: p["quantization_config"]["config_groups"]["FP8_BLOCK"][
+ "input_activations"
+ ].update(num_bits = 4),
+ lambda p: p["quantization_config"]["config_groups"]["FP8_BLOCK"].update(
+ targets = ["re:.*experts.*"]
+ ), # vLLM scales attention differently
+ ],
+)
+def test_unsupported_params_decline(edit):
+ params = json.loads(json.dumps(LARGE3_PARAMS))
+ edit(params)
+ assert mf.mistral_params_to_mistral4_config(params) is None
+
+
+def test_tiny_names_match_the_real_large3_names():
+ names = _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS)
+ folded = {
+ re.sub(r"experts\.\d+\.", "experts.N.", re.sub(r"^layers\.\d+\.", "layers.L.", k))
+ for k in names
+ }
+ real_without_scales = {k for k in LARGE3_TEXT_KEYS if not k.endswith("_scale")}
+ assert folded == real_without_scales
+
+
+def test_bf16_checkpoint_loads_exactly(tmp_path, hub_cache):
+ reference = _reference(TINY_PARAMS)
+ with torch.no_grad():
+ for p in reference.parameters():
+ p.copy_(p.to(torch.bfloat16).float())
+ source = tmp_path / "large3-tiny"
+ _write(
+ source,
+ {k: v.to(torch.bfloat16) for k, v in _mistral_tensors(reference, TINY_PARAMS).items()},
+ TINY_PARAMS,
+ )
+
+ view = mf.prepare_mistral_format_checkpoint(str(source))
+ assert view is not None and mf.is_mistral_format_view(view)
+ assert str(hub_cache.parent) in view
+ assert not any(n.endswith(".safetensors") for n in os.listdir(view)) # nothing copied
+ assert mf.prepare_mistral_format_checkpoint(str(source)) == view # reused
+
+ with mf._mistral_format_conversions():
+ model = Mistral4ForCausalLM.from_pretrained(view, dtype = torch.float32)
+ mf._forget_load_conversions(model)
+ ref_sd, sd = reference.state_dict(), model.state_dict()
+ assert set(ref_sd) == set(sd)
+ assert all(torch.equal(ref_sd[k], sd[k]) for k in ref_sd)
+ ids = torch.randint(
+ 0, TINY_PARAMS["vocab_size"], (2, 40), generator = torch.Generator().manual_seed(1)
+ )
+ with torch.no_grad():
+ assert torch.equal(model(input_ids = ids).logits, reference(input_ids = ids).logits)
+
+ model.save_pretrained(tmp_path / "resaved")
+ again = Mistral4ForCausalLM.from_pretrained(tmp_path / "resaved", dtype = torch.float32)
+ with torch.no_grad():
+ assert torch.equal(again(input_ids = ids).logits, reference(input_ids = ids).logits)
+
+
+def test_view_skips_vision_tensors(tmp_path, hub_cache):
+ reference = _reference(TINY_PARAMS)
+ tensors = {k: v.to(torch.bfloat16) for k, v in _mistral_tensors(reference, TINY_PARAMS).items()}
+ tensors["vision_encoder.transformer.layers.0.attention.wq.weight"] = torch.zeros(
+ 4, 4, dtype = torch.bfloat16
+ )
+ tensors["patch_merger.merging_layer.weight"] = torch.zeros(4, 4, dtype = torch.bfloat16)
+ _write(tmp_path / "vl", tensors, TINY_PARAMS)
+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "vl"))
+ with open(os.path.join(view, "model.safetensors.index.json")) as f:
+ weight_map = json.load(f)["weight_map"]
+ assert not any(k.startswith(("vision_encoder.", "patch_merger.")) for k in weight_map)
+ assert len(weight_map) == len(tensors) - 2
+
+
+def test_conversions_are_scoped_to_the_load():
+ from transformers import conversion_mapping as cm
+
+ # Class-name lookup came with USER_REGISTERED_MAPPINGS; <= 5.5 looks up model_type only.
+ by_class = hasattr(cm, "USER_REGISTERED_MAPPINGS")
+ key, other = (
+ ("Mistral4ForCausalLM", "mistral4") if by_class else ("mistral4", "Mistral4ForCausalLM")
+ )
+ before = repr(cm.get_checkpoint_conversion_mapping(key))
+ untouched = repr(cm.get_checkpoint_conversion_mapping(other))
+ with mf._mistral_format_conversions():
+ inside = repr(cm.get_checkpoint_conversion_mapping(key))
+ # 5.17 also applies the model_type entry to the inner Mistral4Model: it must stay stock.
+ assert repr(cm.get_checkpoint_conversion_mapping(other)) == untouched
+ assert "tok_embeddings" in inside and "tok_embeddings" not in before
+ assert repr(cm.get_checkpoint_conversion_mapping(key)) == before
+ assert key not in getattr(cm, "USER_REGISTERED_MAPPINGS", ())
+
+
+def test_view_is_published_atomically_and_names_its_source(tmp_path, hub_cache):
+ tensors = {
+ k: v.to(torch.bfloat16)
+ for k, v in _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS).items()
+ }
+ _write(tmp_path / "src", tensors, TINY_PARAMS)
+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "src"))
+ assert not [n for n in os.listdir(view) if n.endswith(".tmp")]
+
+ class _Config:
+ _name_or_path = view
+
+ class _Model:
+ config = _Config()
+ name_or_path = view
+
+ model = _Model()
+ mf._record_source(model) # adapters saved later name the source, not this host's view
+ assert model.name_or_path == str(tmp_path / "src")
+
+
[email protected]("shard", ["../outside.safetensors", "/abs/outside.safetensors"])
+def test_index_shards_outside_the_repo_are_refused(tmp_path, hub_cache, shard):
+ tensors = {
+ k: v.to(torch.bfloat16)
+ for k, v in _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS).items()
+ }
+ src = tmp_path / "src"
+ _write(src, tensors, TINY_PARAMS)
+ with open(src / "consolidated.safetensors.index.json", "w") as f:
+ json.dump({"weight_map": {k: shard for k in tensors}}, f)
+ assert mf.prepare_mistral_format_checkpoint(str(src)) is None
+
+
+def test_redirect_retries_with_the_view(monkeypatch):
+ calls = []
+
+ @mf.mistral_format_redirect
+ def from_pretrained(
+ model_name = None,
+ revision = None,
+ **kwargs,
+ ):
+ calls.append((model_name, revision))
+ if len(calls) == 1:
+ raise mf.MistralFormatRedirect("/views/large3", model_name)
+ return object(), "tokenizer"
+
+ monkeypatch.setattr(mf, "_mistral_format_conversions", _null_context)
+ model, tok = from_pretrained("mistralai/Mistral-Large-3", revision = "abc")
+ assert calls == [("mistralai/Mistral-Large-3", "abc"), ("/views/large3", None)]
+ calls.clear()
+ model, tok = from_pretrained(model_name = "org/other") # keyword form too
+ assert calls == [("org/other", None), ("/views/large3", None)]
+
+
+class _null_context:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *exc):
+ return False
+
+
+def _fp8_blocks(weight):
+ rows, cols = weight.shape
+ blocks = weight.float().reshape(rows // BLOCK, BLOCK, cols // BLOCK, BLOCK)
+ scale = (blocks.abs().amax(dim = (1, 3)) / 448.0).clamp(min = 1e-12).to(torch.bfloat16)
+ q = (blocks / scale.float()[:, None, :, None]).clamp(-448, 448).to(torch.float8_e4m3fn)
+ dequant = (q.float() * scale.float()[:, None, :, None]).reshape(rows, cols)
+ return q.reshape(rows, cols).contiguous(), scale.contiguous(), dequant
+
+
[email protected](not torch.cuda.is_available(), reason = "FP8 loading needs a GPU")
+def test_fp8_checkpoint_dequantizes_exactly_and_loads_natively(tmp_path, hub_cache):
+ from transformers import FineGrainedFP8Config
+
+ params = dict(TINY_PARAMS, quantization_config = LARGE3_PARAMS["quantization_config"])
+ reference = _reference(params)
+ ignored = ("wq_a", "wkv_a_with_mqa", ".gate.weight", "tok_embeddings", "output.weight")
+ tensors, dequant = {}, {}
+ for name, value in _mistral_tensors(reference, params).items():
+ if value.ndim == 2 and not any(s in name for s in ignored):
+ tensors[name], tensors[name + "_scale"], dequant[name] = _fp8_blocks(value)
+ else:
+ tensors[name] = value.to(torch.bfloat16)
+ dequant[name] = tensors[name].float()
+ _write(tmp_path / "fp8", tensors, params)
+ if not mf._fp8_dequantize_on_load_available():
+ # No fp8 dequantize-on-load hook (transformers < 5.8): decline the view, never half-load it.
+ assert mf.prepare_mistral_format_checkpoint(str(tmp_path / "fp8")) is None
+ return
+
+ # Reference weights: the dequantized values put back into the transformers layout.
+ inter = params["moe"]["expert_hidden_dim"]
+ with torch.no_grad():
+ sd = reference.state_dict()
+ for layer in range(params["n_layers"]):
+ hf, ml = f"model.layers.{layer}", f"layers.{layer}"
+ for mistral, name in _ATTN.items():
+ sd[f"{hf}.{name}.weight"].copy_(dequant[f"{ml}.{mistral}.weight"])
+ if layer < params["moe"]["first_k_dense_replace"]:
+ for w, name in _W.items():
+ sd[f"{hf}.mlp.{name}.weight"].copy_(dequant[f"{ml}.feed_forward.{w}.weight"])
+ continue
+ for w, name in _W.items():
+ sd[f"{hf}.mlp.shared_experts.{name}.weight"].copy_(
+ dequant[f"{ml}.shared_experts.{w}.weight"]
+ )
+ for e in range(params["moe"]["num_experts"]):
+ sd[f"{hf}.mlp.experts.gate_up_proj"][e, :inter].copy_(
+ dequant[f"{ml}.experts.{e}.w1.weight"]
+ )
+ sd[f"{hf}.mlp.experts.gate_up_proj"][e, inter:].copy_(
+ dequant[f"{ml}.experts.{e}.w3.weight"]
+ )
+ sd[f"{hf}.mlp.experts.down_proj"][e].copy_(dequant[f"{ml}.experts.{e}.w2.weight"])
+ for p in reference.parameters():
+ p.data = p.data.to(torch.bfloat16)
+ reference = reference.cuda()
+
+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "fp8"))
+ with open(os.path.join(view, "config.json")) as f:
+ quant = dict(json.load(f)["quantization_config"])
+ quant.pop("quant_method")
+ with mf._mistral_format_conversions():
+ dequantized = Mistral4ForCausalLM.from_pretrained(
+ view,
+ dtype = torch.bfloat16,
+ device_map = {"": 0},
+ quantization_config = FineGrainedFP8Config(dequantize = True, **quant),
+ )
+ ref_sd, sd = reference.state_dict(), dequantized.state_dict()
+ assert all(torch.equal(ref_sd[k], sd[k]) for k in ref_sd)
+ ids = torch.randint(
+ 0, params["vocab_size"], (2, 40), generator = torch.Generator().manual_seed(1)
+ ).cuda()
+ with torch.no_grad():
+ assert torch.equal(dequantized(input_ids = ids).logits, reference(input_ids = ids).logits)
+ del dequantized
+
+ with mf._mistral_format_conversions():
+ native = Mistral4ForCausalLM.from_pretrained(view, dtype = torch.bfloat16, device_map = {"": 0})
+ kinds = {type(m).__name__ for m in native.modules()}
+ assert {"FP8Linear", "FP8Experts"} <= kinds
+ sd = native.state_dict()
+ layer = f"model.layers.{params['n_layers'] - 1}.mlp.experts"
+ e = params["moe"]["num_experts"] - 1
+ ml = f"layers.{params['n_layers'] - 1}.experts.{e}"
+ assert torch.equal(sd[f"{layer}.gate_up_proj"][e, :inter].cpu(), tensors[f"{ml}.w1.weight"])
+ assert torch.equal(sd[f"{layer}.gate_up_proj"][e, inter:].cpu(), tensors[f"{ml}.w3.weight"])
+ assert torch.equal(
+ sd[f"{layer}.down_proj_scale_inv"][e].cpu().to(torch.bfloat16),
+ tensors[f"{ml}.w2.weight_scale"],
+ )
+ with torch.no_grad():
+ logits = native(input_ids = ids).logits
+ assert torch.isfinite(logits).all()
+
+
+def test_redirect_without_a_view_retries_the_same_call_with_conversions():
+ # Raised for a view or an adapter trained on one: same arguments, conversions on.
+ calls = []
+
+ @mf.mistral_format_redirect
+ def from_pretrained(
+ model_name = None,
+ revision = None,
+ **kwargs,
+ ):
+ calls.append((model_name, revision, mf.mistral_format_conversions_active()))
+ if not mf.mistral_format_conversions_active():
+ raise mf.MistralFormatRedirect(None, model_name)
+ return object(), "tokenizer"
+
+ from_pretrained("/adapters/large3-lora", revision = "abc")
+ assert calls == [
+ ("/adapters/large3-lora", "abc", False),
+ ("/adapters/large3-lora", "abc", True),
+ ]
+ assert not mf.mistral_format_conversions_active()
+
+
+def test_merged_save_from_a_view_is_refused(tmp_path, hub_cache):
+ _write(tmp_path / "src", _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS), TINY_PARAMS)
+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "src"))
+
+ class _Model:
+ config = type("C", (), {"_name_or_path": view})()
+
+ mf.raise_if_merging_mistral_format_view(_Model(), "lora")
+ for method in ("merged_16bit", "merged_4bit"):
+ with pytest.raises(NotImplementedError, match = "Save the LoRA adapter"):
+ mf.raise_if_merging_mistral_format_view(_Model(), method)
+
+
+_ADAPTER_SCRIPT = r"""
+import os, sys, torch
+from unsloth import FastModel
+src, adapter, stage = sys.argv[1:4]
+ids = torch.arange(3, 43).view(1, -1).cuda()
+if stage == "train":
+ model, tok = FastModel.from_pretrained(src, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16)
+ try:
+ model.save_pretrained_gguf(os.path.join(adapter, "gguf"), tok)
+ raise SystemExit("GGUF export of a view was not refused")
+ except NotImplementedError:
+ pass
+ model = FastModel.get_peft_model(model, r = 8, lora_alpha = 16, target_modules = ["q_b_proj", "o_proj", "gate_proj"])
+ with torch.no_grad():
+ for n, p in model.named_parameters():
+ if "lora_B" in n: p.normal_(0, 0.05)
+ model.save_pretrained(adapter); tok.save_pretrained(adapter)
+ try:
+ model.save_pretrained_merged(os.path.join(adapter, "merged"), tok, save_method = "merged_16bit")
+ raise SystemExit("merged save of a view was not refused")
+ except NotImplementedError:
+ pass
+elif stage == "reload_lm":
+ from unsloth import FastLanguageModel
+ model, tok = FastLanguageModel.from_pretrained(adapter, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16)
+else:
+ model, tok = FastModel.from_pretrained(adapter, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16)
+model.eval()
+with torch.no_grad():
+ torch.save(model(input_ids = ids).logits.float().cpu(), os.path.join(adapter, stage + ".pt"))
+"""
+
+
[email protected](not torch.cuda.is_available(), reason = "FastModel needs a GPU")
+def test_lora_adapter_trained_on_a_view_reloads_in_a_fresh_process(tmp_path):
+ # The adapter's base is the view, whose index still names Mistral's tensors.
+ import subprocess
+ import sys
+
+ from huggingface_hub import snapshot_download
+ from transformers import AutoTokenizer
+
+ tok_dir = snapshot_download(
+ "trl-internal-testing/tiny-Qwen3ForCausalLM",
+ allow_patterns = ["tokenizer*", "vocab*", "merges*", "special*"],
+ )
+ params = json.loads(json.dumps(TINY_PARAMS))
+ params["vocab_size"] = -(-len(AutoTokenizer.from_pretrained(tok_dir)) // 128) * 128
+ src = tmp_path / "src"
+ _write(
+ src,
+ {k: v.to(torch.bfloat16) for k, v in _mistral_tensors(_reference(params), params).items()},
+ params,
+ )
+ AutoTokenizer.from_pretrained(tok_dir).save_pretrained(src)
+ script = tmp_path / "adapter_roundtrip.py"
+ script.write_text(_ADAPTER_SCRIPT)
+ env = dict(
+ os.environ,
+ HF_HUB_CACHE = str(tmp_path / "hub"),
+ UNSLOTH_COMPILE_LOCATION = str(tmp_path / "ucc"),
+ )
+ env["PYTHONPATH"] = os.pathsep.join(
+ [os.path.join(os.path.dirname(_PATH), os.pardir, os.pardir), env.get("PYTHONPATH", "")]
+ )
+ adapter = tmp_path / "adapter"
+ for stage in ("train", "reload", "reload_lm"):
+ run = subprocess.run(
+ [sys.executable, str(script), str(src), str(adapter), stage],
+ env = env,
+ capture_output = True,
+ text = True,
+ )
+ assert run.returncode == 0, run.stdout[-3000:] + run.stderr[-3000:]
+ with open(adapter / "adapter_config.json") as f:
+ assert json.load(f)["base_model_name_or_path"] == str(src) # portable: not the local view
+ trained, reloaded = torch.load(adapter / "train.pt"), torch.load(adapter / "reload.pt")
+ assert torch.equal(trained, torch.load(adapter / "reload_lm.pt"))
+ assert torch.equal(trained, reloaded)
+
+
+def _tiny_tekken(path):
+ import base64
+
+ words = [b"he", b"ll", b"hell", b"hello", b" w", b"or", b" wor", b"ld", b" world", b"12"]
+ vocab = [bytes([b]) for b in range(256)] + words
+ specials = ["<unk>", "<s>", "</s>", "[INST]", "[/INST]", "<pad>"] + [
+ f"<SPECIAL_{i}>" for i in range(6, 16)
+ ]
+ tekken = {
+ "config": {
+ "pattern": LARGE3_TEKKEN_PATTERN,
+ "num_vocab_tokens": len(vocab) + len(specials),
+ "default_vocab_size": len(vocab) + len(specials),
+ "default_num_special_tokens": len(specials),
+ "version": "v13",
+ },
+ "vocab": [
+ {"rank": i, "token_bytes": base64.b64encode(t).decode(), "token_str": None}
+ for i, t in enumerate(vocab)
+ ],
+ "special_tokens": [
+ {"rank": i, "token_str": s, "is_control": True} for i, s in enumerate(specials)
+ ],
+ }
+ with open(path, "w") as f:
+ json.dump(tekken, f)
+
+
+# config.pattern of the Large-3 tekken.json.
+LARGE3_TEKKEN_PATTERN = (
+ r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|"
+ r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|"
+ r"\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+"
+)
+
+
+def test_tekken_only_checkpoint_gets_a_bos_tokenizer(tmp_path, hub_cache):
+ # Large-3 BF16 ships tekken.json alone; transformers' own conversion drops BOS or shifts ids.
+ from transformers import AutoTokenizer
+
+ tensors = {
+ k: v.to(torch.bfloat16)
+ for k, v in _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS).items()
+ }
+ _write(tmp_path / "src", tensors, TINY_PARAMS)
+ _tiny_tekken(tmp_path / "src" / "tekken.json")
+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "src"))
+ assert {"tokenizer.json", "tokenizer_config.json"} <= set(os.listdir(view))
+ tok = AutoTokenizer.from_pretrained(view)
+ text = "hello world 12 é"
+ ids = tok(text).input_ids
+ n_special = 16
+ # BOS, then byte-level BPE over the inner vocab shifted past the special tokens.
+ assert ids[:4] == [1, n_special + 256 + 3, n_special + 256 + 8, n_special + ord(" ")]
+ assert tok.decode(ids, skip_special_tokens = True) == text
+ assert (tok.bos_token, tok.eos_token, tok.pad_token) == ("<s>", "</s>", "<pad>")
+ mistral_common = pytest.importorskip("mistral_common.tokens.tokenizers.tekken")
+ reference = mistral_common.Tekkenizer.from_file(str(tmp_path / "src" / "tekken.json"))
+ for sample in (text, " hell\n\nworld12345", "wor ld 🚀"):
+ assert tok(sample).input_ids == [1] + reference.encode(sample, bos = False, eos = False)
+
+
+def _redirecting_loader(calls):
+ @mf.mistral_format_redirect
+ def from_pretrained(
+ model_name = None,
+ max_seq_length = 2048,
+ dtype = None,
+ load_in_4bit = True,
+ load_in_8bit = False,
+ load_in_16bit = False,
+ fast_inference = False,
+ revision = None,
+ quantization_config = None,
+ **kwargs,
+ ):
+ calls.append(
+ dict(model_name = model_name, load_in_4bit = load_in_4bit, load_in_16bit = load_in_16bit)
+ )
+ if len(calls) == 1:
+ raise mf.MistralFormatRedirect("/views/large3", model_name)
+ return object(), "tokenizer"
+
+ return from_pretrained
+
+
+def test_redirect_keeps_the_bf16_repo_rule_and_refuses_fast_inference(monkeypatch):
+ monkeypatch.setattr(mf, "_mistral_format_conversions", _null_context)
+ calls = []
+ # The loader loads '-BF16' hub repos in 16bit; the view path has lost that suffix.
+ _redirecting_loader(calls)(
+ "mistralai/Mistral-Large-3-675B-Instruct-2512-BF16", 4096, None, True
+ )
+ assert calls[1] == dict(model_name = "/views/large3", load_in_4bit = False, load_in_16bit = True)
+ calls.clear()
+ _redirecting_loader(calls)("mistralai/Mistral-Large-3-675B-Instruct-2512")
+ assert calls[1] == dict(model_name = "/views/large3", load_in_4bit = True, load_in_16bit = False)
+ with pytest.raises(NotImplementedError, match = "fast_inference = False"):
+ _redirecting_loader([])("mistralai/Mistral-Large-3-675B-Instruct-2512", fast_inference = True)
+
+
+def test_view_downloads_honour_cache_dir(tmp_path, monkeypatch):
+ seen = []
+ monkeypatch.setattr(
+ mf,
+ "_fetch",
+ lambda *args, **kwargs: seen.append(args[5] if len(args) > 5 else kwargs) or None,
+ )
+ assert mf.prepare_mistral_format_checkpoint("org/repo", cache_dir = str(tmp_path)) is None
+ assert seen == [str(tmp_path)]
+
+
+def test_adapter_retry_on_a_view_base_refuses_fast_inference(tmp_path, monkeypatch):
+ # An adapter whose base became a view retries with no new path: vLLM cannot read it either.
+ monkeypatch.setattr(mf, "_mistral_format_conversions", _null_context)
+ (tmp_path / mf._VIEW_MARKER).write_text("{}")
+
+ @mf.mistral_format_redirect
+ def from_pretrained(
+ model_name = None,
+ fast_inference = False,
+ **kwargs,
+ ):
+ raise mf.MistralFormatRedirect(None, str(tmp_path))
+
+ with pytest.raises(NotImplementedError, match = "fast_inference = False"):
+ from_pretrained("org/adapter", fast_inference = True)
diff --git a/unsloth/models/fp8_to_nf4.py b/unsloth/models/fp8_to_nf4.py
index 4122a6a1dfa..3474038ce84 100644
--- a/unsloth/models/fp8_to_nf4.py
+++ b/unsloth/models/fp8_to_nf4.py
@@ -252,6 +252,16 @@ def maybe_arm_fp8_to_nf4(
return False
if fp8_to_nf4_disabled():
return False
+ from .mistral_format import mistral_format_conversions_active
+
+ if mistral_format_conversions_active():
+ # Mistral-named scales go through the view's own renames, which this converter does not compose with.
+ if verbose and explicit_4bit_requested():
+ print(
+ "Unsloth: 4bit loading of a Mistral-format fp8 checkpoint is not supported yet. "
+ "Loading the fp8 weights instead."
+ )
+ return False
if str(quant.get("expert_dtype") or "").lower() in _PACKED_EXPERT_DTYPES:
# DeepSeek-V4-Flash: fp8 attention, packed FP4 experts. Not handled here.
if verbose and explicit_4bit_requested():
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 670a469c0c0..dfa5b525567 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -34,6 +34,13 @@ from transformers import __version__ as transformers_version
from peft import PeftConfig, PeftModel
from .grouped_linear_lora import register_grouped_linear_lora_for_adapter
from .fp8_to_nf4 import track_explicit_4bit_request
+from .mistral_format import (
+ MistralFormatRedirect,
+ is_mistral_format_view,
+ mistral_format_conversions_active,
+ mistral_format_redirect,
+ prepare_mistral_format_checkpoint,
+)
from .loader_utils import (
DEFAULT_DEVICE_MAP,
OFFLOAD_EMBEDDING_AUTO,
@@ -324,6 +331,29 @@ def _is_mistral_format_checkpoint(
return False
+def _adapter_base_is_mistral_format(
+ model_name,
+ token,
+ revision,
+ local_files_only,
+ cache_dir = None,
+):
+ # A cached config.json settles it without a Hub request (most adapter bases).
+ if not os.path.isdir(model_name):
+ try:
+ from huggingface_hub import try_to_load_from_cache
+ if isinstance(
+ try_to_load_from_cache(
+ model_name, "config.json", revision = revision, cache_dir = cache_dir
+ ),
+ str,
+ ):
+ return False
+ except Exception:
+ pass
+ return _is_mistral_format_checkpoint(model_name, token, revision, local_files_only)
+
+
def _mistral_format_error(model_name):
return (
f"Unsloth: `{model_name}` is a checkpoint in Mistral's own format: it ships `params.json` "
@@ -701,6 +731,7 @@ def _fix_rope_inv_freq(model):
class FastLanguageModel(FastLlamaModel):
@staticmethod
@_offline_aware_load
+ @mistral_format_redirect
@track_explicit_4bit_request
def from_pretrained(
model_name = "unsloth/Llama-3.2-1B-Instruct",
@@ -1052,6 +1083,18 @@ class FastLanguageModel(FastLlamaModel):
f"to obtain the latest transformers build, then restart this session."
)
if _is_mistral_format_checkpoint(model_name, token, base_revision, local_files_only):
+ # Known architectures load via a translated view; others keep the error.
+ _view = prepare_mistral_format_checkpoint(
+ model_name,
+ token,
+ base_revision,
+ local_files_only,
+ cache_dir = kwargs.get("cache_dir", None),
+ )
+ if _view is not None:
+ if not was_disabled:
+ enable_progress_bars()
+ raise MistralFormatRedirect(_view, model_name)
raise RuntimeError(_mistral_format_error(model_name)) from autoconfig_exc
combined_error = (
"Unsloth: Failed to load model. Both AutoConfig and PeftConfig loading failed.\n\n"
@@ -1132,6 +1175,18 @@ class FastLanguageModel(FastLlamaModel):
load_in_8bit = False
load_in_fp8 = False
load_in_16bit = True
+ # After the -bf16 rule: the view path no longer carries the source's suffix.
+ # No revision: the caller's ref names the adapter repo, and the base loads unpinned below.
+ _cache_dir = kwargs.get("cache_dir", None)
+ if _adapter_base_is_mistral_format(
+ model_name, token, None, local_files_only, _cache_dir
+ ):
+ model_name = (
+ prepare_mistral_format_checkpoint(
+ model_name, token, None, local_files_only, cache_dir = _cache_dir
+ )
+ or model_name
+ )
model_config = AutoConfig.from_pretrained(
model_name,
@@ -1143,6 +1198,9 @@ class FastLanguageModel(FastLlamaModel):
if not was_disabled:
enable_progress_bars()
+ # A view, or an adapter trained on one, still names Mistral's tensors.
+ if is_mistral_format_view(model_name) and not mistral_format_conversions_active():
+ raise MistralFormatRedirect(None, model_name)
if check_precision_flags and _precision_flags_conflict(
load_in_4bit, load_in_8bit, load_in_16bit, load_in_fp8
@@ -1505,6 +1563,7 @@ class FastModel(FastBaseModel):
@staticmethod
@_offline_aware_load
+ @mistral_format_redirect
@track_explicit_4bit_request
def from_pretrained(
model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",
@@ -1870,6 +1929,18 @@ class FastModel(FastBaseModel):
f"to obtain the latest transformers build, then restart this session."
)
if _is_mistral_format_checkpoint(model_name, token, base_revision, local_files_only):
+ # Known architectures load via a translated view; others keep the error.
+ _view = prepare_mistral_format_checkpoint(
+ model_name,
+ token,
+ base_revision,
+ local_files_only,
+ cache_dir = kwargs.get("cache_dir", None),
+ )
+ if _view is not None:
+ if not was_disabled:
+ enable_progress_bars()
+ raise MistralFormatRedirect(_view, model_name)
raise RuntimeError(_mistral_format_error(model_name)) from autoconfig_exc
combined_error = (
"Unsloth: Failed to load model. Both AutoConfig and PeftConfig loading failed.\n\n"
@@ -2086,6 +2157,18 @@ class FastModel(FastBaseModel):
load_in_8bit = False
load_in_fp8 = False
load_in_16bit = True
+ # After the -bf16 rule: the view path no longer carries the source's suffix.
+ # No revision: the caller's ref names the adapter repo, and the base loads unpinned below.
+ _cache_dir = kwargs.get("cache_dir", None)
+ if _adapter_base_is_mistral_format(
+ model_name, token, None, local_files_only, _cache_dir
+ ):
+ model_name = (
+ prepare_mistral_format_checkpoint(
+ model_name, token, None, local_files_only, cache_dir = _cache_dir
+ )
+ or model_name
+ )
if user_config is not None:
model_config = user_config
@@ -2100,6 +2183,9 @@ class FastModel(FastBaseModel):
if not was_disabled:
enable_progress_bars()
+ # A view, or an adapter trained on one, still names Mistral's tensors.
+ if is_mistral_format_view(model_name) and not mistral_format_conversions_active():
+ raise MistralFormatRedirect(None, model_name)
do_logging = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1"
if do_logging:
diff --git a/unsloth/models/mistral_format.py b/unsloth/models/mistral_format.py
new file mode 100644
index 00000000000..3c34233fdf6
--- /dev/null
+++ b/unsloth/models/mistral_format.py
@@ -0,0 +1,737 @@
+# Unsloth Zoo - Utilities for Unsloth
+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Load Mistral-format checkpoints (params.json + consolidated*.safetensors, no config.json,
+e.g. Mistral-Large-3) as transformers `mistral4` via a translated view of the same shards.
+
+Mistral-Small-4 ships in both formats with byte-identical tensors, which fixes the name
+mapping. Conversions are registered for one load only so `save_pretrained` writes
+transformers names. Only the text decoder loads; unrecognised checkpoints return None.
+"""
+
+import contextlib
+import functools
+import hashlib
+import inspect
+import json
+import re
+import uuid
+import os
+from typing import Optional
+
+__all__ = [
+ "MistralFormatRedirect",
+ "mistral_params_to_mistral4_config",
+ "mistral_format_weight_conversions",
+ "prepare_mistral_format_checkpoint",
+ "mistral_format_redirect",
+]
+
+_NON_TEXT_PREFIXES = (
+ "vision_encoder.",
+ "patch_merger.",
+ "pre_mm_projector_norm.",
+ "vision_language_adapter.",
+ "mm_audio_embeddings.",
+ "audio_",
+)
+_TOKENIZER_FILES = (
+ "tekken.json",
+ "tokenizer.json",
+ "tokenizer_config.json",
+ "special_tokens_map.json",
+ "tokenizer.model",
+ "vocab.json",
+ "merges.txt",
+ "added_tokens.json",
+ "chat_template.jinja",
+ "generation_config.json",
+ "SYSTEM_PROMPT.txt",
+)
+_VIEW_MARKER = "unsloth_mistral_format.json"
+
+
+class MistralFormatRedirect(Exception):
+ """Raised by the loader; `mistral_format_redirect` catches it and loads the view."""
+
+ def __init__(self, path, source):
+ super().__init__(f"Unsloth: loading {source} through its transformers view at {path}")
+ self.path = path
+ self.source = source
+
+
+def _fp8_block_quantization(quant) -> Optional[dict]:
+ """compressed-tensors FP8 128x128 block (Mistral-Large-3) as fine-grained FP8 config, else None."""
+ if not isinstance(quant, dict):
+ return None
+ if str(quant.get("quant_method", "")).lower().replace("_", "-") != "compressed-tensors":
+ return None
+ if quant.get("format") != "float-quantized":
+ return None
+ block = None
+ for group in (quant.get("config_groups") or {}).values():
+ weights = (group or {}).get("weights") or {}
+ acts = (group or {}).get("input_activations")
+ if weights.get("type") != "float" or int(weights.get("num_bits", 0)) != 8:
+ return None
+ if weights.get("strategy") != "block" or not weights.get("block_structure"):
+ return None
+ if acts is not None and not (
+ acts.get("dynamic")
+ and acts.get("type") == "float"
+ and int(acts.get("num_bits", 8)) == 8
+ ):
+ return None
+ if group.get("output_activations") is not None:
+ return None
+ # Fine-grained FP8 has no targets: a narrower group would claim unscaled bf16 Linears.
+ if [str(t) for t in group.get("targets") or ["Linear"]] != ["Linear"]:
+ return None
+ this_block = list(weights["block_structure"])
+ if block is not None and this_block != block:
+ return None
+ block = this_block
+ if block is None:
+ return None
+ # The ignore list already uses transformers module names.
+ not_convert = []
+ for entry in quant.get("ignore") or []:
+ name = str(entry)
+ if name.startswith("re:"):
+ name = name[3:].strip("^$").replace(".*", "").replace("\\", "").strip(".")
+ if any(p.rstrip(".") in name for p in _NON_TEXT_PREFIXES) or "patch_merger" in name:
+ continue
+ if name:
+ not_convert.append(name.split(".")[-1] if name.startswith("model.") else name)
+ return {
+ "quant_method": "fp8",
+ "activation_scheme": "dynamic",
+ "weight_block_size": block,
+ "modules_to_not_convert": sorted(set(not_convert)),
+ }
+
+
+def mistral_params_to_mistral4_config(params: dict) -> Optional[dict]:
+ """`params.json` -> `Mistral4Config` kwargs as Mistral did for Small-4; None if not expressible."""
+ if not isinstance(params, dict):
+ return None
+ moe = params.get("moe")
+ if not params.get("qk_nope_head_dim") or not isinstance(moe, dict):
+ return None
+ if not params.get("q_lora_rank"):
+ return None
+ if params.get("quantization"):
+ return None # per-tensor FP8: Small-4 ships config.json
+ if int(moe.get("num_shared_experts") or 0) < 1:
+ return None
+ if int(moe.get("route_every_n", 1)) != 1:
+ return None
+ if str(moe.get("renorm_strategy", "WEIGHTS")).upper() != "WEIGHTS":
+ return None
+ if moe.get("use_load_balancing_bias"):
+ return None # mistral4 routes with plain softmax
+ if params.get("sliding_window"):
+ return None
+
+ rope_theta = float(params.get("rope_theta", 10000.0))
+ llama4 = params.get("llama_4_scaling") or {}
+ yarn = params.get("yarn") or {}
+ if yarn:
+ if yarn.get("apply_scale"):
+ return None # the mscale below is the `apply_scale: false` translation
+ original = int(yarn.get("original_max_position_embeddings", 8192))
+ if llama4 and int(llama4.get("original_max_position_embeddings", original)) != original:
+ return None # mistral4 reads one original length for both
+ rope_parameters = {
+ "rope_type": "yarn",
+ "type": "yarn",
+ "rope_theta": rope_theta,
+ "factor": float(yarn["factor"]),
+ "original_max_position_embeddings": original,
+ "beta_fast": float(yarn.get("beta", 32)),
+ "beta_slow": float(yarn.get("alpha", 1)),
+ # Matches Mistral-Small-4's config.json for `apply_scale: false`.
+ "mscale": 1.0,
+ "mscale_all_dim": 1.0,
+ "llama_4_scaling_beta": float(llama4.get("beta", 0.0)),
+ }
+ else:
+ rope_parameters = {
+ "rope_type": "default",
+ "rope_theta": rope_theta,
+ "original_max_position_embeddings": int(
+ llama4.get("original_max_position_embeddings", 8192)
+ ),
+ "llama_4_scaling_beta": float(llama4.get("beta", 0.0)),
+ }
+
+ config = {
+ "model_type": "mistral4",
+ "architectures": ["Mistral4ForCausalLM"],
+ "hidden_size": int(params["dim"]),
+ "intermediate_size": int(params["hidden_dim"]),
+ "num_hidden_layers": int(params["n_layers"]),
+ "num_attention_heads": int(params["n_heads"]),
+ "num_key_value_heads": int(params.get("n_kv_heads", params["n_heads"])),
+ "rms_norm_eps": float(params.get("norm_eps", 1e-6)),
+ "vocab_size": int(params["vocab_size"]),
+ "tie_word_embeddings": bool(params.get("tied_embeddings", False)),
+ "max_position_embeddings": int(params.get("max_position_embeddings", 128_000)),
+ "hidden_act": params.get("activation", "silu"),
+ "q_lora_rank": int(params["q_lora_rank"]),
+ "kv_lora_rank": int(params["kv_lora_rank"]),
+ "qk_nope_head_dim": int(params["qk_nope_head_dim"]),
+ "qk_rope_head_dim": int(params["qk_rope_head_dim"]),
+ "v_head_dim": int(params["v_head_dim"]),
+ "first_k_dense_replace": int(moe.get("first_k_dense_replace", 0)),
+ "n_routed_experts": int(moe["num_experts"]),
+ "num_experts_per_tok": int(moe["num_experts_per_tok"]),
+ "moe_intermediate_size": int(moe["expert_hidden_dim"]),
+ "n_shared_experts": int(moe["num_shared_experts"]),
+ "routed_scaling_factor": float(moe.get("routed_scale", 1.0)),
+ "n_group": int(moe.get("num_expert_groups", 1)),
+ "topk_group": int(moe.get("num_expert_groups_per_tok", 1)),
+ "norm_topk_prob": True,
+ "rope_interleave": True,
+ "rope_parameters": rope_parameters,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "pad_token_id": 11,
+ "attention_bias": False,
+ "mlp_bias": False,
+ }
+ quant = params.get("quantization_config")
+ if quant is not None:
+ fp8 = _fp8_block_quantization(quant)
+ if fp8 is None:
+ return None
+ config["quantization_config"] = fp8
+ return config
+
+
+def mistral_format_weight_conversions():
+ """Mistral -> `Mistral4ForCausalLM` renames (chained in order), then expert merges."""
+ from transformers.core_model_loading import (
+ Concatenate,
+ MergeModulelist,
+ WeightConverter,
+ WeightRenaming,
+ )
+
+ renames = [
+ (r"^tok_embeddings\.weight$", "model.embed_tokens.weight"),
+ (r"^norm\.weight$", "model.norm.weight"),
+ (r"^output\.weight$", "lm_head.weight"),
+ (r"^layers\.", "model.layers."),
+ (r"\.attention_norm\.", ".input_layernorm."),
+ (r"\.ffn_norm\.", ".post_attention_layernorm."),
+ (r"\.attention\.wq_a\.", ".self_attn.q_a_proj."),
+ (r"\.attention\.q_a_norm\.", ".self_attn.q_a_layernorm."),
+ (r"\.attention\.wq_b\.", ".self_attn.q_b_proj."),
+ (r"\.attention\.wkv_a_with_mqa\.", ".self_attn.kv_a_proj_with_mqa."),
+ (r"\.attention\.kv_a_norm\.", ".self_attn.kv_a_layernorm."),
+ (r"\.attention\.wkv_b\.", ".self_attn.kv_b_proj."),
+ (r"\.attention\.wo\.", ".self_attn.o_proj."),
+ (r"\.feed_forward\.w1\.", ".mlp.gate_proj."),
+ (r"\.feed_forward\.w2\.", ".mlp.down_proj."),
+ (r"\.feed_forward\.w3\.", ".mlp.up_proj."),
+ (r"\.gate\.weight$", ".mlp.gate.weight"),
+ (r"\.shared_experts\.w1\.", ".mlp.shared_experts.gate_proj."),
+ (r"\.shared_experts\.w2\.", ".mlp.shared_experts.down_proj."),
+ (r"\.shared_experts\.w3\.", ".mlp.shared_experts.up_proj."),
+ (r"\.experts\.(\d+)\.w1\.", r".mlp.experts.\1.gate_proj."),
+ (r"\.experts\.(\d+)\.w2\.", r".mlp.experts.\1.down_proj."),
+ (r"\.experts\.(\d+)\.w3\.", r".mlp.experts.\1.up_proj."),
+ # compressed-tensors' block scale is the multiplier transformers calls weight_scale_inv.
+ (r"\.weight_scale$", ".weight_scale_inv"),
+ ]
+ conversions = [WeightRenaming(source_patterns = s, target_patterns = t) for s, t in renames]
+ # Scales first: transformers stops at the first match and `.weight` also matches `.weight_scale_inv`.
+ for suffix, merge in (
+ (".weight_scale_inv", True),
+ (".weight_scale_inv", False),
+ (".weight", True),
+ (".weight", False),
+ ):
+ target_suffix = "_scale_inv" if suffix != ".weight" else ""
+ if merge:
+ converter = WeightConverter(
+ source_patterns = [
+ rf"mlp.experts.*.gate_proj{suffix}$"
+ if target_suffix
+ else f"mlp.experts.*.gate_proj{suffix}",
+ rf"mlp.experts.*.up_proj{suffix}$"
+ if target_suffix
+ else f"mlp.experts.*.up_proj{suffix}",
+ ],
+ target_patterns = f"mlp.experts.gate_up_proj{target_suffix}",
+ operations = [MergeModulelist(dim = 0), Concatenate(dim = 1)],
+ )
+ else:
+ converter = WeightConverter(
+ source_patterns = (
+ rf"mlp.experts.*.down_proj{suffix}$"
+ if target_suffix
+ else f"mlp.experts.*.down_proj{suffix}"
+ ),
+ target_patterns = f"mlp.experts.down_proj{target_suffix}",
+ operations = [MergeModulelist(dim = 0)],
+ )
+ conversions.append(converter)
+ return conversions
+
+
+def _mistral4_available() -> bool:
+ try:
+ from transformers import Mistral4Config, Mistral4ForCausalLM # noqa: F401
+ from transformers.conversion_mapping import register_checkpoint_conversion_mapping # noqa: F401
+ return True
+ except Exception:
+ return False
+
+
+def _fp8_dequantize_on_load_available() -> bool:
+ try:
+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer
+ except Exception:
+ return False
+ return callable(getattr(FineGrainedFP8HfQuantizer, "update_weight_conversions", None))
+
+
+def _fetch(
+ model_name,
+ filename,
+ token,
+ revision,
+ local_files_only,
+ cache_dir = None,
+):
+ if os.path.isdir(model_name):
+ path = os.path.join(model_name, filename)
+ return path if os.path.isfile(path) else None
+ from huggingface_hub import hf_hub_download
+ try:
+ return hf_hub_download(
+ model_name,
+ filename,
+ token = token,
+ revision = revision,
+ local_files_only = local_files_only,
+ cache_dir = cache_dir,
+ )
+ except Exception:
+ return None
+
+
+def _view_root():
+ from huggingface_hub import constants
+ return os.path.join(os.path.dirname(constants.HF_HUB_CACHE), "unsloth_mistral_format")
+
+
+def prepare_mistral_format_checkpoint(
+ model_name,
+ token = None,
+ revision = None,
+ local_files_only = False,
+ cache_dir = None,
+) -> Optional[str]:
+ """Write (or reuse) a transformers view of a Mistral-format checkpoint; None if unsupported."""
+ if not _mistral4_available():
+ return None
+ params_path = _fetch(model_name, "params.json", token, revision, local_files_only, cache_dir)
+ if params_path is None:
+ return None
+ with open(params_path, encoding = "utf-8") as f:
+ params = json.load(f)
+ config = mistral_params_to_mistral4_config(params)
+ if config is None:
+ return None
+ if config.get("quantization_config") and not _fp8_dequantize_on_load_available():
+ print(
+ f"Unsloth: `{model_name}` stores fp8 weights; loading them through transformers needs "
+ "transformers >= 5.8. Please upgrade transformers."
+ )
+ return None
+
+ index_path = _fetch(
+ model_name,
+ "consolidated.safetensors.index.json",
+ token,
+ revision,
+ local_files_only,
+ cache_dir,
+ )
+ if index_path is not None:
+ with open(index_path, encoding = "utf-8") as f:
+ weight_map = json.load(f)["weight_map"]
+ else:
+ single = _fetch(
+ model_name, "consolidated.safetensors", token, revision, local_files_only, cache_dir
+ )
+ if single is None:
+ return None
+ from safetensors import safe_open
+
+ with safe_open(single, framework = "pt") as f:
+ weight_map = {k: "consolidated.safetensors" for k in f.keys()}
+ weight_map = {k: v for k, v in weight_map.items() if not k.startswith(_NON_TEXT_PREFIXES)}
+ shards = sorted(set(weight_map.values()))
+ for shard in shards:
+ # Lexical, not realpath: hub snapshot files are symlinks into the blob store.
+ norm = os.path.normpath(str(shard))
+ if os.path.isabs(norm) or os.path.splitdrive(norm)[0] or norm.split(os.sep)[0] == os.pardir:
+ return None
+
+ shard_paths = {}
+ if os.path.isdir(model_name):
+ for shard in shards:
+ shard_paths[shard] = os.path.abspath(os.path.join(model_name, shard))
+ else:
+ from huggingface_hub import snapshot_download
+ snapshot = snapshot_download(
+ model_name,
+ revision = revision,
+ token = token,
+ local_files_only = local_files_only,
+ cache_dir = cache_dir,
+ allow_patterns = shards + ["params.json", *list(_TOKENIZER_FILES)],
+ )
+ for shard in shards:
+ shard_paths[shard] = os.path.join(snapshot, shard)
+ if not all(os.path.isfile(p) for p in shard_paths.values()):
+ return None
+
+ source_dir = os.path.dirname(shard_paths[shards[0]])
+ # Key on what the view copies or points at, so a checkpoint edited in place gets a fresh view.
+ source_files = {}
+ for name in _TOKENIZER_FILES + ("params.json",):
+ try:
+ stat = os.stat(os.path.join(source_dir, name))
+ source_files[name] = [stat.st_size, stat.st_mtime_ns]
+ except OSError:
+ pass
+ digest = hashlib.sha256(
+ json.dumps(
+ [os.path.realpath(source_dir), config, weight_map, source_files], sort_keys = True
+ ).encode()
+ ).hexdigest()[:16]
+ safe_name = re.sub(r"[\\/:]+", "--", str(model_name).strip("/\\"))[-80:]
+ view = os.path.join(_view_root(), f"{safe_name}-{digest}")
+ marker = os.path.join(view, _VIEW_MARKER)
+ if os.path.isfile(marker):
+ return view
+
+ os.makedirs(view, exist_ok = True)
+ # Per-process temp names: ranks of one launch can build the same view at once.
+ suffix = f".{os.getpid()}.{uuid.uuid4().hex}.tmp"
+ tmp = lambda name: os.path.join(view, name + suffix)
+ with open(tmp("config.json"), "w", encoding = "utf-8") as f:
+ json.dump(config, f, indent = 2)
+ with open(tmp("model.safetensors.index.json"), "w", encoding = "utf-8") as f:
+ json.dump(
+ {"metadata": {}, "weight_map": {k: shard_paths[v] for k, v in weight_map.items()}},
+ f,
+ )
+ for name in ("config.json", "model.safetensors.index.json"):
+ os.replace(tmp(name), os.path.join(view, name))
+ for name in _TOKENIZER_FILES:
+ src = os.path.join(source_dir, name)
+ if not os.path.isfile(src):
+ src = _fetch(model_name, name, token, revision, local_files_only, cache_dir)
+ if src and os.path.isfile(src):
+ import shutil
+ shutil.copyfile(src, tmp(name))
+ os.replace(tmp(name), os.path.join(view, name))
+ tekken = os.path.join(view, "tekken.json")
+ if os.path.isfile(tekken) and not os.path.isfile(os.path.join(view, "tokenizer.json")):
+ # transformers converts a lone tekken.json without BOS (5.17) or with shifted ids (5.5).
+ _write_tokenizer_from_tekken(tekken, view, tmp)
+ with open(tmp(_VIEW_MARKER), "w", encoding = "utf-8") as f:
+ json.dump({"source": str(model_name), "revision": revision}, f)
+ os.replace(tmp(_VIEW_MARKER), marker)
+ return view
+
+
+def _bytes_to_unicode():
+ bs = (
+ list(range(ord("!"), ord("~") + 1))
+ + list(range(ord("¡"), ord("¬") + 1))
+ + list(range(ord("®"), ord("ÿ") + 1))
+ )
+ cs, n = bs[:], 0
+ for b in range(256):
+ if b not in bs:
+ bs.append(b)
+ cs.append(256 + n)
+ n += 1
+ return dict(zip(bs, map(chr, cs)))
+
+
+def _write_tokenizer_from_tekken(tekken_path, view, tmp):
+ """tokenizer.json (+ tokenizer_config.json) from tekken.json, as tiktoken-style byte-level BPE:
+ ids match mistral-common's Tekkenizer, and the tokenizer.json Mistral ships beside it."""
+ import base64
+
+ from tokenizers import Regex, Tokenizer, decoders, pre_tokenizers, processors
+ from tokenizers.models import BPE
+
+ with open(tekken_path, encoding = "utf-8") as f:
+ tekken = json.load(f)
+ config = tekken["config"]
+ n_special = int(config["default_num_special_tokens"])
+ specials = {int(t["rank"]): t["token_str"] for t in tekken.get("special_tokens", [])}
+ special_strs = [specials.get(i, f"<SPECIAL_{i}>") for i in range(n_special)]
+ enc = _bytes_to_unicode()
+ ranks = {}
+ for entry in tekken["vocab"][: int(config["default_vocab_size"]) - n_special]:
+ ranks["".join(enc[b] for b in base64.b64decode(entry["token_bytes"]))] = len(ranks)
+ merges = []
+ for token, rank in ranks.items():
+ local = [
+ (token[:i], token[i:])
+ for i in range(1, len(token))
+ if token[:i] in ranks and token[i:] in ranks
+ ]
+ local.sort(key = lambda pair: (ranks[pair[0]], ranks[pair[1]]))
+ merges.extend((a, b, rank) for a, b in local)
+ merges.sort(key = lambda merge: merge[2])
+ vocab = {token: i for i, token in enumerate(special_strs)}
+ vocab.update({token: rank + n_special for token, rank in ranks.items()})
+ tokenizer = Tokenizer(BPE(vocab, [(a, b) for a, b, _ in merges], ignore_merges = True))
+ tokenizer.pre_tokenizer = pre_tokenizers.Sequence(
+ [
+ pre_tokenizers.Split(Regex(config["pattern"]), behavior = "isolated", invert = False),
+ pre_tokenizers.ByteLevel(add_prefix_space = False, use_regex = False),
+ ]
+ )
+ tokenizer.decoder = decoders.ByteLevel()
+ tokenizer.add_special_tokens(special_strs)
+ bos = special_strs[1]
+ tokenizer.post_processor = processors.TemplateProcessing(
+ single = f"{bos} $A", pair = f"{bos} $A $B", special_tokens = [(bos, 1)]
+ )
+ tokenizer.save(tmp("tokenizer.json"))
+ os.replace(tmp("tokenizer.json"), os.path.join(view, "tokenizer.json"))
+ if not os.path.isfile(os.path.join(view, "tokenizer_config.json")):
+ named = {
+ "<s>": "bos_token",
+ "</s>": "eos_token",
+ "<unk>": "unk_token",
+ "<pad>": "pad_token",
+ }
+ tokenizer_config = {
+ "tokenizer_class": "PreTrainedTokenizerFast",
+ "clean_up_tokenization_spaces": False,
+ **{named[t]: t for t in special_strs if t in named},
+ }
+ with open(tmp("tokenizer_config.json"), "w", encoding = "utf-8") as f:
+ json.dump(tokenizer_config, f, indent = 2)
+ os.replace(tmp("tokenizer_config.json"), os.path.join(view, "tokenizer_config.json"))
+
+
+def is_mistral_format_view(path) -> bool:
+ return isinstance(path, str) and os.path.isfile(os.path.join(path, _VIEW_MARKER))
+
+
+def _record_source(model):
+ """Adapters record `model.name_or_path` as their base: name the source, not this host's view."""
+ for module in (model, getattr(model, "model", None), getattr(model, "base_model", None)):
+ view = getattr(getattr(module, "config", None), "_name_or_path", None)
+ if module is None or not is_mistral_format_view(view):
+ continue
+ try:
+ with open(os.path.join(view, _VIEW_MARKER), encoding = "utf-8") as f:
+ source = json.load(f).get("source")
+ except Exception:
+ source = None
+ if source:
+ try:
+ module.name_or_path = source
+ except Exception:
+ pass
+
+
+def raise_if_merging_mistral_format_view(model, save_method):
+ """Merged saves re-read the base shards, which a view names in Mistral's own layout."""
+ view = getattr(getattr(model, "config", None), "_name_or_path", None)
+ if str(save_method).strip().lower() == "lora" or not is_mistral_format_view(view):
+ return
+ try:
+ with open(os.path.join(view, _VIEW_MARKER), encoding = "utf-8") as f:
+ source = json.load(f).get("source") or view
+ except Exception:
+ source = view
+ raise NotImplementedError(
+ f"Unsloth: `{source}` was loaded from Mistral's own checkpoint format, and "
+ f'`save_method = "{save_method}"` (merged and GGUF exports) cannot read those '
+ f"shards yet. Save the LoRA adapter instead (`model.save_pretrained(...)` or "
+ f'`save_method = "lora"`); it reloads through the same path.'
+ )
+
+
+def _is_expert_scale_merge(conversion):
+ from transformers.core_model_loading import WeightConverter
+ targets = getattr(conversion, "target_patterns", None) or []
+ return isinstance(conversion, WeightConverter) and any(
+ str(t).startswith("mlp.experts.") and str(t).endswith("_scale_inv") for t in targets
+ )
+
+
+def _register_mistral4_causal_lm():
+ # AutoModelForCausalLM lacks Mistral4Config and `register` refuses native configs.
+ from transformers import Mistral4Config, Mistral4ForCausalLM
+ from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING
+ if MODEL_FOR_CAUSAL_LM_MAPPING.get(Mistral4Config, None) is None:
+ MODEL_FOR_CAUSAL_LM_MAPPING._extra_content[Mistral4Config] = Mistral4ForCausalLM
+
+
+_active_conversions = 0
+
+
+def mistral_format_conversions_active() -> bool:
+ return _active_conversions > 0
+
+
+def _mistral_format_conversions():
+ """Register Mistral-name conversions for one load, then restore the previous ones."""
+ global _active_conversions
+ from transformers import conversion_mapping as cm
+
+ _register_mistral4_causal_lm()
+ cm.get_checkpoint_conversion_mapping("mistral4")
+ # Class-name lookup arrived with USER_REGISTERED_MAPPINGS; older releases look up model_type only.
+ # Never both: 5.17 would also apply a "mistral4" entry to the inner Mistral4Model.
+ user_registered = getattr(cm, "USER_REGISTERED_MAPPINGS", None)
+ keys = ("Mistral4ForCausalLM",) if user_registered is not None else ("mistral4",)
+ cache = cm._checkpoint_conversion_mapping_cache
+ previous = {key: (key in cache, cache.get(key)) for key in keys}
+ was_user = {key: user_registered is not None and key in user_registered for key in keys}
+ conversions = mistral_format_weight_conversions()
+ for key in keys:
+ cm.register_checkpoint_conversion_mapping(key, conversions, overwrite = True)
+ # Dequantizing FP8 folds scales in transformers' `.weight` converters; scale merges must not claim them.
+ fp8_quantizer, original_update = None, None
+ try:
+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer
+ fp8_quantizer = FineGrainedFP8HfQuantizer
+ original_update = FineGrainedFP8HfQuantizer.__dict__.get("update_weight_conversions")
+ except Exception:
+ pass
+ if original_update is not None:
+
+ def update_weight_conversions(self, weight_conversions):
+ if getattr(self.quantization_config, "dequantize", False):
+ weight_conversions = [
+ c for c in weight_conversions if not _is_expert_scale_merge(c)
+ ]
+ return original_update(self, weight_conversions)
+
+ fp8_quantizer.update_weight_conversions = update_weight_conversions
+ _active_conversions += 1
+ try:
+ yield
+ finally:
+ _active_conversions -= 1
+ if original_update is not None:
+ fp8_quantizer.update_weight_conversions = original_update
+ for key in keys:
+ had_entry, entry = previous[key]
+ if had_entry:
+ cache[key] = entry
+ else:
+ cache.pop(key, None)
+ if user_registered is not None and not was_user[key]:
+ user_registered.discard(key)
+
+
+def _forget_load_conversions(model):
+ seen = set()
+ for module in (model, getattr(model, "model", None), getattr(model, "base_model", None)):
+ if module is not None and id(module) not in seen:
+ seen.add(id(module))
+ try:
+ module._weight_conversions = []
+ except Exception:
+ pass
+
+
+def mistral_format_redirect(fn):
+ """Retry `from_pretrained` against the Mistral-format view the loader found."""
+
+ try:
+ signature = inspect.signature(fn)
+ except (TypeError, ValueError):
+ signature = None
+
+ def _argument(
+ args,
+ kwargs,
+ name,
+ default = None,
+ ):
+ if name in kwargs:
+ return kwargs[name]
+ try:
+ return signature.bind_partial(*args, **kwargs).arguments.get(name, default)
+ except Exception:
+ return default
+
+ @functools.wraps(fn)
+ def _wrapper(*args, **kwargs):
+ try:
+ return fn(*args, **kwargs)
+ except MistralFormatRedirect as redirect:
+ view, source = redirect.path, redirect.source
+ # An adapter whose base is a view retries with view None: vLLM cannot read either.
+ if (view is not None or is_mistral_format_view(source)) and _argument(
+ args, kwargs, "fast_inference", False
+ ):
+ raise NotImplementedError(
+ f"Unsloth: `{source}` is in Mistral's own format, which Unsloth loads through "
+ "transformers only. Use `fast_inference = False`, or serve it with vLLM directly."
+ )
+ if (
+ view is not None
+ and str(source).lower().endswith("-bf16")
+ and not os.path.isdir(os.path.expanduser(str(source)))
+ and _argument(args, kwargs, "quantization_config") is None
+ and signature is not None
+ ):
+ # The loader's -bf16 rule already switched this load to 16bit; the view path lost the suffix.
+ bound = signature.bind_partial(*args, **kwargs)
+ bound.arguments.update(load_in_4bit = False, load_in_8bit = False, load_in_16bit = True)
+ if "load_in_fp8" in signature.parameters:
+ bound.arguments["load_in_fp8"] = False
+ args, kwargs = bound.args, bound.kwargs
+ if view is not None:
+ if "model_name" in kwargs or not args:
+ kwargs["model_name"] = view
+ else:
+ args = (view,) + tuple(args[1:])
+ kwargs.pop("revision", None)
+ print(
+ f"Unsloth: `{source}` is in Mistral's own format. Loading its text decoder "
+ f"as transformers' Mistral4 through {view} (no weights are copied)."
+ )
+ with _mistral_format_conversions():
+ result = fn(*args, **kwargs)
+ model = result[0] if isinstance(result, tuple) else result
+ _forget_load_conversions(model)
+ _record_source(model)
+ return result
+
+ return _wrapper
diff --git a/unsloth/save.py b/unsloth/save.py
index da92f36592a..31eb7b12cde 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -69,6 +69,7 @@ from .models.loader_utils import (
_tokenizer_wants_local_only,
)
from .models._utils import _convert_torchao_model
+from .models.mistral_format import raise_if_merging_mistral_format_view
from .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPER
from transformers import ProcessorMixin, PreTrainedTokenizerBase
from huggingface_hub import HfApi
@@ -997,6 +998,7 @@ def unsloth_save_model(
gc.collect()
save_method = save_method.lower().replace(" ", "_")
+ raise_if_merging_mistral_format_view(model, save_method)
if save_method != "lora" and save_method != "merged_16bit" and save_method != "merged_4bit":
raise RuntimeError(
"Unsloth: You must select one of 3 options when saving models:\n"
@@ -3928,6 +3930,7 @@ def unsloth_save_pretrained_gguf(
"iq3_xxs" : "3.06 bpw quantization",
"q3_k_xs" : "3-bit extra small quantization",
"""
+ raise_if_merging_mistral_format_view(self, "gguf") # the converter would read Mistral names
_assert_export_target_is_not_base_with_lora_layers(self)
if tokenizer is None:
@@ -4539,6 +4542,7 @@ def unsloth_push_to_hub_gguf(
`quantization_method` may be an alias -- "not_quantized" (fast conversion, big files), "fast_quantized" (fast conversion, OK size), "quantized" (slow conversion, small files) -- or a llama.cpp ftype: f32, f16, q8_0, q4_0, q4_1, q5_0, q5_1, or a k-quant q2_k / q3_k_s / q3_k_m / q3_k_l / q4_k_s / q4_k_m / q5_k_s / q5_k_m / q6_k. The _m and _l k-quants keep the attention and feed_forward.w2 tensors a level or two above the nominal width; q2_k_l is the Unsloth preset adding --output-tensor-type q8_0 --token-embedding-type q8_0.
"""
+ raise_if_merging_mistral_format_view(self, "gguf") # the converter would read Mistral names
_assert_export_target_is_not_base_with_lora_layers(self)
if tokenizer is None:
raise ValueError("Unsloth: Saving to GGUF must have a tokenizer.")
@@ -5685,8 +5689,10 @@ def unsloth_generic_save(
datasets = datasets,
)
else:
+ raise_if_merging_mistral_format_view(model, save_method)
_prewarm_base_model_hub_cache(model, save_method = save_method, token = token)
from unsloth_zoo.saving_utils import merge_and_overwrite_lora
+
merge_and_overwrite_lora(
get_model_name,
model = model,