Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Load Mistral-format checkpoints (params.json only) through transformers' Mistral4 (Mistral-Large-3) (#12144)
Daniel Han78afe4445e5f
diff --git a/tests/test_fp8_to_nf4_load.py b/tests/test_fp8_to_nf4_load.pyindex 34437b0c565..09a526b640c 100644--- a/tests/test_fp8_to_nf4_load.py+++ b/tests/test_fp8_to_nf4_load.py@@ -625,3 +625,23 @@ def test_planner_sizes_an_armed_load_expert_merge_in_the_load_dtype():assert measured[False][0] and not measured[True][0]assert measured[True][1] == measured[False][1] # weights still sized as 4bitassert measured[True][2] == 4 * measured[False][2] # bf16 stack, not packed nibbles+++def test_mistral_format_views_are_not_armed(monkeypatch):+ # A Mistral-format view renames its fp8 scales itself; fp8 -> 4bit keeps the fp8 load there.+ from unsloth.models import fp8_to_nf4, mistral_format++ config = SimpleNamespace(+ quantization_config = {+ "quant_method": "fp8",+ "weight_block_size": [128, 128],+ "activation_scheme": "dynamic",+ }+ )+ monkeypatch.setattr(mistral_format, "_active_conversions", 1)+ token = fp8_to_nf4._EXPLICIT_4BIT.set(True)+ try:+ assert not fp8_to_nf4.maybe_arm_fp8_to_nf4(config, True, False, verbose = False)+ finally:+ fp8_to_nf4._EXPLICIT_4BIT.reset(token)+ assert config.quantization_config["quant_method"] == "fp8"diff --git a/tests/test_mistral_format_mistral4.py b/tests/test_mistral_format_mistral4.pynew file mode 100644index 00000000000..8b715a4d07b--- /dev/null+++ b/tests/test_mistral_format_mistral4.py@@ -0,0 +1,818 @@+# SPDX-License-Identifier: AGPL-3.0-only+"""Tiny checkpoints written under Mistral's tensor names (as in vLLM's MistralLarge3 mapper+and the real Large-3 shard headers) load through the view and match the source model.+The module is exec'd from its file so `import unsloth` (needs an accelerator) is avoided.+"""++import importlib.util+import json+import os+import re++import pytest++torch = pytest.importorskip("torch")+transformers = pytest.importorskip("transformers")+if not hasattr(transformers, "Mistral4ForCausalLM"):+ pytest.skip("this transformers has no mistral4", allow_module_level = True)++from safetensors.torch import save_file # noqa: E402+from transformers import Mistral4Config, Mistral4ForCausalLM # noqa: E402++_PATH = os.path.join(os.path.dirname(__file__), os.pardir, "unsloth", "models", "mistral_format.py")+_spec = importlib.util.spec_from_file_location("_unsloth_mistral_format", _PATH)+mf = importlib.util.module_from_spec(_spec)+_spec.loader.exec_module(mf)++# params.json of mistralai/Mistral-Large-3-675B-Instruct-2512 without vision_encoder.+LARGE3_PARAMS = {+ "dim": 7168,+ "head_dim": 192,+ "hidden_dim": 16384,+ "kv_lora_rank": 512,+ "llama_4_scaling": {"beta": 0.1, "original_max_position_embeddings": 8192},+ "max_position_embeddings": 294912,+ "moe": {+ "expert_hidden_dim": 4096,+ "expert_model_parallel": 1,+ "expert_parallel": 1,+ "first_k_dense_replace": 3,+ "num_expert_groups": 1,+ "num_expert_groups_per_tok": 1,+ "num_experts": 128,+ "num_experts_per_tok": 4,+ "num_shared_experts": 1,+ "route_every_n": 1,+ "routed_scale": 1.0,+ },+ "n_heads": 128,+ "n_kv_heads": 128,+ "n_layers": 61,+ "norm_eps": 1e-06,+ "q_lora_rank": 1536,+ "qk_nope_head_dim": 128,+ "qk_rope_head_dim": 64,+ "quantization_config": {+ "config_groups": {+ "FP8_BLOCK": {+ "format": "float-quantized",+ "input_activations": {+ "dynamic": True,+ "group_size": 128,+ "num_bits": 8,+ "strategy": "group",+ "symmetric": True,+ "type": "float",+ },+ "output_activations": None,+ "targets": ["Linear"],+ "weights": {+ "block_structure": [128, 128],+ "dynamic": False,+ "num_bits": 8,+ "strategy": "block",+ "symmetric": True,+ "type": "float",+ },+ }+ },+ "format": "float-quantized",+ "ignore": [+ "model.embed_tokens",+ "re:patch_merger.*",+ "re:vision_encoder.*",+ "re:vision_language_adapter.*",+ "re:.*kv_a_proj_with_mqa$",+ "re:.*q_a_proj$",+ "re:.*gate$",+ "lm_head",+ ],+ "quant_method": "compressed-tensors",+ "quantization_status": "compressed",+ },+ "rope_theta": 10000.0,+ "tied_embeddings": False,+ "v_head_dim": 128,+ "vocab_size": 131072,+ "yarn": {+ "alpha": 1,+ "apply_scale": False,+ "beta": 32,+ "factor": 36,+ "original_max_position_embeddings": 8192,+ },+}++# Text tensor names of Large-3 FP8 dense layer 0 and MoE layer 3, from the shard headers.+LARGE3_TEXT_KEYS = {+ "layers.L.attention.kv_a_norm.weight",+ "layers.L.attention.q_a_norm.weight",+ "layers.L.attention.wkv_a_with_mqa.weight",+ "layers.L.attention.wkv_b.weight",+ "layers.L.attention.wkv_b.weight_scale",+ "layers.L.attention.wo.weight",+ "layers.L.attention.wo.weight_scale",+ "layers.L.attention.wq_a.weight",+ "layers.L.attention.wq_b.weight",+ "layers.L.attention.wq_b.weight_scale",+ "layers.L.attention_norm.weight",+ "layers.L.experts.N.w1.weight",+ "layers.L.experts.N.w1.weight_scale",+ "layers.L.experts.N.w2.weight",+ "layers.L.experts.N.w2.weight_scale",+ "layers.L.experts.N.w3.weight",+ "layers.L.experts.N.w3.weight_scale",+ "layers.L.feed_forward.w1.weight",+ "layers.L.feed_forward.w1.weight_scale",+ "layers.L.feed_forward.w2.weight",+ "layers.L.feed_forward.w2.weight_scale",+ "layers.L.feed_forward.w3.weight",+ "layers.L.feed_forward.w3.weight_scale",+ "layers.L.ffn_norm.weight",+ "layers.L.gate.weight",+ "layers.L.shared_experts.w1.weight",+ "layers.L.shared_experts.w1.weight_scale",+ "layers.L.shared_experts.w2.weight",+ "layers.L.shared_experts.w2.weight_scale",+ "layers.L.shared_experts.w3.weight",+ "layers.L.shared_experts.w3.weight_scale",+ "norm.weight",+ "output.weight",+ "tok_embeddings.weight",+}++BLOCK = 128+TINY_PARAMS = {+ "dim": 256,+ "head_dim": 128,+ "hidden_dim": 256,+ "kv_lora_rank": 128,+ "llama_4_scaling": {"beta": 0.1, "original_max_position_embeddings": 16},+ "max_position_embeddings": 4096,+ "moe": {+ "expert_hidden_dim": 128,+ "expert_model_parallel": 1,+ "expert_parallel": 1,+ "first_k_dense_replace": 1,+ "num_expert_groups": 1,+ "num_expert_groups_per_tok": 1,+ "num_experts": 4,+ "num_experts_per_tok": 2,+ "num_shared_experts": 1,+ "route_every_n": 1,+ "routed_scale": 1.0,+ },+ "n_heads": 2,+ "n_kv_heads": 2,+ "n_layers": 3,+ "norm_eps": 1e-06,+ "q_lora_rank": 128,+ "qk_nope_head_dim": 64,+ "qk_rope_head_dim": 64,+ "rope_theta": 10000.0,+ "tied_embeddings": False,+ "v_head_dim": 128,+ "vocab_size": 512,+ "yarn": {+ "alpha": 1,+ "apply_scale": False,+ "beta": 32,+ "factor": 36,+ "original_max_position_embeddings": 16,+ },+}++# Mistral name -> transformers name, written out independently of the module under test.+_ATTN = {+ "attention.wq_a": "self_attn.q_a_proj",+ "attention.q_a_norm": "self_attn.q_a_layernorm",+ "attention.wq_b": "self_attn.q_b_proj",+ "attention.wkv_a_with_mqa": "self_attn.kv_a_proj_with_mqa",+ "attention.kv_a_norm": "self_attn.kv_a_layernorm",+ "attention.wkv_b": "self_attn.kv_b_proj",+ "attention.wo": "self_attn.o_proj",+ "attention_norm": "input_layernorm",+ "ffn_norm": "post_attention_layernorm",+}+_W = {"w1": "gate_proj", "w2": "down_proj", "w3": "up_proj"}+++def _reference(params, seed = 0):+ torch.manual_seed(seed)+ kwargs = dict(mf.mistral_params_to_mistral4_config(params))+ kwargs.pop("architectures", None)+ kwargs.pop("quantization_config", None)+ model = Mistral4ForCausalLM(Mistral4Config(**kwargs)).float().eval()+ with torch.no_grad():+ for name, p in model.named_parameters():+ if "norm" in name and p.ndim == 1:+ p.copy_(1 + 0.1 * torch.randn_like(p))+ else:+ p.copy_(torch.randn_like(p) * 0.05)+ return model+++def _mistral_tensors(model, params):+ """The reference's weights under Mistral's names (per-expert w1 / w2 / w3)."""+ sd = model.state_dict()+ inter = params["moe"]["expert_hidden_dim"]+ out = {+ "tok_embeddings.weight": sd["model.embed_tokens.weight"],+ "norm.weight": sd["model.norm.weight"],+ "output.weight": sd["lm_head.weight"],+ }+ for layer in range(params["n_layers"]):+ hf = f"model.layers.{layer}"+ for mistral, name in _ATTN.items():+ out[f"layers.{layer}.{mistral}.weight"] = sd[f"{hf}.{name}.weight"]+ if layer < params["moe"]["first_k_dense_replace"]:+ for w, name in _W.items():+ out[f"layers.{layer}.feed_forward.{w}.weight"] = sd[f"{hf}.mlp.{name}.weight"]+ continue+ out[f"layers.{layer}.gate.weight"] = sd[f"{hf}.mlp.gate.weight"]+ for w, name in _W.items():+ out[f"layers.{layer}.shared_experts.{w}.weight"] = sd[+ f"{hf}.mlp.shared_experts.{name}.weight"+ ]+ gate_up, down = sd[f"{hf}.mlp.experts.gate_up_proj"], sd[f"{hf}.mlp.experts.down_proj"]+ for e in range(params["moe"]["num_experts"]):+ out[f"layers.{layer}.experts.{e}.w1.weight"] = gate_up[e, :inter]+ out[f"layers.{layer}.experts.{e}.w3.weight"] = gate_up[e, inter:]+ out[f"layers.{layer}.experts.{e}.w2.weight"] = down[e]+ return {k: v.contiguous().clone() for k, v in out.items()}+++def _write(path, tensors, params):+ os.makedirs(path, exist_ok = True)+ save_file(tensors, os.path.join(path, "consolidated.safetensors"))+ with open(os.path.join(path, "params.json"), "w") as f:+ json.dump(params, f)+++def hub_cache(tmp_path, monkeypatch):+ from huggingface_hub import constants++ cache = tmp_path / "hub"+ monkeypatch.setattr(constants, "HF_HUB_CACHE", str(cache))+ return cache+++def test_large3_params_translate_to_mistral4():+ cfg = mf.mistral_params_to_mistral4_config(LARGE3_PARAMS)+ assert cfg["model_type"] == "mistral4"+ assert cfg["architectures"] == ["Mistral4ForCausalLM"]+ expect = {+ "hidden_size": 7168,+ "num_hidden_layers": 61,+ "num_attention_heads": 128,+ "q_lora_rank": 1536,+ "kv_lora_rank": 512,+ "qk_nope_head_dim": 128,+ "qk_rope_head_dim": 64,+ "v_head_dim": 128,+ "n_routed_experts": 128,+ "num_experts_per_tok": 4,+ "moe_intermediate_size": 4096,+ "n_shared_experts": 1,+ "first_k_dense_replace": 3,+ "intermediate_size": 16384,+ "vocab_size": 131072,+ "norm_topk_prob": True,+ "rope_interleave": True,+ "tie_word_embeddings": False,+ }+ assert {k: cfg[k] for k in expect} == expect+ rope = cfg["rope_parameters"]+ assert rope["rope_type"] == "yarn" and rope["factor"] == 36+ assert rope["original_max_position_embeddings"] == 8192+ assert rope["llama_4_scaling_beta"] == 0.1+ assert cfg["quantization_config"] == {+ "quant_method": "fp8",+ "activation_scheme": "dynamic",+ "weight_block_size": [128, 128],+ "modules_to_not_convert": [+ "embed_tokens",+ "gate",+ "kv_a_proj_with_mqa",+ "lm_head",+ "q_a_proj",+ ],+ }+ # transformers accepts it and derives the MLA head sizes Mistral lists.+ config = Mistral4Config(**{k: v for k, v in cfg.items() if k != "quantization_config"})+ assert config.head_dim == LARGE3_PARAMS["head_dim"] == config.qk_head_dim+++ "edit",+ [+ lambda p: p.pop("q_lora_rank"), # no MLA+ lambda p: p["moe"].update(num_shared_experts = 0),+ lambda p: p["moe"].update(route_every_n = 2),+ lambda p: p.update(sliding_window = 4096),+ lambda p: p.update(+ quantization = {"qformat_weight": "fp8_e4m3"}+ ), # per-tensor, Small-4 style+ lambda p: p["quantization_config"]["config_groups"]["FP8_BLOCK"]["weights"].update(+ num_bits = 4+ ),+ lambda p: p["yarn"].update(apply_scale = True),+ lambda p: p["quantization_config"]["config_groups"]["FP8_BLOCK"][+ "input_activations"+ ].update(num_bits = 4),+ lambda p: p["quantization_config"]["config_groups"]["FP8_BLOCK"].update(+ targets = ["re:.*experts.*"]+ ), # vLLM scales attention differently+ ],+)+def test_unsupported_params_decline(edit):+ params = json.loads(json.dumps(LARGE3_PARAMS))+ edit(params)+ assert mf.mistral_params_to_mistral4_config(params) is None+++def test_tiny_names_match_the_real_large3_names():+ names = _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS)+ folded = {+ re.sub(r"experts\.\d+\.", "experts.N.", re.sub(r"^layers\.\d+\.", "layers.L.", k))+ for k in names+ }+ real_without_scales = {k for k in LARGE3_TEXT_KEYS if not k.endswith("_scale")}+ assert folded == real_without_scales+++def test_bf16_checkpoint_loads_exactly(tmp_path, hub_cache):+ reference = _reference(TINY_PARAMS)+ with torch.no_grad():+ for p in reference.parameters():+ p.copy_(p.to(torch.bfloat16).float())+ source = tmp_path / "large3-tiny"+ _write(+ source,+ {k: v.to(torch.bfloat16) for k, v in _mistral_tensors(reference, TINY_PARAMS).items()},+ TINY_PARAMS,+ )++ view = mf.prepare_mistral_format_checkpoint(str(source))+ assert view is not None and mf.is_mistral_format_view(view)+ assert str(hub_cache.parent) in view+ assert not any(n.endswith(".safetensors") for n in os.listdir(view)) # nothing copied+ assert mf.prepare_mistral_format_checkpoint(str(source)) == view # reused++ with mf._mistral_format_conversions():+ model = Mistral4ForCausalLM.from_pretrained(view, dtype = torch.float32)+ mf._forget_load_conversions(model)+ ref_sd, sd = reference.state_dict(), model.state_dict()+ assert set(ref_sd) == set(sd)+ assert all(torch.equal(ref_sd[k], sd[k]) for k in ref_sd)+ ids = torch.randint(+ 0, TINY_PARAMS["vocab_size"], (2, 40), generator = torch.Generator().manual_seed(1)+ )+ with torch.no_grad():+ assert torch.equal(model(input_ids = ids).logits, reference(input_ids = ids).logits)++ model.save_pretrained(tmp_path / "resaved")+ again = Mistral4ForCausalLM.from_pretrained(tmp_path / "resaved", dtype = torch.float32)+ with torch.no_grad():+ assert torch.equal(again(input_ids = ids).logits, reference(input_ids = ids).logits)+++def test_view_skips_vision_tensors(tmp_path, hub_cache):+ reference = _reference(TINY_PARAMS)+ tensors = {k: v.to(torch.bfloat16) for k, v in _mistral_tensors(reference, TINY_PARAMS).items()}+ tensors["vision_encoder.transformer.layers.0.attention.wq.weight"] = torch.zeros(+ 4, 4, dtype = torch.bfloat16+ )+ tensors["patch_merger.merging_layer.weight"] = torch.zeros(4, 4, dtype = torch.bfloat16)+ _write(tmp_path / "vl", tensors, TINY_PARAMS)+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "vl"))+ with open(os.path.join(view, "model.safetensors.index.json")) as f:+ weight_map = json.load(f)["weight_map"]+ assert not any(k.startswith(("vision_encoder.", "patch_merger.")) for k in weight_map)+ assert len(weight_map) == len(tensors) - 2+++def test_conversions_are_scoped_to_the_load():+ from transformers import conversion_mapping as cm++ # Class-name lookup came with USER_REGISTERED_MAPPINGS; <= 5.5 looks up model_type only.+ by_class = hasattr(cm, "USER_REGISTERED_MAPPINGS")+ key, other = (+ ("Mistral4ForCausalLM", "mistral4") if by_class else ("mistral4", "Mistral4ForCausalLM")+ )+ before = repr(cm.get_checkpoint_conversion_mapping(key))+ untouched = repr(cm.get_checkpoint_conversion_mapping(other))+ with mf._mistral_format_conversions():+ inside = repr(cm.get_checkpoint_conversion_mapping(key))+ # 5.17 also applies the model_type entry to the inner Mistral4Model: it must stay stock.+ assert repr(cm.get_checkpoint_conversion_mapping(other)) == untouched+ assert "tok_embeddings" in inside and "tok_embeddings" not in before+ assert repr(cm.get_checkpoint_conversion_mapping(key)) == before+ assert key not in getattr(cm, "USER_REGISTERED_MAPPINGS", ())+++def test_view_is_published_atomically_and_names_its_source(tmp_path, hub_cache):+ tensors = {+ k: v.to(torch.bfloat16)+ for k, v in _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS).items()+ }+ _write(tmp_path / "src", tensors, TINY_PARAMS)+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "src"))+ assert not [n for n in os.listdir(view) if n.endswith(".tmp")]++ class _Config:+ _name_or_path = view++ class _Model:+ config = _Config()+ name_or_path = view++ model = _Model()+ mf._record_source(model) # adapters saved later name the source, not this host's view+ assert model.name_or_path == str(tmp_path / "src")++[email protected]("shard", ["../outside.safetensors", "/abs/outside.safetensors"])+def test_index_shards_outside_the_repo_are_refused(tmp_path, hub_cache, shard):+ tensors = {+ k: v.to(torch.bfloat16)+ for k, v in _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS).items()+ }+ src = tmp_path / "src"+ _write(src, tensors, TINY_PARAMS)+ with open(src / "consolidated.safetensors.index.json", "w") as f:+ json.dump({"weight_map": {k: shard for k in tensors}}, f)+ assert mf.prepare_mistral_format_checkpoint(str(src)) is None+++def test_redirect_retries_with_the_view(monkeypatch):+ calls = []++ @mf.mistral_format_redirect+ def from_pretrained(+ model_name = None,+ revision = None,+ **kwargs,+ ):+ calls.append((model_name, revision))+ if len(calls) == 1:+ raise mf.MistralFormatRedirect("/views/large3", model_name)+ return object(), "tokenizer"++ monkeypatch.setattr(mf, "_mistral_format_conversions", _null_context)+ model, tok = from_pretrained("mistralai/Mistral-Large-3", revision = "abc")+ assert calls == [("mistralai/Mistral-Large-3", "abc"), ("/views/large3", None)]+ calls.clear()+ model, tok = from_pretrained(model_name = "org/other") # keyword form too+ assert calls == [("org/other", None), ("/views/large3", None)]+++class _null_context:+ def __enter__(self):+ return self++ def __exit__(self, *exc):+ return False+++def _fp8_blocks(weight):+ rows, cols = weight.shape+ blocks = weight.float().reshape(rows // BLOCK, BLOCK, cols // BLOCK, BLOCK)+ scale = (blocks.abs().amax(dim = (1, 3)) / 448.0).clamp(min = 1e-12).to(torch.bfloat16)+ q = (blocks / scale.float()[:, None, :, None]).clamp(-448, 448).to(torch.float8_e4m3fn)+ dequant = (q.float() * scale.float()[:, None, :, None]).reshape(rows, cols)+ return q.reshape(rows, cols).contiguous(), scale.contiguous(), dequant++[email protected](not torch.cuda.is_available(), reason = "FP8 loading needs a GPU")+def test_fp8_checkpoint_dequantizes_exactly_and_loads_natively(tmp_path, hub_cache):+ from transformers import FineGrainedFP8Config++ params = dict(TINY_PARAMS, quantization_config = LARGE3_PARAMS["quantization_config"])+ reference = _reference(params)+ ignored = ("wq_a", "wkv_a_with_mqa", ".gate.weight", "tok_embeddings", "output.weight")+ tensors, dequant = {}, {}+ for name, value in _mistral_tensors(reference, params).items():+ if value.ndim == 2 and not any(s in name for s in ignored):+ tensors[name], tensors[name + "_scale"], dequant[name] = _fp8_blocks(value)+ else:+ tensors[name] = value.to(torch.bfloat16)+ dequant[name] = tensors[name].float()+ _write(tmp_path / "fp8", tensors, params)+ if not mf._fp8_dequantize_on_load_available():+ # No fp8 dequantize-on-load hook (transformers < 5.8): decline the view, never half-load it.+ assert mf.prepare_mistral_format_checkpoint(str(tmp_path / "fp8")) is None+ return++ # Reference weights: the dequantized values put back into the transformers layout.+ inter = params["moe"]["expert_hidden_dim"]+ with torch.no_grad():+ sd = reference.state_dict()+ for layer in range(params["n_layers"]):+ hf, ml = f"model.layers.{layer}", f"layers.{layer}"+ for mistral, name in _ATTN.items():+ sd[f"{hf}.{name}.weight"].copy_(dequant[f"{ml}.{mistral}.weight"])+ if layer < params["moe"]["first_k_dense_replace"]:+ for w, name in _W.items():+ sd[f"{hf}.mlp.{name}.weight"].copy_(dequant[f"{ml}.feed_forward.{w}.weight"])+ continue+ for w, name in _W.items():+ sd[f"{hf}.mlp.shared_experts.{name}.weight"].copy_(+ dequant[f"{ml}.shared_experts.{w}.weight"]+ )+ for e in range(params["moe"]["num_experts"]):+ sd[f"{hf}.mlp.experts.gate_up_proj"][e, :inter].copy_(+ dequant[f"{ml}.experts.{e}.w1.weight"]+ )+ sd[f"{hf}.mlp.experts.gate_up_proj"][e, inter:].copy_(+ dequant[f"{ml}.experts.{e}.w3.weight"]+ )+ sd[f"{hf}.mlp.experts.down_proj"][e].copy_(dequant[f"{ml}.experts.{e}.w2.weight"])+ for p in reference.parameters():+ p.data = p.data.to(torch.bfloat16)+ reference = reference.cuda()++ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "fp8"))+ with open(os.path.join(view, "config.json")) as f:+ quant = dict(json.load(f)["quantization_config"])+ quant.pop("quant_method")+ with mf._mistral_format_conversions():+ dequantized = Mistral4ForCausalLM.from_pretrained(+ view,+ dtype = torch.bfloat16,+ device_map = {"": 0},+ quantization_config = FineGrainedFP8Config(dequantize = True, **quant),+ )+ ref_sd, sd = reference.state_dict(), dequantized.state_dict()+ assert all(torch.equal(ref_sd[k], sd[k]) for k in ref_sd)+ ids = torch.randint(+ 0, params["vocab_size"], (2, 40), generator = torch.Generator().manual_seed(1)+ ).cuda()+ with torch.no_grad():+ assert torch.equal(dequantized(input_ids = ids).logits, reference(input_ids = ids).logits)+ del dequantized++ with mf._mistral_format_conversions():+ native = Mistral4ForCausalLM.from_pretrained(view, dtype = torch.bfloat16, device_map = {"": 0})+ kinds = {type(m).__name__ for m in native.modules()}+ assert {"FP8Linear", "FP8Experts"} <= kinds+ sd = native.state_dict()+ layer = f"model.layers.{params['n_layers'] - 1}.mlp.experts"+ e = params["moe"]["num_experts"] - 1+ ml = f"layers.{params['n_layers'] - 1}.experts.{e}"+ assert torch.equal(sd[f"{layer}.gate_up_proj"][e, :inter].cpu(), tensors[f"{ml}.w1.weight"])+ assert torch.equal(sd[f"{layer}.gate_up_proj"][e, inter:].cpu(), tensors[f"{ml}.w3.weight"])+ assert torch.equal(+ sd[f"{layer}.down_proj_scale_inv"][e].cpu().to(torch.bfloat16),+ tensors[f"{ml}.w2.weight_scale"],+ )+ with torch.no_grad():+ logits = native(input_ids = ids).logits+ assert torch.isfinite(logits).all()+++def test_redirect_without_a_view_retries_the_same_call_with_conversions():+ # Raised for a view or an adapter trained on one: same arguments, conversions on.+ calls = []++ @mf.mistral_format_redirect+ def from_pretrained(+ model_name = None,+ revision = None,+ **kwargs,+ ):+ calls.append((model_name, revision, mf.mistral_format_conversions_active()))+ if not mf.mistral_format_conversions_active():+ raise mf.MistralFormatRedirect(None, model_name)+ return object(), "tokenizer"++ from_pretrained("/adapters/large3-lora", revision = "abc")+ assert calls == [+ ("/adapters/large3-lora", "abc", False),+ ("/adapters/large3-lora", "abc", True),+ ]+ assert not mf.mistral_format_conversions_active()+++def test_merged_save_from_a_view_is_refused(tmp_path, hub_cache):+ _write(tmp_path / "src", _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS), TINY_PARAMS)+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "src"))++ class _Model:+ config = type("C", (), {"_name_or_path": view})()++ mf.raise_if_merging_mistral_format_view(_Model(), "lora")+ for method in ("merged_16bit", "merged_4bit"):+ with pytest.raises(NotImplementedError, match = "Save the LoRA adapter"):+ mf.raise_if_merging_mistral_format_view(_Model(), method)+++_ADAPTER_SCRIPT = r"""+import os, sys, torch+from unsloth import FastModel+src, adapter, stage = sys.argv[1:4]+ids = torch.arange(3, 43).view(1, -1).cuda()+if stage == "train":+ model, tok = FastModel.from_pretrained(src, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16)+ try:+ model.save_pretrained_gguf(os.path.join(adapter, "gguf"), tok)+ raise SystemExit("GGUF export of a view was not refused")+ except NotImplementedError:+ pass+ model = FastModel.get_peft_model(model, r = 8, lora_alpha = 16, target_modules = ["q_b_proj", "o_proj", "gate_proj"])+ with torch.no_grad():+ for n, p in model.named_parameters():+ if "lora_B" in n: p.normal_(0, 0.05)+ model.save_pretrained(adapter); tok.save_pretrained(adapter)+ try:+ model.save_pretrained_merged(os.path.join(adapter, "merged"), tok, save_method = "merged_16bit")+ raise SystemExit("merged save of a view was not refused")+ except NotImplementedError:+ pass+elif stage == "reload_lm":+ from unsloth import FastLanguageModel+ model, tok = FastLanguageModel.from_pretrained(adapter, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16)+else:+ model, tok = FastModel.from_pretrained(adapter, max_seq_length = 64, load_in_4bit = False, dtype = torch.bfloat16)+model.eval()+with torch.no_grad():+ torch.save(model(input_ids = ids).logits.float().cpu(), os.path.join(adapter, stage + ".pt"))+"""++[email protected](not torch.cuda.is_available(), reason = "FastModel needs a GPU")+def test_lora_adapter_trained_on_a_view_reloads_in_a_fresh_process(tmp_path):+ # The adapter's base is the view, whose index still names Mistral's tensors.+ import subprocess+ import sys++ from huggingface_hub import snapshot_download+ from transformers import AutoTokenizer++ tok_dir = snapshot_download(+ "trl-internal-testing/tiny-Qwen3ForCausalLM",+ allow_patterns = ["tokenizer*", "vocab*", "merges*", "special*"],+ )+ params = json.loads(json.dumps(TINY_PARAMS))+ params["vocab_size"] = -(-len(AutoTokenizer.from_pretrained(tok_dir)) // 128) * 128+ src = tmp_path / "src"+ _write(+ src,+ {k: v.to(torch.bfloat16) for k, v in _mistral_tensors(_reference(params), params).items()},+ params,+ )+ AutoTokenizer.from_pretrained(tok_dir).save_pretrained(src)+ script = tmp_path / "adapter_roundtrip.py"+ script.write_text(_ADAPTER_SCRIPT)+ env = dict(+ os.environ,+ HF_HUB_CACHE = str(tmp_path / "hub"),+ UNSLOTH_COMPILE_LOCATION = str(tmp_path / "ucc"),+ )+ env["PYTHONPATH"] = os.pathsep.join(+ [os.path.join(os.path.dirname(_PATH), os.pardir, os.pardir), env.get("PYTHONPATH", "")]+ )+ adapter = tmp_path / "adapter"+ for stage in ("train", "reload", "reload_lm"):+ run = subprocess.run(+ [sys.executable, str(script), str(src), str(adapter), stage],+ env = env,+ capture_output = True,+ text = True,+ )+ assert run.returncode == 0, run.stdout[-3000:] + run.stderr[-3000:]+ with open(adapter / "adapter_config.json") as f:+ assert json.load(f)["base_model_name_or_path"] == str(src) # portable: not the local view+ trained, reloaded = torch.load(adapter / "train.pt"), torch.load(adapter / "reload.pt")+ assert torch.equal(trained, torch.load(adapter / "reload_lm.pt"))+ assert torch.equal(trained, reloaded)+++def _tiny_tekken(path):+ import base64++ words = [b"he", b"ll", b"hell", b"hello", b" w", b"or", b" wor", b"ld", b" world", b"12"]+ vocab = [bytes([b]) for b in range(256)] + words+ specials = ["<unk>", "<s>", "</s>", "[INST]", "[/INST]", "<pad>"] + [+ f"<SPECIAL_{i}>" for i in range(6, 16)+ ]+ tekken = {+ "config": {+ "pattern": LARGE3_TEKKEN_PATTERN,+ "num_vocab_tokens": len(vocab) + len(specials),+ "default_vocab_size": len(vocab) + len(specials),+ "default_num_special_tokens": len(specials),+ "version": "v13",+ },+ "vocab": [+ {"rank": i, "token_bytes": base64.b64encode(t).decode(), "token_str": None}+ for i, t in enumerate(vocab)+ ],+ "special_tokens": [+ {"rank": i, "token_str": s, "is_control": True} for i, s in enumerate(specials)+ ],+ }+ with open(path, "w") as f:+ json.dump(tekken, f)+++# config.pattern of the Large-3 tekken.json.+LARGE3_TEKKEN_PATTERN = (+ r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+|"+ r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*|"+ r"\p{N}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+"+)+++def test_tekken_only_checkpoint_gets_a_bos_tokenizer(tmp_path, hub_cache):+ # Large-3 BF16 ships tekken.json alone; transformers' own conversion drops BOS or shifts ids.+ from transformers import AutoTokenizer++ tensors = {+ k: v.to(torch.bfloat16)+ for k, v in _mistral_tensors(_reference(TINY_PARAMS), TINY_PARAMS).items()+ }+ _write(tmp_path / "src", tensors, TINY_PARAMS)+ _tiny_tekken(tmp_path / "src" / "tekken.json")+ view = mf.prepare_mistral_format_checkpoint(str(tmp_path / "src"))+ assert {"tokenizer.json", "tokenizer_config.json"} <= set(os.listdir(view))+ tok = AutoTokenizer.from_pretrained(view)+ text = "hello world 12 é"+ ids = tok(text).input_ids+ n_special = 16+ # BOS, then byte-level BPE over the inner vocab shifted past the special tokens.+ assert ids[:4] == [1, n_special + 256 + 3, n_special + 256 + 8, n_special + ord(" ")]+ assert tok.decode(ids, skip_special_tokens = True) == text+ assert (tok.bos_token, tok.eos_token, tok.pad_token) == ("<s>", "</s>", "<pad>")+ mistral_common = pytest.importorskip("mistral_common.tokens.tokenizers.tekken")+ reference = mistral_common.Tekkenizer.from_file(str(tmp_path / "src" / "tekken.json"))+ for sample in (text, " hell\n\nworld12345", "wor ld 🚀"):+ assert tok(sample).input_ids == [1] + reference.encode(sample, bos = False, eos = False)+++def _redirecting_loader(calls):+ @mf.mistral_format_redirect+ def from_pretrained(+ model_name = None,+ max_seq_length = 2048,+ dtype = None,+ load_in_4bit = True,+ load_in_8bit = False,+ load_in_16bit = False,+ fast_inference = False,+ revision = None,+ quantization_config = None,+ **kwargs,+ ):+ calls.append(+ dict(model_name = model_name, load_in_4bit = load_in_4bit, load_in_16bit = load_in_16bit)+ )+ if len(calls) == 1:+ raise mf.MistralFormatRedirect("/views/large3", model_name)+ return object(), "tokenizer"++ return from_pretrained+++def test_redirect_keeps_the_bf16_repo_rule_and_refuses_fast_inference(monkeypatch):+ monkeypatch.setattr(mf, "_mistral_format_conversions", _null_context)+ calls = []+ # The loader loads '-BF16' hub repos in 16bit; the view path has lost that suffix.+ _redirecting_loader(calls)(+ "mistralai/Mistral-Large-3-675B-Instruct-2512-BF16", 4096, None, True+ )+ assert calls[1] == dict(model_name = "/views/large3", load_in_4bit = False, load_in_16bit = True)+ calls.clear()+ _redirecting_loader(calls)("mistralai/Mistral-Large-3-675B-Instruct-2512")+ assert calls[1] == dict(model_name = "/views/large3", load_in_4bit = True, load_in_16bit = False)+ with pytest.raises(NotImplementedError, match = "fast_inference = False"):+ _redirecting_loader([])("mistralai/Mistral-Large-3-675B-Instruct-2512", fast_inference = True)+++def test_view_downloads_honour_cache_dir(tmp_path, monkeypatch):+ seen = []+ monkeypatch.setattr(+ mf,+ "_fetch",+ lambda *args, **kwargs: seen.append(args[5] if len(args) > 5 else kwargs) or None,+ )+ assert mf.prepare_mistral_format_checkpoint("org/repo", cache_dir = str(tmp_path)) is None+ assert seen == [str(tmp_path)]+++def test_adapter_retry_on_a_view_base_refuses_fast_inference(tmp_path, monkeypatch):+ # An adapter whose base became a view retries with no new path: vLLM cannot read it either.+ monkeypatch.setattr(mf, "_mistral_format_conversions", _null_context)+ (tmp_path / mf._VIEW_MARKER).write_text("{}")++ @mf.mistral_format_redirect+ def from_pretrained(+ model_name = None,+ fast_inference = False,+ **kwargs,+ ):+ raise mf.MistralFormatRedirect(None, str(tmp_path))++ with pytest.raises(NotImplementedError, match = "fast_inference = False"):+ from_pretrained("org/adapter", fast_inference = True)diff --git a/unsloth/models/fp8_to_nf4.py b/unsloth/models/fp8_to_nf4.pyindex 4122a6a1dfa..3474038ce84 100644--- a/unsloth/models/fp8_to_nf4.py+++ b/unsloth/models/fp8_to_nf4.py@@ -252,6 +252,16 @@ def maybe_arm_fp8_to_nf4(return Falseif fp8_to_nf4_disabled():return False+ from .mistral_format import mistral_format_conversions_active++ if mistral_format_conversions_active():+ # Mistral-named scales go through the view's own renames, which this converter does not compose with.+ if verbose and explicit_4bit_requested():+ print(+ "Unsloth: 4bit loading of a Mistral-format fp8 checkpoint is not supported yet. "+ "Loading the fp8 weights instead."+ )+ return Falseif str(quant.get("expert_dtype") or "").lower() in _PACKED_EXPERT_DTYPES:# DeepSeek-V4-Flash: fp8 attention, packed FP4 experts. Not handled here.if verbose and explicit_4bit_requested():diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 670a469c0c0..dfa5b525567 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -34,6 +34,13 @@ from transformers import __version__ as transformers_versionfrom peft import PeftConfig, PeftModelfrom .grouped_linear_lora import register_grouped_linear_lora_for_adapterfrom .fp8_to_nf4 import track_explicit_4bit_request+from .mistral_format import (+ MistralFormatRedirect,+ is_mistral_format_view,+ mistral_format_conversions_active,+ mistral_format_redirect,+ prepare_mistral_format_checkpoint,+)from .loader_utils import (DEFAULT_DEVICE_MAP,OFFLOAD_EMBEDDING_AUTO,@@ -324,6 +331,29 @@ def _is_mistral_format_checkpoint(return False+def _adapter_base_is_mistral_format(+ model_name,+ token,+ revision,+ local_files_only,+ cache_dir = None,+):+ # A cached config.json settles it without a Hub request (most adapter bases).+ if not os.path.isdir(model_name):+ try:+ from huggingface_hub import try_to_load_from_cache+ if isinstance(+ try_to_load_from_cache(+ model_name, "config.json", revision = revision, cache_dir = cache_dir+ ),+ str,+ ):+ return False+ except Exception:+ pass+ return _is_mistral_format_checkpoint(model_name, token, revision, local_files_only)++def _mistral_format_error(model_name):return (f"Unsloth: `{model_name}` is a checkpoint in Mistral's own format: it ships `params.json` "@@ -701,6 +731,7 @@ def _fix_rope_inv_freq(model):class FastLanguageModel(FastLlamaModel):@staticmethod@_offline_aware_load+ @mistral_format_redirect@track_explicit_4bit_requestdef from_pretrained(model_name = "unsloth/Llama-3.2-1B-Instruct",@@ -1052,6 +1083,18 @@ class FastLanguageModel(FastLlamaModel):f"to obtain the latest transformers build, then restart this session.")if _is_mistral_format_checkpoint(model_name, token, base_revision, local_files_only):+ # Known architectures load via a translated view; others keep the error.+ _view = prepare_mistral_format_checkpoint(+ model_name,+ token,+ base_revision,+ local_files_only,+ cache_dir = kwargs.get("cache_dir", None),+ )+ if _view is not None:+ if not was_disabled:+ enable_progress_bars()+ raise MistralFormatRedirect(_view, model_name)raise RuntimeError(_mistral_format_error(model_name)) from autoconfig_exccombined_error = ("Unsloth: Failed to load model. Both AutoConfig and PeftConfig loading failed.\n\n"@@ -1132,6 +1175,18 @@ class FastLanguageModel(FastLlamaModel):load_in_8bit = Falseload_in_fp8 = Falseload_in_16bit = True+ # After the -bf16 rule: the view path no longer carries the source's suffix.+ # No revision: the caller's ref names the adapter repo, and the base loads unpinned below.+ _cache_dir = kwargs.get("cache_dir", None)+ if _adapter_base_is_mistral_format(+ model_name, token, None, local_files_only, _cache_dir+ ):+ model_name = (+ prepare_mistral_format_checkpoint(+ model_name, token, None, local_files_only, cache_dir = _cache_dir+ )+ or model_name+ )model_config = AutoConfig.from_pretrained(model_name,@@ -1143,6 +1198,9 @@ class FastLanguageModel(FastLlamaModel):if not was_disabled:enable_progress_bars()+ # A view, or an adapter trained on one, still names Mistral's tensors.+ if is_mistral_format_view(model_name) and not mistral_format_conversions_active():+ raise MistralFormatRedirect(None, model_name)if check_precision_flags and _precision_flags_conflict(load_in_4bit, load_in_8bit, load_in_16bit, load_in_fp8@@ -1505,6 +1563,7 @@ class FastModel(FastBaseModel):@staticmethod@_offline_aware_load+ @mistral_format_redirect@track_explicit_4bit_requestdef from_pretrained(model_name = "unsloth/Llama-3.2-11B-Vision-Instruct-bnb-4bit",@@ -1870,6 +1929,18 @@ class FastModel(FastBaseModel):f"to obtain the latest transformers build, then restart this session.")if _is_mistral_format_checkpoint(model_name, token, base_revision, local_files_only):+ # Known architectures load via a translated view; others keep the error.+ _view = prepare_mistral_format_checkpoint(+ model_name,+ token,+ base_revision,+ local_files_only,+ cache_dir = kwargs.get("cache_dir", None),+ )+ if _view is not None:+ if not was_disabled:+ enable_progress_bars()+ raise MistralFormatRedirect(_view, model_name)raise RuntimeError(_mistral_format_error(model_name)) from autoconfig_exccombined_error = ("Unsloth: Failed to load model. Both AutoConfig and PeftConfig loading failed.\n\n"@@ -2086,6 +2157,18 @@ class FastModel(FastBaseModel):load_in_8bit = Falseload_in_fp8 = Falseload_in_16bit = True+ # After the -bf16 rule: the view path no longer carries the source's suffix.+ # No revision: the caller's ref names the adapter repo, and the base loads unpinned below.+ _cache_dir = kwargs.get("cache_dir", None)+ if _adapter_base_is_mistral_format(+ model_name, token, None, local_files_only, _cache_dir+ ):+ model_name = (+ prepare_mistral_format_checkpoint(+ model_name, token, None, local_files_only, cache_dir = _cache_dir+ )+ or model_name+ )if user_config is not None:model_config = user_config@@ -2100,6 +2183,9 @@ class FastModel(FastBaseModel):if not was_disabled:enable_progress_bars()+ # A view, or an adapter trained on one, still names Mistral's tensors.+ if is_mistral_format_view(model_name) and not mistral_format_conversions_active():+ raise MistralFormatRedirect(None, model_name)do_logging = os.environ.get("UNSLOTH_ENABLE_LOGGING", "0") == "1"if do_logging:diff --git a/unsloth/models/mistral_format.py b/unsloth/models/mistral_format.pynew file mode 100644index 00000000000..3c34233fdf6--- /dev/null+++ b/unsloth/models/mistral_format.py@@ -0,0 +1,737 @@+# Unsloth Zoo - Utilities for Unsloth+# Copyright 2023-present Daniel Han-Chen, Michael Han-Chen & the Unsloth team. All rights reserved.+#+# This program is free software: you can redistribute it and/or modify+# it under the terms of the GNU Affero General Public License as published+# by the Free Software Foundation, either version 3 of the License, or+# (at your option) any later version.+#+# This program is distributed in the hope that it will be useful,+# but WITHOUT ANY WARRANTY; without even the implied warranty of+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+# GNU Affero General Public License for more details.+#+# You should have received a copy of the GNU Affero General Public License+# along with this program. If not, see <https://www.gnu.org/licenses/>.++"""Load Mistral-format checkpoints (params.json + consolidated*.safetensors, no config.json,+e.g. Mistral-Large-3) as transformers `mistral4` via a translated view of the same shards.++Mistral-Small-4 ships in both formats with byte-identical tensors, which fixes the name+mapping. Conversions are registered for one load only so `save_pretrained` writes+transformers names. Only the text decoder loads; unrecognised checkpoints return None.+"""++import contextlib+import functools+import hashlib+import inspect+import json+import re+import uuid+import os+from typing import Optional++__all__ = [+ "MistralFormatRedirect",+ "mistral_params_to_mistral4_config",+ "mistral_format_weight_conversions",+ "prepare_mistral_format_checkpoint",+ "mistral_format_redirect",+]++_NON_TEXT_PREFIXES = (+ "vision_encoder.",+ "patch_merger.",+ "pre_mm_projector_norm.",+ "vision_language_adapter.",+ "mm_audio_embeddings.",+ "audio_",+)+_TOKENIZER_FILES = (+ "tekken.json",+ "tokenizer.json",+ "tokenizer_config.json",+ "special_tokens_map.json",+ "tokenizer.model",+ "vocab.json",+ "merges.txt",+ "added_tokens.json",+ "chat_template.jinja",+ "generation_config.json",+ "SYSTEM_PROMPT.txt",+)+_VIEW_MARKER = "unsloth_mistral_format.json"+++class MistralFormatRedirect(Exception):+ """Raised by the loader; `mistral_format_redirect` catches it and loads the view."""++ def __init__(self, path, source):+ super().__init__(f"Unsloth: loading {source} through its transformers view at {path}")+ self.path = path+ self.source = source+++def _fp8_block_quantization(quant) -> Optional[dict]:+ """compressed-tensors FP8 128x128 block (Mistral-Large-3) as fine-grained FP8 config, else None."""+ if not isinstance(quant, dict):+ return None+ if str(quant.get("quant_method", "")).lower().replace("_", "-") != "compressed-tensors":+ return None+ if quant.get("format") != "float-quantized":+ return None+ block = None+ for group in (quant.get("config_groups") or {}).values():+ weights = (group or {}).get("weights") or {}+ acts = (group or {}).get("input_activations")+ if weights.get("type") != "float" or int(weights.get("num_bits", 0)) != 8:+ return None+ if weights.get("strategy") != "block" or not weights.get("block_structure"):+ return None+ if acts is not None and not (+ acts.get("dynamic")+ and acts.get("type") == "float"+ and int(acts.get("num_bits", 8)) == 8+ ):+ return None+ if group.get("output_activations") is not None:+ return None+ # Fine-grained FP8 has no targets: a narrower group would claim unscaled bf16 Linears.+ if [str(t) for t in group.get("targets") or ["Linear"]] != ["Linear"]:+ return None+ this_block = list(weights["block_structure"])+ if block is not None and this_block != block:+ return None+ block = this_block+ if block is None:+ return None+ # The ignore list already uses transformers module names.+ not_convert = []+ for entry in quant.get("ignore") or []:+ name = str(entry)+ if name.startswith("re:"):+ name = name[3:].strip("^$").replace(".*", "").replace("\\", "").strip(".")+ if any(p.rstrip(".") in name for p in _NON_TEXT_PREFIXES) or "patch_merger" in name:+ continue+ if name:+ not_convert.append(name.split(".")[-1] if name.startswith("model.") else name)+ return {+ "quant_method": "fp8",+ "activation_scheme": "dynamic",+ "weight_block_size": block,+ "modules_to_not_convert": sorted(set(not_convert)),+ }+++def mistral_params_to_mistral4_config(params: dict) -> Optional[dict]:+ """`params.json` -> `Mistral4Config` kwargs as Mistral did for Small-4; None if not expressible."""+ if not isinstance(params, dict):+ return None+ moe = params.get("moe")+ if not params.get("qk_nope_head_dim") or not isinstance(moe, dict):+ return None+ if not params.get("q_lora_rank"):+ return None+ if params.get("quantization"):+ return None # per-tensor FP8: Small-4 ships config.json+ if int(moe.get("num_shared_experts") or 0) < 1:+ return None+ if int(moe.get("route_every_n", 1)) != 1:+ return None+ if str(moe.get("renorm_strategy", "WEIGHTS")).upper() != "WEIGHTS":+ return None+ if moe.get("use_load_balancing_bias"):+ return None # mistral4 routes with plain softmax+ if params.get("sliding_window"):+ return None++ rope_theta = float(params.get("rope_theta", 10000.0))+ llama4 = params.get("llama_4_scaling") or {}+ yarn = params.get("yarn") or {}+ if yarn:+ if yarn.get("apply_scale"):+ return None # the mscale below is the `apply_scale: false` translation+ original = int(yarn.get("original_max_position_embeddings", 8192))+ if llama4 and int(llama4.get("original_max_position_embeddings", original)) != original:+ return None # mistral4 reads one original length for both+ rope_parameters = {+ "rope_type": "yarn",+ "type": "yarn",+ "rope_theta": rope_theta,+ "factor": float(yarn["factor"]),+ "original_max_position_embeddings": original,+ "beta_fast": float(yarn.get("beta", 32)),+ "beta_slow": float(yarn.get("alpha", 1)),+ # Matches Mistral-Small-4's config.json for `apply_scale: false`.+ "mscale": 1.0,+ "mscale_all_dim": 1.0,+ "llama_4_scaling_beta": float(llama4.get("beta", 0.0)),+ }+ else:+ rope_parameters = {+ "rope_type": "default",+ "rope_theta": rope_theta,+ "original_max_position_embeddings": int(+ llama4.get("original_max_position_embeddings", 8192)+ ),+ "llama_4_scaling_beta": float(llama4.get("beta", 0.0)),+ }++ config = {+ "model_type": "mistral4",+ "architectures": ["Mistral4ForCausalLM"],+ "hidden_size": int(params["dim"]),+ "intermediate_size": int(params["hidden_dim"]),+ "num_hidden_layers": int(params["n_layers"]),+ "num_attention_heads": int(params["n_heads"]),+ "num_key_value_heads": int(params.get("n_kv_heads", params["n_heads"])),+ "rms_norm_eps": float(params.get("norm_eps", 1e-6)),+ "vocab_size": int(params["vocab_size"]),+ "tie_word_embeddings": bool(params.get("tied_embeddings", False)),+ "max_position_embeddings": int(params.get("max_position_embeddings", 128_000)),+ "hidden_act": params.get("activation", "silu"),+ "q_lora_rank": int(params["q_lora_rank"]),+ "kv_lora_rank": int(params["kv_lora_rank"]),+ "qk_nope_head_dim": int(params["qk_nope_head_dim"]),+ "qk_rope_head_dim": int(params["qk_rope_head_dim"]),+ "v_head_dim": int(params["v_head_dim"]),+ "first_k_dense_replace": int(moe.get("first_k_dense_replace", 0)),+ "n_routed_experts": int(moe["num_experts"]),+ "num_experts_per_tok": int(moe["num_experts_per_tok"]),+ "moe_intermediate_size": int(moe["expert_hidden_dim"]),+ "n_shared_experts": int(moe["num_shared_experts"]),+ "routed_scaling_factor": float(moe.get("routed_scale", 1.0)),+ "n_group": int(moe.get("num_expert_groups", 1)),+ "topk_group": int(moe.get("num_expert_groups_per_tok", 1)),+ "norm_topk_prob": True,+ "rope_interleave": True,+ "rope_parameters": rope_parameters,+ "bos_token_id": 1,+ "eos_token_id": 2,+ "pad_token_id": 11,+ "attention_bias": False,+ "mlp_bias": False,+ }+ quant = params.get("quantization_config")+ if quant is not None:+ fp8 = _fp8_block_quantization(quant)+ if fp8 is None:+ return None+ config["quantization_config"] = fp8+ return config+++def mistral_format_weight_conversions():+ """Mistral -> `Mistral4ForCausalLM` renames (chained in order), then expert merges."""+ from transformers.core_model_loading import (+ Concatenate,+ MergeModulelist,+ WeightConverter,+ WeightRenaming,+ )++ renames = [+ (r"^tok_embeddings\.weight$", "model.embed_tokens.weight"),+ (r"^norm\.weight$", "model.norm.weight"),+ (r"^output\.weight$", "lm_head.weight"),+ (r"^layers\.", "model.layers."),+ (r"\.attention_norm\.", ".input_layernorm."),+ (r"\.ffn_norm\.", ".post_attention_layernorm."),+ (r"\.attention\.wq_a\.", ".self_attn.q_a_proj."),+ (r"\.attention\.q_a_norm\.", ".self_attn.q_a_layernorm."),+ (r"\.attention\.wq_b\.", ".self_attn.q_b_proj."),+ (r"\.attention\.wkv_a_with_mqa\.", ".self_attn.kv_a_proj_with_mqa."),+ (r"\.attention\.kv_a_norm\.", ".self_attn.kv_a_layernorm."),+ (r"\.attention\.wkv_b\.", ".self_attn.kv_b_proj."),+ (r"\.attention\.wo\.", ".self_attn.o_proj."),+ (r"\.feed_forward\.w1\.", ".mlp.gate_proj."),+ (r"\.feed_forward\.w2\.", ".mlp.down_proj."),+ (r"\.feed_forward\.w3\.", ".mlp.up_proj."),+ (r"\.gate\.weight$", ".mlp.gate.weight"),+ (r"\.shared_experts\.w1\.", ".mlp.shared_experts.gate_proj."),+ (r"\.shared_experts\.w2\.", ".mlp.shared_experts.down_proj."),+ (r"\.shared_experts\.w3\.", ".mlp.shared_experts.up_proj."),+ (r"\.experts\.(\d+)\.w1\.", r".mlp.experts.\1.gate_proj."),+ (r"\.experts\.(\d+)\.w2\.", r".mlp.experts.\1.down_proj."),+ (r"\.experts\.(\d+)\.w3\.", r".mlp.experts.\1.up_proj."),+ # compressed-tensors' block scale is the multiplier transformers calls weight_scale_inv.+ (r"\.weight_scale$", ".weight_scale_inv"),+ ]+ conversions = [WeightRenaming(source_patterns = s, target_patterns = t) for s, t in renames]+ # Scales first: transformers stops at the first match and `.weight` also matches `.weight_scale_inv`.+ for suffix, merge in (+ (".weight_scale_inv", True),+ (".weight_scale_inv", False),+ (".weight", True),+ (".weight", False),+ ):+ target_suffix = "_scale_inv" if suffix != ".weight" else ""+ if merge:+ converter = WeightConverter(+ source_patterns = [+ rf"mlp.experts.*.gate_proj{suffix}$"+ if target_suffix+ else f"mlp.experts.*.gate_proj{suffix}",+ rf"mlp.experts.*.up_proj{suffix}$"+ if target_suffix+ else f"mlp.experts.*.up_proj{suffix}",+ ],+ target_patterns = f"mlp.experts.gate_up_proj{target_suffix}",+ operations = [MergeModulelist(dim = 0), Concatenate(dim = 1)],+ )+ else:+ converter = WeightConverter(+ source_patterns = (+ rf"mlp.experts.*.down_proj{suffix}$"+ if target_suffix+ else f"mlp.experts.*.down_proj{suffix}"+ ),+ target_patterns = f"mlp.experts.down_proj{target_suffix}",+ operations = [MergeModulelist(dim = 0)],+ )+ conversions.append(converter)+ return conversions+++def _mistral4_available() -> bool:+ try:+ from transformers import Mistral4Config, Mistral4ForCausalLM # noqa: F401+ from transformers.conversion_mapping import register_checkpoint_conversion_mapping # noqa: F401+ return True+ except Exception:+ return False+++def _fp8_dequantize_on_load_available() -> bool:+ try:+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer+ except Exception:+ return False+ return callable(getattr(FineGrainedFP8HfQuantizer, "update_weight_conversions", None))+++def _fetch(+ model_name,+ filename,+ token,+ revision,+ local_files_only,+ cache_dir = None,+):+ if os.path.isdir(model_name):+ path = os.path.join(model_name, filename)+ return path if os.path.isfile(path) else None+ from huggingface_hub import hf_hub_download+ try:+ return hf_hub_download(+ model_name,+ filename,+ token = token,+ revision = revision,+ local_files_only = local_files_only,+ cache_dir = cache_dir,+ )+ except Exception:+ return None+++def _view_root():+ from huggingface_hub import constants+ return os.path.join(os.path.dirname(constants.HF_HUB_CACHE), "unsloth_mistral_format")+++def prepare_mistral_format_checkpoint(+ model_name,+ token = None,+ revision = None,+ local_files_only = False,+ cache_dir = None,+) -> Optional[str]:+ """Write (or reuse) a transformers view of a Mistral-format checkpoint; None if unsupported."""+ if not _mistral4_available():+ return None+ params_path = _fetch(model_name, "params.json", token, revision, local_files_only, cache_dir)+ if params_path is None:+ return None+ with open(params_path, encoding = "utf-8") as f:+ params = json.load(f)+ config = mistral_params_to_mistral4_config(params)+ if config is None:+ return None+ if config.get("quantization_config") and not _fp8_dequantize_on_load_available():+ print(+ f"Unsloth: `{model_name}` stores fp8 weights; loading them through transformers needs "+ "transformers >= 5.8. Please upgrade transformers."+ )+ return None++ index_path = _fetch(+ model_name,+ "consolidated.safetensors.index.json",+ token,+ revision,+ local_files_only,+ cache_dir,+ )+ if index_path is not None:+ with open(index_path, encoding = "utf-8") as f:+ weight_map = json.load(f)["weight_map"]+ else:+ single = _fetch(+ model_name, "consolidated.safetensors", token, revision, local_files_only, cache_dir+ )+ if single is None:+ return None+ from safetensors import safe_open++ with safe_open(single, framework = "pt") as f:+ weight_map = {k: "consolidated.safetensors" for k in f.keys()}+ weight_map = {k: v for k, v in weight_map.items() if not k.startswith(_NON_TEXT_PREFIXES)}+ shards = sorted(set(weight_map.values()))+ for shard in shards:+ # Lexical, not realpath: hub snapshot files are symlinks into the blob store.+ norm = os.path.normpath(str(shard))+ if os.path.isabs(norm) or os.path.splitdrive(norm)[0] or norm.split(os.sep)[0] == os.pardir:+ return None++ shard_paths = {}+ if os.path.isdir(model_name):+ for shard in shards:+ shard_paths[shard] = os.path.abspath(os.path.join(model_name, shard))+ else:+ from huggingface_hub import snapshot_download+ snapshot = snapshot_download(+ model_name,+ revision = revision,+ token = token,+ local_files_only = local_files_only,+ cache_dir = cache_dir,+ allow_patterns = shards + ["params.json", *list(_TOKENIZER_FILES)],+ )+ for shard in shards:+ shard_paths[shard] = os.path.join(snapshot, shard)+ if not all(os.path.isfile(p) for p in shard_paths.values()):+ return None++ source_dir = os.path.dirname(shard_paths[shards[0]])+ # Key on what the view copies or points at, so a checkpoint edited in place gets a fresh view.+ source_files = {}+ for name in _TOKENIZER_FILES + ("params.json",):+ try:+ stat = os.stat(os.path.join(source_dir, name))+ source_files[name] = [stat.st_size, stat.st_mtime_ns]+ except OSError:+ pass+ digest = hashlib.sha256(+ json.dumps(+ [os.path.realpath(source_dir), config, weight_map, source_files], sort_keys = True+ ).encode()+ ).hexdigest()[:16]+ safe_name = re.sub(r"[\\/:]+", "--", str(model_name).strip("/\\"))[-80:]+ view = os.path.join(_view_root(), f"{safe_name}-{digest}")+ marker = os.path.join(view, _VIEW_MARKER)+ if os.path.isfile(marker):+ return view++ os.makedirs(view, exist_ok = True)+ # Per-process temp names: ranks of one launch can build the same view at once.+ suffix = f".{os.getpid()}.{uuid.uuid4().hex}.tmp"+ tmp = lambda name: os.path.join(view, name + suffix)+ with open(tmp("config.json"), "w", encoding = "utf-8") as f:+ json.dump(config, f, indent = 2)+ with open(tmp("model.safetensors.index.json"), "w", encoding = "utf-8") as f:+ json.dump(+ {"metadata": {}, "weight_map": {k: shard_paths[v] for k, v in weight_map.items()}},+ f,+ )+ for name in ("config.json", "model.safetensors.index.json"):+ os.replace(tmp(name), os.path.join(view, name))+ for name in _TOKENIZER_FILES:+ src = os.path.join(source_dir, name)+ if not os.path.isfile(src):+ src = _fetch(model_name, name, token, revision, local_files_only, cache_dir)+ if src and os.path.isfile(src):+ import shutil+ shutil.copyfile(src, tmp(name))+ os.replace(tmp(name), os.path.join(view, name))+ tekken = os.path.join(view, "tekken.json")+ if os.path.isfile(tekken) and not os.path.isfile(os.path.join(view, "tokenizer.json")):+ # transformers converts a lone tekken.json without BOS (5.17) or with shifted ids (5.5).+ _write_tokenizer_from_tekken(tekken, view, tmp)+ with open(tmp(_VIEW_MARKER), "w", encoding = "utf-8") as f:+ json.dump({"source": str(model_name), "revision": revision}, f)+ os.replace(tmp(_VIEW_MARKER), marker)+ return view+++def _bytes_to_unicode():+ bs = (+ list(range(ord("!"), ord("~") + 1))+ + list(range(ord("¡"), ord("¬") + 1))+ + list(range(ord("®"), ord("ÿ") + 1))+ )+ cs, n = bs[:], 0+ for b in range(256):+ if b not in bs:+ bs.append(b)+ cs.append(256 + n)+ n += 1+ return dict(zip(bs, map(chr, cs)))+++def _write_tokenizer_from_tekken(tekken_path, view, tmp):+ """tokenizer.json (+ tokenizer_config.json) from tekken.json, as tiktoken-style byte-level BPE:+ ids match mistral-common's Tekkenizer, and the tokenizer.json Mistral ships beside it."""+ import base64++ from tokenizers import Regex, Tokenizer, decoders, pre_tokenizers, processors+ from tokenizers.models import BPE++ with open(tekken_path, encoding = "utf-8") as f:+ tekken = json.load(f)+ config = tekken["config"]+ n_special = int(config["default_num_special_tokens"])+ specials = {int(t["rank"]): t["token_str"] for t in tekken.get("special_tokens", [])}+ special_strs = [specials.get(i, f"<SPECIAL_{i}>") for i in range(n_special)]+ enc = _bytes_to_unicode()+ ranks = {}+ for entry in tekken["vocab"][: int(config["default_vocab_size"]) - n_special]:+ ranks["".join(enc[b] for b in base64.b64decode(entry["token_bytes"]))] = len(ranks)+ merges = []+ for token, rank in ranks.items():+ local = [+ (token[:i], token[i:])+ for i in range(1, len(token))+ if token[:i] in ranks and token[i:] in ranks+ ]+ local.sort(key = lambda pair: (ranks[pair[0]], ranks[pair[1]]))+ merges.extend((a, b, rank) for a, b in local)+ merges.sort(key = lambda merge: merge[2])+ vocab = {token: i for i, token in enumerate(special_strs)}+ vocab.update({token: rank + n_special for token, rank in ranks.items()})+ tokenizer = Tokenizer(BPE(vocab, [(a, b) for a, b, _ in merges], ignore_merges = True))+ tokenizer.pre_tokenizer = pre_tokenizers.Sequence(+ [+ pre_tokenizers.Split(Regex(config["pattern"]), behavior = "isolated", invert = False),+ pre_tokenizers.ByteLevel(add_prefix_space = False, use_regex = False),+ ]+ )+ tokenizer.decoder = decoders.ByteLevel()+ tokenizer.add_special_tokens(special_strs)+ bos = special_strs[1]+ tokenizer.post_processor = processors.TemplateProcessing(+ single = f"{bos} $A", pair = f"{bos} $A $B", special_tokens = [(bos, 1)]+ )+ tokenizer.save(tmp("tokenizer.json"))+ os.replace(tmp("tokenizer.json"), os.path.join(view, "tokenizer.json"))+ if not os.path.isfile(os.path.join(view, "tokenizer_config.json")):+ named = {+ "<s>": "bos_token",+ "</s>": "eos_token",+ "<unk>": "unk_token",+ "<pad>": "pad_token",+ }+ tokenizer_config = {+ "tokenizer_class": "PreTrainedTokenizerFast",+ "clean_up_tokenization_spaces": False,+ **{named[t]: t for t in special_strs if t in named},+ }+ with open(tmp("tokenizer_config.json"), "w", encoding = "utf-8") as f:+ json.dump(tokenizer_config, f, indent = 2)+ os.replace(tmp("tokenizer_config.json"), os.path.join(view, "tokenizer_config.json"))+++def is_mistral_format_view(path) -> bool:+ return isinstance(path, str) and os.path.isfile(os.path.join(path, _VIEW_MARKER))+++def _record_source(model):+ """Adapters record `model.name_or_path` as their base: name the source, not this host's view."""+ for module in (model, getattr(model, "model", None), getattr(model, "base_model", None)):+ view = getattr(getattr(module, "config", None), "_name_or_path", None)+ if module is None or not is_mistral_format_view(view):+ continue+ try:+ with open(os.path.join(view, _VIEW_MARKER), encoding = "utf-8") as f:+ source = json.load(f).get("source")+ except Exception:+ source = None+ if source:+ try:+ module.name_or_path = source+ except Exception:+ pass+++def raise_if_merging_mistral_format_view(model, save_method):+ """Merged saves re-read the base shards, which a view names in Mistral's own layout."""+ view = getattr(getattr(model, "config", None), "_name_or_path", None)+ if str(save_method).strip().lower() == "lora" or not is_mistral_format_view(view):+ return+ try:+ with open(os.path.join(view, _VIEW_MARKER), encoding = "utf-8") as f:+ source = json.load(f).get("source") or view+ except Exception:+ source = view+ raise NotImplementedError(+ f"Unsloth: `{source}` was loaded from Mistral's own checkpoint format, and "+ f'`save_method = "{save_method}"` (merged and GGUF exports) cannot read those '+ f"shards yet. Save the LoRA adapter instead (`model.save_pretrained(...)` or "+ f'`save_method = "lora"`); it reloads through the same path.'+ )+++def _is_expert_scale_merge(conversion):+ from transformers.core_model_loading import WeightConverter+ targets = getattr(conversion, "target_patterns", None) or []+ return isinstance(conversion, WeightConverter) and any(+ str(t).startswith("mlp.experts.") and str(t).endswith("_scale_inv") for t in targets+ )+++def _register_mistral4_causal_lm():+ # AutoModelForCausalLM lacks Mistral4Config and `register` refuses native configs.+ from transformers import Mistral4Config, Mistral4ForCausalLM+ from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING+ if MODEL_FOR_CAUSAL_LM_MAPPING.get(Mistral4Config, None) is None:+ MODEL_FOR_CAUSAL_LM_MAPPING._extra_content[Mistral4Config] = Mistral4ForCausalLM+++_active_conversions = 0+++def mistral_format_conversions_active() -> bool:+ return _active_conversions > 0+++def _mistral_format_conversions():+ """Register Mistral-name conversions for one load, then restore the previous ones."""+ global _active_conversions+ from transformers import conversion_mapping as cm++ _register_mistral4_causal_lm()+ cm.get_checkpoint_conversion_mapping("mistral4")+ # Class-name lookup arrived with USER_REGISTERED_MAPPINGS; older releases look up model_type only.+ # Never both: 5.17 would also apply a "mistral4" entry to the inner Mistral4Model.+ user_registered = getattr(cm, "USER_REGISTERED_MAPPINGS", None)+ keys = ("Mistral4ForCausalLM",) if user_registered is not None else ("mistral4",)+ cache = cm._checkpoint_conversion_mapping_cache+ previous = {key: (key in cache, cache.get(key)) for key in keys}+ was_user = {key: user_registered is not None and key in user_registered for key in keys}+ conversions = mistral_format_weight_conversions()+ for key in keys:+ cm.register_checkpoint_conversion_mapping(key, conversions, overwrite = True)+ # Dequantizing FP8 folds scales in transformers' `.weight` converters; scale merges must not claim them.+ fp8_quantizer, original_update = None, None+ try:+ from transformers.quantizers.quantizer_finegrained_fp8 import FineGrainedFP8HfQuantizer+ fp8_quantizer = FineGrainedFP8HfQuantizer+ original_update = FineGrainedFP8HfQuantizer.__dict__.get("update_weight_conversions")+ except Exception:+ pass+ if original_update is not None:++ def update_weight_conversions(self, weight_conversions):+ if getattr(self.quantization_config, "dequantize", False):+ weight_conversions = [+ c for c in weight_conversions if not _is_expert_scale_merge(c)+ ]+ return original_update(self, weight_conversions)++ fp8_quantizer.update_weight_conversions = update_weight_conversions+ _active_conversions += 1+ try:+ yield+ finally:+ _active_conversions -= 1+ if original_update is not None:+ fp8_quantizer.update_weight_conversions = original_update+ for key in keys:+ had_entry, entry = previous[key]+ if had_entry:+ cache[key] = entry+ else:+ cache.pop(key, None)+ if user_registered is not None and not was_user[key]:+ user_registered.discard(key)+++def _forget_load_conversions(model):+ seen = set()+ for module in (model, getattr(model, "model", None), getattr(model, "base_model", None)):+ if module is not None and id(module) not in seen:+ seen.add(id(module))+ try:+ module._weight_conversions = []+ except Exception:+ pass+++def mistral_format_redirect(fn):+ """Retry `from_pretrained` against the Mistral-format view the loader found."""++ try:+ signature = inspect.signature(fn)+ except (TypeError, ValueError):+ signature = None++ def _argument(+ args,+ kwargs,+ name,+ default = None,+ ):+ if name in kwargs:+ return kwargs[name]+ try:+ return signature.bind_partial(*args, **kwargs).arguments.get(name, default)+ except Exception:+ return default++ @functools.wraps(fn)+ def _wrapper(*args, **kwargs):+ try:+ return fn(*args, **kwargs)+ except MistralFormatRedirect as redirect:+ view, source = redirect.path, redirect.source+ # An adapter whose base is a view retries with view None: vLLM cannot read either.+ if (view is not None or is_mistral_format_view(source)) and _argument(+ args, kwargs, "fast_inference", False+ ):+ raise NotImplementedError(+ f"Unsloth: `{source}` is in Mistral's own format, which Unsloth loads through "+ "transformers only. Use `fast_inference = False`, or serve it with vLLM directly."+ )+ if (+ view is not None+ and str(source).lower().endswith("-bf16")+ and not os.path.isdir(os.path.expanduser(str(source)))+ and _argument(args, kwargs, "quantization_config") is None+ and signature is not None+ ):+ # The loader's -bf16 rule already switched this load to 16bit; the view path lost the suffix.+ bound = signature.bind_partial(*args, **kwargs)+ bound.arguments.update(load_in_4bit = False, load_in_8bit = False, load_in_16bit = True)+ if "load_in_fp8" in signature.parameters:+ bound.arguments["load_in_fp8"] = False+ args, kwargs = bound.args, bound.kwargs+ if view is not None:+ if "model_name" in kwargs or not args:+ kwargs["model_name"] = view+ else:+ args = (view,) + tuple(args[1:])+ kwargs.pop("revision", None)+ print(+ f"Unsloth: `{source}` is in Mistral's own format. Loading its text decoder "+ f"as transformers' Mistral4 through {view} (no weights are copied)."+ )+ with _mistral_format_conversions():+ result = fn(*args, **kwargs)+ model = result[0] if isinstance(result, tuple) else result+ _forget_load_conversions(model)+ _record_source(model)+ return result++ return _wrapperdiff --git a/unsloth/save.py b/unsloth/save.pyindex da92f36592a..31eb7b12cde 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -69,6 +69,7 @@ from .models.loader_utils import (_tokenizer_wants_local_only,)from .models._utils import _convert_torchao_model+from .models.mistral_format import raise_if_merging_mistral_format_viewfrom .ollama_template_mappers import OLLAMA_TEMPLATES, MODEL_TO_OLLAMA_TEMPLATE_MAPPERfrom transformers import ProcessorMixin, PreTrainedTokenizerBasefrom huggingface_hub import HfApi@@ -997,6 +998,7 @@ def unsloth_save_model(gc.collect()save_method = save_method.lower().replace(" ", "_")+ raise_if_merging_mistral_format_view(model, save_method)if save_method != "lora" and save_method != "merged_16bit" and save_method != "merged_4bit":raise RuntimeError("Unsloth: You must select one of 3 options when saving models:\n"@@ -3928,6 +3930,7 @@ def unsloth_save_pretrained_gguf("iq3_xxs" : "3.06 bpw quantization","q3_k_xs" : "3-bit extra small quantization","""+ raise_if_merging_mistral_format_view(self, "gguf") # the converter would read Mistral names_assert_export_target_is_not_base_with_lora_layers(self)if tokenizer is None:@@ -4539,6 +4542,7 @@ def unsloth_push_to_hub_gguf(`quantization_method` may be an alias -- "not_quantized" (fast conversion, big files), "fast_quantized" (fast conversion, OK size), "quantized" (slow conversion, small files) -- or a llama.cpp ftype: f32, f16, q8_0, q4_0, q4_1, q5_0, q5_1, or a k-quant q2_k / q3_k_s / q3_k_m / q3_k_l / q4_k_s / q4_k_m / q5_k_s / q5_k_m / q6_k. The _m and _l k-quants keep the attention and feed_forward.w2 tensors a level or two above the nominal width; q2_k_l is the Unsloth preset adding --output-tensor-type q8_0 --token-embedding-type q8_0."""+ raise_if_merging_mistral_format_view(self, "gguf") # the converter would read Mistral names_assert_export_target_is_not_base_with_lora_layers(self)if tokenizer is None:raise ValueError("Unsloth: Saving to GGUF must have a tokenizer.")@@ -5685,8 +5689,10 @@ def unsloth_generic_save(datasets = datasets,)else:+ raise_if_merging_mistral_format_view(model, save_method)_prewarm_base_model_hub_cache(model, save_method = save_method, token = token)from unsloth_zoo.saving_utils import merge_and_overwrite_lora+merge_and_overwrite_lora(get_model_name,model = model,