Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Trim comments in the DeepSeek-V4 grouped LoRA, remote-code shims and Mistral-format loader code (#12089)
Daniel Han10f2575495bf
diff --git a/tests/test_grouped_linear_lora.py b/tests/test_grouped_linear_lora.pyindex 8c4fa44103e..b2e0449dc35 100644--- a/tests/test_grouped_linear_lora.py+++ b/tests/test_grouped_linear_lora.py@@ -8,8 +8,6 @@ peft = pytest.importorskip("peft")class GroupedLinear(torch.nn.Linear):- """The shape contract of `DeepseekV4GroupedLinear` and `FP8GroupedLinear`."""-def __init__(self, in_per_group, out_features, n_groups):super().__init__(in_per_group, out_features, bias = False)self.n_groups = n_groups@@ -58,7 +56,6 @@ def test_is_grouped_linear_wants_n_groups_and_an_overridden_forward():def test_dense_lora_fails_on_the_grouped_linear():- """The arm that fails on main."""model = _peft_model(register = False)with pytest.raises(RuntimeError, match = "must match the size"):model(torch.randn(2, 5, 4, 16))@@ -74,7 +71,6 @@ def test_grouped_lora_trains_and_matches_the_merged_weight():assert type(layer).__name__ == "GroupedLinearLoRA"assert layer.lora_A["default"].weight.grad is not Noneassert layer.lora_B["default"].weight.grad is not None- # Merging lora_B @ lora_A into the block-diagonal weight gives the same forward.with torch.no_grad():merged = model.merge_and_unload()merged_out = merged(x)@@ -82,8 +78,7 @@ def test_grouped_lora_trains_and_matches_the_merged_weight():def test_merge_into_an_fp8_grouped_weight_is_refused():- """FP8GroupedLinear keeps an fp8 weight plus weight_scale_inv; B @ A cannot be added in place.- PEFT merges layer by layer, so the dense layer ahead of it must not be merged either."""+ """PEFT merges layer by layer, so the dense layer ahead of the fp8 one must not be merged either."""if not hasattr(torch, "float8_e4m3fn"):pytest.skip("torch without float8")from peft import LoraConfig, get_peft_model@@ -92,7 +87,7 @@ def test_merge_into_an_fp8_grouped_weight_is_refused():class DenseFirst(torch.nn.Module):def __init__(self):super().__init__()- self.q_proj = torch.nn.Linear(16, 16, bias = False) # merged first by PEFT+ self.q_proj = torch.nn.Linear(16, 16, bias = False)self.o_a_proj = GroupedLinear(16, 4 * 8, 4)def forward(self, x):@@ -140,7 +135,6 @@ def test_state_dict_is_a_plain_lora_checkpoint():def test_dora_is_refused_on_a_grouped_linear():- """The grouped forward computes the plain LoRA sum only, so DoRA is refused."""from peft import LoraConfigfrom unsloth.models.grouped_linear_lora import register_grouped_linear_lora@@ -168,7 +162,6 @@ def test_a_variant_reaching_the_forward_is_refused():def test_dora_is_allowed_when_no_grouped_linear_is_targeted():- """DoRA on the dense o_b_proj alone is valid: the grouped forward is never built."""from peft import LoraConfigfrom unsloth.models.grouped_linear_lora import register_grouped_linear_lora@@ -183,7 +176,6 @@ def test_dora_is_allowed_when_no_grouped_linear_is_targeted():def test_a_saved_adapter_reloads_onto_the_grouped_forward(tmp_path):- """The custom mapping is not in adapter_config.json, so a reload must register it again."""from peft import PeftModelfrom unsloth.models.grouped_linear_lora import register_grouped_linear_lora_for_adapter@@ -207,12 +199,10 @@ def test_a_saved_adapter_reloads_onto_the_grouped_forward(tmp_path):with torch.no_grad():got = reloaded(x)torch.testing.assert_close(got, want)- # A model with no grouped linear reloads exactly as before.assert register_grouped_linear_lora_for_adapter(torch.nn.Linear(4, 4), str(tmp_path)) is Nonedef test_fan_in_fan_out_is_reset_like_peft_does_for_nn_linear():- """PEFT resets fan_in_fan_out for nn.Linear; the custom mapping skips that dispatcher."""from peft import LoraConfig, get_peft_modelfrom unsloth.models.grouped_linear_lora import register_grouped_linear_loradiff --git a/tests/test_mistral_format_checkpoint.py b/tests/test_mistral_format_checkpoint.pyindex 7fe5e870e70..d152b3df9e9 100644--- a/tests/test_mistral_format_checkpoint.py+++ b/tests/test_mistral_format_checkpoint.py@@ -1,6 +1,5 @@# SPDX-License-Identifier: AGPL-3.0-only-"""A Mistral-format checkpoint (params.json, no config.json, e.g. Mistral-Large-3) gets a-message that says what it is, instead of the generic "both configs failed" one."""+"""A Mistral-format checkpoint (params.json, no config.json) gets its own error message."""import jsonimport os@@ -8,7 +7,7 @@ import osimport pytestfrom real_accelerator import (has_real_accelerator,-) # tests/_shared, on sys.path via tests/conftest.py+)import torch@@ -24,9 +23,7 @@ def test_params_json_without_config_json_is_mistral_format(tmp_path):def test_a_params_json_without_a_mistral_marker_is_not_claimed(tmp_path):- """Meta's original Llama layout also has params.json; the generic message must stay."""from unsloth.models.loader import _is_mistral_format_checkpoint-assert (_is_mistral_format_checkpoint(_write(tmp_path, ["params.json", "consolidated.00.pth", "tokenizer.model"])@@ -66,7 +63,6 @@ def test_the_message_names_the_file_and_the_route():@pytest.mark.skipif(not has_real_accelerator(), reason = "import unsloth needs an accelerator")def test_loader_raises_the_specific_message(tmp_path):- """The arm that fails on main: the loader used to raise the generic message."""import unsloth # noqa: F401from unsloth import FastLanguageModeldiff --git a/tests/test_remote_code_shims.py b/tests/test_remote_code_shims.pyindex f7f7f4d35d4..fe729568e7f 100644--- a/tests/test_remote_code_shims.py+++ b/tests/test_remote_code_shims.py@@ -58,7 +58,7 @@ class Outer(PreTrainedModel):return self.model.get_input_embeddings()def get_output_embeddings(self):- return self.model.get_output_embeddings() # the inner model has no head: None+ return self.model.get_output_embeddings()def forward(self,@@ -68,14 +68,12 @@ class Outer(PreTrainedModel):):logits = self.lm_head(self.model(input_ids))if labels is not None:- self.config.text_config.vocab_size # the port's own loss code trips here+ self.config.text_config.vocab_sizereturn CausalLMOutputWithPast(logits = logits)-# Remote code lives under transformers_modules; the loss shim keys on that.Outer.__module__ = "transformers_modules.tiny_remote.modeling_tiny"-# Newer transformers reads sys.modules[cls.__module__] while building a model-# (the experts implementation probe), so the name must resolve to a real module.+# Newer transformers reads sys.modules[cls.__module__] while building a model.sys.modules.setdefault(Outer.__module__, sys.modules[__name__])@@ -86,7 +84,6 @@ def model():def test_the_defect_before_the_shim(model):- """The arm that fails on main."""with pytest.raises(TypeError, match = "missing 1 required positional argument"):model.get_input_embeddings()@@ -94,9 +91,7 @@ def test_the_defect_before_the_shim(model):def test_accessor_serves_both_contracts_after_the_shim(model):from unsloth.models.remote_code_shims import apply_remote_code_shims- apply_remote_code_shims(- model- ) # class-level, so a second call in the same process repairs nothing new+ apply_remote_code_shims(model)assert Inner._unsloth_original_get_input_embeddings is not Noneassert model.get_input_embeddings() is model.model.embed_tokensassert model.model.get_input_embeddings() is model.model.embed_tokens@@ -153,8 +148,6 @@ def test_a_forward_that_returns_its_own_loss_is_left_alone():def test_a_tuple_output_with_its_own_loss_is_left_alone():- """return_dict = False gives (loss, logits); the model's own loss must survive the probe."""-class TupleGood(Outer):def forward(self,@@ -178,8 +171,6 @@ def test_a_tuple_output_with_its_own_loss_is_left_alone():@pytest.mark.parametrize("healthy_first", [True, False])def test_loss_support_is_decided_per_instance(healthy_first):- """One remote class, two configs: only the broken one gets the synthesized loss."""-class Mixed(Outer):def forward(self,@@ -208,7 +199,7 @@ def test_loss_support_is_decided_per_instance(healthy_first):def test_a_subclass_override_of_a_repaired_accessor_is_repaired_too(model):from unsloth.models.remote_code_shims import apply_remote_code_shims- apply_remote_code_shims(model) # repairs Inner.get_input_embeddings+ apply_remote_code_shims(model)class SubInner(Inner):def get_input_embeddings(self, input_ids):@@ -250,7 +241,6 @@ def test_transformers_own_classes_are_not_touched():def test_the_repaired_forward_is_reached_through_an_accelerate_hook():- """device_map loading hooks forward before the shims run (Step-3.7 in 16-bit)."""accelerate = pytest.importorskip("accelerate")from accelerate.hooks import add_hook_to_module, ModelHookfrom unsloth.models.remote_code_shims import apply_remote_code_shims@@ -265,7 +255,6 @@ def test_the_repaired_forward_is_reached_through_an_accelerate_hook():def test_the_synthesized_loss_is_the_first_ordered_entry(model):- """Positional readers take output[0] and to_tuple()[0] as the loss when labels were given."""from unsloth.models.remote_code_shims import apply_remote_code_shimsapply_remote_code_shims(model)@@ -276,7 +265,6 @@ def test_the_synthesized_loss_is_the_first_ordered_entry(model):def test_positional_labels_reach_the_synthesized_loss(model):- """Positional labels get the same loss as keyword labels."""from unsloth.models.remote_code_shims import apply_remote_code_shimsapply_remote_code_shims(model)@@ -288,7 +276,6 @@ def test_positional_labels_reach_the_synthesized_loss(model):def test_a_forward_without_a_loss_runs_once_on_the_probing_call():- """A second forward would hold both autograd graphs at once."""from unsloth.models.remote_code_shims import apply_remote_code_shimscalls = []@@ -318,7 +305,6 @@ def test_a_forward_without_a_loss_runs_once_on_the_probing_call():def test_non_token_logits_never_get_a_causal_loss():- """A classification head's (batch, n_labels) logits must not be shifted across examples."""from unsloth.models.remote_code_shims import apply_remote_code_shimsclass Classifier(Outer):@@ -339,7 +325,6 @@ def test_non_token_logits_never_get_a_causal_loss():def test_an_unrelated_first_call_error_does_not_switch_the_objective():- """A failing first call must not pin the fallback: the model's own loss wins once calls work."""from unsloth.models.remote_code_shims import apply_remote_code_shimsclass Picky(Outer):diff --git a/unsloth/models/grouped_linear_lora.py b/unsloth/models/grouped_linear_lora.pyindex c51678f02a1..1cfd2b99512 100644--- a/unsloth/models/grouped_linear_lora.py+++ b/unsloth/models/grouped_linear_lora.py@@ -1,11 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.-"""LoRA for block-diagonal grouped linears (DeepSeek-V4 `o_a_proj`, `FP8GroupedLinear`).--PEFT's dense LoRA sum has the wrong shape for them; only the forward changes (group g uses-its own row block of `lora_B`), so saving, loading and merging are unchanged.-"""+"""LoRA for block-diagonal grouped linears (DeepSeek-V4 `o_a_proj`): only the forward differs from PEFT's."""import functools@@ -70,7 +66,6 @@ def _grouped_lora_layer():if active_adapter not in self.lora_A:continueif active_adapter in getattr(self, "lora_variant", {}):- # Variants (DoRA) replace the plain LoRA sum, which is all this layer computes.raise NotImplementedError("Unsloth: DoRA and other LoRA variants are not supported on grouped ""linears (block-diagonal `o_a_proj`). Use plain LoRA for these layers."@@ -81,7 +76,6 @@ def _grouped_lora_layer():scaling = self.scaling[active_adapter]x_cast = self._cast_input_dtype(x, lora_A.weight.dtype)low_rank = lora_A(dropout(x_cast))- # lora_B.weight is (n_groups * out_per_group, r): block g is rows of group g.weight_B = lora_B.weight.view(n_groups, out_per_group, -1)delta = torch.einsum("...gr,gor->...go", low_rank, weight_B)if lora_B.bias is not None:@@ -93,8 +87,6 @@ def _grouped_lora_layer():def _refuse_fp8_grouped_merge(modules):- """PEFT adds B @ A to the stored weight; an FP8GroupedLinear's weight is fp8 read with an- unchanged weight_scale_inv, so that merge is wrong or raises."""for module in modules:if not getattr(module, "_unsloth_grouped_lora", False):continue@@ -107,8 +99,7 @@ def _refuse_fp8_grouped_merge(modules):def _preflight_peft_merges():- """Refuse before PEFT merges anything: it merges layer by layer, so a refusal from the- grouped layer itself would leave the earlier layers already merged."""+ # PEFT merges layer by layer: refusing only in the grouped layer leaves earlier layers merged.try:from peft.tuners.tuners_utils import BaseTunerexcept Exception:@@ -132,10 +123,6 @@ def _preflight_peft_merges():def _targets_module(lora_config, name):- """Whether `lora_config` selects the module called `name`, by PEFT's own matcher.-- `target_modules = None` or an unreadable config counts as targeted.- """target_modules = getattr(lora_config, "target_modules", None)if target_modules is None or target_modules == "all-linear":return True@@ -159,7 +146,6 @@ def targeted_grouped_linear_classes(lora_config, model):def register_grouped_linear_lora(lora_config, model):- """Map every targeted grouped-linear class in `model` to `GroupedLinearLoRA` on `lora_config`; returns them."""classes = targeted_grouped_linear_classes(lora_config, model)if not classes or not hasattr(lora_config, "_register_custom_module"):return []@@ -177,10 +163,7 @@ def register_grouped_linear_lora(lora_config, model):def register_grouped_linear_lora_for_adapter(model, adapter_path, **hub_kwargs):- """The `PeftConfig` of a saved adapter with the grouped mapping registered, or None.-- The mapping holds class objects, so it is never in the saved adapter config.- """+ # The mapping holds class objects, so it is never in the saved adapter config.if not grouped_linear_classes(model):return Nonetry:diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 944fe66a5c8..aae7b25eb5d 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -308,8 +308,6 @@ def _is_mistral_format_checkpoint(revision = None,local_files_only = False,):- """True for a checkpoint in Mistral's own format (`params.json`, no `config.json`), which- AutoConfig cannot read. Answers False on any doubt, including offline."""# Meta's original Llama checkpoints also ship `params.json`, so require a Mistral-only file.markers = ("tekken.json", "consolidated.safetensors", "consolidated.safetensors.index.json")try:@@ -1350,7 +1348,6 @@ class FastLanguageModel(FastLlamaModel):peft_load_kwargs = {}if kwargs.get("cache_dir") is not None:peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]- # Grouped linears (DeepSeek-V4 o_a_proj): the LoRA mapping is not saved, re-register it._grouped_config = register_grouped_linear_lora_for_adapter(model,old_model_name,diff --git a/unsloth/models/remote_code_shims.py b/unsloth/models/remote_code_shims.pyindex 25dd38b63f6..123ae32f5dc 100644--- a/unsloth/models/remote_code_shims.py+++ b/unsloth/models/remote_code_shims.py@@ -1,13 +1,7 @@# SPDX-License-Identifier: AGPL-3.0-only# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.-"""Repairs for remote modeling code (vLLM ports such as Step-3.7-Flash) that breaks training:--* `get_input_embeddings(self, input_ids)` returns embedded tokens, not the embedding module.-* `forward(..., labels=...)` accepts labels but returns no loss, or fails in its loss code.--Only the checkpoint's own classes are patched, and the original behaviour stays reachable.-"""+"""Repairs for remote modeling code (vLLM ports such as Step-3.7-Flash) that breaks training."""import functoolsimport inspect@@ -24,7 +18,6 @@ _EMBEDDING_ATTRIBUTES = ("embed_tokens", "wte", "word_embeddings", "tok_embeddindef accessor_requires_arguments(function):- """True when `function(self)` cannot be called: a required parameter follows self."""try:parameters = list(inspect.signature(function).parameters.values())except (TypeError, ValueError):@@ -112,7 +105,6 @@ def find_output_head(module):def _repair_output_accessor(cls):- """`get_output_embeddings` that returns None while the class owns an `lm_head` (Step-3.7 delegates to a headless inner model)."""original = cls.__dict__.get("get_output_embeddings")if original is None or "_unsloth_original_get_output_embeddings" in cls.__dict__:return False@@ -142,11 +134,6 @@ def _output_accessor_is_broken(model):def _fill_missing_loss(cls):- """Wrap `cls.forward` so a call with labels always yields a loss.-- The first labelled call probes the original; if it gives no loss, labels are withheld- from then on and the causal LM loss is computed from its logits.- """original = cls.__dict__.get("forward")# Own dict only: a subclass of an already repaired class has its own unwrapped forward.if original is None or "_unsloth_original_forward" in cls.__dict__:@@ -201,7 +188,6 @@ def _fill_missing_loss(cls):self_placeholder = object()def _bind(args, kwargs):- """Move a positional `labels` into kwargs; anything unbindable is left as it came."""if not args or signature is None:return args, kwargstry:@@ -250,7 +236,7 @@ def _fill_missing_loss(cls):f"Unsloth: `{cls.__name__}.forward` returned neither a loss nor logits, so no loss can be trained on.")if returns_loss is None:- # Cached only once a no-label forward worked: a first call failing for another reason must not pin it.+ # Cache only after a no-label forward worked, so an unrelated first failure cannot pin it.self.__dict__[state_key] = Falseprint(f"Unsloth: `{cls.__name__}.forward` accepts `labels` but returns no loss, "@@ -275,10 +261,7 @@ def _fill_missing_loss(cls):def _rebind_accelerate_hook(model):- """Point an accelerate hook attached during loading at the repaired forward.-- `device_map` loading keeps the bound original as `model._old_forward`, bypassing class repairs.- """+ # device_map loading keeps the bound original as `_old_forward`, bypassing class repairs.if getattr(type(model), "_unsloth_original_forward", None) is None:returnif "_old_forward" not in vars(model):diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 6f81385f6fb..d778ca9aa0a 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -3169,7 +3169,6 @@ class FastBaseModel:lora_config = LoraConfig(**{k: v for k, v in local_variables.items() if k in allowed_parameters},)- # Block-diagonal grouped linears (DeepSeek-V4's o_a_proj) need a LoRA forward that is grouped too._grouped_classes = register_grouped_linear_lora(lora_config, model)if _grouped_classes:print(