Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Trim comments in the DeepSeek-V4 grouped LoRA, remote-code shims and Mistral-format loader code (#12089)

Daniel Han10f2575495bf
diff --git a/tests/test_grouped_linear_lora.py b/tests/test_grouped_linear_lora.py
index 8c4fa44103e..b2e0449dc35 100644
--- a/tests/test_grouped_linear_lora.py
+++ b/tests/test_grouped_linear_lora.py
@@ -8,8 +8,6 @@ peft = pytest.importorskip("peft")
class GroupedLinear(torch.nn.Linear):
- """The shape contract of `DeepseekV4GroupedLinear` and `FP8GroupedLinear`."""
-
def __init__(self, in_per_group, out_features, n_groups):
super().__init__(in_per_group, out_features, bias = False)
self.n_groups = n_groups
@@ -58,7 +56,6 @@ def test_is_grouped_linear_wants_n_groups_and_an_overridden_forward():
def test_dense_lora_fails_on_the_grouped_linear():
- """The arm that fails on main."""
model = _peft_model(register = False)
with pytest.raises(RuntimeError, match = "must match the size"):
model(torch.randn(2, 5, 4, 16))
@@ -74,7 +71,6 @@ def test_grouped_lora_trains_and_matches_the_merged_weight():
assert type(layer).__name__ == "GroupedLinearLoRA"
assert layer.lora_A["default"].weight.grad is not None
assert layer.lora_B["default"].weight.grad is not None
- # Merging lora_B @ lora_A into the block-diagonal weight gives the same forward.
with torch.no_grad():
merged = model.merge_and_unload()
merged_out = merged(x)
@@ -82,8 +78,7 @@ def test_grouped_lora_trains_and_matches_the_merged_weight():
def test_merge_into_an_fp8_grouped_weight_is_refused():
- """FP8GroupedLinear keeps an fp8 weight plus weight_scale_inv; B @ A cannot be added in place.
- PEFT merges layer by layer, so the dense layer ahead of it must not be merged either."""
+ """PEFT merges layer by layer, so the dense layer ahead of the fp8 one must not be merged either."""
if not hasattr(torch, "float8_e4m3fn"):
pytest.skip("torch without float8")
from peft import LoraConfig, get_peft_model
@@ -92,7 +87,7 @@ def test_merge_into_an_fp8_grouped_weight_is_refused():
class DenseFirst(torch.nn.Module):
def __init__(self):
super().__init__()
- self.q_proj = torch.nn.Linear(16, 16, bias = False) # merged first by PEFT
+ self.q_proj = torch.nn.Linear(16, 16, bias = False)
self.o_a_proj = GroupedLinear(16, 4 * 8, 4)
def forward(self, x):
@@ -140,7 +135,6 @@ def test_state_dict_is_a_plain_lora_checkpoint():
def test_dora_is_refused_on_a_grouped_linear():
- """The grouped forward computes the plain LoRA sum only, so DoRA is refused."""
from peft import LoraConfig
from unsloth.models.grouped_linear_lora import register_grouped_linear_lora
@@ -168,7 +162,6 @@ def test_a_variant_reaching_the_forward_is_refused():
def test_dora_is_allowed_when_no_grouped_linear_is_targeted():
- """DoRA on the dense o_b_proj alone is valid: the grouped forward is never built."""
from peft import LoraConfig
from unsloth.models.grouped_linear_lora import register_grouped_linear_lora
@@ -183,7 +176,6 @@ def test_dora_is_allowed_when_no_grouped_linear_is_targeted():
def test_a_saved_adapter_reloads_onto_the_grouped_forward(tmp_path):
- """The custom mapping is not in adapter_config.json, so a reload must register it again."""
from peft import PeftModel
from unsloth.models.grouped_linear_lora import register_grouped_linear_lora_for_adapter
@@ -207,12 +199,10 @@ def test_a_saved_adapter_reloads_onto_the_grouped_forward(tmp_path):
with torch.no_grad():
got = reloaded(x)
torch.testing.assert_close(got, want)
- # A model with no grouped linear reloads exactly as before.
assert register_grouped_linear_lora_for_adapter(torch.nn.Linear(4, 4), str(tmp_path)) is None
def test_fan_in_fan_out_is_reset_like_peft_does_for_nn_linear():
- """PEFT resets fan_in_fan_out for nn.Linear; the custom mapping skips that dispatcher."""
from peft import LoraConfig, get_peft_model
from unsloth.models.grouped_linear_lora import register_grouped_linear_lora
diff --git a/tests/test_mistral_format_checkpoint.py b/tests/test_mistral_format_checkpoint.py
index 7fe5e870e70..d152b3df9e9 100644
--- a/tests/test_mistral_format_checkpoint.py
+++ b/tests/test_mistral_format_checkpoint.py
@@ -1,6 +1,5 @@
# SPDX-License-Identifier: AGPL-3.0-only
-"""A Mistral-format checkpoint (params.json, no config.json, e.g. Mistral-Large-3) gets a
-message that says what it is, instead of the generic "both configs failed" one."""
+"""A Mistral-format checkpoint (params.json, no config.json) gets its own error message."""
import json
import os
@@ -8,7 +7,7 @@ import os
import pytest
from real_accelerator import (
has_real_accelerator,
-) # tests/_shared, on sys.path via tests/conftest.py
+)
import torch
@@ -24,9 +23,7 @@ def test_params_json_without_config_json_is_mistral_format(tmp_path):
def test_a_params_json_without_a_mistral_marker_is_not_claimed(tmp_path):
- """Meta's original Llama layout also has params.json; the generic message must stay."""
from unsloth.models.loader import _is_mistral_format_checkpoint
-
assert (
_is_mistral_format_checkpoint(
_write(tmp_path, ["params.json", "consolidated.00.pth", "tokenizer.model"])
@@ -66,7 +63,6 @@ def test_the_message_names_the_file_and_the_route():
@pytest.mark.skipif(not has_real_accelerator(), reason = "import unsloth needs an accelerator")
def test_loader_raises_the_specific_message(tmp_path):
- """The arm that fails on main: the loader used to raise the generic message."""
import unsloth # noqa: F401
from unsloth import FastLanguageModel
diff --git a/tests/test_remote_code_shims.py b/tests/test_remote_code_shims.py
index f7f7f4d35d4..fe729568e7f 100644
--- a/tests/test_remote_code_shims.py
+++ b/tests/test_remote_code_shims.py
@@ -58,7 +58,7 @@ class Outer(PreTrainedModel):
return self.model.get_input_embeddings()
def get_output_embeddings(self):
- return self.model.get_output_embeddings() # the inner model has no head: None
+ return self.model.get_output_embeddings()
def forward(
self,
@@ -68,14 +68,12 @@ class Outer(PreTrainedModel):
):
logits = self.lm_head(self.model(input_ids))
if labels is not None:
- self.config.text_config.vocab_size # the port's own loss code trips here
+ self.config.text_config.vocab_size
return CausalLMOutputWithPast(logits = logits)
-# Remote code lives under transformers_modules; the loss shim keys on that.
Outer.__module__ = "transformers_modules.tiny_remote.modeling_tiny"
-# Newer transformers reads sys.modules[cls.__module__] while building a model
-# (the experts implementation probe), so the name must resolve to a real module.
+# Newer transformers reads sys.modules[cls.__module__] while building a model.
sys.modules.setdefault(Outer.__module__, sys.modules[__name__])
@@ -86,7 +84,6 @@ def model():
def test_the_defect_before_the_shim(model):
- """The arm that fails on main."""
with pytest.raises(TypeError, match = "missing 1 required positional argument"):
model.get_input_embeddings()
@@ -94,9 +91,7 @@ def test_the_defect_before_the_shim(model):
def test_accessor_serves_both_contracts_after_the_shim(model):
from unsloth.models.remote_code_shims import apply_remote_code_shims
- apply_remote_code_shims(
- model
- ) # class-level, so a second call in the same process repairs nothing new
+ apply_remote_code_shims(model)
assert Inner._unsloth_original_get_input_embeddings is not None
assert model.get_input_embeddings() is model.model.embed_tokens
assert model.model.get_input_embeddings() is model.model.embed_tokens
@@ -153,8 +148,6 @@ def test_a_forward_that_returns_its_own_loss_is_left_alone():
def test_a_tuple_output_with_its_own_loss_is_left_alone():
- """return_dict = False gives (loss, logits); the model's own loss must survive the probe."""
-
class TupleGood(Outer):
def forward(
self,
@@ -178,8 +171,6 @@ def test_a_tuple_output_with_its_own_loss_is_left_alone():
@pytest.mark.parametrize("healthy_first", [True, False])
def test_loss_support_is_decided_per_instance(healthy_first):
- """One remote class, two configs: only the broken one gets the synthesized loss."""
-
class Mixed(Outer):
def forward(
self,
@@ -208,7 +199,7 @@ def test_loss_support_is_decided_per_instance(healthy_first):
def test_a_subclass_override_of_a_repaired_accessor_is_repaired_too(model):
from unsloth.models.remote_code_shims import apply_remote_code_shims
- apply_remote_code_shims(model) # repairs Inner.get_input_embeddings
+ apply_remote_code_shims(model)
class SubInner(Inner):
def get_input_embeddings(self, input_ids):
@@ -250,7 +241,6 @@ def test_transformers_own_classes_are_not_touched():
def test_the_repaired_forward_is_reached_through_an_accelerate_hook():
- """device_map loading hooks forward before the shims run (Step-3.7 in 16-bit)."""
accelerate = pytest.importorskip("accelerate")
from accelerate.hooks import add_hook_to_module, ModelHook
from unsloth.models.remote_code_shims import apply_remote_code_shims
@@ -265,7 +255,6 @@ def test_the_repaired_forward_is_reached_through_an_accelerate_hook():
def test_the_synthesized_loss_is_the_first_ordered_entry(model):
- """Positional readers take output[0] and to_tuple()[0] as the loss when labels were given."""
from unsloth.models.remote_code_shims import apply_remote_code_shims
apply_remote_code_shims(model)
@@ -276,7 +265,6 @@ def test_the_synthesized_loss_is_the_first_ordered_entry(model):
def test_positional_labels_reach_the_synthesized_loss(model):
- """Positional labels get the same loss as keyword labels."""
from unsloth.models.remote_code_shims import apply_remote_code_shims
apply_remote_code_shims(model)
@@ -288,7 +276,6 @@ def test_positional_labels_reach_the_synthesized_loss(model):
def test_a_forward_without_a_loss_runs_once_on_the_probing_call():
- """A second forward would hold both autograd graphs at once."""
from unsloth.models.remote_code_shims import apply_remote_code_shims
calls = []
@@ -318,7 +305,6 @@ def test_a_forward_without_a_loss_runs_once_on_the_probing_call():
def test_non_token_logits_never_get_a_causal_loss():
- """A classification head's (batch, n_labels) logits must not be shifted across examples."""
from unsloth.models.remote_code_shims import apply_remote_code_shims
class Classifier(Outer):
@@ -339,7 +325,6 @@ def test_non_token_logits_never_get_a_causal_loss():
def test_an_unrelated_first_call_error_does_not_switch_the_objective():
- """A failing first call must not pin the fallback: the model's own loss wins once calls work."""
from unsloth.models.remote_code_shims import apply_remote_code_shims
class Picky(Outer):
diff --git a/unsloth/models/grouped_linear_lora.py b/unsloth/models/grouped_linear_lora.py
index c51678f02a1..1cfd2b99512 100644
--- a/unsloth/models/grouped_linear_lora.py
+++ b/unsloth/models/grouped_linear_lora.py
@@ -1,11 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
-"""LoRA for block-diagonal grouped linears (DeepSeek-V4 `o_a_proj`, `FP8GroupedLinear`).
-
-PEFT's dense LoRA sum has the wrong shape for them; only the forward changes (group g uses
-its own row block of `lora_B`), so saving, loading and merging are unchanged.
-"""
+"""LoRA for block-diagonal grouped linears (DeepSeek-V4 `o_a_proj`): only the forward differs from PEFT's."""
import functools
@@ -70,7 +66,6 @@ def _grouped_lora_layer():
if active_adapter not in self.lora_A:
continue
if active_adapter in getattr(self, "lora_variant", {}):
- # Variants (DoRA) replace the plain LoRA sum, which is all this layer computes.
raise NotImplementedError(
"Unsloth: DoRA and other LoRA variants are not supported on grouped "
"linears (block-diagonal `o_a_proj`). Use plain LoRA for these layers."
@@ -81,7 +76,6 @@ def _grouped_lora_layer():
scaling = self.scaling[active_adapter]
x_cast = self._cast_input_dtype(x, lora_A.weight.dtype)
low_rank = lora_A(dropout(x_cast))
- # lora_B.weight is (n_groups * out_per_group, r): block g is rows of group g.
weight_B = lora_B.weight.view(n_groups, out_per_group, -1)
delta = torch.einsum("...gr,gor->...go", low_rank, weight_B)
if lora_B.bias is not None:
@@ -93,8 +87,6 @@ def _grouped_lora_layer():
def _refuse_fp8_grouped_merge(modules):
- """PEFT adds B @ A to the stored weight; an FP8GroupedLinear's weight is fp8 read with an
- unchanged weight_scale_inv, so that merge is wrong or raises."""
for module in modules:
if not getattr(module, "_unsloth_grouped_lora", False):
continue
@@ -107,8 +99,7 @@ def _refuse_fp8_grouped_merge(modules):
def _preflight_peft_merges():
- """Refuse before PEFT merges anything: it merges layer by layer, so a refusal from the
- grouped layer itself would leave the earlier layers already merged."""
+ # PEFT merges layer by layer: refusing only in the grouped layer leaves earlier layers merged.
try:
from peft.tuners.tuners_utils import BaseTuner
except Exception:
@@ -132,10 +123,6 @@ def _preflight_peft_merges():
def _targets_module(lora_config, name):
- """Whether `lora_config` selects the module called `name`, by PEFT's own matcher.
-
- `target_modules = None` or an unreadable config counts as targeted.
- """
target_modules = getattr(lora_config, "target_modules", None)
if target_modules is None or target_modules == "all-linear":
return True
@@ -159,7 +146,6 @@ def targeted_grouped_linear_classes(lora_config, model):
def register_grouped_linear_lora(lora_config, model):
- """Map every targeted grouped-linear class in `model` to `GroupedLinearLoRA` on `lora_config`; returns them."""
classes = targeted_grouped_linear_classes(lora_config, model)
if not classes or not hasattr(lora_config, "_register_custom_module"):
return []
@@ -177,10 +163,7 @@ def register_grouped_linear_lora(lora_config, model):
def register_grouped_linear_lora_for_adapter(model, adapter_path, **hub_kwargs):
- """The `PeftConfig` of a saved adapter with the grouped mapping registered, or None.
-
- The mapping holds class objects, so it is never in the saved adapter config.
- """
+ # The mapping holds class objects, so it is never in the saved adapter config.
if not grouped_linear_classes(model):
return None
try:
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 944fe66a5c8..aae7b25eb5d 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -308,8 +308,6 @@ def _is_mistral_format_checkpoint(
revision = None,
local_files_only = False,
):
- """True for a checkpoint in Mistral's own format (`params.json`, no `config.json`), which
- AutoConfig cannot read. Answers False on any doubt, including offline."""
# Meta's original Llama checkpoints also ship `params.json`, so require a Mistral-only file.
markers = ("tekken.json", "consolidated.safetensors", "consolidated.safetensors.index.json")
try:
@@ -1350,7 +1348,6 @@ class FastLanguageModel(FastLlamaModel):
peft_load_kwargs = {}
if kwargs.get("cache_dir") is not None:
peft_load_kwargs["cache_dir"] = kwargs["cache_dir"]
- # Grouped linears (DeepSeek-V4 o_a_proj): the LoRA mapping is not saved, re-register it.
_grouped_config = register_grouped_linear_lora_for_adapter(
model,
old_model_name,
diff --git a/unsloth/models/remote_code_shims.py b/unsloth/models/remote_code_shims.py
index 25dd38b63f6..123ae32f5dc 100644
--- a/unsloth/models/remote_code_shims.py
+++ b/unsloth/models/remote_code_shims.py
@@ -1,13 +1,7 @@
# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
-"""Repairs for remote modeling code (vLLM ports such as Step-3.7-Flash) that breaks training:
-
-* `get_input_embeddings(self, input_ids)` returns embedded tokens, not the embedding module.
-* `forward(..., labels=...)` accepts labels but returns no loss, or fails in its loss code.
-
-Only the checkpoint's own classes are patched, and the original behaviour stays reachable.
-"""
+"""Repairs for remote modeling code (vLLM ports such as Step-3.7-Flash) that breaks training."""
import functools
import inspect
@@ -24,7 +18,6 @@ _EMBEDDING_ATTRIBUTES = ("embed_tokens", "wte", "word_embeddings", "tok_embeddin
def accessor_requires_arguments(function):
- """True when `function(self)` cannot be called: a required parameter follows self."""
try:
parameters = list(inspect.signature(function).parameters.values())
except (TypeError, ValueError):
@@ -112,7 +105,6 @@ def find_output_head(module):
def _repair_output_accessor(cls):
- """`get_output_embeddings` that returns None while the class owns an `lm_head` (Step-3.7 delegates to a headless inner model)."""
original = cls.__dict__.get("get_output_embeddings")
if original is None or "_unsloth_original_get_output_embeddings" in cls.__dict__:
return False
@@ -142,11 +134,6 @@ def _output_accessor_is_broken(model):
def _fill_missing_loss(cls):
- """Wrap `cls.forward` so a call with labels always yields a loss.
-
- The first labelled call probes the original; if it gives no loss, labels are withheld
- from then on and the causal LM loss is computed from its logits.
- """
original = cls.__dict__.get("forward")
# Own dict only: a subclass of an already repaired class has its own unwrapped forward.
if original is None or "_unsloth_original_forward" in cls.__dict__:
@@ -201,7 +188,6 @@ def _fill_missing_loss(cls):
self_placeholder = object()
def _bind(args, kwargs):
- """Move a positional `labels` into kwargs; anything unbindable is left as it came."""
if not args or signature is None:
return args, kwargs
try:
@@ -250,7 +236,7 @@ def _fill_missing_loss(cls):
f"Unsloth: `{cls.__name__}.forward` returned neither a loss nor logits, so no loss can be trained on."
)
if returns_loss is None:
- # Cached only once a no-label forward worked: a first call failing for another reason must not pin it.
+ # Cache only after a no-label forward worked, so an unrelated first failure cannot pin it.
self.__dict__[state_key] = False
print(
f"Unsloth: `{cls.__name__}.forward` accepts `labels` but returns no loss, "
@@ -275,10 +261,7 @@ def _fill_missing_loss(cls):
def _rebind_accelerate_hook(model):
- """Point an accelerate hook attached during loading at the repaired forward.
-
- `device_map` loading keeps the bound original as `model._old_forward`, bypassing class repairs.
- """
+ # device_map loading keeps the bound original as `_old_forward`, bypassing class repairs.
if getattr(type(model), "_unsloth_original_forward", None) is None:
return
if "_old_forward" not in vars(model):
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 6f81385f6fb..d778ca9aa0a 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -3169,7 +3169,6 @@ class FastBaseModel:
lora_config = LoraConfig(
**{k: v for k, v in local_variables.items() if k in allowed_parameters},
)
- # Block-diagonal grouped linears (DeepSeek-V4's o_a_proj) need a LoRA forward that is grouped too.
_grouped_classes = register_grouped_linear_lora(lora_config, model)
if _grouped_classes:
print(