Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Do not auto-enable padding-free for a model whose forward cannot take it (#11468)
Daniel Han2390f4f86893
diff --git a/tests/test_padding_free_signature_gate.py b/tests/test_padding_free_signature_gate.pynew file mode 100644index 00000000000..fa1869a174c--- /dev/null+++ b/tests/test_padding_free_signature_gate.py@@ -0,0 +1,644 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Padding-free is auto-enabled, and it hands the model `packed_seq_lengths`.++A forward declaring neither that nor `**kwargs` raises TypeError on the first step+(Phi4ForCausalLMV). The gate asks the signature, and only ever turns padding-free OFF+where the signature positively shows it cannot work.+"""++import pytest++torch = pytest.importorskip("torch")+from torch import nn # noqa: E402++try:+ from unsloth.trainer import _forward_accepts_packing_kwargs # noqa: E402+except ImportError:+ # On Apple Silicon with MLX, `unsloth/__init__.py` swaps `unsloth.trainer` for a shim+ # carrying only the MLX trainer names, so these helpers are unreachable and padding-free+ # does not apply. Skip rather than fail collection, which aborts more than itself.+ import unsloth+ if getattr(unsloth, "DEVICE_TYPE", None) != "mlx":+ raise+ pytest.skip("unsloth.trainer is the MLX shim here", allow_module_level = True)+++class _NoKwargs(nn.Module):+ """microsoft/Phi-4-reasoning-vision-15B's shape: no **kwargs, no packing."""++ def forward(+ self,+ input_ids = None,+ attention_mask = None,+ labels = None,+ ):+ return None+++class _TakesKwargs(nn.Module):+ def forward(+ self,+ input_ids = None,+ **kwargs,+ ):+ return None+++class _NamesPacking(nn.Module):+ def forward(+ self,+ input_ids = None,+ packed_seq_lengths = None,+ ):+ return None+++class _FakePeft(nn.Module):+ """PEFT forwards **kwargs through, so the wrapper says yes for a checkpoint that does not."""++ def __init__(self, inner):+ super().__init__()+ self.inner = inner++ def get_base_model(self):+ return self.inner++ def forward(self, *args, **kwargs):+ return self.inner(*args, **kwargs)+++class _InnerDecoderTakesKwargs(nn.Module):+ """`PreTrainedModel.base_model` reaches the inner decoder, which usually does take+ **kwargs, so following it answers for the wrong module."""++ def __init__(self):+ super().__init__()+ self.model = _TakesKwargs()++ @property+ def base_model(self):+ return self.model++ def forward(+ self,+ input_ids = None,+ attention_mask = None,+ ):+ return None+++class _SelfReturningUnwrap(nn.Module):+ def get_base_model(self):+ return self++ def forward(self, input_ids = None):+ return None+++class _RaisingUnwrap(nn.Module):+ def get_base_model(self):+ raise RuntimeError("adapter not ready")++ def forward(self, input_ids = None):+ return None+++ "model, expected, why",+ [+ (_NoKwargs(), False, "the real defect"),+ (_TakesKwargs(), True, "**kwargs absorbs it"),+ (_NamesPacking(), True, "names it explicitly"),+ (_FakePeft(_NoKwargs()), False, "unwrap past the adapter"),+ (_FakePeft(_TakesKwargs()), True, "unwrapped model accepts it"),+ (_InnerDecoderTakesKwargs(), False, "base_model must NOT be followed"),+ (_SelfReturningUnwrap(), False, "self-returning unwrap terminates"),+ (_RaisingUnwrap(), False, "a raising unwrap falls back to the wrapper"),+ (None, True, "unknown fails open"),+ ("meta-llama/Llama-3.1-8B", True, "a name, not a model: fails open"),+ (object(), True, "no forward at all: fails open"),+ ],+)+def test_gate(model, expected, why):+ assert _forward_accepts_packing_kwargs(model) is expected, why+++def test_the_defect_is_real():+ """Negative control: the rejected shape really does raise."""+ with pytest.raises(TypeError, match = "packed_seq_lengths"):+ _NoKwargs()(input_ids = torch.zeros(1, 4).long(), packed_seq_lengths = [4])+++def test_the_accepted_shape_really_accepts_it():+ """And the shape the gate allows really does tolerate the argument."""+ _TakesKwargs()(input_ids = torch.zeros(1, 4).long(), packed_seq_lengths = [4])+++def test_the_blocker_names_itself_in_the_warning():+ """The chain must not fall through to UNSLOTH_RETURN_LOGITS=1, a flag the user never set."""+ import inspect as _inspect++ from unsloth import trainer as trainer_module++ source = _inspect.getsource(trainer_module._patch_sft_trainer_auto_packing)+ assert "forward_rejects_packing" in source+ # the new branch must come BEFORE the catch-all env-var branch+ assert source.index("elif forward_rejects_packing") < source.index(+ 'reason = "UNSLOTH_RETURN_LOGITS=1"'+ )+ # and the predicate is evaluated once, not twice+ assert source.count("_forward_accepts_packing_kwargs(model)") == 1++[email protected]("packing", [False, True])+def test_a_string_model_is_rechecked_once_trl_has_built_it(monkeypatch, packing):+ """A string the class could not be resolved for is rechecked after init, and REFUSED.++ Clearing the flags there would leave batches flattened with nothing naming the+ boundaries; re-running `__init__` would materialize the checkpoint twice.+ """+ from types import SimpleNamespace++ import unsloth.trainer as trainer_module++ built = _NoKwargs()+ inits = []++ class _StubSFTTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ # What TRL does with a string: materialize it, then expose it as self.model.+ self.model = built if isinstance(model, str) else model+ self.args = args+ inits.append(model)++ injected = []+ for _name in ("enable_padding_free_metadata", "enable_sample_packing"):+ monkeypatch.setattr(trainer_module, _name, lambda model, trainer: injected.append(model))+ # No hub access: the config is irrelevant to the signature question.+ monkeypatch.setattr(trainer_module, "_resolve_string_model_config", lambda *a, **k: None)++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)++ config = SimpleNamespace(packing = packing, padding_free = True, max_length = 512)+ with pytest.raises(ValueError, match = "packed_seq_lengths"):+ module.SFTTrainer(model = "microsoft/Phi-4-reasoning-vision-15B", args = config)++ assert injected == [], "nothing may wrap the collator for this forward"+ assert len(inits) == 1, "the checkpoint must not be materialized a second time"+++def test_a_string_model_that_can_take_the_metadata_is_left_alone(monkeypatch):+ """The refusal must fire only on the signature, never on the string itself."""+ from types import SimpleNamespace++ import unsloth.trainer as trainer_module++ class _StubSFTTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ self.model = _TakesKwargs() if isinstance(model, str) else model+ self.args = args++ injected = []+ monkeypatch.setattr(+ trainer_module,+ "enable_padding_free_metadata",+ lambda model, trainer: injected.append(model),+ )+ monkeypatch.setattr(trainer_module, "_resolve_string_model_config", lambda *a, **k: None)++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)++ config = SimpleNamespace(packing = False, padding_free = True, max_length = 512)+ instance = module.SFTTrainer(model = "meta-llama/Llama-3.1-8B", args = config)++ assert len(injected) == 1+ assert instance.args.padding_free is True+++def _fake_config():+ """A config just real enough for the model-type lookup the wrapper does first."""+ from types import SimpleNamespace+ return SimpleNamespace(+ model_type = "llama",+ architectures = ["LlamaForCausalLM"],+ auto_map = None,+ is_encoder_decoder = False,+ to_dict = lambda: {"model_type": "llama"},+ )+++def test_the_class_behind_a_string_is_resolved_without_downloading_weights():+ """`from_pretrained` is never called, so a config built in memory is enough."""+ from transformers import LlamaConfig, LlamaForCausalLM++ from unsloth.trainer import _resolve_string_model_class++ config = LlamaConfig(architectures = ["LlamaForCausalLM"])+ assert _resolve_string_model_class("any/name", config, None) is LlamaForCausalLM++ # and with no `architectures` recorded, the auto mappings answer from the config class+ bare = LlamaConfig()+ bare.architectures = None+ assert _resolve_string_model_class("any/name", bare, None) is LlamaForCausalLM+++def test_a_native_architecture_is_answered_without_touching_remote_code():+ """Native `architectures` wins over a remote `auto_map`, so the loader is never called.++ TRL resolves `getattr(transformers, config.architectures[0])` and never reads+ `auto_map`, so reaching for it first would run code nothing else would.+ """+ from types import SimpleNamespace++ import transformers+ from transformers import LlamaForCausalLM++ from unsloth.trainer import _resolve_string_model_class++ called = []++ class _Config:+ model_type = "llama"+ architectures = ["LlamaForCausalLM"]+ auto_map = {"AutoModelForCausalLM": "payload.Model"}++ import transformers.dynamic_module_utils as dmu++ original = dmu.get_class_from_dynamic_module+ dmu.get_class_from_dynamic_module = lambda *a, **k: called.append(a) or _NoKwargs+ try:+ resolved = _resolve_string_model_class(+ "evil/repo", _Config(), SimpleNamespace(trust_remote_code = True)+ )+ finally:+ dmu.get_class_from_dynamic_module = original++ assert resolved is LlamaForCausalLM+ assert called == [], "the remote module must not be imported when a native name answers"+++ "init_kwargs, top_level, may_execute",+ [+ ({}, True, True),+ ({}, False, False),+ ({}, None, False),+ # an explicit None in model_init_kwargs is NOT a grant, and must not be+ # overwritten by a truthy top-level attribute: `_resolve_string_model_config`+ # reads it by membership too, and the two must agree or the module would run+ # under a grant the config load did not accept+ ({"trust_remote_code": None}, True, False),+ ({"trust_remote_code": False}, True, False),+ ({"trust_remote_code": True}, False, True),+ ],+)+def test_the_remote_code_grant_is_read_by_membership(init_kwargs, top_level, may_execute):+ from types import SimpleNamespace++ from unsloth.trainer import _resolve_string_model_class++ called = []++ class _RemoteOnlyConfig:+ model_type = "not-a-real-model-type"+ architectures = ["NoSuchClassInTransformers"]+ auto_map = {"AutoModelForCausalLM": "payload.Model"}++ import transformers.dynamic_module_utils as dmu++ original = dmu.get_class_from_dynamic_module+ dmu.get_class_from_dynamic_module = lambda *a, **k: called.append(a) or _NoKwargs+ try:+ _resolve_string_model_class(+ "some/repo",+ _RemoteOnlyConfig(),+ SimpleNamespace(model_init_kwargs = init_kwargs, trust_remote_code = top_level),+ )+ finally:+ dmu.get_class_from_dynamic_module = original++ assert bool(called) is may_execute+++def test_the_same_auth_keys_reach_both_fetches():+ """The config fetch and the class fetch must authenticate identically.++ transformers honours `use_auth_token` as a deprecated alias for `token` across the+ supported range, so dropping it would authenticate one fetch and not the other.+ Asserted against the sibling's own key list so the two cannot drift apart.+ """+ import inspect as _inspect+ from types import SimpleNamespace++ from unsloth import trainer as trainer_module+ from unsloth.trainer import _resolve_string_model_class++ sibling = _inspect.getsource(trainer_module._resolve_string_model_config)+ resolver = _inspect.getsource(_resolve_string_model_class)+ # trust_remote_code is the grant and is handled separately; every other key the+ # config fetch forwards must also be forwarded here+ for key in ("revision", "subfolder", "token", "use_auth_token", "cache_dir", "code_revision"):+ assert f'"{key}"' in sibling, f"{key} is not forwarded by the config fetch"+ assert f'"{key}"' in resolver, f"{key} is not forwarded by the class fetch"++ captured = {}++ class _RemoteOnlyConfig:+ model_type = "not-a-real-model-type"+ architectures = ["NoSuchClassInTransformers"]+ auto_map = {"AutoModelForCausalLM": "payload.Model"}++ import transformers.dynamic_module_utils as dmu++ original = dmu.get_class_from_dynamic_module++ def _capture(*args, **kwargs):+ captured.update(kwargs)+ return _NoKwargs++ dmu.get_class_from_dynamic_module = _capture+ try:+ _resolve_string_model_class(+ "private/repo",+ _RemoteOnlyConfig(),+ SimpleNamespace(+ model_init_kwargs = {+ "trust_remote_code": True,+ "use_auth_token": "hf_legacy",+ "revision": "abc123",+ }+ ),+ )+ finally:+ dmu.get_class_from_dynamic_module = original++ assert captured.get("use_auth_token") == "hf_legacy"+ assert captured.get("revision") == "abc123"+ # the grant itself is not an auth key and must not be forwarded as one+ assert "trust_remote_code" not in captured+++def test_an_unresolvable_string_returns_none_rather_than_guessing():+ """None leaves the post-init backstop in charge, which is the safe answer."""+ from transformers import LlamaConfig++ from unsloth.trainer import _resolve_string_model_class++ # no config at all+ assert _resolve_string_model_class("any/name", None, None) is None+ # not a string+ assert _resolve_string_model_class(_NoKwargs(), LlamaConfig(), None) is None++ # an architecture name that is not in the transformers namespace, and a config class+ # that is in no auto mapping+ class _UnknownConfig:+ architectures = ["NoSuchModelForCausalLM"]+ auto_map = None++ assert _resolve_string_model_class("any/name", _UnknownConfig(), None) is None+++def test_a_resolvable_string_is_blocked_silently_instead_of_raising(monkeypatch):+ """No exception, just padding-free turned off: the user configured nothing."""+ from types import SimpleNamespace++ import unsloth.trainer as trainer_module++ class _StubSFTTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ self.model = _NoKwargs() if isinstance(model, str) else model+ self.args = args++ injected = []+ for _name in ("enable_padding_free_metadata", "enable_sample_packing"):+ monkeypatch.setattr(trainer_module, _name, lambda model, trainer: injected.append(model))+ monkeypatch.setattr(+ trainer_module, "_resolve_string_model_config", lambda *a, **k: _fake_config()+ )+ monkeypatch.setattr(trainer_module, "_resolve_string_model_class", lambda *a, **k: _NoKwargs)++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)++ config = SimpleNamespace(packing = False, padding_free = None, max_length = 512)+ instance = module.SFTTrainer(model = "microsoft/Phi-4-reasoning-vision-15B", args = config)++ assert instance.args.padding_free is False, "padding-free must be off, not fatal"+ assert injected == [], "nothing may wrap the collator for this forward"+++def test_a_resolvable_string_that_accepts_the_metadata_keeps_padding_free(monkeypatch):+ """The control: resolving early must not cost a normal checkpoint its padding-free."""+ from types import SimpleNamespace++ import unsloth.trainer as trainer_module++ class _StubSFTTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ self.model = _TakesKwargs() if isinstance(model, str) else model+ self.args = args++ injected = []+ monkeypatch.setattr(+ trainer_module,+ "enable_padding_free_metadata",+ lambda model, trainer: injected.append(model),+ )+ monkeypatch.setattr(+ trainer_module, "_resolve_string_model_config", lambda *a, **k: _fake_config()+ )+ monkeypatch.setattr(trainer_module, "_resolve_string_model_class", lambda *a, **k: _TakesKwargs)++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)++ config = SimpleNamespace(packing = False, padding_free = True, max_length = 512)+ instance = module.SFTTrainer(model = "meta-llama/Llama-3.1-8B", args = config)++ assert instance.args.padding_free is True+ assert len(injected) == 1+++def test_the_warning_names_the_resolved_class_not_str(monkeypatch, caplog):+ """`str.forward()` would name the spelling rather than the checkpoint."""+ import logging+ from types import SimpleNamespace++ import unsloth.trainer as trainer_module++ class _StubSFTTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ self.model = _NoKwargs() if isinstance(model, str) else model+ self.args = args++ for _name in ("enable_padding_free_metadata", "enable_sample_packing"):+ monkeypatch.setattr(trainer_module, _name, lambda model, trainer: None)+ monkeypatch.setattr(+ trainer_module, "_resolve_string_model_config", lambda *a, **k: _fake_config()+ )+ monkeypatch.setattr(trainer_module, "_resolve_string_model_class", lambda *a, **k: _NoKwargs)++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)++ # packing=True so the reason chain actually emits its warning+ config = SimpleNamespace(packing = True, padding_free = None, max_length = 512)+ with caplog.at_level(logging.WARNING, logger = trainer_module.logger.name):+ module.SFTTrainer(model = "microsoft/Phi-4-reasoning-vision-15B", args = config)++ assert "_NoKwargs.forward()" in caplog.text+ assert "str.forward()" not in caplog.text+++def test_a_class_that_disagrees_with_the_built_model_is_still_caught(monkeypatch):+ """`auto_map` can name several classes, so a resolved "yes" is not proof about the+ instance. The post-init check stays armed; a correct block already turns both flags+ off, which is the condition it skips on.+ """+ from types import SimpleNamespace++ import unsloth.trainer as trainer_module++ class _StubSFTTrainer:+ def __init__(self, model = None, args = None, **kwargs):+ # what actually gets built disagrees with what the config advertised+ self.model = _NoKwargs() if isinstance(model, str) else model+ self.args = args++ for _name in ("enable_padding_free_metadata", "enable_sample_packing"):+ monkeypatch.setattr(trainer_module, _name, lambda model, trainer: None)+ monkeypatch.setattr(+ trainer_module, "_resolve_string_model_config", lambda *a, **k: _fake_config()+ )+ # the optimistic, and wrong, answer+ monkeypatch.setattr(trainer_module, "_resolve_string_model_class", lambda *a, **k: _TakesKwargs)++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)++ config = SimpleNamespace(packing = False, padding_free = True, max_length = 512)+ with pytest.raises(ValueError, match = "packed_seq_lengths"):+ module.SFTTrainer(model = "some/multi-headed-checkpoint", args = config)+++def test_a_resolver_that_explodes_falls_back_to_the_backstop(monkeypatch):+ """Resolution is best-effort; a failure must not become the user's problem."""+ from types import SimpleNamespace++ import unsloth.trainer as trainer_module++ class _StubSFTTrainer:+ def __init__(+ self,+ model = None,+ args = None,+ **kwargs,+ ):+ self.model = _NoKwargs() if isinstance(model, str) else model+ self.args = args++ def _explode(*a, **k):+ raise RuntimeError("hub is down")++ for _name in ("enable_padding_free_metadata", "enable_sample_packing"):+ monkeypatch.setattr(trainer_module, _name, lambda model, trainer: None)+ monkeypatch.setattr(+ trainer_module, "_resolve_string_model_config", lambda *a, **k: _fake_config()+ )+ monkeypatch.setattr(trainer_module, "_resolve_string_model_class", _explode)++ module = SimpleNamespace(SFTTrainer = _StubSFTTrainer)+ trainer_module._patch_sft_trainer_auto_packing(module)++ config = SimpleNamespace(packing = False, padding_free = True, max_length = 512)+ # still caught, just later and loudly, exactly as before this resolution existed+ with pytest.raises(ValueError, match = "packed_seq_lengths"):+ module.SFTTrainer(model = "microsoft/Phi-4-reasoning-vision-15B", args = config)+++def test_a_mixed_adapter_wrapper_is_unwrapped_to_the_checkpoint():+ """`PeftMixedModel` has no `get_base_model`, and its variadic forward only delegates.++ Driven through real PEFT rather than a stand-in, because the point is which+ attribute the real wrapper exposes.+ """+ peft = pytest.importorskip("peft")+ transformers = pytest.importorskip("transformers")++ from unsloth.trainer import _forward_accepts_packing_kwargs as gate++ def _config():+ return transformers.LlamaConfig(+ hidden_size = 32,+ intermediate_size = 64,+ num_hidden_layers = 1,+ num_attention_heads = 4,+ num_key_value_heads = 4,+ vocab_size = 64,+ )++ class _NarrowLlama(transformers.LlamaForCausalLM):+ def forward(self, input_ids = None, attention_mask = None, labels = None):+ return super().forward(+ input_ids = input_ids, attention_mask = attention_mask, labels = labels+ )++ lora = peft.LoraConfig(target_modules = ["q_proj"])+ mixed_narrow = peft.get_peft_model(_NarrowLlama(_config()), lora, mixed = True)+ mixed_stock = peft.get_peft_model(transformers.LlamaForCausalLM(_config()), lora, mixed = True)++ # the wrapper really is the shape described above, so this test cannot quietly+ # stop testing anything if PEFT changes+ assert type(mixed_narrow).__name__ == "PeftMixedModel"+ assert not hasattr(mixed_narrow, "get_base_model")++ assert gate(mixed_narrow) is False, "must see through to the narrow checkpoint"+ assert gate(mixed_stock) is True, "and must not cost a stock checkpoint its padding-free"+++def test_every_delegating_wrapper_is_unwrapped():+ """Driven over the whole registered family, so a wrapper added without an unwrap fails here."""+ from unsloth.trainer import _DELEGATING_MODULE_WRAPPERS+ from unsloth.trainer import _forward_accepts_packing_kwargs as gate++ assert _DELEGATING_MODULE_WRAPPERS, "nothing resolved: the gate would read the wrapper"+ for wrapper, attribute in _DELEGATING_MODULE_WRAPPERS:+ for inner, expected in ((_NoKwargs(), False), (_TakesKwargs(), True)):+ # A subclass, because FSDP exposes `module` as a read-only property and a class+ # attribute shadows it. Uninitialised on purpose: the gate reads only the type+ # and the held model, so device placement, process groups and a compile step are+ # all beside the question, and FSDP's __setattr__ rejects a stub anyway.+ stub = type("_Stub", (wrapper,), {attribute: inner})+ wrapped = stub.__new__(stub)++ assert gate(inner) is expected, f"{wrapper.__name__}: control"+ assert gate(wrapped) is expected, wrapper.__name__diff --git a/unsloth/trainer.py b/unsloth/trainer.pyindex 5ce1bcb0fae..4b983104172 100644--- a/unsloth/trainer.py+++ b/unsloth/trainer.py@@ -19,8 +19,10 @@ from types import SimpleNamespacefrom typing import Optional, Listfrom functools import wraps-import trl+import importlibimport inspect++import trlfrom trl import SFTTrainer# Bypass the partially-initialised unsloth namespace during the _gpu_init load.@@ -284,6 +286,224 @@ _HYBRID_CONFIG_MARKERS = ()+def _delegating_module_wrappers():+ """Wrappers whose forward is variadic over a real model, and the attribute holding it.++ Safe to follow where `PreTrainedModel.base_model` is not, because the attribute+ below IS the checkpoint rather than an inner decoder. Resolved tolerantly: FSDP+ and dynamo have both moved between torch versions.+ """+ found = []+ for module_path, name, attribute in (+ ("torch.nn.parallel", "DataParallel", "module"),+ ("torch.nn.parallel", "DistributedDataParallel", "module"),+ ("torch.distributed.fsdp", "FullyShardedDataParallel", "module"),+ ("torch._dynamo.eval_frame", "OptimizedModule", "_orig_mod"),+ ):+ try:+ found.append((getattr(importlib.import_module(module_path), name), attribute))+ except Exception:+ continue+ return tuple(found)+++_DELEGATING_MODULE_WRAPPERS = _delegating_module_wrappers()+++def _mixed_adapter_wrappers():+ """PEFT's mixed-adapter wrapper, and the path to the checkpoint under it.++ `PeftMixedModel` has no `get_base_model()`, so the unwrap below stops on its+ variadic forward. The path is PEFT's own accessor (peft_model.py:1083) for any+ config that is not prompt-learning, which mixed adapters cannot be.+ """+ try:+ from peft import PeftMixedModel++ return ((PeftMixedModel, ("base_model", "model")),)+ except Exception:+ return ()+++_MIXED_ADAPTER_WRAPPERS = _mixed_adapter_wrappers()+++def _attribute_path(obj, path):+ for attribute in path:+ obj = getattr(obj, attribute, None)+ if obj is None:+ return None+ return obj+++def _forward_accepts_packing_kwargs(model) -> bool:+ """Can this model's forward be handed the packing metadata at all?++ Padding-free passes `packed_seq_lengths` down, and a forward naming neither it+ nor **kwargs raises TypeError on the first step (microsoft/Phi-4-reasoning-vision-15B).+ Unknown or uninspectable answers True: refusing on an unreadable signature would+ disable padding-free for models that support it.+ """+ if model is None or isinstance(model, str):+ return True+ # Adapter wrappers only: they forward **kwargs through, so the wrapper always+ # answers yes while the checkpoint underneath is the one that raises. Stop there,+ # since the inner decoder may take **kwargs and answer for a model that does not.+ target = model+ for _ in range(4):+ # Delegating wrappers (DataParallel, DDP, FSDP, torch.compile) first. Exact+ # isinstance, so nothing else carrying the same attribute name is followed.+ inner = next(+ (+ held+ for wrapper, attribute in _DELEGATING_MODULE_WRAPPERS+ if isinstance(target, wrapper)+ for held in (getattr(target, attribute, None),)+ if held is not None and held is not target+ ),+ None,+ )+ if inner is not None:+ target = inner+ continue+ # `PeftMixedModel` next: no `get_base_model`, so the unwrap below would stop on+ # a variadic forward that merely delegates.+ mixed = next(+ (+ held+ for wrapper, path in _MIXED_ADAPTER_WRAPPERS+ if isinstance(target, wrapper)+ for held in (_attribute_path(target, path),)+ if held is not None and held is not target+ ),+ None,+ )+ if mixed is not None:+ target = mixed+ continue+ # PEFT's own unwrap only: `PreTrainedModel.base_model` is a property returning+ # the inner decoder, which usually does take **kwargs, so it answers for the+ # wrong module.+ unwrap = getattr(target, "get_base_model", None)+ if not callable(unwrap):+ break+ try:+ unwrapped = unwrap()+ except Exception:+ break+ if unwrapped is None or unwrapped is target:+ break+ target = unwrapped++ return _forward_signature_accepts_packing(getattr(target, "forward", None))+++def _forward_signature_accepts_packing(forward) -> bool:+ """The signature question alone, so a class can be asked it as well as an instance.++ Shared, not duplicated: a disagreement would turn padding-free on for one+ spelling of a checkpoint and off for the other.+ """+ if forward is None:+ return True+ try:+ parameters = inspect.signature(forward).parameters+ except (TypeError, ValueError):+ return True+ if any(p.kind is inspect.Parameter.VAR_KEYWORD for p in parameters.values()):+ return True+ return "packed_seq_lengths" in parameters+++def _resolve_string_model_class(model_name, model_config, config_arg):+ """The class TRL is about to build, resolved from the config and no weights.++ A string `model=` has no forward to read, so without this the gate can only fail+ open and then refuse after `__init__`. `from_pretrained` is never called, so this+ costs at most the modeling module. Returns None when the class cannot be named,+ leaving the post-init check as the backstop rather than guessing.+ """+ if not isinstance(model_name, str) or model_config is None:+ return None++ # In-tree first, which is also TRL's order: `create_model_from_path` does+ # `getattr(transformers, config.architectures[0])` and never consults `auto_map`.+ for architecture in getattr(model_config, "architectures", None) or ():+ try:+ import transformers++ resolved = getattr(transformers, architecture, None)+ except Exception:+ resolved = None+ if isinstance(resolved, type):+ return resolved++ # Then the auto mappings keyed by config class: lazy, and in-tree only.+ try:+ from transformers.models.auto import modeling_auto+ except Exception:+ modeling_auto = None+ for mapping_name in (+ "MODEL_FOR_CAUSAL_LM_MAPPING",+ "MODEL_FOR_IMAGE_TEXT_TO_TEXT_MAPPING",+ "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING",+ "MODEL_MAPPING",+ ):+ mapping = getattr(modeling_auto, mapping_name, None) if modeling_auto else None+ if mapping is None:+ continue+ try:+ resolved = mapping[type(model_config)]+ except Exception:+ continue+ if isinstance(resolved, type):+ return resolved++ # Remote code LAST: resolving `auto_map` imports and executes a module from the+ # repo, and a checkpoint can carry a native `architectures` alongside it, so trying+ # it first would run code nothing else in the stack would have run.+ #+ # The grant is read by membership, as `_resolve_string_model_config` reads it, so an+ # explicit `model_init_kwargs["trust_remote_code"] = None` keeps its falsy meaning.+ # Disagreeing with that function would execute a module under a grant the config+ # load itself did not accept.+ init_kwargs = getattr(config_arg, "model_init_kwargs", None) or {}+ if "trust_remote_code" in init_kwargs:+ trust_remote_code = init_kwargs["trust_remote_code"]+ else:+ trust_remote_code = getattr(config_arg, "trust_remote_code", None)++ auto_map = getattr(model_config, "auto_map", None) or {}+ if not trust_remote_code or not isinstance(auto_map, dict):+ return None+ # The same auth keys the config fetch forwards, `use_auth_token` included:+ # transformers honours it as a deprecated alias for `token` across the supported+ # range (dynamic_module_utils.py:584 in 4.57.6), so dropping it would authenticate+ # one fetch and not the other.+ forward = {+ key: init_kwargs[key]+ for key in ("revision", "subfolder", "token", "use_auth_token", "cache_dir", "code_revision")+ if key in init_kwargs+ }+ for auto_class in (+ "AutoModelForImageTextToText",+ "AutoModelForCausalLM",+ "AutoModelForVision2Seq",+ "AutoModelForSeq2SeqLM",+ "AutoModel",+ ):+ reference = auto_map.get(auto_class)+ if not isinstance(reference, str):+ continue+ try:+ from transformers.dynamic_module_utils import get_class_from_dynamic_module++ return get_class_from_dynamic_module(reference, model_name, **forward)+ except Exception:+ continue+ return None++def _is_hybrid_linear_attention_model(model) -> bool:"""Detect models mixing linear-attention / state-space mixers (gated-delta,Mamba-style) with a causal conv1d, e.g. Qwen3.5 / Qwen3-Next. Packing and@@ -929,6 +1149,7 @@ def _patch_sft_trainer_auto_packing(trl_module):config_arg = kwargs.get("args")model = args[0] if len(args) >= 1 else kwargs.get("model")+ model_config = Noneis_vlm = Falseis_unsupported_model = Falseis_hybrid = False@@ -980,6 +1201,25 @@ def _patch_sft_trainer_auto_packing(trl_module):)# Disable padding-free for VLMs / custom collators / blocklisted models+ forward_rejects_packing = not _forward_accepts_packing_kwargs(model)+ _packing_gate_deferred = model is None or isinstance(model, str)+ # Resolve the class a string names, so that case is an ordinary silent block like+ # every other reason: after __init__ the only move left is to refuse, the collator+ # and transformed datasets having been built from the flags by then.+ _resolved_class = None+ if _packing_gate_deferred and isinstance(model, str):+ try:+ _resolved_class = _resolve_string_model_class(model, model_config, config_arg)+ except Exception:+ _resolved_class = None+ if _resolved_class is not None:+ forward_rejects_packing = not _forward_signature_accepts_packing(+ getattr(_resolved_class, "forward", None)+ )+ # `_packing_gate_deferred` deliberately stays True: `auto_map` can name+ # several classes, so a "yes" from the resolved one is not proof about the+ # instance. A correct "no" has already turned both flags off, which is the+ # condition the post-init check skips on, so leaving it armed is free.blocked = ((data_collator is not None)or is_processor@@ -989,6 +1229,7 @@ def _patch_sft_trainer_auto_packing(trl_module):or is_encoder_decoderor (is_hybrid and not hybrid_varlen_active)or (os.environ.get("UNSLOTH_RETURN_LOGITS", "0") == "1")+ or forward_rejects_packing)requested_pack = bool(getattr(config_arg, "packing", False))if blocked:@@ -1011,6 +1252,16 @@ def _patch_sft_trainer_auto_packing(trl_module):reason = "hybrid linear-attention model"elif is_unsupported_model:reason = f"unsupported model type(s): {', '.join(model_types)}"+ elif forward_rejects_packing:+ # Name the real blocker, else this falls through to the+ # UNSLOTH_RETURN_LOGITS branch and points at an unset flag. For a string+ # `model=` that is the resolved class, not `str`.+ blocker = (+ _resolved_class.__name__+ if _resolved_class is not None+ else type(model).__name__+ )+ reason = f"{blocker}.forward() does not accept packed_seq_lengths"elif data_collator is None:# compute_metrics, preprocess_logits_for_metrics, for_inference() and the user can all set it, so# name the flag and not a setter.@@ -1059,6 +1310,27 @@ def _patch_sft_trainer_auto_packing(trl_module):else:raise+ # Backstop for the deferred cases: a `model_init=`, or a string whose class could+ # not be resolved. It refuses rather than undoing the flags, because TRL has built+ # its collator from them and (under packing) transformed the datasets, so clearing+ # them would leave batches flattened with nothing naming the sequence boundaries --+ # attention and loss crossing examples silently, worse than the TypeError. Re-running+ # `__init__` would rebuild them but materialize the checkpoint twice, an OOM in+ # exactly the large-model case this is for.+ if _packing_gate_deferred and not _forward_accepts_packing_kwargs(+ getattr(self, "model", None)+ ):+ if packing_active or getattr(config_arg, "padding_free", False):+ raise ValueError(+ f"Unsloth: {type(getattr(self, 'model', None)).__name__}.forward cannot "+ "take `packed_seq_lengths`, which packing and padding-free both pass, so "+ "training would fail on the first step. This could not be detected before "+ "the trainer was built because `model` was not given as a model and its "+ "class could not be resolved from its config. Either load it first and pass "+ "the model itself, and Unsloth will turn the two off for you, or set "+ "`packing = False` and `padding_free = False` on the config."+ )+trainer_args = getattr(self, "args", None)trainer_packing = bool(trainer_args and getattr(trainer_args, "packing", False))trainer_padding_free = bool(trainer_args and getattr(trainer_args, "padding_free", False))