Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Hand a composition with no forward of its own to its thinker (Qwen3-Omni) (#11523)

Daniel Han3395708dbd15
diff --git a/tests/test_omni_text_only_load.py b/tests/test_omni_text_only_load.py
new file mode 100644
index 00000000000..0dd16cf98ce
--- /dev/null
+++ b/tests/test_omni_text_only_load.py
@@ -0,0 +1,603 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""Qwen3-Omni loads with text_only = True, and a kept wrapper still takes a text forward.
+
+Qwen3OmniMoeConfig has no top-level vision_config (it lives under thinker_config) and no
+causal-LM class, so text_only = True used to send it to AutoModelForCausalLM, which raised
+"Unrecognized configuration class". A default load kept the wrapper, whose missing forward
+made model(input_ids = ...) reach nn.Module.forward.
+"""
+
+import pytest
+import torch
+from real_accelerator import has_real_cuda # tests/_shared, on sys.path via tests/conftest.py
+
+transformers = pytest.importorskip("transformers")
+omni_config = pytest.importorskip(
+ "transformers.models.qwen3_omni_moe.configuration_qwen3_omni_moe",
+ reason = "this transformers has no qwen3_omni_moe",
+)
+omni_modeling = pytest.importorskip("transformers.models.qwen3_omni_moe.modeling_qwen3_omni_moe")
+
+VOCAB = 64
+# Unsloth's patched kernels are Triton on a GPU host, so the tiny model runs where they do.
+DEVICE = "cuda" if has_real_cuda() else "cpu"
+
+
+def _tiny_config():
+ text = dict(
+ vocab_size = VOCAB,
+ hidden_size = 16,
+ intermediate_size = 32,
+ moe_intermediate_size = 8,
+ shared_expert_intermediate_size = 8,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 8,
+ num_experts = 2,
+ num_experts_per_tok = 1,
+ max_position_embeddings = 64,
+ rope_scaling = {"rope_type": "default", "mrope_section": [1, 1, 2], "interleaved": True},
+ )
+ config = omni_config.Qwen3OmniMoeConfig(
+ enable_audio_output = True,
+ thinker_config = dict(
+ text_config = text,
+ # Special ids above anything the tests feed in, so a text prompt stays text.
+ audio_start_token_id = 56,
+ audio_end_token_id = 57,
+ audio_token_id = 58,
+ image_token_id = 59,
+ video_token_id = 60,
+ vision_start_token_id = 61,
+ vision_end_token_id = 62,
+ audio_config = dict(
+ num_mel_bins = 8,
+ encoder_layers = 1,
+ encoder_attention_heads = 2,
+ encoder_ffn_dim = 16,
+ d_model = 16,
+ output_dim = 16,
+ n_window = 2,
+ n_window_infer = 4,
+ downsample_hidden_size = 8,
+ conv_chunksize = 10,
+ max_source_positions = 16,
+ ),
+ vision_config = dict(
+ depth = 1,
+ hidden_size = 16,
+ intermediate_size = 16,
+ num_heads = 2,
+ out_hidden_size = 16,
+ patch_size = 2,
+ spatial_merge_size = 1,
+ temporal_patch_size = 1,
+ deepstack_visual_indexes = [0],
+ ),
+ ),
+ talker_config = dict(
+ text_config = text,
+ code_predictor_config = dict(
+ vocab_size = 16,
+ hidden_size = 16,
+ intermediate_size = 16,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ head_dim = 8,
+ num_code_groups = 2,
+ ),
+ thinker_hidden_size = 16,
+ num_code_groups = 2,
+ spatial_merge_size = 1,
+ ),
+ code2wav_config = dict(
+ hidden_size = 16,
+ num_hidden_layers = 1,
+ num_attention_heads = 2,
+ num_key_value_heads = 1,
+ intermediate_size = 16,
+ codebook_size = 16,
+ decoder_dim = 16,
+ num_quantizers = 2,
+ upsample_rates = (2,),
+ upsampling_ratios = (2,),
+ head_dim = 8,
+ sliding_window = 8,
+ ),
+ )
+ config.architectures = ["Qwen3OmniMoeForConditionalGeneration"]
+ # transformers 5.4's top-level config does not declare the field its _init_weights reads.
+ if getattr(config, "initializer_range", None) is None:
+ config.initializer_range = 0.02
+ return config
+
+
+def _tiny_omni():
+ torch.manual_seed(0)
+ try:
+ return omni_modeling.Qwen3OmniMoeForConditionalGeneration(_tiny_config()).to(DEVICE).eval()
+ except Exception as error: # a transformers whose Omni config takes other names
+ pytest.skip(f"cannot build a tiny Qwen3-Omni here: {error}")
+
+
+def _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only):
+ from unsloth import FastModel
+ from unsloth.models.vision import FastBaseModel
+
+ _tiny_config().save_pretrained(tmp_path)
+ seen = {}
+
+ def capture(*args, **kwargs):
+ seen.update(kwargs)
+ raise RuntimeError("captured")
+
+ monkeypatch.setattr(FastBaseModel, "from_pretrained", staticmethod(capture))
+ try:
+ FastModel.from_pretrained(str(tmp_path), text_only = text_only, load_in_4bit = False)
+ except RuntimeError as error:
+ if str(error) != "captured":
+ if not has_real_cuda():
+ pytest.skip(f"FastModel.from_pretrained needs a GPU here: {error}")
+ raise
+ else:
+ pytest.fail("FastModel.from_pretrained did not reach FastBaseModel.from_pretrained")
+ return seen
+
+
+def test_text_only_loads_qwen3_omni_through_its_own_auto_class(monkeypatch, tmp_path):
+ from transformers import AutoModelForCausalLM
+ from unsloth.models._utils import resolve_model_class
+
+ seen = _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only = True)
+ auto_model = seen["auto_model"]
+ assert auto_model is not AutoModelForCausalLM
+ assert resolve_model_class(auto_model, _tiny_config()) is not None
+ # The full composition loads; the caller's text intent hands it to the thinker.
+ assert seen["text_only"] is False
+ assert seen["text_intent"] is True
+ assert seen["text_only_decoder"] is False
+
+
+def test_the_default_load_resolves_the_same_class(monkeypatch, tmp_path):
+ default = _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only = False)
+ text = _capture_fast_base_kwargs(monkeypatch, tmp_path, text_only = True)
+ assert default["auto_model"] is text["auto_model"]
+ assert default["text_intent"] is False
+
+
+def test_text_intent_hands_qwen3_omni_to_its_thinker(monkeypatch):
+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
+ from unsloth.models.vision import _text_trainable_core
+
+ model = _tiny_omni()
+ core = _text_trainable_core(model, text_intent = True)
+ assert type(core).__name__ == "Qwen3OmniMoeThinkerForConditionalGeneration"
+ # generate and save read the architecture off the core; a sub-config names none.
+ assert core.config.architectures == ["Qwen3OmniMoeThinkerForConditionalGeneration"]
+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)
+ out = core(input_ids = ids, labels = ids)
+ assert out.logits.shape == (1, 4, VOCAB)
+ assert torch.isfinite(out.loss)
+ generated = core.generate(input_ids = ids, max_new_tokens = 2, do_sample = False)
+ assert generated.shape == (1, 6)
+
+
+def test_a_kept_qwen3_omni_wrapper_forwards_through_its_thinker(capsys, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
+ from unsloth.models.vision import _text_trainable_core
+
+ model = _tiny_omni()
+ kept = _text_trainable_core(model, text_intent = False)
+ assert kept is model
+ # Nothing dropped: the talker still generates audio.
+ assert hasattr(model, "talker") and hasattr(model, "code2wav")
+ assert "text_only = True" in capsys.readouterr().out
+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)
+ with torch.no_grad():
+ out = model(input_ids = ids, labels = ids)
+ reference = model.thinker(input_ids = ids, labels = ids)
+ assert torch.equal(out.logits, reference.logits)
+ assert torch.isfinite(out.loss)
+ assert model.get_output_embeddings() is model.thinker.lm_head
+ assert model.get_input_embeddings() is model.thinker.get_input_embeddings()
+ text = model.generate(
+ input_ids = ids, return_audio = False, thinker_max_new_tokens = 2, do_sample = False
+ )
+ if isinstance(text, tuple): # transformers 4.x returns (sequences, None) without audio
+ text = text[0]
+ assert text.shape == (1, 6)
+ # deepcopy (and so a reference model for KD or DPO) follows the copy's own thinker.
+ import copy
+
+ clone = copy.deepcopy(model)
+ assert clone.get_output_embeddings() is clone.thinker.lm_head
+ assert clone.forward.__self__ is clone.thinker
+ # Only this instance changed; the class still has no forward.
+ assert type(model).forward is torch.nn.Module.forward
+ fresh = _tiny_omni()
+ with pytest.raises(TypeError, match = "input_ids"):
+ fresh(input_ids = ids)
+
+
+def test_a_kept_wrapper_trains_through_peft():
+ peft = pytest.importorskip("peft")
+ from unsloth.models.vision import _text_trainable_core
+
+ model = _text_trainable_core(_tiny_omni(), text_intent = False)
+ # Reentrant checkpointing with frozen embeddings trains LoRA only when the embedding
+ # output requires grad, which goes through the wrapper's get_input_embeddings.
+ model.gradient_checkpointing_enable(gradient_checkpointing_kwargs = {"use_reentrant": True})
+ model.enable_input_require_grads()
+ config = peft.LoraConfig(
+ r = 2, target_modules = r"thinker\.model\.layers\.\d+\.self_attn\.q_proj", lora_alpha = 4
+ )
+ peft_model = peft.get_peft_model(model, config)
+ peft_model.train()
+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)
+ for name, parameter in peft_model.named_parameters():
+ if "lora_B" in name:
+ torch.nn.init.normal_(parameter, std = 0.1)
+ loss = peft_model(input_ids = ids, labels = ids).loss
+ loss.backward()
+ grads = [p.grad for n, p in peft_model.named_parameters() if "lora_A" in n]
+ assert grads and all(g is not None and g.abs().sum() > 0 for g in grads)
+ assert peft_model.get_output_embeddings() is model.thinker.lm_head
+
+
+def test_the_off_switch_leaves_the_wrapper_without_a_forward(monkeypatch):
+ from unsloth.models.vision import _text_trainable_core
+
+ monkeypatch.setenv("UNSLOTH_KEEP_COMPOSED_WRAPPER", "1")
+ model = _tiny_omni()
+ assert _text_trainable_core(model, text_intent = False) is model
+ assert "forward" not in vars(model)
+
+
+def test_a_kept_wrapper_scopes_lora_to_the_thinker_decoder():
+ """get_peft_model's regex must reach thinker.model.layers, not only the vision tower."""
+ import inspect
+ import re
+ from unsloth.models.vision import _text_core_decoder_prefix, _text_trainable_core
+ from unsloth_zoo.peft_utils import get_peft_regex
+
+ if "language_tags" not in inspect.signature(get_peft_regex).parameters:
+ pytest.skip("this unsloth_zoo's get_peft_regex takes no language_tags")
+ model = _text_trainable_core(_tiny_omni(), text_intent = False)
+ assert _text_core_decoder_prefix(model) == "thinker.model"
+ default_tags = inspect.signature(get_peft_regex).parameters["language_tags"].default
+ regex = get_peft_regex(model, language_tags = list(default_tags) + [re.escape("thinker.model")])
+ targets = [name for name, _ in model.named_modules() if re.fullmatch(regex, name)]
+ assert any(name.startswith("thinker.model.layers.") for name in targets)
+ assert not any(
+ name.startswith(("talker.", "code2wav.", "thinker.audio_tower.")) for name in targets
+ )
+ # Without the core's prefix only the vision tower matched, so a text batch trained nothing.
+ before = get_peft_regex(model)
+ assert not any(
+ name.startswith("thinker.model.layers.")
+ for name, _ in model.named_modules()
+ if re.fullmatch(before, name)
+ )
+ assert _text_core_decoder_prefix(_tiny_omni()) is None
+
+
+def test_get_peft_model_passes_the_thinker_decoder_as_a_language_tag(monkeypatch):
+ import inspect
+ from unsloth.models import vision
+
+ if "language_tags" not in inspect.signature(vision.get_peft_regex).parameters:
+ pytest.skip("this unsloth_zoo's get_peft_regex takes no language_tags")
+ import functools
+
+ seen = {}
+
+ # functools.wraps keeps the real signature, which get_peft_model inspects.
+ @functools.wraps(vision.get_peft_regex)
+ def capture(model, **kwargs):
+ seen.update(kwargs)
+ raise RuntimeError("captured")
+
+ monkeypatch.setattr(vision, "get_peft_regex", capture)
+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)
+ with pytest.raises(RuntimeError, match = "captured"):
+ vision.FastBaseModel.get_peft_model(model, r = 2)
+ assert "thinker\\.model" in seen["language_tags"]
+ seen.clear()
+ thinker = vision._text_trainable_core(_tiny_omni(), text_intent = True)
+ with pytest.raises(RuntimeError, match = "captured"):
+ vision.FastBaseModel.get_peft_model(thinker, r = 2)
+ assert "language_tags" not in seen
+
+
+def test_a_kept_wrapper_with_an_accelerate_hook_forwards_through_the_hook(monkeypatch):
+ """A device-mapped load wraps forward on the instance; the hook must reach the thinker."""
+ hooks = pytest.importorskip("accelerate.hooks")
+ from unsloth.models.vision import _text_trainable_core
+
+ monkeypatch.setenv("UNSLOTH_RETURN_LOGITS", "1")
+ calls = []
+
+ class Counting(hooks.ModelHook):
+ def pre_forward(self, module, *args, **kwargs):
+ calls.append(type(module).__name__)
+ return args, kwargs
+
+ model = _tiny_omni()
+ hooks.add_hook_to_module(model, Counting())
+ model = _text_trainable_core(model, text_intent = False)
+ ids = torch.tensor([[1, 2, 3, 4]], device = DEVICE)
+ with torch.no_grad():
+ out = model(input_ids = ids, labels = ids)
+ assert torch.isfinite(out.loss)
+ assert calls == ["Qwen3OmniMoeForConditionalGeneration"]
+
+
+def _capture_adapter_reload(monkeypatch, tmp_path, target_modules):
+ peft = pytest.importorskip("peft")
+ from unsloth import FastModel
+ from unsloth.models.vision import FastBaseModel
+
+ base = tmp_path / "base"
+ adapter = tmp_path / "adapter"
+ _tiny_config().save_pretrained(base)
+ peft.LoraConfig(
+ r = 2, lora_alpha = 2, target_modules = target_modules, base_model_name_or_path = str(base)
+ ).save_pretrained(adapter)
+ seen = {}
+
+ def capture(*args, **kwargs):
+ seen.update(kwargs)
+ raise RuntimeError("captured")
+
+ monkeypatch.setattr(FastBaseModel, "from_pretrained", staticmethod(capture))
+ try:
+ FastModel.from_pretrained(str(adapter), load_in_4bit = False)
+ except RuntimeError as error:
+ if str(error) != "captured":
+ if not has_real_cuda():
+ pytest.skip(f"FastModel.from_pretrained needs a GPU here: {error}")
+ raise
+ return seen
+
+
+def test_an_adapter_trained_on_the_thinker_reloads_onto_the_thinker(monkeypatch, tmp_path):
+ # A text_only adapter's regex is rooted at model.layers; the composition names them
+ # thinker.model.layers, so a default reload could not find its targets.
+ seen = _capture_adapter_reload(
+ monkeypatch, tmp_path, r"(?:\bmodel\.layers\.[\d]{1,}\.(?:self_attn)\.(?:q_proj))"
+ )
+ assert seen["text_intent"] is True
+
+
+def test_an_adapter_trained_on_the_kept_wrapper_keeps_the_wrapper(monkeypatch, tmp_path):
+ seen = _capture_adapter_reload(
+ monkeypatch, tmp_path, r"(?:.*?(?:thinker\.model).*?(?:self_attn).*?(?:q_proj))"
+ )
+ assert seen["text_intent"] is False
+
+
+def _tiny_omni_checkpoint(path):
+ tokenizers = pytest.importorskip("tokenizers")
+ _tiny_omni().save_pretrained(path)
+ vocab = {f"w{i}": i for i in range(48)}
+ vocab.update({"<unk>": 48, "<pad>": 49, "<eos>": 50})
+ backend = tokenizers.Tokenizer(tokenizers.models.WordLevel(vocab, unk_token = "<unk>"))
+ backend.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()
+ transformers.PreTrainedTokenizerFast(
+ tokenizer_object = backend, unk_token = "<unk>", pad_token = "<pad>", eos_token = "<eos>"
+ ).save_pretrained(path)
+
+
[email protected]("text_only", [True, False], ids = ["thinker", "kept_wrapper"])
+ "target_modules", [None, ["q_proj", "v_proj"]], ids = ["unsloth_regex", "leaf_list"]
+)
+def test_an_omni_adapter_reloads_onto_the_model_it_was_trained_on(
+ tmp_path, text_only, target_modules
+):
+ # A leaf-name list reloads as a set that matches either layout, so the saved weight
+ # keys (model.layers vs thinker.model.layers) are what decide.
+ if not has_real_cuda():
+ pytest.skip("FastModel needs a GPU")
+ from unsloth import FastModel
+
+ base = tmp_path / "base"
+ _tiny_omni_checkpoint(base)
+ model, _ = FastModel.from_pretrained(str(base), text_only = text_only, load_in_4bit = False)
+ kwargs = {} if target_modules is None else {"target_modules": target_modules}
+ model = FastModel.get_peft_model(model, r = 2, lora_alpha = 2, random_state = 0, **kwargs)
+ trained_on = type(model.get_base_model()).__name__
+ with torch.no_grad():
+ for name, param in model.named_parameters():
+ if "lora_B" in name:
+ param.normal_(0, 0.1)
+ saved = {
+ n: p.detach().float().cpu().clone() for n, p in model.named_parameters() if "lora_" in n
+ }
+ assert saved
+ model.save_pretrained(tmp_path / "adapter")
+ del model
+ reloaded, _ = FastModel.from_pretrained(str(tmp_path / "adapter"), load_in_4bit = False)
+ assert type(reloaded.get_base_model()).__name__ == trained_on
+ loaded = {n: p.detach().float().cpu() for n, p in reloaded.named_parameters() if "lora_" in n}
+ assert loaded.keys() == saved.keys()
+ for name, value in saved.items():
+ assert torch.equal(loaded[name], value), name
+
+
+def test_saved_weight_keys_decide_over_the_target_regex():
+ from unsloth.models.loader import _adapter_targets_text_core
+
+ config = type("Config", (), {"target_modules": {"q_proj", "v_proj"}})()
+ thinker_keys = ["base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight"]
+ wrapper_keys = ["base_model.model.thinker.model.layers.0.self_attn.q_proj.lora_A.weight"]
+ assert _adapter_targets_text_core(config, thinker_keys) is True
+ assert _adapter_targets_text_core(config, wrapper_keys) is False
+ # Without keys a leaf list cannot tell the layouts apart, so the composition is kept.
+ assert _adapter_targets_text_core(config, None) is False
+ regex = type("Config", (), {"target_modules": r".*\.q_proj"})()
+ assert _adapter_targets_text_core(regex, wrapper_keys) is False
+
+
[email protected]("saved_as", ["adapter_model.safetensors", "adapter_model.bin"])
+def test_hub_adapter_keys_are_read_from_the_saved_file(monkeypatch, tmp_path, saved_as):
+ # Resolved through hf_hub_download (cache first, offline honoured) in either format.
+ huggingface_hub = pytest.importorskip("huggingface_hub")
+ safetensors_torch = pytest.importorskip("safetensors.torch")
+ from unsloth.models.loader import _adapter_weight_keys
+
+ key = "base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight"
+ path = tmp_path / saved_as
+ if saved_as.endswith(".safetensors"):
+ safetensors_torch.save_file({key: torch.ones(2, 2)}, str(path))
+ else:
+ torch.save({key: torch.ones(2, 2)}, path)
+ asked = []
+
+ def fake_download(repo_id, filename, **kwargs):
+ asked.append((repo_id, filename, kwargs.get("local_files_only"), kwargs.get("revision")))
+ if filename != saved_as:
+ raise FileNotFoundError(filename)
+ return str(path)
+
+ monkeypatch.setattr(huggingface_hub, "hf_hub_download", fake_download)
+ assert _adapter_weight_keys("someone/omni-adapter", revision = "r", local_files_only = True) == [
+ key
+ ]
+ assert asked[0] == ("someone/omni-adapter", "adapter_model.safetensors", True, "r")
+ assert asked[-1][1] == saved_as
+
+
+def test_local_bin_adapter_keys_are_read_without_weights(tmp_path):
+ from unsloth.models.loader import _adapter_weight_keys
+
+ key = "base_model.model.thinker.model.layers.0.q_proj.lora_A.weight"
+ torch.save({key: torch.ones(2, 2)}, tmp_path / "adapter_model.bin")
+ assert _adapter_weight_keys(str(tmp_path)) == [key]
+ assert _adapter_weight_keys(str(tmp_path / "missing")) is None
+
+
+def test_an_audio_only_wrapper_adapter_stays_on_the_wrapper():
+ # Trained on the kept wrapper's talker only: no thinker. key, but still wrapper-rooted.
+ from unsloth.models.loader import _adapter_targets_text_core, _composition_children
+
+ children = _composition_children(_tiny_config())
+ assert set(children) == {"thinker", "talker", "code2wav"}
+ config = type("Config", (), {"target_modules": {"q_proj"}})()
+ talker = ["base_model.model.talker.model.layers.0.self_attn.q_proj.lora_A.weight"]
+ code2wav = ["base_model.model.code2wav.decoder.0.conv.lora_A.weight"]
+ thinker_only = ["base_model.model.model.layers.0.self_attn.q_proj.lora_A.weight"]
+ assert _adapter_targets_text_core(config, talker, children) is False
+ assert _adapter_targets_text_core(config, code2wav, children) is False
+ assert _adapter_targets_text_core(config, thinker_only, children) is True
+ regex = type("Config", (), {"target_modules": r"(?:.*?(?:talker).*?(?:q_proj))"})()
+ assert _adapter_targets_text_core(regex, None, children) is False
+
+
+def test_only_a_forwardless_composition_redirects_adapter_reloads():
+ """Every VLM maps to an image-text auto class; routing their adapters to text_only dropped the vision tower."""
+ from transformers import Gemma3Config, Qwen2_5_VLConfig
+ from unsloth.models.loader import _is_forwardless_composition
+
+ assert _is_forwardless_composition(_tiny_config()) is True
+ assert _is_forwardless_composition(Gemma3Config()) is False
+ assert _is_forwardless_composition(Qwen2_5_VLConfig()) is False
+
+
+def test_a_kept_wrapper_freezes_the_siblings_its_forward_never_uses():
+ """DDP (find_unused_parameters=False) fails on trainable talker / code2wav weights with no gradient."""
+ from unsloth.models.vision import _text_trainable_core
+
+ model = _text_trainable_core(_tiny_omni(), text_intent = False)
+ assert all(not p.requires_grad for p in model.talker.parameters())
+ assert all(not p.requires_grad for p in model.code2wav.parameters())
+ assert any(p.requires_grad for p in model.thinker.parameters())
+
+
+def test_a_kept_wrapper_resolves_its_layer_count_through_the_thinker():
+ from unsloth.models.vision import _get_total_transformer_layers, _text_trainable_core
+
+ model = _text_trainable_core(_tiny_omni(), text_intent = False)
+ assert _get_total_transformer_layers(model) is None
+ assert _get_total_transformer_layers(model.thinker) == model.thinker.config.text_config.num_hidden_layers
+
+
+def test_an_explicit_leaf_list_on_a_kept_wrapper_is_scoped_to_the_thinker(monkeypatch):
+ """PEFT suffix-matches ["q_proj", "v_proj"] in the talker too; those adapters never train."""
+ import functools
+ from unsloth.models import vision
+
+ seen = {}
+
+ @functools.wraps(vision.get_peft_regex)
+ def capture(model, **kwargs):
+ seen.update(kwargs)
+ raise RuntimeError("captured")
+
+ monkeypatch.setattr(vision, "get_peft_regex", capture)
+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)
+ with pytest.raises(RuntimeError, match = "captured"):
+ vision.FastBaseModel.get_peft_model(model, r = 2, target_modules = ["q_proj", "v_proj"])
+ assert seen["target_modules"] == ["q_proj", "v_proj"]
+
+
+def test_a_kept_wrapper_resizes_its_vocabulary_through_the_thinker():
+ from unsloth.models.vision import _text_trainable_core
+
+ model = _text_trainable_core(_tiny_omni(), text_intent = False)
+ old = model.get_input_embeddings().weight.shape[0]
+ model.resize_token_embeddings(old + 8)
+ assert model.thinker.get_input_embeddings().weight.shape[0] == old + 8
+ assert model.thinker.lm_head.weight.shape[0] == old + 8
+
+
+def _kept_wrapper_peft(monkeypatch, **peft_kwargs):
+ from unsloth.models import vision
+
+ monkeypatch.setattr(vision.FastBaseModel, "post_patch_model", staticmethod(lambda model, *a, **k: model))
+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)
+ model.max_seq_length = 64
+ return vision.FastBaseModel.get_peft_model(model, r = 2, **peft_kwargs)
+
+
+def test_a_qualified_target_on_a_kept_wrapper_reaches_peft_unchanged(monkeypatch):
+ """get_peft_regex treats entries as leaves, so a full path raised "No layers to finetune"."""
+ target = "thinker.model.layers.0.self_attn.q_proj"
+ peft_model = _kept_wrapper_peft(monkeypatch, target_modules = [target])
+ trained = {n for n, p in peft_model.named_parameters() if p.requires_grad}
+ assert trained and all(target in n for n in trained)
+
+
+def test_saved_modules_on_a_kept_wrapper_skip_the_talker(monkeypatch):
+ """An unqualified "lm_head" also suffix-matched talker.code_predictor.lm_head, a ModuleList PEFT refuses."""
+ peft_model = _kept_wrapper_peft(monkeypatch, target_modules = ["q_proj"], modules_to_save = ["lm_head"])
+ trained = [n for n, p in peft_model.named_parameters() if p.requires_grad]
+ assert any("thinker.lm_head" in n for n in trained)
+ assert not any(".talker." in n for n in trained)
+
+
+def test_full_finetuning_keeps_a_kept_wrappers_siblings_frozen(monkeypatch):
+ """prepare_model_for_training(full_finetuning = True) turned every parameter back on."""
+ from unsloth.models import vision
+
+ monkeypatch.setenv("UNSLOTH_ENABLE_FULL_FINETUNING", "1")
+ model = vision._text_trainable_core(_tiny_omni(), text_intent = False)
+ try:
+ vision.FastBaseModel.post_patch_model(model, use_gradient_checkpointing = False)
+ except RuntimeError as error:
+ if "inner_training_loop" not in str(error):
+ raise
+ assert any(p.requires_grad for p in model.thinker.parameters())
+ assert not any(p.requires_grad for p in model.talker.parameters())
+
+
+def test_suffix_target_parameters_on_a_kept_wrapper_skip_the_talker(monkeypatch):
+ """PEFT suffix-matches target_parameters; bare expert names also adapted the talker's experts."""
+ peft_model = _kept_wrapper_peft(
+ monkeypatch, target_modules = ["q_proj"], target_parameters = ["mlp.experts.gate_up_proj"]
+ )
+ trained = [n for n, p in peft_model.named_parameters() if p.requires_grad]
+ assert any("thinker." in n and "experts" in n for n in trained)
+ assert not any(".talker." in n for n in trained)
diff --git a/tests/test_omni_trains_through_thinker.py b/tests/test_omni_trains_through_thinker.py
new file mode 100644
index 00000000000..dcf3a65295d
--- /dev/null
+++ b/tests/test_omni_trains_through_thinker.py
@@ -0,0 +1,192 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+"""A composition with no forward of its own (Qwen3-Omni) trains through its thinker."""
+
+import pytest
+import torch
+
+transformers = pytest.importorskip("transformers")
+from transformers import PreTrainedModel, PretrainedConfig
+
+
+class TinyConfig(PretrainedConfig):
+ model_type = "tiny_composed"
+
+ def __init__(
+ self,
+ vocab_size = 16,
+ hidden_size = 8,
+ **kwargs,
+ ):
+ self.vocab_size = vocab_size
+ self.hidden_size = hidden_size
+ super().__init__(**kwargs)
+
+
+class Thinker(PreTrainedModel):
+ config_class = TinyConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.embed_tokens = torch.nn.Embedding(config.vocab_size, config.hidden_size)
+ self.lm_head = torch.nn.Linear(config.hidden_size, config.vocab_size, bias = False)
+
+ def get_input_embeddings(self):
+ return self.embed_tokens
+
+ def get_output_embeddings(self):
+ return self.lm_head
+
+ def forward(
+ self,
+ input_ids = None,
+ **kwargs,
+ ):
+ return self.lm_head(self.embed_tokens(input_ids))
+
+
+class Talker(PreTrainedModel):
+ config_class = TinyConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.proj = torch.nn.Linear(config.hidden_size, config.hidden_size)
+
+ def forward(
+ self,
+ hidden_states = None,
+ **kwargs,
+ ):
+ return self.proj(hidden_states)
+
+
+class Composed(PreTrainedModel):
+ """No forward of its own, like Qwen3OmniMoeForConditionalGeneration."""
+
+ config_class = TinyConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.thinker = Thinker(config)
+ self.talker = Talker(config)
+ self.code2wav = torch.nn.Linear(config.hidden_size, 1)
+
+
+def test_the_defect_before_the_fix():
+ """The arm that fails on main."""
+ model = Composed(TinyConfig())
+ with pytest.raises(TypeError, match = "unexpected keyword argument 'input_ids'"):
+ model(input_ids = torch.tensor([[1, 2]]))
+
+
+def test_composition_without_forward_hands_off_to_the_thinker():
+ from unsloth.models.vision import _text_trainable_core
+
+ model = Composed(TinyConfig())
+ core = _text_trainable_core(model)
+ assert isinstance(core, Thinker)
+ assert core._unsloth_composed_parent == "Composed"
+ assert not hasattr(model, "talker") and not hasattr(model, "code2wav")
+ logits = core(input_ids = torch.tensor([[1, 2, 3]]))
+ assert logits.shape == (1, 3, 16)
+
+
+def test_a_model_with_a_text_forward_is_returned_unchanged():
+ from unsloth.models.vision import _text_trainable_core
+ model = Thinker(TinyConfig())
+ assert _text_trainable_core(model) is model
+
+
+def test_ambiguous_compositions_are_left_alone():
+ from unsloth.models.vision import _text_trainable_core
+
+ class TwoCores(PreTrainedModel):
+ config_class = TinyConfig
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.encoder = Thinker(config)
+ self.decoder = Thinker(config)
+
+ model = TwoCores(TinyConfig())
+ assert _text_trainable_core(model) is model
+ assert hasattr(model, "encoder") and hasattr(model, "decoder")
+
+
+def test_the_off_switch_keeps_the_wrapper(monkeypatch):
+ from unsloth.models.vision import _text_trainable_core
+
+ monkeypatch.setenv("UNSLOTH_KEEP_COMPOSED_WRAPPER", "1")
+ model = Composed(TinyConfig())
+ assert _text_trainable_core(model) is model
+
+
+def test_a_multimodal_load_keeps_the_composition_for_generation(capsys):
+ """text_intent = False keeps the composition whole and prints the text_only hint."""
+ from unsloth.models.vision import _text_trainable_core
+
+ model = Composed(TinyConfig())
+ assert _text_trainable_core(model, text_intent = False) is model
+ assert hasattr(model, "talker")
+ assert "text_only = True" in capsys.readouterr().out
+
+
+def test_the_thinker_carries_the_checkpoint_identity():
+ """PEFT writes name_or_path into the adapter's base_model_name_or_path; a thinker built
+ from thinker_config has none of its own, so it takes the wrapper's."""
+ from types import SimpleNamespace
+ from unsloth.models.vision import _carry_loader_state_to_core
+
+ core = torch.nn.Module()
+ core.config = SimpleNamespace(_name_or_path = "")
+ core.name_or_path = ""
+ model = torch.nn.Module()
+ model.config = SimpleNamespace(_name_or_path = "Qwen/Qwen3-Omni-30B-A3B-Instruct")
+ model.name_or_path = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+ model.thinker = core
+ _carry_loader_state_to_core(model, core, "thinker")
+ assert core.name_or_path == "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+ assert core.config._name_or_path == "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+
+
+def test_a_thinker_whose_output_accessor_says_none_is_still_found_by_its_lm_head():
+ """transformers 4.x returns None from get_output_embeddings unless a class overrides it;
+ Qwen3-Omni's thinker does not, but owns an lm_head."""
+ from transformers import PreTrainedModel, PretrainedConfig
+ from unsloth.models.vision import _text_trainable_core
+
+ class Cfg(PretrainedConfig):
+ model_type = "thinker_for_test"
+
+ class Thinker(PreTrainedModel):
+ config_class = Cfg
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.embed = torch.nn.Embedding(8, 4)
+ self.lm_head = torch.nn.Linear(4, 8)
+
+ def get_input_embeddings(self):
+ return self.embed
+
+ def get_output_embeddings(self):
+ return None
+
+ def forward(
+ self,
+ input_ids = None,
+ labels = None,
+ **kwargs,
+ ):
+ return self.lm_head(self.embed(input_ids))
+
+ class Wrapper(PreTrainedModel):
+ config_class = Cfg
+
+ def __init__(self, config):
+ super().__init__(config)
+ self.thinker = Thinker(config)
+ self.talker = torch.nn.Linear(2, 2)
+
+ wrapper = Wrapper(Cfg())
+ core = _text_trainable_core(wrapper, text_intent = True)
+ assert isinstance(core, Thinker)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 0fce2d6152e..8d8c8e2d59e 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -311,6 +311,97 @@ _OMNI_AUTO_CLASS_NAMES = (
)
+def _adapter_file_keys(path):
+ if path.endswith(".safetensors"):
+ from safetensors import safe_open
+ with safe_open(path, framework = "pt") as handle:
+ return list(handle.keys())
+ return list(torch.load(path, map_location = "meta", weights_only = True).keys())
+
+
+def _adapter_weight_keys(
+ adapter_name,
+ token = None,
+ revision = None,
+ local_files_only = False,
+ cache_dir = None,
+):
+ """Tensor names of a saved adapter without loading weights, or None."""
+ filenames = ("adapter_model.safetensors", "adapter_model.bin")
+ try:
+ local = os.path.expanduser(adapter_name)
+ if os.path.isdir(local):
+ for filename in filenames:
+ path = os.path.join(local, filename)
+ if os.path.exists(path):
+ return _adapter_file_keys(path)
+ return None
+ from huggingface_hub import hf_hub_download
+
+ for filename in filenames:
+ try:
+ path = hf_hub_download(
+ adapter_name,
+ filename,
+ revision = revision,
+ token = token,
+ cache_dir = cache_dir,
+ local_files_only = local_files_only,
+ )
+ except Exception:
+ continue
+ return _adapter_file_keys(path)
+ except Exception:
+ pass
+ return None
+
+
+def _composition_children(model_config):
+ names = [
+ name[: -len("_config")]
+ for name in (getattr(model_config, "sub_configs", None) or {})
+ if name.endswith("_config")
+ ]
+ return tuple(names) or ("thinker",)
+
+
+def _adapter_targets_text_core(
+ peft_config,
+ weight_keys = None,
+ wrapper_children = ("thinker",),
+):
+ """True when an adapter was trained on an extracted thinker (`text_only = True`).
+
+ Saved keys decide (wrapper-trained keys are rooted at a wrapper child); else the saved
+ regex. A leaf-name list matches either layout, so it cannot decide alone.
+ """
+ children = set(wrapper_children)
+ if weight_keys:
+ roots = {
+ (key[len("base_model.model.") :] if key.startswith("base_model.model.") else key).split(
+ ".", 1
+ )[0]
+ for key in weight_keys
+ }
+ return not (roots & children)
+ targets = getattr(peft_config, "target_modules", None)
+ return isinstance(targets, str) and not any(child in targets for child in children)
+
+
+def _is_forwardless_composition(model_config, trust_remote_code = None, **hub_kwargs):
+ # Only a wrapper with no forward (Qwen3-Omni) can have a thinker-trained adapter; VLM adapters never.
+ auto_class = _resolve_omni_auto_model(model_config, trust_remote_code = trust_remote_code, **hub_kwargs)
+ model_class = (
+ resolve_model_class(auto_class, model_config, trust_remote_code = trust_remote_code, **hub_kwargs)
+ if auto_class is not None
+ else None
+ )
+ if model_class is None:
+ return False
+ forward = getattr(model_class, "forward", None)
+ return forward is None or forward is torch.nn.Module.forward
+
+
def _config_has_native_class(auto_class, config):
"""True when transformers itself maps ``type(config)`` in ``auto_class`` (no repo code needed)."""
try:
@@ -1965,12 +2056,6 @@ class FastModel(FastBaseModel):
for _cfg_key, _cfg_val in task_config_attrs.items():
set_task_config_attr(model_config, _cfg_key, _cfg_val)
- architectures = getattr(model_config, "architectures", None)
- if architectures is None:
- architectures = []
- is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)
- is_vlm = is_vlm or hasattr(model_config, "vision_config")
- load_text_only = text_only and auto_model is None
# Class probes below fetch remote modeling code exactly as the load will.
_probe_hub_kwargs = dict(
trust_remote_code = trust_remote_code,
@@ -1982,6 +2067,40 @@ class FastModel(FastBaseModel):
force_download = kwargs.get("force_download", None),
proxies = kwargs.get("proxies", None),
)
+ architectures = getattr(model_config, "architectures", None)
+ if architectures is None:
+ architectures = []
+ is_vlm = any(x.endswith("ForConditionalGeneration") for x in architectures)
+ is_vlm = is_vlm or hasattr(model_config, "vision_config")
+ if (
+ is_peft
+ and not text_only
+ and auto_model is None
+ and _is_forwardless_composition(model_config, **_probe_hub_kwargs)
+ ):
+ _adapter_keys = _adapter_weight_keys(
+ old_model_name,
+ token = token,
+ revision = adapter_revision,
+ local_files_only = local_files_only,
+ cache_dir = kwargs.get("cache_dir"),
+ )
+ if _adapter_targets_text_core(
+ peft_config, _adapter_keys, _composition_children(model_config)
+ ):
+ print(
+ "Unsloth: this adapter was trained on the thinker alone (`text_only = True`), "
+ "so its base is loaded the same way."
+ )
+ text_only = True
+ elif not _adapter_keys and not isinstance(
+ getattr(peft_config, "target_modules", None), str
+ ):
+ print(
+ "Unsloth: could not read this adapter's weight names, so the full model is "
+ "loaded. If it was trained with `text_only = True`, pass `text_only = True` here too."
+ )
+ load_text_only = text_only and auto_model is None
text_only_decoder = False
if load_text_only:
if hasattr(model_config, "vision_config"):
@@ -2006,6 +2125,13 @@ class FastModel(FastBaseModel):
is_vlm = False
# model_config is no longer the repo's config, so anything rebuilding it from model_name (the device-map planner) sees a different model.
text_only_decoder = True
+ elif (
+ is_vlm
+ and resolve_model_class(AutoModelForCausalLM, model_config, **_probe_hub_kwargs) is None
+ and _resolve_omni_auto_model(model_config, **_probe_hub_kwargs) is not None
+ ):
+ # Qwen3-Omni has no causal-LM class; load the composition, text_intent picks the thinker.
+ load_text_only = False
else:
is_vlm = False
for _cfg_key, _cfg_val in task_config_attrs.items():
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 11bf9a9d097..0e13f5ad968 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -944,7 +944,7 @@ def unsloth_base_fast_generate(self, *args, **kwargs):
if do_bfloat16_mixed_precision:
dtype = torch.bfloat16
- # text_only configs have architectures=None.
+ # A core built from a sub-config (Qwen3-Omni's thinker) carries no architectures.
architectures = getattr(self.config, "architectures", None) or [type(self).__name__]
is_vlm = any(
x.endswith(("ForConditionalGeneration", "ForVisionText2Text")) for x in architectures
@@ -1364,27 +1364,25 @@ def _required_non_text_inputs(forward):
]
-def _text_trainable_core(model, text_intent = True):
- """Nemotron-3-Nano-Omni's forward requires `pixel_values`: with text_only, train the one text child instead."""
- if os.environ.get("UNSLOTH_KEEP_COMPOSED_WRAPPER", "0") == "1":
- return model
- forward = getattr(type(model), "forward", None)
- if forward is None or forward is torch.nn.Module.forward:
- return model
- required = _required_non_text_inputs(forward)
- if not required:
- return model
- if not text_intent:
- print(
- f"Unsloth: `{type(model).__name__}.forward` requires {', '.join(required)}, so it "
- "trains on multimodal batches only. Pass `text_only = True` to from_pretrained to "
- "train its language model on text batches instead."
- )
- return model
+def _output_embeddings_of(module):
+ # transformers 4.x returns None unless overridden; Qwen3-Omni's thinker does not override.
+ try:
+ output_embeddings = module.get_output_embeddings()
+ except Exception:
+ output_embeddings = None
+ if output_embeddings is None:
+ output_embeddings = getattr(module, "lm_head", None)
+ if not isinstance(output_embeddings, torch.nn.Module):
+ output_embeddings = None
+ return output_embeddings
+
+
+def _find_text_core(model):
+ """(name, child) of the one direct child a text batch can run through, else None."""
try:
from transformers import PreTrainedModel
except Exception:
- return model
+ return None
cores = []
for name, child in model.named_children():
if not isinstance(child, PreTrainedModel):
@@ -1397,7 +1395,7 @@ def _text_trainable_core(model, text_intent = True):
try:
has_embeddings = (
child.get_input_embeddings() is not None
- and child.get_output_embeddings() is not None
+ and _output_embeddings_of(child) is not None
)
except Exception:
has_embeddings = False
@@ -1407,16 +1405,180 @@ def _text_trainable_core(model, text_intent = True):
if len(preferred) == 1:
cores = preferred
if len(cores) != 1:
+ return None
+ return cores[0]
+
+
+def _text_core_output_embeddings(model):
+ return _output_embeddings_of(getattr(model, model._unsloth_text_core))
+
+
+def _delegate_text_forward(model, name, core):
+ """Instance-only (class and generate untouched): forward + embeddings from the core."""
+ if "_old_forward" in vars(model):
+ # accelerate's hook calls _old_forward.
+ model._old_forward = core.forward
+ else:
+ model.forward = core.forward
+ try:
+ wrapper_output = model.get_output_embeddings()
+ except Exception:
+ wrapper_output = None
+ if wrapper_output is None:
+ # Bound method, not a closure, so deepcopy and pickle follow it.
+ model.get_output_embeddings = types.MethodType(_text_core_output_embeddings, model)
+ # Gradient checkpointing needs this, else transformers 5 raises and LoRA gets no gradient.
+ try:
+ wrapper_input = model.get_input_embeddings()
+ except Exception:
+ wrapper_input = None
+ if wrapper_input is None:
+ model.get_input_embeddings = core.get_input_embeddings
+ # resize_token_embeddings pairs the getters with these setters.
+ model.set_input_embeddings = core.set_input_embeddings
+ if wrapper_output is None:
+ model.set_output_embeddings = core.set_output_embeddings
+ model._unsloth_text_core = name
+ _freeze_unused_siblings(model)
+ return model
+
+
+def _freeze_unused_siblings(model):
+ """Siblings never reach the loss: freeze them, else DDP full finetuning fails on unused parameters."""
+ name = getattr(model, "_unsloth_text_core", None)
+ core = getattr(model, name, None) if isinstance(name, str) else None
+ if core is None:
+ return
+ core_parameters = {id(parameter) for parameter in core.parameters()}
+ for child in model.children():
+ if child is core:
+ continue
+ for parameter in child.parameters():
+ if id(parameter) not in core_parameters:
+ parameter.requires_grad_(False)
+
+
+def _scope_parameter_targets_to_core(model, target_parameters):
+ """PEFT suffix-matches target_parameters too: on a kept wrapper "mlp.experts.gate_up_proj"
+ also hits the talker's experts, which never train."""
+ name = getattr(model, "_unsloth_text_core", None)
+ core = getattr(model, name, None) if isinstance(name, str) else None
+ if core is None or not target_parameters or isinstance(target_parameters, str):
+ return target_parameters
+ scoped = []
+ for entry in target_parameters:
+ if entry.startswith(name + "."):
+ scoped.append(entry)
+ continue
+ paths = [
+ f"{name}.{path}"
+ for path, _ in core.named_parameters()
+ if path == entry or path.endswith("." + entry)
+ ]
+ scoped.extend(paths or [entry])
+ return scoped
+
+
+def _scope_modules_to_save_to_core(model, modules_to_save):
+ """PEFT suffix-matches modules_to_save over the whole model: on a kept wrapper an unqualified
+ "lm_head" also hits the talker's code_predictor.lm_head, a ModuleList PEFT refuses."""
+ name = getattr(model, "_unsloth_text_core", None)
+ core = getattr(model, name, None) if isinstance(name, str) else None
+ if core is None or not modules_to_save:
+ return modules_to_save
+ scoped = []
+ for entry in modules_to_save:
+ paths = [
+ f"{name}.{child}"
+ for child, _ in core.named_modules()
+ if child and (child == entry or child.endswith("." + entry))
+ ]
+ scoped.extend(paths or [entry])
+ return scoped
+
+
+def _text_core_decoder_prefix(model):
+ name = getattr(model, "_unsloth_text_core", None)
+ core = getattr(model, name, None) if isinstance(name, str) else None
+ if core is None:
+ return None
+ try:
+ decoder = core.get_decoder()
+ except Exception:
+ decoder = None
+ if isinstance(decoder, torch.nn.Module) and decoder is not core:
+ for child_name, module in core.named_modules():
+ if module is decoder and child_name:
+ return f"{name}.{child_name}"
+ # transformers 5.4's thinker returns itself from get_decoder: use the embeddings' owner.
+ try:
+ embeddings = core.get_input_embeddings()
+ except Exception:
+ embeddings = None
+ if embeddings is not None:
+ for child_name, child in core.named_children():
+ try:
+ # A vision tower raises here on transformers 5.
+ if child.get_input_embeddings() is embeddings:
+ return f"{name}.{child_name}"
+ except Exception:
+ continue
+ return name
+
+
+def _text_trainable_core(model, text_intent = True):
+ """With text_intent, the text child to train when the wrapper's forward needs non-text
+ inputs (Nemotron-3-Nano-Omni) or has none (Qwen3-Omni); else a hint, and a forward-less
+ wrapper gets its core's forward on the instance."""
+ if os.environ.get("UNSLOTH_KEEP_COMPOSED_WRAPPER", "0") == "1":
return model
- name, core = cores[0]
+ forward = getattr(type(model), "forward", None)
+ has_no_forward = forward is None or forward is torch.nn.Module.forward
+ required = [] if has_no_forward else _required_non_text_inputs(forward)
+ if not has_no_forward and not required:
+ return model
+ if not text_intent:
+ if has_no_forward:
+ found = _find_text_core(model)
+ if found is not None:
+ _delegate_text_forward(model, *found)
+ print(
+ f"Unsloth: `{type(model).__name__}` has no forward of its own, so a text "
+ f"forward runs through its `{found[0]}` and the rest is kept for generation. "
+ "Pass `text_only = True` to from_pretrained to train the "
+ f"`{found[0]}` alone and free the rest."
+ )
+ else:
+ print(
+ f"Unsloth: `{type(model).__name__}` has no forward of its own, so it cannot be "
+ "trained as loaded. Pass `text_only = True` to from_pretrained to train its "
+ "thinker on text batches instead."
+ )
+ else:
+ print(
+ f"Unsloth: `{type(model).__name__}.forward` requires {', '.join(required)}, so it "
+ "trains on multimodal batches only. Pass `text_only = True` to from_pretrained to "
+ "train its language model on text batches instead."
+ )
+ return model
+ found = _find_text_core(model)
+ if found is None:
+ return model
+ name, core = found
dropped = [child_name for child_name, _ in model.named_children() if child_name != name]
for child_name in dropped:
delattr(model, child_name)
gc.collect()
clean_gpu_cache()
+ if has_no_forward:
+ reason = f"`{type(model).__name__}` has no forward of its own"
+ else:
+ reason = (
+ f"`{type(model).__name__}.forward` requires {', '.join(required)}, which a text "
+ "batch does not carry"
+ )
print(
- f"Unsloth: `{type(model).__name__}.forward` requires {', '.join(required)}, which a text "
- f"batch does not carry, so training uses its `{name}` (`{type(core).__name__}`)."
+ f"Unsloth: {reason}, so training uses its `{name}` (`{type(core).__name__}`)."
+ (f" Dropped {', '.join(dropped)}: not used by text training." if dropped else "")
+ " Saving writes a checkpoint of that module."
)
@@ -1458,20 +1620,26 @@ def _carry_loader_state_to_core(model, core, name):
carried[key[len(prefix) :]] = device
if carried:
core.hf_device_map = carried
+ # PEFT writes name_or_path into base_model_name_or_path; sub-config children lack one.
+ wrapper_name = getattr(model, "name_or_path", None) or getattr(
+ getattr(model, "config", None), "_name_or_path", None
+ )
+ if wrapper_name:
+ if not getattr(core, "name_or_path", None):
+ try:
+ core.name_or_path = wrapper_name
+ except Exception:
+ pass
+ _core_cfg = getattr(core, "config", None)
+ if _core_cfg is not None and not getattr(_core_cfg, "_name_or_path", None):
+ try:
+ _core_cfg._name_or_path = wrapper_name
+ except Exception:
+ pass
wrapper_config = getattr(model, "config", None)
core_config = getattr(core, "config", None)
# A sub-config without its own dtype leaves correct_dtype None, so bnb quant_state.dtype becomes None.
wrapper_dtype = getattr(wrapper_config, "dtype", None)
- # A child built in __init__ has no _name_or_path: adapter_config and merged saves need the repo.
- wrapper_path = getattr(wrapper_config, "_name_or_path", "") or getattr(
- model, "name_or_path", ""
- )
- if core_config is not None and wrapper_path and not getattr(core_config, "_name_or_path", ""):
- core_config._name_or_path = wrapper_path
- try:
- core.name_or_path = wrapper_path
- except Exception:
- pass
if (
core_config is not None
and wrapper_dtype is not None
@@ -1481,6 +1649,12 @@ def _carry_loader_state_to_core(model, core, name):
core_config.dtype = wrapper_dtype
except Exception:
pass
+ # A sub-config names no architecture; generate and save read it off the core.
+ if core_config is not None and not getattr(core_config, "architectures", None):
+ try:
+ core_config.architectures = [type(core).__name__]
+ except Exception:
+ pass
# generation_config.json is loaded onto the wrapper only; the child has config defaults (EOS etc).
wrapper_generation_config = getattr(model, "generation_config", None)
if wrapper_generation_config is not None:
@@ -2686,6 +2860,16 @@ class FastBaseModel:
)
else:
_audio_kwargs = {}
+ # No language tag matches `thinker.model`; without this LoRA hits only the vision tower.
+ _core_prefix = _text_core_decoder_prefix(model)
+ if _core_prefix and "language_tags" in inspect.signature(get_peft_regex).parameters:
+ _language_tag_default = (
+ inspect.signature(get_peft_regex).parameters["language_tags"].default
+ )
+ if isinstance(_language_tag_default, (list, tuple)):
+ _audio_kwargs["language_tags"] = list(_language_tag_default) + [
+ re.escape(_core_prefix)
+ ]
# Remember the caller's ORIGINAL explicit leaf list for MoE expert detection: routing it through get_peft_regex adds the full "mlp|feed_forward|ffn|dense" block even for attention-only leaves, so keying on that regex would train the experts. Only the auto path relies on the regex.
_moe_detect_target = target_modules if type(target_modules) in (list, tuple) else None
@@ -2713,6 +2897,7 @@ class FastBaseModel:
f"`modules_to_save`, so they are trained as full weight matrices.\n"
f"This uses more VRAM than LoRA. Please list them in `modules_to_save` directly."
)
+ modules_to_save = _scope_modules_to_save_to_core(model, modules_to_save)
_raise_if_fast_inference_modules_to_save(model, modules_to_save)
# Only a regex generated here may be widened to expert submodules below.
@@ -2736,7 +2921,14 @@ class FastBaseModel:
or not finetune_attention_modules
or not finetune_mlp_modules
)
- if type(target_modules) in (list, tuple) and (_scoping or finetune_audio_layers):
+ # A kept wrapper (Qwen3-Omni) also holds a talker the forward never reaches: scope leaves to the text core.
+ _leaf_list = type(target_modules) in (list, tuple) and all(
+ "." not in str(name) for name in target_modules
+ )
+ # Qualified paths already name their module; get_peft_regex treats every entry as a leaf.
+ if type(target_modules) in (list, tuple) and (
+ _scoping or finetune_audio_layers or (_core_prefix and _leaf_list)
+ ):
if _scoping:
print(
"Unsloth: Explicit target_modules are constrained by the "
@@ -2809,12 +3001,26 @@ class FastBaseModel:
_moe_module_targets = get_moe_target_modules(model, _moe_module_detect)
# Auto-detect MoE models and populate target_parameters for expert layers.
- if target_parameters is None:
+ if target_parameters is not None:
+ target_parameters = _scope_parameter_targets_to_core(model, target_parameters)
+ else:
target_parameters = get_moe_target_parameters(
model,
_moe_module_detect,
moe_module_targets = _moe_module_targets,
)
+ # A forward-less wrapper's config hides the thinker's experts; resolve on the core and
+ # name full paths, since PEFT suffix-matches and would also hit the talker's experts.
+ _core_name = getattr(model, "_unsloth_text_core", None)
+ _core = getattr(model, _core_name, None) if isinstance(_core_name, str) else None
+ if target_parameters is None and _core is not None:
+ _core_parameters = get_moe_target_parameters(_core, _moe_module_detect)
+ if _core_parameters:
+ target_parameters = [
+ f"{_core_name}.{name}"
+ for name, _ in _core.named_parameters()
+ if any(name == t or name.endswith("." + t) for t in _core_parameters)
+ ] or None
if _moe_module_targets:
if isinstance(target_modules, (list, tuple)):
@@ -2837,6 +3043,10 @@ class FastBaseModel:
if finetune_last_n_layers is not None and layers_to_transform is None:
_total_layers = _get_total_transformer_layers(model)
+ _core_name = getattr(model, "_unsloth_text_core", None)
+ if _total_layers is None and isinstance(_core_name, str):
+ # A kept wrapper's config nests the count (Qwen3-Omni: thinker.config.text_config).
+ _total_layers = _get_total_transformer_layers(getattr(model, _core_name, None))
if _total_layers is not None and _total_layers > 0:
n = max(1, min(int(finetune_last_n_layers), _total_layers))
layers_to_transform = list(range(_total_layers - n, _total_layers))
@@ -2991,6 +3201,9 @@ class FastBaseModel:
float32_mixed_precision = float32_mixed_precision,
patch_modules_to_save = True,
)
+ if full_finetuning:
+ # prepare_model_for_training re-enabled every parameter, a kept wrapper's siblings too.
+ _freeze_unused_siblings(model)
# Persist the configured GC mode so the trainer restores it verbatim: for_inference() clears the module flags every GRPO generation step, and TrainingArguments defaults gradient_checkpointing=False, which would silently disable it at train time (#4735).
model._unsloth_gradient_checkpointing = use_gradient_checkpointing